1. Patchew
  2. linux
  3. [PATCH v3 00/14] Introduce AMD Pensando RDMA driver
  4. View patch

[PATCH v3 08/14] RDMA/ionic: Register auxiliary module for ionic ethernet adapter

Abhijit Gangurde posted 14 patches 7 months, 2 weeks ago
There is a newer version of this series
[PATCH v3 08/14] RDMA/ionic: Register auxiliary module for ionic ethernet adapter
Posted by Abhijit Gangurde 7 months, 2 weeks ago 
Register auxiliary module to create ibdevice for ionic
ethernet adapter.

Co-developed-by: Andrew Boyer <andrew.boyer@amd.com>
Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
Co-developed-by: Allen Hubbe <allen.hubbe@amd.com>
Signed-off-by: Allen Hubbe <allen.hubbe@amd.com>
Signed-off-by: Abhijit Gangurde <abhijit.gangurde@amd.com>
---
v1->v2
  - Removed netdev references from ionic RDMA driver
  - Moved to ionic_lif* instead of void* to convey information between
    aux devices and drivers.

 drivers/infiniband/hw/ionic/ionic_ibdev.c   | 133 ++++++++++++++++++++
 drivers/infiniband/hw/ionic/ionic_ibdev.h   |  21 ++++
 drivers/infiniband/hw/ionic/ionic_lif_cfg.c | 118 +++++++++++++++++
 drivers/infiniband/hw/ionic/ionic_lif_cfg.h |  65 ++++++++++
 4 files changed, 337 insertions(+)
 create mode 100644 drivers/infiniband/hw/ionic/ionic_ibdev.c
 create mode 100644 drivers/infiniband/hw/ionic/ionic_ibdev.h
 create mode 100644 drivers/infiniband/hw/ionic/ionic_lif_cfg.c
 create mode 100644 drivers/infiniband/hw/ionic/ionic_lif_cfg.h

diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.c b/drivers/infiniband/hw/ionic/ionic_ibdev.c
new file mode 100644
index 000000000000..6bd0175cd7b1
--- /dev/null
+++ b/drivers/infiniband/hw/ionic/ionic_ibdev.c
@@ -0,0 +1,133 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
+
+#include <linux/module.h>
+#include <linux/printk.h>
+#include <net/addrconf.h>
+
+#include "ionic_ibdev.h"
+
+#define DRIVER_DESCRIPTION "AMD Pensando RoCE HCA driver"
+#define DEVICE_DESCRIPTION "AMD Pensando RoCE HCA"
+
+MODULE_AUTHOR("Allen Hubbe <allen.hubbe@amd.com>");
+MODULE_DESCRIPTION(DRIVER_DESCRIPTION);
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("NET_IONIC");
+
+static void ionic_destroy_ibdev(struct ionic_ibdev *dev)
+{
+	ib_unregister_device(&dev->ibdev);
+	ib_dealloc_device(&dev->ibdev);
+}
+
+static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev)
+{
+	struct ib_device *ibdev;
+	struct ionic_ibdev *dev;
+	int rc;
+
+	rc = ionic_version_check(&ionic_adev->adev.dev, ionic_adev->lif);
+	if (rc)
+		return ERR_PTR(rc);
+
+	dev = ib_alloc_device(ionic_ibdev, ibdev);
+	if (!dev)
+		return ERR_PTR(-EINVAL);
+
+	ionic_fill_lif_cfg(ionic_adev->lif, &dev->lif_cfg);
+
+	ibdev = &dev->ibdev;
+	ibdev->dev.parent = dev->lif_cfg.hwdev;
+
+	strscpy(ibdev->name, "ionic_%d", IB_DEVICE_NAME_MAX);
+	strscpy(ibdev->node_desc, DEVICE_DESCRIPTION, IB_DEVICE_NODE_DESC_MAX);
+
+	ibdev->node_type = RDMA_NODE_IB_CA;
+	ibdev->phys_port_cnt = 1;
+
+	/* the first two eq are reserved for async events */
+	ibdev->num_comp_vectors = dev->lif_cfg.eq_count - 2;
+
+	addrconf_ifid_eui48((u8 *)&ibdev->node_guid,
+			    ionic_lif_netdev(ionic_adev->lif));
+
+	rc = ib_device_set_netdev(ibdev, ionic_lif_netdev(ionic_adev->lif), 1);
+	if (rc)
+		goto err_admin;
+
+	rc = ib_register_device(ibdev, "ionic_%d", ibdev->dev.parent);
+	if (rc)
+		goto err_register;
+
+	return dev;
+
+err_register:
+err_admin:
+	ib_dealloc_device(&dev->ibdev);
+
+	return ERR_PTR(rc);
+}
+
+static int ionic_aux_probe(struct auxiliary_device *adev,
+			   const struct auxiliary_device_id *id)
+{
+	struct ionic_aux_dev *ionic_adev;
+	struct ionic_ibdev *dev;
+
+	ionic_adev = container_of(adev, struct ionic_aux_dev, adev);
+	dev = ionic_create_ibdev(ionic_adev);
+	if (IS_ERR(dev))
+		return dev_err_probe(&adev->dev, PTR_ERR(dev),
+				     "Failed to register ibdev\n");
+
+	auxiliary_set_drvdata(adev, dev);
+	ibdev_dbg(&dev->ibdev, "registered\n");
+
+	return 0;
+}
+
+static void ionic_aux_remove(struct auxiliary_device *adev)
+{
+	struct ionic_ibdev *dev = auxiliary_get_drvdata(adev);
+
+	dev_dbg(&adev->dev, "unregister ibdev\n");
+	ionic_destroy_ibdev(dev);
+	dev_dbg(&adev->dev, "unregistered\n");
+}
+
+static const struct auxiliary_device_id ionic_aux_id_table[] = {
+	{ .name = "ionic.rdma", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(auxiliary, ionic_aux_id_table);
+
+static struct auxiliary_driver ionic_aux_r_driver = {
+	.name = "rdma",
+	.probe = ionic_aux_probe,
+	.remove = ionic_aux_remove,
+	.id_table = ionic_aux_id_table,
+};
+
+static int __init ionic_mod_init(void)
+{
+	int rc;
+
+	rc = auxiliary_driver_register(&ionic_aux_r_driver);
+	if (rc)
+		goto err_aux;
+
+	return 0;
+
+err_aux:
+	return rc;
+}
+
+static void __exit ionic_mod_exit(void)
+{
+	auxiliary_driver_unregister(&ionic_aux_r_driver);
+}
+
+module_init(ionic_mod_init);
+module_exit(ionic_mod_exit);
diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.h b/drivers/infiniband/hw/ionic/ionic_ibdev.h
new file mode 100644
index 000000000000..e13adff390d7
--- /dev/null
+++ b/drivers/infiniband/hw/ionic/ionic_ibdev.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
+
+#ifndef _IONIC_IBDEV_H_
+#define _IONIC_IBDEV_H_
+
+#include <rdma/ib_verbs.h>
+#include <ionic_api.h>
+
+#include "ionic_lif_cfg.h"
+
+#define IONIC_MIN_RDMA_VERSION	0
+#define IONIC_MAX_RDMA_VERSION	2
+
+struct ionic_ibdev {
+	struct ib_device	ibdev;
+
+	struct ionic_lif_cfg	lif_cfg;
+};
+
+#endif /* _IONIC_IBDEV_H_ */
diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c
new file mode 100644
index 000000000000..386af7d351d7
--- /dev/null
+++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c
@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
+
+#include <linux/kernel.h>
+
+#include <ionic.h>
+#include <ionic_lif.h>
+
+#include "ionic_lif_cfg.h"
+
+#define IONIC_MIN_RDMA_VERSION	0
+#define IONIC_MAX_RDMA_VERSION	2
+
+static u8 ionic_get_expdb(struct ionic_lif *lif)
+{
+	u8 expdb_support = 0;
+
+	if (lif->ionic->idev.phy_cmb_expdb64_pages)
+		expdb_support |= IONIC_EXPDB_64B_WQE;
+	if (lif->ionic->idev.phy_cmb_expdb128_pages)
+		expdb_support |= IONIC_EXPDB_128B_WQE;
+	if (lif->ionic->idev.phy_cmb_expdb256_pages)
+		expdb_support |= IONIC_EXPDB_256B_WQE;
+	if (lif->ionic->idev.phy_cmb_expdb512_pages)
+		expdb_support |= IONIC_EXPDB_512B_WQE;
+
+	return expdb_support;
+}
+
+void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg)
+{
+	union ionic_lif_identity *ident = &lif->ionic->ident.lif;
+
+	cfg->lif = lif;
+	cfg->hwdev = &lif->ionic->pdev->dev;
+	cfg->lif_index = lif->index;
+	cfg->lif_hw_index = lif->hw_index;
+
+	cfg->dbid = lif->kern_pid;
+	cfg->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
+	cfg->dbpage = lif->kern_dbpage;
+	cfg->intr_ctrl = lif->ionic->idev.intr_ctrl;
+
+	cfg->db_phys = lif->ionic->bars[IONIC_PCI_BAR_DBELL].bus_addr;
+
+	if (IONIC_VERSION(ident->rdma.version, ident->rdma.minor_version) >=
+	    IONIC_VERSION(2, 1))
+		cfg->page_size_supported =
+		    le64_to_cpu(ident->rdma.page_size_cap);
+	else
+		cfg->page_size_supported = IONIC_PAGE_SIZE_SUPPORTED;
+
+	cfg->rdma_version = ident->rdma.version;
+	cfg->qp_opcodes = ident->rdma.qp_opcodes;
+	cfg->admin_opcodes = ident->rdma.admin_opcodes;
+
+	cfg->stats_type = le16_to_cpu(ident->rdma.stats_type);
+	cfg->npts_per_lif = le32_to_cpu(ident->rdma.npts_per_lif);
+	cfg->nmrs_per_lif = le32_to_cpu(ident->rdma.nmrs_per_lif);
+	cfg->nahs_per_lif = le32_to_cpu(ident->rdma.nahs_per_lif);
+
+	cfg->aq_base = le32_to_cpu(ident->rdma.aq_qtype.qid_base);
+	cfg->cq_base = le32_to_cpu(ident->rdma.cq_qtype.qid_base);
+	cfg->eq_base = le32_to_cpu(ident->rdma.eq_qtype.qid_base);
+
+	/*
+	 * ionic_create_rdma_admin() may reduce aq_count or eq_count if
+	 * it is unable to allocate all that were requested.
+	 * aq_count is tunable; see ionic_aq_count
+	 * eq_count is tunable; see ionic_eq_count
+	 */
+	cfg->aq_count = le32_to_cpu(ident->rdma.aq_qtype.qid_count);
+	cfg->eq_count = le32_to_cpu(ident->rdma.eq_qtype.qid_count);
+	cfg->cq_count = le32_to_cpu(ident->rdma.cq_qtype.qid_count);
+	cfg->qp_count = le32_to_cpu(ident->rdma.sq_qtype.qid_count);
+	cfg->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
+
+	cfg->aq_qtype = ident->rdma.aq_qtype.qtype;
+	cfg->sq_qtype = ident->rdma.sq_qtype.qtype;
+	cfg->rq_qtype = ident->rdma.rq_qtype.qtype;
+	cfg->cq_qtype = ident->rdma.cq_qtype.qtype;
+	cfg->eq_qtype = ident->rdma.eq_qtype.qtype;
+	cfg->udma_qgrp_shift = ident->rdma.udma_shift;
+	cfg->udma_count = 2;
+
+	cfg->max_stride = ident->rdma.max_stride;
+	cfg->expdb_mask = ionic_get_expdb(lif);
+
+	cfg->sq_expdb =
+	    !!(lif->qtype_info[IONIC_QTYPE_TXQ].features & IONIC_QIDENT_F_EXPDB);
+	cfg->rq_expdb =
+	    !!(lif->qtype_info[IONIC_QTYPE_RXQ].features & IONIC_QIDENT_F_EXPDB);
+}
+
+struct net_device *ionic_lif_netdev(struct ionic_lif *lif)
+{
+	return lif->netdev;
+}
+
+int ionic_version_check(const struct device *dev, struct ionic_lif *lif)
+{
+	union ionic_lif_identity *ident = &lif->ionic->ident.lif;
+
+	if (ident->rdma.version < IONIC_MIN_RDMA_VERSION ||
+	    ident->rdma.version > IONIC_MAX_RDMA_VERSION) {
+		dev_err_probe(dev, -EINVAL,
+			      "ionic_rdma: incompatible version, fw ver %u\n",
+			      ident->rdma.version);
+		dev_err_probe(dev, -EINVAL,
+			      "ionic_rdma: Driver Min Version %u\n",
+			      IONIC_MIN_RDMA_VERSION);
+		dev_err_probe(dev, -EINVAL,
+			      "ionic_rdma: Driver Max Version %u\n",
+			      IONIC_MAX_RDMA_VERSION);
+	}
+
+	return 0;
+}
diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.h b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h
new file mode 100644
index 000000000000..b095637c54cf
--- /dev/null
+++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h
@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
+
+#ifndef _IONIC_LIF_CFG_H_
+
+#define IONIC_VERSION(a, b) (((a) << 16) + ((b) << 8))
+#define IONIC_PAGE_SIZE_SUPPORTED	0x40201000 /* 4kb, 2Mb, 1Gb */
+
+#define IONIC_EXPDB_64B_WQE	BIT(0)
+#define IONIC_EXPDB_128B_WQE	BIT(1)
+#define IONIC_EXPDB_256B_WQE	BIT(2)
+#define IONIC_EXPDB_512B_WQE	BIT(3)
+
+struct ionic_lif_cfg {
+	struct device *hwdev;
+	struct ionic_lif *lif;
+
+	int lif_index;
+	int lif_hw_index;
+
+	u32 dbid;
+	int dbid_count;
+	u64 __iomem *dbpage;
+	struct ionic_intr __iomem *intr_ctrl;
+	phys_addr_t db_phys;
+
+	u64 page_size_supported;
+	u32 npts_per_lif;
+	u32 nmrs_per_lif;
+	u32 nahs_per_lif;
+
+	u32 aq_base;
+	u32 cq_base;
+	u32 eq_base;
+
+	int aq_count;
+	int eq_count;
+	int cq_count;
+	int qp_count;
+
+	u16 stats_type;
+	u8 aq_qtype;
+	u8 sq_qtype;
+	u8 rq_qtype;
+	u8 cq_qtype;
+	u8 eq_qtype;
+
+	u8 udma_count;
+	u8 udma_qgrp_shift;
+
+	u8 rdma_version;
+	u8 qp_opcodes;
+	u8 admin_opcodes;
+
+	u8 max_stride;
+	bool sq_expdb;
+	bool rq_expdb;
+	u8 expdb_mask;
+};
+
+int ionic_version_check(const struct device *dev, struct ionic_lif *lif);
+void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg);
+struct net_device *ionic_lif_netdev(struct ionic_lif *lif);
+
+#endif /* _IONIC_LIF_CFG_H_ */
-- 
2.43.0
Re: [PATCH v3 08/14] RDMA/ionic: Register auxiliary module for ionic ethernet adapter
Posted by Leon Romanovsky 7 months, 2 weeks ago 
On Tue, Jun 24, 2025 at 05:43:09PM +0530, Abhijit Gangurde wrote:
> Register auxiliary module to create ibdevice for ionic
> ethernet adapter.
> 
> Co-developed-by: Andrew Boyer <andrew.boyer@amd.com>
> Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
> Co-developed-by: Allen Hubbe <allen.hubbe@amd.com>
> Signed-off-by: Allen Hubbe <allen.hubbe@amd.com>
> Signed-off-by: Abhijit Gangurde <abhijit.gangurde@amd.com>
> ---
> v1->v2
>   - Removed netdev references from ionic RDMA driver
>   - Moved to ionic_lif* instead of void* to convey information between
>     aux devices and drivers.
> 
>  drivers/infiniband/hw/ionic/ionic_ibdev.c   | 133 ++++++++++++++++++++
>  drivers/infiniband/hw/ionic/ionic_ibdev.h   |  21 ++++
>  drivers/infiniband/hw/ionic/ionic_lif_cfg.c | 118 +++++++++++++++++
>  drivers/infiniband/hw/ionic/ionic_lif_cfg.h |  65 ++++++++++
>  4 files changed, 337 insertions(+)
>  create mode 100644 drivers/infiniband/hw/ionic/ionic_ibdev.c
>  create mode 100644 drivers/infiniband/hw/ionic/ionic_ibdev.h
>  create mode 100644 drivers/infiniband/hw/ionic/ionic_lif_cfg.c
>  create mode 100644 drivers/infiniband/hw/ionic/ionic_lif_cfg.h

<...>

> +	rc = ionic_version_check(&ionic_adev->adev.dev, ionic_adev->lif);
> +	if (rc)
> +		return ERR_PTR(rc);

<...>

> +struct net_device *ionic_lif_netdev(struct ionic_lif *lif)
> +{
> +	return lif->netdev;
> +}

Why do you need to store netdev pointer?
Why can't you use existing ib_device_get_netdev/ib_device_set_netdev?

> +
> +int ionic_version_check(const struct device *dev, struct ionic_lif *lif)
> +{
> +	union ionic_lif_identity *ident = &lif->ionic->ident.lif;
> +
> +	if (ident->rdma.version < IONIC_MIN_RDMA_VERSION ||
> +	    ident->rdma.version > IONIC_MAX_RDMA_VERSION) {
> +		dev_err_probe(dev, -EINVAL,
> +			      "ionic_rdma: incompatible version, fw ver %u\n",
> +			      ident->rdma.version);
> +		dev_err_probe(dev, -EINVAL,
> +			      "ionic_rdma: Driver Min Version %u\n",
> +			      IONIC_MIN_RDMA_VERSION);
> +		dev_err_probe(dev, -EINVAL,
> +			      "ionic_rdma: Driver Max Version %u\n",
> +			      IONIC_MAX_RDMA_VERSION);
> +	}
> +
> +	return 0;
> +}

Upstream code has all subsystems in sync, and RDMA driver is always
compatible with its netdev counterpart. Please remove this part.

This is not full review yet, please wait till next week, so we will
review more deeply.

Thanks
Re: [PATCH v3 08/14] RDMA/ionic: Register auxiliary module for ionic ethernet adapter
Posted by Abhijit Gangurde 7 months, 2 weeks ago 
On 6/26/25 12:56, Leon Romanovsky wrote:
> On Tue, Jun 24, 2025 at 05:43:09PM +0530, Abhijit Gangurde wrote:
>> Register auxiliary module to create ibdevice for ionic
>> ethernet adapter.
>>
>> Co-developed-by: Andrew Boyer <andrew.boyer@amd.com>
>> Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
>> Co-developed-by: Allen Hubbe <allen.hubbe@amd.com>
>> Signed-off-by: Allen Hubbe <allen.hubbe@amd.com>
>> Signed-off-by: Abhijit Gangurde <abhijit.gangurde@amd.com>
>> ---
>> v1->v2
>>    - Removed netdev references from ionic RDMA driver
>>    - Moved to ionic_lif* instead of void* to convey information between
>>      aux devices and drivers.
>>
>>   drivers/infiniband/hw/ionic/ionic_ibdev.c   | 133 ++++++++++++++++++++
>>   drivers/infiniband/hw/ionic/ionic_ibdev.h   |  21 ++++
>>   drivers/infiniband/hw/ionic/ionic_lif_cfg.c | 118 +++++++++++++++++
>>   drivers/infiniband/hw/ionic/ionic_lif_cfg.h |  65 ++++++++++
>>   4 files changed, 337 insertions(+)
>>   create mode 100644 drivers/infiniband/hw/ionic/ionic_ibdev.c
>>   create mode 100644 drivers/infiniband/hw/ionic/ionic_ibdev.h
>>   create mode 100644 drivers/infiniband/hw/ionic/ionic_lif_cfg.c
>>   create mode 100644 drivers/infiniband/hw/ionic/ionic_lif_cfg.h
> <...>
>
>> +	rc = ionic_version_check(&ionic_adev->adev.dev, ionic_adev->lif);
>> +	if (rc)
>> +		return ERR_PTR(rc);
> <...>
>
>> +struct net_device *ionic_lif_netdev(struct ionic_lif *lif)
>> +{
>> +	return lif->netdev;
>> +}
> Why do you need to store netdev pointer?
> Why can't you use existing ib_device_get_netdev/ib_device_set_netdev?

We are not storing the netdev in RDMA driver. This function is accessing
the ethernet copy to set netdev and guid. I can add dev_hold() here
till netdev is registered with ib device.

>
>> +
>> +int ionic_version_check(const struct device *dev, struct ionic_lif *lif)
>> +{
>> +	union ionic_lif_identity *ident = &lif->ionic->ident.lif;
>> +
>> +	if (ident->rdma.version < IONIC_MIN_RDMA_VERSION ||
>> +	    ident->rdma.version > IONIC_MAX_RDMA_VERSION) {
>> +		dev_err_probe(dev, -EINVAL,
>> +			      "ionic_rdma: incompatible version, fw ver %u\n",
>> +			      ident->rdma.version);
>> +		dev_err_probe(dev, -EINVAL,
>> +			      "ionic_rdma: Driver Min Version %u\n",
>> +			      IONIC_MIN_RDMA_VERSION);
>> +		dev_err_probe(dev, -EINVAL,
>> +			      "ionic_rdma: Driver Max Version %u\n",
>> +			      IONIC_MAX_RDMA_VERSION);
>> +	}
>> +
>> +	return 0;
>> +}
> Upstream code has all subsystems in sync, and RDMA driver is always
> compatible with its netdev counterpart. Please remove this part.
>
> This is not full review yet, please wait till next week, so we will
> review more deeply.
>
> Thanks
I will remove this.

Thanks,
Abhijit

Patchew 2.1-devel-b67e4c2

© 2016 - 2026 Red Hat, Inc.