From nobody Sat Apr 4 03:19:57 2026 Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 68A8D3A7F57; Fri, 20 Mar 2026 23:54:34 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=13.77.154.182 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1774050875; cv=none; b=l2ks2G3BdndNxGXC5+oRS7+Xa2CUbSmolajBQMIKgaYkM0lGyldqVgzixPyC9VcX8wSfw015ocyT0JD8sS3hQM2HJ6hmrlgUCc6CQz/XzIS3oOnQ4Rg1uRYt8Hi+gHeLSUBxcjHOtUAZALGnVVLFns25tQWdBTfOiXCWhkYKk4A= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1774050875; c=relaxed/simple; bh=bx4TxW0kLmD+UsJZ3TF9KQwztd8ZY1iD8K9uIiWbb0g=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=k9bdWG+QPWc1v42azhc04xPnYeow1mkQmMb1PfFQCQnKU9UQPMNdP8t1PKjuXdlzmYWMQc95GtlLGAHBZasL39Tbd3CSHZ74nXZ6/9dY/Wb+aKybfeHw3n/yz1VvboZz+DM2i9sZk4mMrZD5l8fuBsfeswl45XTl6e+L/oF6tMQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=reject dis=none) header.from=microsoft.com; spf=pass smtp.mailfrom=linux.microsoft.com; arc=none smtp.client-ip=13.77.154.182 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=reject dis=none) header.from=microsoft.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.microsoft.com Received: by linux.microsoft.com (Postfix, from userid 1202) id 5928320B6F01; Fri, 20 Mar 2026 16:54:34 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com 5928320B6F01 From: Long Li To: Long Li , Konstantin Taranov , Jakub Kicinski , "David S . Miller" , Paolo Abeni , Eric Dumazet , Andrew Lunn , Jason Gunthorpe , Leon Romanovsky , Haiyang Zhang , "K . Y . Srinivasan" , Wei Liu , Dexuan Cui Cc: Simon Horman , netdev@vger.kernel.org, linux-rdma@vger.kernel.org, linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org Subject: [PATCH net-next v4 1/6] net: mana: Create separate EQs for each vPort Date: Fri, 20 Mar 2026 16:54:14 -0700 Message-ID: X-Mailer: git-send-email 2.43.7 In-Reply-To: References: Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" To prepare for assigning vPorts to dedicated MSI-X vectors, remove EQ sharing among the vPorts and create dedicated EQs for each vPort. Move the EQ definition from struct mana_context to struct mana_port_context and update related support functions. Export mana_create_eq() and mana_destroy_eq() for use by the MANA RDMA driver. Signed-off-by: Long Li --- Changes in v3: - Added NULL check for mpc->eqs in mana_ib_create_qp_rss() to prevent kernel crash when RSS QP is created before EQs are allocated --- drivers/infiniband/hw/mana/main.c | 14 ++- drivers/infiniband/hw/mana/qp.c | 16 ++- drivers/net/ethernet/microsoft/mana/mana_en.c | 109 ++++++++++-------- include/net/mana/mana.h | 7 +- 4 files changed, 94 insertions(+), 52 deletions(-) diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana= /main.c index 8d99cd00f002..d51dd0ee85f4 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -20,8 +20,10 @@ void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct= mana_ib_pd *pd, pd->vport_use_count--; WARN_ON(pd->vport_use_count < 0); =20 - if (!pd->vport_use_count) + if (!pd->vport_use_count) { + mana_destroy_eq(mpc); mana_uncfg_vport(mpc); + } =20 mutex_unlock(&pd->vport_mutex); } @@ -55,15 +57,21 @@ int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port= , struct mana_ib_pd *pd, return err; } =20 - mutex_unlock(&pd->vport_mutex); =20 pd->tx_shortform_allowed =3D mpc->tx_shortform_allowed; pd->tx_vp_offset =3D mpc->tx_vp_offset; + err =3D mana_create_eq(mpc); + if (err) { + mana_uncfg_vport(mpc); + pd->vport_use_count--; + } + + mutex_unlock(&pd->vport_mutex); =20 ibdev_dbg(&dev->ib_dev, "vport handle %llx pdid %x doorbell_id %x\n", mpc->port_handle, pd->pdn, doorbell_id); =20 - return 0; + return err; } =20 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/q= p.c index 82f84f7ad37a..80cf4ade4b75 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -188,7 +188,15 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, s= truct ib_pd *pd, cq_spec.gdma_region =3D cq->queue.gdma_region; cq_spec.queue_size =3D cq->cqe * COMP_ENTRY_SIZE; cq_spec.modr_ctx_id =3D 0; - eq =3D &mpc->ac->eqs[cq->comp_vector]; + /* EQs are created when a raw QP configures the vport. + * A raw QP must be created before creating rwq_ind_tbl. + */ + if (!mpc->eqs) { + ret =3D -EINVAL; + i--; + goto fail; + } + eq =3D &mpc->eqs[cq->comp_vector % mpc->num_queues]; cq_spec.attached_eq =3D eq->eq->id; =20 ret =3D mana_create_wq_obj(mpc, mpc->port_handle, GDMA_RQ, @@ -340,7 +348,11 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, s= truct ib_pd *ibpd, cq_spec.queue_size =3D send_cq->cqe * COMP_ENTRY_SIZE; cq_spec.modr_ctx_id =3D 0; eq_vec =3D send_cq->comp_vector; - eq =3D &mpc->ac->eqs[eq_vec]; + if (!mpc->eqs) { + err =3D -EINVAL; + goto err_destroy_queue; + } + eq =3D &mpc->eqs[eq_vec % mpc->num_queues]; cq_spec.attached_eq =3D eq->eq->id; =20 err =3D mana_create_wq_obj(mpc, mpc->port_handle, GDMA_SQ, &wq_spec, diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/et= hernet/microsoft/mana/mana_en.c index 7cae8a7b9f31..32f924d2a99b 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -1596,78 +1596,82 @@ void mana_destroy_wq_obj(struct mana_port_context *= apc, u32 wq_type, } EXPORT_SYMBOL_NS(mana_destroy_wq_obj, "NET_MANA"); =20 -static void mana_destroy_eq(struct mana_context *ac) +void mana_destroy_eq(struct mana_port_context *apc) { + struct mana_context *ac =3D apc->ac; struct gdma_context *gc =3D ac->gdma_dev->gdma_context; struct gdma_queue *eq; int i; =20 - if (!ac->eqs) + if (!apc->eqs) return; =20 - debugfs_remove_recursive(ac->mana_eqs_debugfs); - ac->mana_eqs_debugfs =3D NULL; + debugfs_remove_recursive(apc->mana_eqs_debugfs); + apc->mana_eqs_debugfs =3D NULL; =20 - for (i =3D 0; i < gc->max_num_queues; i++) { - eq =3D ac->eqs[i].eq; + for (i =3D 0; i < apc->num_queues; i++) { + eq =3D apc->eqs[i].eq; if (!eq) continue; =20 mana_gd_destroy_queue(gc, eq); } =20 - kfree(ac->eqs); - ac->eqs =3D NULL; + kfree(apc->eqs); + apc->eqs =3D NULL; } +EXPORT_SYMBOL_NS(mana_destroy_eq, "NET_MANA"); =20 -static void mana_create_eq_debugfs(struct mana_context *ac, int i) +static void mana_create_eq_debugfs(struct mana_port_context *apc, int i) { - struct mana_eq eq =3D ac->eqs[i]; + struct mana_eq eq =3D apc->eqs[i]; char eqnum[32]; =20 sprintf(eqnum, "eq%d", i); - eq.mana_eq_debugfs =3D debugfs_create_dir(eqnum, ac->mana_eqs_debugfs); + eq.mana_eq_debugfs =3D debugfs_create_dir(eqnum, apc->mana_eqs_debugfs); debugfs_create_u32("head", 0400, eq.mana_eq_debugfs, &eq.eq->head); debugfs_create_u32("tail", 0400, eq.mana_eq_debugfs, &eq.eq->tail); debugfs_create_file("eq_dump", 0400, eq.mana_eq_debugfs, eq.eq, &mana_dbg= _q_fops); } =20 -static int mana_create_eq(struct mana_context *ac) +int mana_create_eq(struct mana_port_context *apc) { - struct gdma_dev *gd =3D ac->gdma_dev; + struct gdma_dev *gd =3D apc->ac->gdma_dev; struct gdma_context *gc =3D gd->gdma_context; struct gdma_queue_spec spec =3D {}; int err; int i; =20 - ac->eqs =3D kzalloc_objs(struct mana_eq, gc->max_num_queues); - if (!ac->eqs) + WARN_ON(apc->eqs); + apc->eqs =3D kzalloc_objs(struct mana_eq, apc->num_queues); + if (!apc->eqs) return -ENOMEM; =20 spec.type =3D GDMA_EQ; spec.monitor_avl_buf =3D false; spec.queue_size =3D EQ_SIZE; spec.eq.callback =3D NULL; - spec.eq.context =3D ac->eqs; + spec.eq.context =3D apc->eqs; spec.eq.log2_throttle_limit =3D LOG2_EQ_THROTTLE; =20 - ac->mana_eqs_debugfs =3D debugfs_create_dir("EQs", gc->mana_pci_debugfs); + apc->mana_eqs_debugfs =3D debugfs_create_dir("EQs", apc->mana_port_debugf= s); =20 - for (i =3D 0; i < gc->max_num_queues; i++) { + for (i =3D 0; i < apc->num_queues; i++) { spec.eq.msix_index =3D (i + 1) % gc->num_msix_usable; - err =3D mana_gd_create_mana_eq(gd, &spec, &ac->eqs[i].eq); + err =3D mana_gd_create_mana_eq(gd, &spec, &apc->eqs[i].eq); if (err) { dev_err(gc->dev, "Failed to create EQ %d : %d\n", i, err); goto out; } - mana_create_eq_debugfs(ac, i); + mana_create_eq_debugfs(apc, i); } =20 return 0; out: - mana_destroy_eq(ac); + mana_destroy_eq(apc); return err; } +EXPORT_SYMBOL_NS(mana_create_eq, "NET_MANA"); =20 static int mana_fence_rq(struct mana_port_context *apc, struct mana_rxq *r= xq) { @@ -2421,7 +2425,7 @@ static int mana_create_txq(struct mana_port_context *= apc, spec.monitor_avl_buf =3D false; spec.queue_size =3D cq_size; spec.cq.callback =3D mana_schedule_napi; - spec.cq.parent_eq =3D ac->eqs[i].eq; + spec.cq.parent_eq =3D apc->eqs[i].eq; spec.cq.context =3D cq; err =3D mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq); if (err) @@ -2814,13 +2818,12 @@ static void mana_create_rxq_debugfs(struct mana_por= t_context *apc, int idx) static int mana_add_rx_queues(struct mana_port_context *apc, struct net_device *ndev) { - struct mana_context *ac =3D apc->ac; struct mana_rxq *rxq; int err =3D 0; int i; =20 for (i =3D 0; i < apc->num_queues; i++) { - rxq =3D mana_create_rxq(apc, i, &ac->eqs[i], ndev); + rxq =3D mana_create_rxq(apc, i, &apc->eqs[i], ndev); if (!rxq) { err =3D -ENOMEM; netdev_err(ndev, "Failed to create rxq %d : %d\n", i, err); @@ -2839,9 +2842,8 @@ static int mana_add_rx_queues(struct mana_port_contex= t *apc, return err; } =20 -static void mana_destroy_vport(struct mana_port_context *apc) +static void mana_destroy_rxqs(struct mana_port_context *apc) { - struct gdma_dev *gd =3D apc->ac->gdma_dev; struct mana_rxq *rxq; u32 rxq_idx; =20 @@ -2853,8 +2855,12 @@ static void mana_destroy_vport(struct mana_port_cont= ext *apc) mana_destroy_rxq(apc, rxq, true); apc->rxqs[rxq_idx] =3D NULL; } +} + +static void mana_destroy_vport(struct mana_port_context *apc) +{ + struct gdma_dev *gd =3D apc->ac->gdma_dev; =20 - mana_destroy_txq(apc); mana_uncfg_vport(apc); =20 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) @@ -2875,11 +2881,7 @@ static int mana_create_vport(struct mana_port_contex= t *apc, return err; } =20 - err =3D mana_cfg_vport(apc, gd->pdid, gd->doorbell); - if (err) - return err; - - return mana_create_txq(apc, net); + return mana_cfg_vport(apc, gd->pdid, gd->doorbell); } =20 static int mana_rss_table_alloc(struct mana_port_context *apc) @@ -3156,21 +3158,36 @@ int mana_alloc_queues(struct net_device *ndev) =20 err =3D mana_create_vport(apc, ndev); if (err) { - netdev_err(ndev, "Failed to create vPort %u : %d\n", apc->port_idx, err); + netdev_err(ndev, "Failed to create vPort %u : %d\n", + apc->port_idx, err); return err; } =20 + err =3D mana_create_eq(apc); + if (err) { + netdev_err(ndev, "Failed to create EQ on vPort %u: %d\n", + apc->port_idx, err); + goto destroy_vport; + } + + err =3D mana_create_txq(apc, ndev); + if (err) { + netdev_err(ndev, "Failed to create TXQ on vPort %u: %d\n", + apc->port_idx, err); + goto destroy_eq; + } + err =3D netif_set_real_num_tx_queues(ndev, apc->num_queues); if (err) { netdev_err(ndev, "netif_set_real_num_tx_queues () failed for ndev with num_queues %u = : %d\n", apc->num_queues, err); - goto destroy_vport; + goto destroy_txq; } =20 err =3D mana_add_rx_queues(apc, ndev); if (err) - goto destroy_vport; + goto destroy_rxq; =20 apc->rss_state =3D apc->num_queues > 1 ? TRI_STATE_TRUE : TRI_STATE_FALSE; =20 @@ -3179,7 +3196,7 @@ int mana_alloc_queues(struct net_device *ndev) netdev_err(ndev, "netif_set_real_num_rx_queues () failed for ndev with num_queues %u = : %d\n", apc->num_queues, err); - goto destroy_vport; + goto destroy_rxq; } =20 mana_rss_table_init(apc); @@ -3187,19 +3204,25 @@ int mana_alloc_queues(struct net_device *ndev) err =3D mana_config_rss(apc, TRI_STATE_TRUE, true, true); if (err) { netdev_err(ndev, "Failed to configure RSS table: %d\n", err); - goto destroy_vport; + goto destroy_rxq; } =20 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) { err =3D mana_pf_register_filter(apc); if (err) - goto destroy_vport; + goto destroy_rxq; } =20 mana_chn_setxdp(apc, mana_xdp_get(apc)); =20 return 0; =20 +destroy_rxq: + mana_destroy_rxqs(apc); +destroy_txq: + mana_destroy_txq(apc); +destroy_eq: + mana_destroy_eq(apc); destroy_vport: mana_destroy_vport(apc); return err; @@ -3302,6 +3325,9 @@ static int mana_dealloc_queues(struct net_device *nde= v) netdev_err(ndev, "Failed to disable vPort: %d\n", err); =20 /* Even in err case, still need to cleanup the vPort */ + mana_destroy_rxqs(apc); + mana_destroy_txq(apc); + mana_destroy_eq(apc); mana_destroy_vport(apc); =20 return 0; @@ -3617,12 +3643,6 @@ int mana_probe(struct gdma_dev *gd, bool resuming) gd->driver_data =3D ac; } =20 - err =3D mana_create_eq(ac); - if (err) { - dev_err(dev, "Failed to create EQs: %d\n", err); - goto out; - } - err =3D mana_query_device_cfg(ac, MANA_MAJOR_VERSION, MANA_MINOR_VERSION, MANA_MICRO_VERSION, &num_ports, &bm_hostmode); if (err) @@ -3761,7 +3781,6 @@ void mana_remove(struct gdma_dev *gd, bool suspending) free_netdev(ndev); } =20 - mana_destroy_eq(ac); out: if (ac->per_port_queue_reset_wq) { destroy_workqueue(ac->per_port_queue_reset_wq); diff --git a/include/net/mana/mana.h b/include/net/mana/mana.h index 96d21cbbdee2..204c2b612a62 100644 --- a/include/net/mana/mana.h +++ b/include/net/mana/mana.h @@ -480,8 +480,6 @@ struct mana_context { u8 bm_hostmode; =20 struct mana_ethtool_hc_stats hc_stats; - struct mana_eq *eqs; - struct dentry *mana_eqs_debugfs; struct workqueue_struct *per_port_queue_reset_wq; /* Workqueue for querying hardware stats */ struct delayed_work gf_stats_work; @@ -501,6 +499,9 @@ struct mana_port_context { =20 u8 mac_addr[ETH_ALEN]; =20 + struct mana_eq *eqs; + struct dentry *mana_eqs_debugfs; + enum TRI_STATE rss_state; =20 mana_handle_t default_rxobj; @@ -1033,6 +1034,8 @@ void mana_destroy_wq_obj(struct mana_port_context *ap= c, u32 wq_type, int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id, u32 doorbell_pg_id); void mana_uncfg_vport(struct mana_port_context *apc); +int mana_create_eq(struct mana_port_context *apc); +void mana_destroy_eq(struct mana_port_context *apc); =20 struct net_device *mana_get_primary_netdev(struct mana_context *ac, u32 port_index, --=20 2.43.0