From nobody Wed Dec 17 08:51:10 2025 Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 7878428E5E4; Fri, 9 May 2025 10:13:10 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=13.77.154.182 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746785592; cv=none; b=Gfs1CCVsUEPUZirbSD76vpDsNnts95JSxWnx04SMWI8TS4J5WLcQRNJaOxqxX70XJnUTo+JjxvDMA7MfQzSP0odE0b88vER9tHPA+Zs7j7DF4235Fg5QqFx28yCs58Q2QDNyzszxh6NGO9KiQPrDfRHs4P3/jgpoG8EbUxx7zDc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746785592; c=relaxed/simple; bh=q15WeObahKJHHfymajlCTXtEDne8pMoDfBNA+7AhkEU=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References; b=pIISrTEpAQMNFCcwNUyauXvXxcU/vPuLZsJwkzY1hTxdtwMl/PaqhcH+RAwaaT975VU7UrpJiaHtOfxjR+qERSqkXWn4EFj3oe8EBRg9CXpF4aw/2Hb+iGbGq7QCVWs9C5O7rG+Yeead+p0pJJTdKyuqxAMorG72S/I+gnnJVhQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.microsoft.com; spf=pass smtp.mailfrom=linux.microsoft.com; dkim=pass (1024-bit key) header.d=linux.microsoft.com header.i=@linux.microsoft.com header.b=TsGDuRn5; arc=none smtp.client-ip=13.77.154.182 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.microsoft.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.microsoft.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.microsoft.com header.i=@linux.microsoft.com header.b="TsGDuRn5" Received: by linux.microsoft.com (Postfix, from userid 1134) id 13A102098462; Fri, 9 May 2025 03:13:10 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com 13A102098462 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.microsoft.com; s=default; t=1746785590; bh=ck7OlH3iLM2zajPne9r3rXRVKyZNS5D+OnC1eJ41hxs=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=TsGDuRn5Hb7+OUimyftXQz+zWa71E06UcCg8Tl/Pk/wg+b3AJkHuYAvt/TGQ5NHtO 0Q+kTBpYBorhzOOIE3aS1xbrC3RwFq9YsXtfF1MZ6F6N6VAo5Hsf8nzHH1Ni82X4Fv SMbdTMcTDexIZq8eRLzDC4NC/0OlP07nnljp4uIE= From: Shradha Gupta To: Jason Gunthorpe , Jonathan Cameron , Anna-Maria Behnsen , Thomas Gleixner , Bjorn Helgaas Cc: Shradha Gupta , linux-hyperv@vger.kernel.org, linux-pci@vger.kernel.org, linux-kernel@vger.kernel.org, Nipun Gupta , Yury Norov , Kevin Tian , Long Li , Rob Herring , Manivannan Sadhasivam , =?UTF-8?q?Krzysztof=20Wilczy=EF=BF=BD=7EDski?= , Lorenzo Pieralisi , Dexuan Cui , Wei Liu , Haiyang Zhang , "K. Y. Srinivasan" , Andrew Lunn , "David S. Miller" , Eric Dumazet , Jakub Kicinski , Paolo Abeni , Konstantin Taranov , Simon Horman , Leon Romanovsky , Maxim Levitsky , Erni Sri Satya Vennela , Peter Zijlstra , netdev@vger.kernel.org, linux-rdma@vger.kernel.org, Paul Rosswurm , Shradha Gupta Subject: [PATCH v3 1/4] PCI/MSI: Export pci_msix_prepare_desc() for dynamic MSI-X allocations Date: Fri, 9 May 2025 03:13:07 -0700 Message-Id: <1746785587-4494-1-git-send-email-shradhagupta@linux.microsoft.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1746785566-4337-1-git-send-email-shradhagupta@linux.microsoft.com> References: <1746785566-4337-1-git-send-email-shradhagupta@linux.microsoft.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" For supporting dynamic MSI-X vector allocation by PCI controllers, enabling the flag MSI_FLAG_PCI_MSIX_ALLOC_DYN is not enough, msix_prepare_msi_desc() to prepare the MSI descriptor is also needed. Export pci_msix_prepare_desc() to allow PCI controllers to support dynamic MSI-X vector allocation. Signed-off-by: Shradha Gupta Reviewed-by: Haiyang Zhang Reviewed-by: Thomas Gleixner --- Changes in v3 * Improved the patch description by removing abbreviations=20 --- drivers/pci/msi/irqdomain.c | 5 +++-- include/linux/msi.h | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/pci/msi/irqdomain.c b/drivers/pci/msi/irqdomain.c index d7ba8795d60f..43129aa6d6c7 100644 --- a/drivers/pci/msi/irqdomain.c +++ b/drivers/pci/msi/irqdomain.c @@ -222,13 +222,14 @@ static void pci_irq_unmask_msix(struct irq_data *data) pci_msix_unmask(irq_data_get_msi_desc(data)); } =20 -static void pci_msix_prepare_desc(struct irq_domain *domain, msi_alloc_inf= o_t *arg, - struct msi_desc *desc) +void pci_msix_prepare_desc(struct irq_domain *domain, msi_alloc_info_t *ar= g, + struct msi_desc *desc) { /* Don't fiddle with preallocated MSI descriptors */ if (!desc->pci.mask_base) msix_prepare_msi_desc(to_pci_dev(desc->dev), desc); } +EXPORT_SYMBOL_GPL(pci_msix_prepare_desc); =20 static const struct msi_domain_template pci_msix_template =3D { .chip =3D { diff --git a/include/linux/msi.h b/include/linux/msi.h index 86e42742fd0f..d5864d5e75c2 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -691,6 +691,8 @@ struct irq_domain *pci_msi_create_irq_domain(struct fwn= ode_handle *fwnode, struct irq_domain *parent); u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *= pdev); struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev); +void pci_msix_prepare_desc(struct irq_domain *domain, msi_alloc_info_t *ar= g, + struct msi_desc *desc); #else /* CONFIG_PCI_MSI */ static inline struct irq_domain *pci_msi_get_device_domain(struct pci_dev = *pdev) { --=20 2.34.1 From nobody Wed Dec 17 08:51:10 2025 Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 1ABCA28EA44; Fri, 9 May 2025 10:13:24 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=13.77.154.182 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746785606; cv=none; b=D/jLvQfKDn++qupEdBweWgY6IB+Tg7eS9N+SnALJdMz5Y3+M0WBxq3Xv0mZE0qoWDppJ1etqSdcQeddwSLSKr4uq+qOWVLFHTb3zr27F9KEay7dkLVOMrRMspJXvEqz3njx+GdinIAYFrfuzkA9ncOF87hG+ZOFVlSGR30xuo/c= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746785606; c=relaxed/simple; bh=eheSknn2THWCqMhu/QSVH4frT7M4Uuq3n5+Qbd+NYcE=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References; b=TPSwD/x1Wue15Ft1l6NGd7NT1DOhUkCkuDc2kbJEBUDMYG4Tozynsxyx1uOL6Z/WxnBN5KAPjxqwbz0vFVbWhQs8IDnAHkstxAdSVRXmvYRN6hfxgFbJM8t7WVBSa2HVl1Iyz7vkAMP1yDvX/P100kA4pVX2rMwxef4fLjX4Hd4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.microsoft.com; spf=pass smtp.mailfrom=linux.microsoft.com; dkim=pass (1024-bit key) header.d=linux.microsoft.com header.i=@linux.microsoft.com header.b=BefuLuEY; arc=none smtp.client-ip=13.77.154.182 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.microsoft.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.microsoft.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.microsoft.com header.i=@linux.microsoft.com header.b="BefuLuEY" Received: by linux.microsoft.com (Postfix, from userid 1134) id B079E2111571; Fri, 9 May 2025 03:13:24 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com B079E2111571 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.microsoft.com; s=default; t=1746785604; bh=km0lQIF3NA+f3D6n5s967/qB/wGbEZ8pGu9m1a7cwt0=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=BefuLuEY7t1KJ8cb7O5BZ3ssurzZrp2ed+2ZhNgirvLSygKKshwV9fHIzgd2Y3G5q D2vScBhsEM0ykCLMtJVyDTcqrCD04Dxgk/8DWvEc3fg2WJ6yXG7Sk1FcKSn+VmzNP/ NH1M0tvhB3MmqZ+FE6JkkEYa5Movp5B15Qu7NTm4= From: Shradha Gupta To: Bjorn Helgaas , Rob Herring , Manivannan Sadhasivam , =?UTF-8?q?Krzysztof=20Wilczy=EF=BF=BD=7EDski?= , Lorenzo Pieralisi , Dexuan Cui , Wei Liu , Haiyang Zhang , "K. Y. Srinivasan" Cc: Shradha Gupta , linux-hyperv@vger.kernel.org, linux-pci@vger.kernel.org, linux-kernel@vger.kernel.org, Nipun Gupta , Yury Norov , Jason Gunthorpe , Jonathan Cameron , Anna-Maria Behnsen , Kevin Tian , Long Li , Thomas Gleixner , Andrew Lunn , "David S. Miller" , Eric Dumazet , Jakub Kicinski , Paolo Abeni , Konstantin Taranov , Simon Horman , Leon Romanovsky , Maxim Levitsky , Erni Sri Satya Vennela , Peter Zijlstra , netdev@vger.kernel.org, linux-rdma@vger.kernel.org, Paul Rosswurm , Shradha Gupta Subject: [PATCH v3 2/4] PCI: hv: Allow dynamic MSI-X vector allocation Date: Fri, 9 May 2025 03:13:22 -0700 Message-Id: <1746785602-4600-1-git-send-email-shradhagupta@linux.microsoft.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1746785566-4337-1-git-send-email-shradhagupta@linux.microsoft.com> References: <1746785566-4337-1-git-send-email-shradhagupta@linux.microsoft.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Allow dynamic MSI-X vector allocation for pci_hyperv PCI controller by adding support for the flag MSI_FLAG_PCI_MSIX_ALLOC_DYN and using pci_msix_prepare_desc() to prepare the MSI-X descriptors. Feature support added for both x86 and ARM64 Signed-off-by: Shradha Gupta Reviewed-by: Haiyang Zhang --- Changes in v3: * Add arm64 support --- Changes in v2: * split the patch to keep changes in PCI and pci_hyperv controller seperate * replace strings "pci vectors" by "MSI-X vectors" --- drivers/pci/controller/pci-hyperv.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/p= ci-hyperv.c index ac27bda5ba26..8c8882cb0ad2 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c @@ -598,7 +598,8 @@ static unsigned int hv_msi_get_int_vector(struct irq_da= ta *data) return cfg->vector; } =20 -#define hv_msi_prepare pci_msi_prepare +#define hv_msi_prepare pci_msi_prepare +#define hv_msix_prepare_desc pci_msix_prepare_desc =20 /** * hv_arch_irq_unmask() - "Unmask" the IRQ by setting its current @@ -727,6 +728,7 @@ static void hv_arch_irq_unmask(struct irq_data *data) #define FLOW_HANDLER NULL #define FLOW_NAME NULL #define hv_msi_prepare NULL +#define hv_msix_prepare_desc pci_msix_prepare_desc =20 struct hv_pci_chip_data { DECLARE_BITMAP(spi_map, HV_PCI_MSI_SPI_NR); @@ -2063,6 +2065,7 @@ static struct irq_chip hv_msi_irq_chip =3D { static struct msi_domain_ops hv_msi_ops =3D { .msi_prepare =3D hv_msi_prepare, .msi_free =3D hv_msi_free, + .prepare_desc =3D hv_msix_prepare_desc, }; =20 /** @@ -2084,7 +2087,7 @@ static int hv_pcie_init_irq_domain(struct hv_pcibus_d= evice *hbus) hbus->msi_info.ops =3D &hv_msi_ops; hbus->msi_info.flags =3D (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | MSI_FLAG_MULTI_PCI_MSI | - MSI_FLAG_PCI_MSIX); + MSI_FLAG_PCI_MSIX | MSI_FLAG_PCI_MSIX_ALLOC_DYN); hbus->msi_info.handler =3D FLOW_HANDLER; hbus->msi_info.handler_name =3D FLOW_NAME; hbus->msi_info.data =3D hbus; --=20 2.34.1 From nobody Wed Dec 17 08:51:10 2025 Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 9485E274674; Fri, 9 May 2025 10:13:47 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=13.77.154.182 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746785629; cv=none; b=VgeE8kEj+FEiYSlnS5ti8ARDDECGVj6deXskM0mScxP5P8D6ss9SAU+cmzkH50uR96UtrAD8Om5FB7YPDucCmlNDH1TaKdK8+SHihp56HDzBqg3ZuFut5/ieELS5jn+IOzkEKleCfeEXGICnS0XsW1vSllvPB1A2qI+d1/n5WdE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746785629; c=relaxed/simple; bh=BUJ8x0JLhkS7DoHd6lHAsVOFU72/Ls1DqhLRpDoFWOM=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References; b=FLTvGYVVtAZj46JS7UYppmjS9YTK1Uko9wZ+UrtGvL49y4sRhFwm7oWUeF61hssRevLwbxTIKQ7rd9vWVnTx4xzXCG1+uylBxuk5aaYG7ZMqusNDGJl3dlOLXodg9RgOMmTuA5rZFAUViSzy+hjZi4LhCYXI69UlyA12wgdbkDU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.microsoft.com; spf=pass smtp.mailfrom=linux.microsoft.com; dkim=pass (1024-bit key) header.d=linux.microsoft.com header.i=@linux.microsoft.com header.b=Tkbqnbb3; arc=none smtp.client-ip=13.77.154.182 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.microsoft.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.microsoft.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.microsoft.com header.i=@linux.microsoft.com header.b="Tkbqnbb3" Received: by linux.microsoft.com (Postfix, from userid 1134) id 4972C2098462; Fri, 9 May 2025 03:13:47 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com 4972C2098462 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.microsoft.com; s=default; t=1746785627; bh=gEW7wXPmdwYptJzvTzaNmB9Cse5HO3g+2gZnzmwQfaU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Tkbqnbb3uFxEnjLH/M27IoWSeGc2Mhez63YAaoeMgyGQfhJ7w2ZOWOusJfZ7sl5K3 DaqwZoH5E96HJXMBfgwMjsjvv4Vq9GnbPmJrZjaQguKKDbqqb+TSLQeHCw2f3gKsK3 LXU3DmiXeD7b5jnhTUTiXLikUNECTMJ5BTYwHlKw= From: Shradha Gupta To: Dexuan Cui , Wei Liu , Haiyang Zhang , "K. Y. Srinivasan" , Andrew Lunn , "David S. Miller" , Eric Dumazet , Jakub Kicinski , Paolo Abeni , Konstantin Taranov , Simon Horman , Leon Romanovsky , Maxim Levitsky , Erni Sri Satya Vennela , Peter Zijlstra Cc: Shradha Gupta , linux-hyperv@vger.kernel.org, linux-pci@vger.kernel.org, linux-kernel@vger.kernel.org, Nipun Gupta , Yury Norov , Jason Gunthorpe , Jonathan Cameron , Anna-Maria Behnsen , Kevin Tian , Long Li , Thomas Gleixner , Bjorn Helgaas , Rob Herring , Manivannan Sadhasivam , =?UTF-8?q?Krzysztof=20Wilczy=EF=BF=BD=7EDski?= , Lorenzo Pieralisi , netdev@vger.kernel.org, linux-rdma@vger.kernel.org, Paul Rosswurm , Shradha Gupta Subject: [PATCH v3 3/4] net: mana: Allow irq_setup() to skip cpus for affinity Date: Fri, 9 May 2025 03:13:45 -0700 Message-Id: <1746785625-4753-1-git-send-email-shradhagupta@linux.microsoft.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1746785566-4337-1-git-send-email-shradhagupta@linux.microsoft.com> References: <1746785566-4337-1-git-send-email-shradhagupta@linux.microsoft.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" In order to prepare the MANA driver to allocate the MSI-X IRQs dynamically, we need to prepare the irq_setup() to allow skipping affinitizing IRQs to first CPU sibling group. This would be for cases when number of IRQs is less than or equal to number of online CPUs. In such cases for dynamically added IRQs the first CPU sibling group would already be affinitized with HWC IRQ Signed-off-by: Shradha Gupta Reviewed-by: Haiyang Zhang Reviewed-by: Yury Norov [NVIDIA] --- drivers/net/ethernet/microsoft/mana/gdma_main.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/= ethernet/microsoft/mana/gdma_main.c index 4ffaf7588885..2de42ce43373 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -1288,7 +1288,8 @@ void mana_gd_free_res_map(struct gdma_resource *r) r->size =3D 0; } =20 -static int irq_setup(unsigned int *irqs, unsigned int len, int node) +static int irq_setup(unsigned int *irqs, unsigned int len, int node, + bool skip_first_cpu) { const struct cpumask *next, *prev =3D cpu_none_mask; cpumask_var_t cpus __free(free_cpumask_var); @@ -1303,9 +1304,20 @@ static int irq_setup(unsigned int *irqs, unsigned in= t len, int node) while (weight > 0) { cpumask_andnot(cpus, next, prev); for_each_cpu(cpu, cpus) { + /* + * if the CPU sibling set is to be skipped we + * just move on to the next CPUs without len-- + */ + if (unlikely(skip_first_cpu)) { + skip_first_cpu =3D false; + goto next_cpumask; + } + if (len-- =3D=3D 0) goto done; + irq_set_affinity_and_hint(*irqs++, topology_sibling_cpumask(cpu)); +next_cpumask: cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu)); --weight; } @@ -1403,7 +1415,7 @@ static int mana_gd_setup_irqs(struct pci_dev *pdev) } } =20 - err =3D irq_setup(irqs, (nvec - start_irq_index), gc->numa_node); + err =3D irq_setup(irqs, (nvec - start_irq_index), gc->numa_node, false); if (err) goto free_irq; =20 --=20 2.34.1 From nobody Wed Dec 17 08:51:10 2025 Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 7678C28F51D; Fri, 9 May 2025 10:13:59 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=13.77.154.182 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746785642; cv=none; b=NYyj3dSbLhL7O4K/Mw021M8OcafcFyxX5k0IV4OXE4jAZEJIM7G+Jp/aNvq6YBe8732nCsbWyLj+5fD75corWxj+Tr6ntd/49XYxCCj2i8b1ZOeW8SjeRjiGrMYo9ZhuoYyZNZnEtA21zQbN7ukZ8OfNMP5zvL+YPbOaLPsBKeQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746785642; c=relaxed/simple; bh=dj2bntkdl7xEPsU3O8kNrVVqiaY9ql9KfTEsUCfgmbQ=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References; b=D5zuVj1Zn6vgiL6/HloNpDrO3s8RNRDJELlUIkoja9Gltw4VByjlILPA3fsedwarG4ziR74nPRrfjj0PXiZpiMseK/FBbcPKHtmml2HhG9mq80CtoqundJvBPbcQL57+fNZO/6YkqiRDi3CugEeE7Wa6VtQ5BuDQE4dDwmWYvKY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.microsoft.com; spf=pass smtp.mailfrom=linux.microsoft.com; dkim=pass (1024-bit key) header.d=linux.microsoft.com header.i=@linux.microsoft.com header.b=SBTI+6qT; arc=none smtp.client-ip=13.77.154.182 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.microsoft.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.microsoft.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.microsoft.com header.i=@linux.microsoft.com header.b="SBTI+6qT" Received: by linux.microsoft.com (Postfix, from userid 1134) id 091A92098462; Fri, 9 May 2025 03:13:59 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com 091A92098462 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.microsoft.com; s=default; t=1746785639; bh=j0hv02lMNf3ygxb/Gu2IuZyCzEMjtz+gbRdiT+OLPPg=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=SBTI+6qThlS97fRCryGco2kspD3TzuiKHPkm152kXMIQstFup7tonhUeWdp8Wi8aj xUNjTJoXgr07lzcMF02GNOND9ijoXfFJq6Bup6omNO6Yw6pGDpEifn/mO82NORbeAm M92uanuJVaxYNsY8rO++MsuOlwiXB0gCOx27044w= From: Shradha Gupta To: Dexuan Cui , Wei Liu , Haiyang Zhang , "K. Y. Srinivasan" , Andrew Lunn , "David S. Miller" , Eric Dumazet , Jakub Kicinski , Paolo Abeni , Konstantin Taranov , Simon Horman , Leon Romanovsky , Maxim Levitsky , Erni Sri Satya Vennela , Peter Zijlstra Cc: Shradha Gupta , linux-hyperv@vger.kernel.org, linux-pci@vger.kernel.org, linux-kernel@vger.kernel.org, Nipun Gupta , Yury Norov , Jason Gunthorpe , Jonathan Cameron , Anna-Maria Behnsen , Kevin Tian , Long Li , Thomas Gleixner , Bjorn Helgaas , Rob Herring , Manivannan Sadhasivam , =?UTF-8?q?Krzysztof=20Wilczy=EF=BF=BD=7EDski?= , Lorenzo Pieralisi , netdev@vger.kernel.org, linux-rdma@vger.kernel.org, Paul Rosswurm , Shradha Gupta Subject: [PATCH v3 4/4] net: mana: Allocate MSI-X vectors dynamically Date: Fri, 9 May 2025 03:13:57 -0700 Message-Id: <1746785637-4881-1-git-send-email-shradhagupta@linux.microsoft.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1746785566-4337-1-git-send-email-shradhagupta@linux.microsoft.com> References: <1746785566-4337-1-git-send-email-shradhagupta@linux.microsoft.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Currently, the MANA driver allocates MSI-X vectors statically based on MANA_MAX_NUM_QUEUES and num_online_cpus() values and in some cases ends up allocating more vectors than it needs. This is because, by this time we do not have a HW channel and do not know how many IRQs should be allocated. To avoid this, we allocate 1 MSI-X vector during the creation of HWC and after getting the value supported by hardware, dynamically add the remaining MSI-X vectors. Signed-off-by: Shradha Gupta Reviewed-by: Haiyang Zhang --- Changes in v3: * implemented irq_contexts as xarrays rather than list * split the patch to create a perparation patch around irq_setup() * add log when IRQ allocation/setup for remaining IRQs fails --- Changes in v2: * Use string 'MSI-X vectors' instead of 'pci vectors' * make skip-cpu a bool instead of int * rearrange the comment arout skip_cpu variable appropriately * update the capability bit for driver indicating dynamic IRQ allocation * enforced max line length to 80 * enforced RCT convention * initialized gic to NULL, for when there is a possibility of gic not being populated correctly --- .../net/ethernet/microsoft/mana/gdma_main.c | 248 +++++++++++++++--- include/net/mana/gdma.h | 8 +- 2 files changed, 211 insertions(+), 45 deletions(-) diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/= ethernet/microsoft/mana/gdma_main.c index 2de42ce43373..f07cebffc30d 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -6,6 +6,8 @@ #include #include #include +#include +#include =20 #include =20 @@ -80,8 +82,15 @@ static int mana_gd_query_max_resources(struct pci_dev *p= dev) return err ? err : -EPROTO; } =20 - if (gc->num_msix_usable > resp.max_msix) - gc->num_msix_usable =3D resp.max_msix; + if (!pci_msix_can_alloc_dyn(pdev)) { + if (gc->num_msix_usable > resp.max_msix) + gc->num_msix_usable =3D resp.max_msix; + } else { + /* If dynamic allocation is enabled we have already allocated + * hwc msi + */ + gc->num_msix_usable =3D min(resp.max_msix, num_online_cpus() + 1); + } =20 if (gc->num_msix_usable <=3D 1) return -ENOSPC; @@ -482,7 +491,9 @@ static int mana_gd_register_irq(struct gdma_queue *queu= e, } =20 queue->eq.msix_index =3D msi_index; - gic =3D &gc->irq_contexts[msi_index]; + gic =3D xa_load(&gc->irq_contexts, msi_index); + if (!gic) + return -EINVAL; =20 spin_lock_irqsave(&gic->lock, flags); list_add_rcu(&queue->entry, &gic->eq_list); @@ -507,7 +518,10 @@ static void mana_gd_deregiser_irq(struct gdma_queue *q= ueue) if (WARN_ON(msix_index >=3D gc->num_msix_usable)) return; =20 - gic =3D &gc->irq_contexts[msix_index]; + gic =3D xa_load(&gc->irq_contexts, msix_index); + if (!gic) + return; + spin_lock_irqsave(&gic->lock, flags); list_for_each_entry_rcu(eq, &gic->eq_list, entry) { if (queue =3D=3D eq) { @@ -1329,29 +1343,96 @@ static int irq_setup(unsigned int *irqs, unsigned i= nt len, int node, return 0; } =20 -static int mana_gd_setup_irqs(struct pci_dev *pdev) +static int mana_gd_setup_dyn_irqs(struct pci_dev *pdev, int nvec) { struct gdma_context *gc =3D pci_get_drvdata(pdev); - unsigned int max_queues_per_port; struct gdma_irq_context *gic; - unsigned int max_irqs, cpu; - int start_irq_index =3D 1; - int nvec, *irqs, irq; + bool skip_first_cpu =3D false; int err, i =3D 0, j; + int *irqs, irq; =20 cpus_read_lock(); - max_queues_per_port =3D num_online_cpus(); - if (max_queues_per_port > MANA_MAX_NUM_QUEUES) - max_queues_per_port =3D MANA_MAX_NUM_QUEUES; =20 - /* Need 1 interrupt for the Hardware communication Channel (HWC) */ - max_irqs =3D max_queues_per_port + 1; + irqs =3D kmalloc_array(nvec, sizeof(int), GFP_KERNEL); + if (!irqs) { + err =3D -ENOMEM; + goto free_irq_vector; + } =20 - nvec =3D pci_alloc_irq_vectors(pdev, 2, max_irqs, PCI_IRQ_MSIX); - if (nvec < 0) { - cpus_read_unlock(); - return nvec; + for (i =3D 0; i < nvec; i++) { + gic =3D kcalloc(1, sizeof(struct gdma_irq_context), GFP_KERNEL); + if (!gic) { + err =3D -ENOMEM; + goto free_irq; + } + gic->handler =3D mana_gd_process_eq_events; + INIT_LIST_HEAD(&gic->eq_list); + spin_lock_init(&gic->lock); + + snprintf(gic->name, MANA_IRQ_NAME_SZ, "mana_q%d@pci:%s", + i, pci_name(pdev)); + + /* one pci vector is already allocated for HWC */ + irqs[i] =3D pci_irq_vector(pdev, i + 1); + if (irqs[i] < 0) { + err =3D irqs[i]; + goto free_current_gic; + } + + err =3D request_irq(irqs[i], mana_gd_intr, 0, gic->name, gic); + if (err) + goto free_current_gic; + + xa_store(&gc->irq_contexts, i + 1, gic, GFP_KERNEL); } + + /* + * When calling irq_setup() for dynamically added IRQs, if number of + * CPUs is more than or equal to allocated MSI-X, we need to skip the + * first CPU sibling group since they are already affinitized to HWC IRQ + */ + if (gc->num_msix_usable <=3D num_online_cpus()) + skip_first_cpu =3D true; + + err =3D irq_setup(irqs, nvec, gc->numa_node, skip_first_cpu); + if (err) + goto free_irq; + + cpus_read_unlock(); + kfree(irqs); + return 0; + +free_current_gic: + kfree(gic); +free_irq: + for (j =3D i; j >=3D 0; j--) { + irq =3D pci_irq_vector(pdev, j); + gic =3D xa_load(&gc->irq_contexts, j); + if (!gic) + continue; + + irq_update_affinity_hint(irq, NULL); + free_irq(irq, gic); + xa_erase(&gc->irq_contexts, j); + kfree(gic); + } + kfree(irqs); +free_irq_vector: + cpus_read_unlock(); + return err; +} + +static int mana_gd_setup_irqs(struct pci_dev *pdev, int nvec) +{ + struct gdma_context *gc =3D pci_get_drvdata(pdev); + struct gdma_irq_context *gic; + int start_irq_index =3D 1; + unsigned int cpu; + int *irqs, irq; + int err, i =3D 0, j; + + cpus_read_lock(); + if (nvec <=3D num_online_cpus()) start_irq_index =3D 0; =20 @@ -1361,15 +1442,13 @@ static int mana_gd_setup_irqs(struct pci_dev *pdev) goto free_irq_vector; } =20 - gc->irq_contexts =3D kcalloc(nvec, sizeof(struct gdma_irq_context), - GFP_KERNEL); - if (!gc->irq_contexts) { - err =3D -ENOMEM; - goto free_irq_array; - } - for (i =3D 0; i < nvec; i++) { - gic =3D &gc->irq_contexts[i]; + gic =3D kcalloc(1, sizeof(struct gdma_irq_context), GFP_KERNEL); + if (!gic) { + err =3D -ENOMEM; + goto free_irq; + } + gic->handler =3D mana_gd_process_eq_events; INIT_LIST_HEAD(&gic->eq_list); spin_lock_init(&gic->lock); @@ -1384,13 +1463,13 @@ static int mana_gd_setup_irqs(struct pci_dev *pdev) irq =3D pci_irq_vector(pdev, i); if (irq < 0) { err =3D irq; - goto free_irq; + goto free_current_gic; } =20 if (!i) { err =3D request_irq(irq, mana_gd_intr, 0, gic->name, gic); if (err) - goto free_irq; + goto free_current_gic; =20 /* If number of IRQ is one extra than number of online CPUs, * then we need to assign IRQ0 (hwc irq) and IRQ1 to @@ -1408,39 +1487,110 @@ static int mana_gd_setup_irqs(struct pci_dev *pdev) } } else { irqs[i - start_irq_index] =3D irq; - err =3D request_irq(irqs[i - start_irq_index], mana_gd_intr, 0, - gic->name, gic); + err =3D request_irq(irqs[i - start_irq_index], + mana_gd_intr, 0, gic->name, gic); if (err) - goto free_irq; + goto free_current_gic; } + + xa_store(&gc->irq_contexts, i, gic, GFP_KERNEL); } =20 err =3D irq_setup(irqs, (nvec - start_irq_index), gc->numa_node, false); if (err) goto free_irq; =20 - gc->max_num_msix =3D nvec; - gc->num_msix_usable =3D nvec; cpus_read_unlock(); kfree(irqs); return 0; =20 +free_current_gic: + kfree(gic); free_irq: for (j =3D i - 1; j >=3D 0; j--) { irq =3D pci_irq_vector(pdev, j); - gic =3D &gc->irq_contexts[j]; + gic =3D xa_load(&gc->irq_contexts, j); + if (!gic) + continue; =20 irq_update_affinity_hint(irq, NULL); free_irq(irq, gic); + xa_erase(&gc->irq_contexts, j); + kfree(gic); } =20 - kfree(gc->irq_contexts); - gc->irq_contexts =3D NULL; -free_irq_array: kfree(irqs); free_irq_vector: + xa_destroy(&gc->irq_contexts); cpus_read_unlock(); - pci_free_irq_vectors(pdev); + return err; +} + +static int mana_gd_setup_hwc_irqs(struct pci_dev *pdev) +{ + struct gdma_context *gc =3D pci_get_drvdata(pdev); + unsigned int max_irqs, min_irqs; + int max_queues_per_port; + int nvec, err; + + if (pci_msix_can_alloc_dyn(pdev)) { + max_irqs =3D 1; + min_irqs =3D 1; + } else { + max_queues_per_port =3D num_online_cpus(); + if (max_queues_per_port > MANA_MAX_NUM_QUEUES) + max_queues_per_port =3D MANA_MAX_NUM_QUEUES; + /* Need 1 interrupt for HWC */ + max_irqs =3D max_queues_per_port + 1; + min_irqs =3D 2; + } + + nvec =3D pci_alloc_irq_vectors(pdev, min_irqs, max_irqs, PCI_IRQ_MSIX); + if (nvec < 0) + return nvec; + + err =3D mana_gd_setup_irqs(pdev, nvec); + if (err) { + pci_free_irq_vectors(pdev); + return err; + } + + gc->num_msix_usable =3D nvec; + gc->max_num_msix =3D nvec; + + return err; +} + +static int mana_gd_setup_remaining_irqs(struct pci_dev *pdev) +{ + struct gdma_context *gc =3D pci_get_drvdata(pdev); + int max_irqs, i, err =3D 0; + struct msi_map irq_map; + + if (!pci_msix_can_alloc_dyn(pdev)) + /* remain irqs are already allocated with HWC IRQ */ + return 0; + + /* allocate only remaining IRQs*/ + max_irqs =3D gc->num_msix_usable - 1; + + for (i =3D 1; i <=3D max_irqs; i++) { + irq_map =3D pci_msix_alloc_irq_at(pdev, i, NULL); + if (!irq_map.virq) { + err =3D irq_map.index; + /* caller will handle cleaning up all allocated + * irqs, after HWC is destroyed + */ + return err; + } + } + + err =3D mana_gd_setup_dyn_irqs(pdev, max_irqs); + if (err) + return err; + + gc->max_num_msix =3D gc->max_num_msix + max_irqs; + return err; } =20 @@ -1458,19 +1608,22 @@ static void mana_gd_remove_irqs(struct pci_dev *pde= v) if (irq < 0) continue; =20 - gic =3D &gc->irq_contexts[i]; + gic =3D xa_load(&gc->irq_contexts, i); + if (!gic) + continue; =20 /* Need to clear the hint before free_irq */ irq_update_affinity_hint(irq, NULL); free_irq(irq, gic); + xa_erase(&gc->irq_contexts, i); + kfree(gic); } =20 pci_free_irq_vectors(pdev); =20 gc->max_num_msix =3D 0; gc->num_msix_usable =3D 0; - kfree(gc->irq_contexts); - gc->irq_contexts =3D NULL; + xa_destroy(&gc->irq_contexts); } =20 static int mana_gd_setup(struct pci_dev *pdev) @@ -1481,9 +1634,10 @@ static int mana_gd_setup(struct pci_dev *pdev) mana_gd_init_registers(pdev); mana_smc_init(&gc->shm_channel, gc->dev, gc->shm_base); =20 - err =3D mana_gd_setup_irqs(pdev); + err =3D mana_gd_setup_hwc_irqs(pdev); if (err) { - dev_err(gc->dev, "Failed to setup IRQs: %d\n", err); + dev_err(gc->dev, "Failed to setup IRQs for HWC creation: %d\n", + err); return err; } =20 @@ -1499,6 +1653,12 @@ static int mana_gd_setup(struct pci_dev *pdev) if (err) goto destroy_hwc; =20 + err =3D mana_gd_setup_remaining_irqs(pdev); + if (err) { + dev_err(gc->dev, "Failed to setup remaining IRQs: %d", err); + goto destroy_hwc; + } + err =3D mana_gd_detect_devices(pdev); if (err) goto destroy_hwc; @@ -1575,6 +1735,7 @@ static int mana_gd_probe(struct pci_dev *pdev, const = struct pci_device_id *ent) gc->is_pf =3D mana_is_pf(pdev->device); gc->bar0_va =3D bar0_va; gc->dev =3D &pdev->dev; + xa_init(&gc->irq_contexts); =20 if (gc->is_pf) gc->mana_pci_debugfs =3D debugfs_create_dir("0", mana_debugfs_root); diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h index 228603bf03f2..f20d1d1ea5e8 100644 --- a/include/net/mana/gdma.h +++ b/include/net/mana/gdma.h @@ -373,7 +373,7 @@ struct gdma_context { unsigned int max_num_queues; unsigned int max_num_msix; unsigned int num_msix_usable; - struct gdma_irq_context *irq_contexts; + struct xarray irq_contexts; =20 /* L2 MTU */ u16 adapter_mtu; @@ -558,12 +558,16 @@ enum { /* Driver can handle holes (zeros) in the device list */ #define GDMA_DRV_CAP_FLAG_1_DEV_LIST_HOLES_SUP BIT(11) =20 +/* Driver supports dynamic MSI-X vector allocation */ +#define GDMA_DRV_CAP_FLAG_1_DYNAMIC_IRQ_ALLOC_SUPPORT BIT(13) + #define GDMA_DRV_CAP_FLAGS1 \ (GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \ GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX | \ GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG | \ GDMA_DRV_CAP_FLAG_1_VARIABLE_INDIRECTION_TABLE_SUPPORT | \ - GDMA_DRV_CAP_FLAG_1_DEV_LIST_HOLES_SUP) + GDMA_DRV_CAP_FLAG_1_DEV_LIST_HOLES_SUP | \ + GDMA_DRV_CAP_FLAG_1_DYNAMIC_IRQ_ALLOC_SUPPORT) =20 #define GDMA_DRV_CAP_FLAGS2 0 =20 --=20 2.34.1