On RPi5 there is an external MIP MSI-X interrupt controller
which can handle up to 64 interrupts.

Signed-off-by: Stanimir Varbanov <svarbanov@suse.de>
---
v3 -> v4:
 - no changes.

 drivers/pci/controller/pcie-brcmstb.c | 63 +++++++++++++++++++++++++--
 1 file changed, 59 insertions(+), 4 deletions(-)

diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c
index c01462b07eea..af01a7915c94 100644
--- a/drivers/pci/controller/pcie-brcmstb.c
+++ b/drivers/pci/controller/pcie-brcmstb.c
@@ -1318,6 +1318,52 @@ static int brcm_pcie_start_link(struct brcm_pcie *pcie)
 	return 0;
 }
 
+static int brcm_pcie_enable_external_msix(struct brcm_pcie *pcie,
+					  struct device_node *msi_np)
+{
+	struct inbound_win inbound_wins[PCIE_BRCM_MAX_INBOUND_WINS];
+	u64 msi_pci_addr, msi_phys_addr;
+	struct resource r;
+	int mip_bar, ret;
+	u32 val, reg;
+
+	ret = of_property_read_reg(msi_np, 1, &msi_pci_addr, NULL);
+	if (ret)
+		return ret;
+
+	ret = of_address_to_resource(msi_np, 0, &r);
+	if (ret)
+		return ret;
+
+	msi_phys_addr = r.start;
+
+	/* Find free inbound window for MIP access */
+	mip_bar = brcm_pcie_get_inbound_wins(pcie, inbound_wins);
+	if (mip_bar < 0)
+		return mip_bar;
+
+	mip_bar += 1;
+	reg = brcm_bar_reg_offset(mip_bar);
+
+	val = lower_32_bits(msi_pci_addr);
+	val |= brcm_pcie_encode_ibar_size(SZ_4K);
+	writel(val, pcie->base + reg);
+
+	val = upper_32_bits(msi_pci_addr);
+	writel(val, pcie->base + reg + 4);
+
+	reg = brcm_ubus_reg_offset(mip_bar);
+
+	val = lower_32_bits(msi_phys_addr);
+	val |= PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_ACCESS_EN_MASK;
+	writel(val, pcie->base + reg);
+
+	val = upper_32_bits(msi_phys_addr);
+	writel(val, pcie->base + reg + 4);
+
+	return 0;
+}
+
 static const char * const supplies[] = {
 	"vpcie3v3",
 	"vpcie3v3aux",
@@ -1879,11 +1925,20 @@ static int brcm_pcie_probe(struct platform_device *pdev)
 		goto fail;
 	}
 
-	msi_np = of_parse_phandle(pcie->np, "msi-parent", 0);
-	if (pci_msi_enabled() && msi_np == pcie->np) {
-		ret = brcm_pcie_enable_msi(pcie);
+	if (pci_msi_enabled()) {
+		msi_np = of_parse_phandle(pcie->np, "msi-parent", 0);
+		const char *str;
+
+		if (msi_np == pcie->np) {
+			str = "internal MSI";
+			ret = brcm_pcie_enable_msi(pcie);
+		} else {
+			str = "external MSI-X";
+			ret = brcm_pcie_enable_external_msix(pcie, msi_np);
+		}
+
 		if (ret) {
-			dev_err(pcie->dev, "probe of internal MSI failed");
+			dev_err(pcie->dev, "enable of %s failed\n", str);
 			goto fail;
 		}
 	}
-- 
2.43.0

On 10/25/24 08:45, Stanimir Varbanov wrote:
> On RPi5 there is an external MIP MSI-X interrupt controller
> which can handle up to 64 interrupts.
>
> Signed-off-by: Stanimir Varbanov <svarbanov@suse.de>
> ---
> v3 -> v4:
>   - no changes.
>
>   drivers/pci/controller/pcie-brcmstb.c | 63 +++++++++++++++++++++++++--
>   1 file changed, 59 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c
> index c01462b07eea..af01a7915c94 100644
> --- a/drivers/pci/controller/pcie-brcmstb.c
> +++ b/drivers/pci/controller/pcie-brcmstb.c
> @@ -1318,6 +1318,52 @@ static int brcm_pcie_start_link(struct brcm_pcie *pcie)
>   	return 0;
>   }
>   
> +static int brcm_pcie_enable_external_msix(struct brcm_pcie *pcie,
> +					  struct device_node *msi_np)
> +{
> +	struct inbound_win inbound_wins[PCIE_BRCM_MAX_INBOUND_WINS];
> +	u64 msi_pci_addr, msi_phys_addr;
> +	struct resource r;
> +	int mip_bar, ret;
> +	u32 val, reg;
> +
> +	ret = of_property_read_reg(msi_np, 1, &msi_pci_addr, NULL);
> +	if (ret)
> +		return ret;
> +
> +	ret = of_address_to_resource(msi_np, 0, &r);
> +	if (ret)
> +		return ret;
> +
> +	msi_phys_addr = r.start;
> +
> +	/* Find free inbound window for MIP access */
> +	mip_bar = brcm_pcie_get_inbound_wins(pcie, inbound_wins);
> +	if (mip_bar < 0)
> +		return mip_bar;
> +
> +	mip_bar += 1;
> +	reg = brcm_bar_reg_offset(mip_bar);
> +
> +	val = lower_32_bits(msi_pci_addr);
> +	val |= brcm_pcie_encode_ibar_size(SZ_4K);
> +	writel(val, pcie->base + reg);
> +
> +	val = upper_32_bits(msi_pci_addr);
> +	writel(val, pcie->base + reg + 4);
> +
> +	reg = brcm_ubus_reg_offset(mip_bar);
> +
> +	val = lower_32_bits(msi_phys_addr);
> +	val |= PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_ACCESS_EN_MASK;
> +	writel(val, pcie->base + reg);
> +
> +	val = upper_32_bits(msi_phys_addr);
> +	writel(val, pcie->base + reg + 4);

Hi Stan,

It looks like all this is doing is setting up an identity-mapped inbound 
window, is that correct?  If so, couldn't you get the same effect by 
adding an identity-mapped dma-range in the PCIe DT node?

Jim Quinlan  Broadcom STB/CM

> +
> +	return 0;
> +}
> +
>   static const char * const supplies[] = {
>   	"vpcie3v3",
>   	"vpcie3v3aux",
> @@ -1879,11 +1925,20 @@ static int brcm_pcie_probe(struct platform_device *pdev)
>   		goto fail;
>   	}
>   
> -	msi_np = of_parse_phandle(pcie->np, "msi-parent", 0);
> -	if (pci_msi_enabled() && msi_np == pcie->np) {
> -		ret = brcm_pcie_enable_msi(pcie);
> +	if (pci_msi_enabled()) {
> +		msi_np = of_parse_phandle(pcie->np, "msi-parent", 0);
> +		const char *str;
> +
> +		if (msi_np == pcie->np) {
> +			str = "internal MSI";
> +			ret = brcm_pcie_enable_msi(pcie);
> +		} else {
> +			str = "external MSI-X";
> +			ret = brcm_pcie_enable_external_msix(pcie, msi_np);
> +		}
> +
>   		if (ret) {
> -			dev_err(pcie->dev, "probe of internal MSI failed");
> +			dev_err(pcie->dev, "enable of %s failed\n", str);
>   			goto fail;
>   		}
>   	}

Hi Jim,

Thanks for comments!

On 12/11/24 10:01 PM, James Quinlan wrote:
> On 10/25/24 08:45, Stanimir Varbanov wrote:
>> On RPi5 there is an external MIP MSI-X interrupt controller
>> which can handle up to 64 interrupts.
>>
>> Signed-off-by: Stanimir Varbanov <svarbanov@suse.de>
>> ---
>> v3 -> v4:
>>   - no changes.
>>
>>   drivers/pci/controller/pcie-brcmstb.c | 63 +++++++++++++++++++++++++--
>>   1 file changed, 59 insertions(+), 4 deletions(-)
>>
>> diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/
>> controller/pcie-brcmstb.c
>> index c01462b07eea..af01a7915c94 100644
>> --- a/drivers/pci/controller/pcie-brcmstb.c
>> +++ b/drivers/pci/controller/pcie-brcmstb.c
>> @@ -1318,6 +1318,52 @@ static int brcm_pcie_start_link(struct
>> brcm_pcie *pcie)
>>       return 0;
>>   }
>>   +static int brcm_pcie_enable_external_msix(struct brcm_pcie *pcie,
>> +                      struct device_node *msi_np)
>> +{
>> +    struct inbound_win inbound_wins[PCIE_BRCM_MAX_INBOUND_WINS];
>> +    u64 msi_pci_addr, msi_phys_addr;
>> +    struct resource r;
>> +    int mip_bar, ret;
>> +    u32 val, reg;
>> +
>> +    ret = of_property_read_reg(msi_np, 1, &msi_pci_addr, NULL);
>> +    if (ret)
>> +        return ret;
>> +
>> +    ret = of_address_to_resource(msi_np, 0, &r);
>> +    if (ret)
>> +        return ret;
>> +
>> +    msi_phys_addr = r.start;
>> +
>> +    /* Find free inbound window for MIP access */
>> +    mip_bar = brcm_pcie_get_inbound_wins(pcie, inbound_wins);
>> +    if (mip_bar < 0)
>> +        return mip_bar;
>> +
>> +    mip_bar += 1;
>> +    reg = brcm_bar_reg_offset(mip_bar);
>> +
>> +    val = lower_32_bits(msi_pci_addr);
>> +    val |= brcm_pcie_encode_ibar_size(SZ_4K);
>> +    writel(val, pcie->base + reg);
>> +
>> +    val = upper_32_bits(msi_pci_addr);
>> +    writel(val, pcie->base + reg + 4);
>> +
>> +    reg = brcm_ubus_reg_offset(mip_bar);
>> +
>> +    val = lower_32_bits(msi_phys_addr);
>> +    val |= PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_ACCESS_EN_MASK;
>> +    writel(val, pcie->base + reg);
>> +
>> +    val = upper_32_bits(msi_phys_addr);
>> +    writel(val, pcie->base + reg + 4);
> 
> Hi Stan,
> 
> It looks like all this is doing is setting up an identity-mapped inbound
> window, is that correct?  If so, couldn't you get the same effect by
> adding an identity-mapped dma-range in the PCIe DT node?

Yes, that approach could work, I verified it.

Do you want me to drop this patch from the series and make the relevant
changes in PCIe dma-ranges properties?

~Stan

On Wed, Dec 18, 2024 at 9:54 AM Stanimir Varbanov <svarbanov@suse.de> wrote:
>
> Hi Jim,
>
> Thanks for comments!
>
> On 12/11/24 10:01 PM, James Quinlan wrote:
> > On 10/25/24 08:45, Stanimir Varbanov wrote:
> >> On RPi5 there is an external MIP MSI-X interrupt controller
> >> which can handle up to 64 interrupts.
> >>
> >> Signed-off-by: Stanimir Varbanov <svarbanov@suse.de>
> >> ---
> >> v3 -> v4:
> >>   - no changes.
> >>
> >>   drivers/pci/controller/pcie-brcmstb.c | 63 +++++++++++++++++++++++++--
> >>   1 file changed, 59 insertions(+), 4 deletions(-)
> >>
> >> diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/
> >> controller/pcie-brcmstb.c
> >> index c01462b07eea..af01a7915c94 100644
> >> --- a/drivers/pci/controller/pcie-brcmstb.c
> >> +++ b/drivers/pci/controller/pcie-brcmstb.c
> >> @@ -1318,6 +1318,52 @@ static int brcm_pcie_start_link(struct
> >> brcm_pcie *pcie)
> >>       return 0;
> >>   }
> >>   +static int brcm_pcie_enable_external_msix(struct brcm_pcie *pcie,
> >> +                      struct device_node *msi_np)
> >> +{
> >> +    struct inbound_win inbound_wins[PCIE_BRCM_MAX_INBOUND_WINS];
> >> +    u64 msi_pci_addr, msi_phys_addr;
> >> +    struct resource r;
> >> +    int mip_bar, ret;
> >> +    u32 val, reg;
> >> +
> >> +    ret = of_property_read_reg(msi_np, 1, &msi_pci_addr, NULL);
> >> +    if (ret)
> >> +        return ret;
> >> +
> >> +    ret = of_address_to_resource(msi_np, 0, &r);
> >> +    if (ret)
> >> +        return ret;
> >> +
> >> +    msi_phys_addr = r.start;
> >> +
> >> +    /* Find free inbound window for MIP access */
> >> +    mip_bar = brcm_pcie_get_inbound_wins(pcie, inbound_wins);
> >> +    if (mip_bar < 0)
> >> +        return mip_bar;
> >> +
> >> +    mip_bar += 1;
> >> +    reg = brcm_bar_reg_offset(mip_bar);
> >> +
> >> +    val = lower_32_bits(msi_pci_addr);
> >> +    val |= brcm_pcie_encode_ibar_size(SZ_4K);
> >> +    writel(val, pcie->base + reg);
> >> +
> >> +    val = upper_32_bits(msi_pci_addr);
> >> +    writel(val, pcie->base + reg + 4);
> >> +
> >> +    reg = brcm_ubus_reg_offset(mip_bar);
> >> +
> >> +    val = lower_32_bits(msi_phys_addr);
> >> +    val |= PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_ACCESS_EN_MASK;
> >> +    writel(val, pcie->base + reg);
> >> +
> >> +    val = upper_32_bits(msi_phys_addr);
> >> +    writel(val, pcie->base + reg + 4);
> >
> > Hi Stan,
> >
> > It looks like all this is doing is setting up an identity-mapped inbound
> > window, is that correct?  If so, couldn't you get the same effect by
> > adding an identity-mapped dma-range in the PCIe DT node?
>
> Yes, that approach could work, I verified it.
>
> Do you want me to drop this patch from the series and make the relevant
> changes in PCIe dma-ranges properties?
>
Hi Stan,

Yes please, that was what I was hoping for -- the less code the
better, assuming everything still works :-)

Thanks & regards,
Jim Quinlan
Broadcom STB/CM

> ~Stan
>
>
>

Hi,

Jim, Florian there is no review comments on this patch. Could you please
take a look.

regards,
~Stan

On 10/25/24 15:45, Stanimir Varbanov wrote:
> On RPi5 there is an external MIP MSI-X interrupt controller
> which can handle up to 64 interrupts.
> 
> Signed-off-by: Stanimir Varbanov <svarbanov@suse.de>
> ---
> v3 -> v4:
>  - no changes.
> 
>  drivers/pci/controller/pcie-brcmstb.c | 63 +++++++++++++++++++++++++--
>  1 file changed, 59 insertions(+), 4 deletions(-)
> 

<cut>