The current implementation requires iATU for every configuration
space access which increases latency & cpu utilization.

Configuring iATU in config shift mode enables ECAM feature to access the
config space, which avoids iATU configuration for every config access.

Add "ctrl2" into struct dw_pcie_ob_atu_cfg  to enable config shift mode.

As DBI comes under config space, this avoids remapping of DBI space
separately. Instead, it uses the mapped config space address returned from
ECAM initialization. Change the order of dw_pcie_get_resources() execution
to acheive this.

Introduce new ecam_init() function op for the clients to configure after
ecam window creation has been done.

Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
---
 drivers/pci/controller/dwc/pcie-designware-host.c | 114 ++++++++++++++++++----
 drivers/pci/controller/dwc/pcie-designware.c      |   2 +-
 drivers/pci/controller/dwc/pcie-designware.h      |   6 ++
 3 files changed, 102 insertions(+), 20 deletions(-)

diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
index 3e41865c7290..e98cc841a2a9 100644
--- a/drivers/pci/controller/dwc/pcie-designware-host.c
+++ b/drivers/pci/controller/dwc/pcie-designware-host.c
@@ -418,6 +418,62 @@ static void dw_pcie_host_request_msg_tlp_res(struct dw_pcie_rp *pp)
 	}
 }
 
+static int dw_pcie_config_ecam_iatu(struct dw_pcie_rp *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct dw_pcie_ob_atu_cfg atu = {0};
+	struct resource_entry *bus;
+	int ret, bus_range_max;
+
+	bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
+
+	/*
+	 * Bus 1 config space needs type 0 atu configuration
+	 * Remaining buses need type 1 atu configuration
+	 */
+	atu.index = 0;
+	atu.type = PCIE_ATU_TYPE_CFG0;
+	atu.cpu_addr = pp->cfg0_base + SZ_1M;
+	atu.size = SZ_1M;
+	atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE;
+	ret = dw_pcie_prog_outbound_atu(pci, &atu);
+	if (ret)
+		return ret;
+
+	bus_range_max = bus->res->end - bus->res->start + 1;
+
+	/* Configure for bus 2 - bus_range_max in type 1 */
+	atu.index = 1;
+	atu.type = PCIE_ATU_TYPE_CFG1;
+	atu.cpu_addr = pp->cfg0_base + SZ_2M;
+	atu.size = (SZ_1M * (bus_range_max - 2));
+	atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE;
+	return dw_pcie_prog_outbound_atu(pci, &atu);
+}
+
+static int dw_pcie_create_ecam_window(struct dw_pcie_rp *pp, struct resource *res)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct device *dev = pci->dev;
+	struct resource_entry *bus;
+
+	bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
+	if (!bus)
+		return -ENODEV;
+
+	pp->cfg = pci_ecam_create(dev, res, bus->res, &pci_generic_ecam_ops);
+	if (IS_ERR(pp->cfg))
+		return PTR_ERR(pp->cfg);
+
+	pci->dbi_base = pp->cfg->win;
+	pci->dbi_phys_addr = res->start;
+
+	if (pp->ops->ecam_init)
+		pp->ops->ecam_init(pci, pp->cfg);
+
+	return 0;
+}
+
 int dw_pcie_host_init(struct dw_pcie_rp *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
@@ -431,19 +487,8 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
 
 	raw_spin_lock_init(&pp->lock);
 
-	ret = dw_pcie_get_resources(pci);
-	if (ret)
-		return ret;
-
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config");
-	if (res) {
-		pp->cfg0_size = resource_size(res);
-		pp->cfg0_base = res->start;
-
-		pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res);
-		if (IS_ERR(pp->va_cfg0_base))
-			return PTR_ERR(pp->va_cfg0_base);
-	} else {
+	if (!res) {
 		dev_err(dev, "Missing *config* reg space\n");
 		return -ENODEV;
 	}
@@ -454,6 +499,30 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
 
 	pp->bridge = bridge;
 
+	pp->cfg0_size = resource_size(res);
+	pp->cfg0_base = res->start;
+
+	if (!pp->enable_ecam) {
+		pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res);
+		if (IS_ERR(pp->va_cfg0_base))
+			return PTR_ERR(pp->va_cfg0_base);
+
+		/* Set default bus ops */
+		bridge->ops = &dw_pcie_ops;
+		bridge->child_ops = &dw_child_pcie_ops;
+		bridge->sysdata = pp;
+	} else {
+		ret = dw_pcie_create_ecam_window(pp, res);
+		if (ret)
+			return ret;
+		bridge->ops = (struct pci_ops *)&pci_generic_ecam_ops.pci_ops;
+		pp->bridge->sysdata = pp->cfg;
+	}
+
+	ret = dw_pcie_get_resources(pci);
+	if (ret)
+		goto err_free_ecam;
+
 	/* Get the I/O range from DT */
 	win = resource_list_first_type(&bridge->windows, IORESOURCE_IO);
 	if (win) {
@@ -462,14 +531,10 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
 		pp->io_base = pci_pio_to_address(win->res->start);
 	}
 
-	/* Set default bus ops */
-	bridge->ops = &dw_pcie_ops;
-	bridge->child_ops = &dw_child_pcie_ops;
-
 	if (pp->ops->init) {
 		ret = pp->ops->init(pp);
 		if (ret)
-			return ret;
+			goto err_free_ecam;
 	}
 
 	if (pci_msi_enabled()) {
@@ -504,6 +569,12 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
 
 	dw_pcie_iatu_detect(pci);
 
+	if (pp->enable_ecam) {
+		ret = dw_pcie_config_ecam_iatu(pp);
+		if (ret)
+			goto err_free_msi;
+	}
+
 	/*
 	 * Allocate the resource for MSG TLP before programming the iATU
 	 * outbound window in dw_pcie_setup_rc(). Since the allocation depends
@@ -533,8 +604,6 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
 	/* Ignore errors, the link may come up later */
 	dw_pcie_wait_for_link(pci);
 
-	bridge->sysdata = pp;
-
 	ret = pci_host_probe(bridge);
 	if (ret)
 		goto err_stop_link;
@@ -558,6 +627,10 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
 	if (pp->ops->deinit)
 		pp->ops->deinit(pp);
 
+err_free_ecam:
+	if (pp->cfg)
+		pci_ecam_free(pp->cfg);
+
 	return ret;
 }
 EXPORT_SYMBOL_GPL(dw_pcie_host_init);
@@ -578,6 +651,9 @@ void dw_pcie_host_deinit(struct dw_pcie_rp *pp)
 
 	if (pp->ops->deinit)
 		pp->ops->deinit(pp);
+
+	if (pp->cfg)
+		pci_ecam_free(pp->cfg);
 }
 EXPORT_SYMBOL_GPL(dw_pcie_host_deinit);
 
diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c
index 6d6cbc8b5b2c..63d36676f858 100644
--- a/drivers/pci/controller/dwc/pcie-designware.c
+++ b/drivers/pci/controller/dwc/pcie-designware.c
@@ -509,7 +509,7 @@ int dw_pcie_prog_outbound_atu(struct dw_pcie *pci,
 		val = dw_pcie_enable_ecrc(val);
 	dw_pcie_writel_atu_ob(pci, atu->index, PCIE_ATU_REGION_CTRL1, val);
 
-	val = PCIE_ATU_ENABLE;
+	val = PCIE_ATU_ENABLE | atu->ctrl2;
 	if (atu->type == PCIE_ATU_TYPE_MSG) {
 		/* The data-less messages only for now */
 		val |= PCIE_ATU_INHIBIT_PAYLOAD | atu->code;
diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
index 347ab74ac35a..33afa91b402c 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -20,6 +20,7 @@
 #include <linux/irq.h>
 #include <linux/msi.h>
 #include <linux/pci.h>
+#include <linux/pci-ecam.h>
 #include <linux/reset.h>
 
 #include <linux/pci-epc.h>
@@ -171,6 +172,7 @@
 #define PCIE_ATU_REGION_CTRL2		0x004
 #define PCIE_ATU_ENABLE			BIT(31)
 #define PCIE_ATU_BAR_MODE_ENABLE	BIT(30)
+#define PCIE_ATU_CFG_SHIFT_MODE_ENABLE	BIT(28)
 #define PCIE_ATU_INHIBIT_PAYLOAD	BIT(22)
 #define PCIE_ATU_FUNC_NUM_MATCH_EN      BIT(19)
 #define PCIE_ATU_LOWER_BASE		0x008
@@ -342,6 +344,7 @@ struct dw_pcie_ob_atu_cfg {
 	u8 func_no;
 	u8 code;
 	u8 routing;
+	u32 ctrl2;
 	u64 cpu_addr;
 	u64 pci_addr;
 	u64 size;
@@ -353,6 +356,7 @@ struct dw_pcie_host_ops {
 	void (*post_init)(struct dw_pcie_rp *pp);
 	int (*msi_init)(struct dw_pcie_rp *pp);
 	void (*pme_turn_off)(struct dw_pcie_rp *pp);
+	int (*ecam_init)(struct dw_pcie *pcie, struct pci_config_window *cfg);
 };
 
 struct dw_pcie_rp {
@@ -379,6 +383,8 @@ struct dw_pcie_rp {
 	bool			use_atu_msg;
 	int			msg_atu_index;
 	struct resource		*msg_res;
+	bool			enable_ecam;
+	struct pci_config_window *cfg;
 };
 
 struct dw_pcie_ep_ops {

-- 
2.34.1

On Sun, Nov 17, 2024 at 03:30:19AM +0530, Krishna chaitanya chundru wrote:
> The current implementation requires iATU for every configuration
> space access which increases latency & cpu utilization.
> 
> Configuring iATU in config shift mode enables ECAM feature to access the
> config space, which avoids iATU configuration for every config access.
> 
> Add "ctrl2" into struct dw_pcie_ob_atu_cfg  to enable config shift mode.
> 
> As DBI comes under config space, this avoids remapping of DBI space
> separately. Instead, it uses the mapped config space address returned from
> ECAM initialization. Change the order of dw_pcie_get_resources() execution
> to acheive this.

s/acheive/achieve/

> Introduce new ecam_init() function op for the clients to configure after
> ecam window creation has been done.
> 
> Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
> ---
>  drivers/pci/controller/dwc/pcie-designware-host.c | 114 ++++++++++++++++++----
>  drivers/pci/controller/dwc/pcie-designware.c      |   2 +-
>  drivers/pci/controller/dwc/pcie-designware.h      |   6 ++
>  3 files changed, 102 insertions(+), 20 deletions(-)
> 
> diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
> index 3e41865c7290..e98cc841a2a9 100644
> --- a/drivers/pci/controller/dwc/pcie-designware-host.c
> +++ b/drivers/pci/controller/dwc/pcie-designware-host.c
> @@ -418,6 +418,62 @@ static void dw_pcie_host_request_msg_tlp_res(struct dw_pcie_rp *pp)
>  	}
>  }
>  
> +static int dw_pcie_config_ecam_iatu(struct dw_pcie_rp *pp)
> +{
> +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> +	struct dw_pcie_ob_atu_cfg atu = {0};
> +	struct resource_entry *bus;
> +	int ret, bus_range_max;
> +
> +	bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
> +
> +	/*
> +	 * Bus 1 config space needs type 0 atu configuration
> +	 * Remaining buses need type 1 atu configuration

s/atu/ATU/ (initialism, looks like "iATU" might be appropriate here?)

I'm confused about the bus numbering; you refer to "bus 1" and "bus
2".  Is bus 1 the root bus, i.e., the primary bus of a Root Port?

The root bus number would typically be 0, not 1, and is sometimes
programmable.  I don't know how the DesignWare core works, but since
you have "bus" here, referring to "bus 1" and "bus 2" here seems
overly specific.

> +	 */
> +	atu.index = 0;
> +	atu.type = PCIE_ATU_TYPE_CFG0;
> +	atu.cpu_addr = pp->cfg0_base + SZ_1M;
> +	atu.size = SZ_1M;
> +	atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE;
> +	ret = dw_pcie_prog_outbound_atu(pci, &atu);
> +	if (ret)
> +		return ret;
> +
> +	bus_range_max = bus->res->end - bus->res->start + 1;
> +
> +	/* Configure for bus 2 - bus_range_max in type 1 */
> +	atu.index = 1;
> +	atu.type = PCIE_ATU_TYPE_CFG1;
> +	atu.cpu_addr = pp->cfg0_base + SZ_2M;
> +	atu.size = (SZ_1M * (bus_range_max - 2));
> +	atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE;
> +	return dw_pcie_prog_outbound_atu(pci, &atu);
> +}
> +
> +static int dw_pcie_create_ecam_window(struct dw_pcie_rp *pp, struct resource *res)
> +{
> +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> +	struct device *dev = pci->dev;
> +	struct resource_entry *bus;
> +
> +	bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
> +	if (!bus)
> +		return -ENODEV;
> +
> +	pp->cfg = pci_ecam_create(dev, res, bus->res, &pci_generic_ecam_ops);
> +	if (IS_ERR(pp->cfg))
> +		return PTR_ERR(pp->cfg);
> +
> +	pci->dbi_base = pp->cfg->win;
> +	pci->dbi_phys_addr = res->start;
> +
> +	if (pp->ops->ecam_init)
> +		pp->ops->ecam_init(pci, pp->cfg);

.ecam_init() is defined to return int, but you ignore the return value.
If it's practical, I think it would be nicer if you could manage to:

  - Drop .enable_ecam.

  - Have .ecam_init() return failure if there's not enough ECAM space
    or whatever, i.e., move the qcom_pcie_check_ecam_support() code
    there.

  - Handle .ecam_init() failure here by doing whatever we did before.

If there's no useful return value from .ecam_init(), make it void.

> +	return 0;
> +}

> @@ -454,6 +499,30 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
>  
>  	pp->bridge = bridge;
>  
> +	pp->cfg0_size = resource_size(res);
> +	pp->cfg0_base = res->start;
> +
> +	if (!pp->enable_ecam) {

If you can't get rid of .enable_ecam, reverse order so this uses
positive logic:

  if (pp->enable_ecam) {
    ...
  } else {
    ...
  }

> +		pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res);
> +		if (IS_ERR(pp->va_cfg0_base))
> +			return PTR_ERR(pp->va_cfg0_base);
> +
> +		/* Set default bus ops */
> +		bridge->ops = &dw_pcie_ops;
> +		bridge->child_ops = &dw_child_pcie_ops;
> +		bridge->sysdata = pp;
> +	} else {
> +		ret = dw_pcie_create_ecam_window(pp, res);
> +		if (ret)
> +			return ret;
> +		bridge->ops = (struct pci_ops *)&pci_generic_ecam_ops.pci_ops;
> +		pp->bridge->sysdata = pp->cfg;
> +	}

On 12/4/2024 12:25 AM, Bjorn Helgaas wrote:
> On Sun, Nov 17, 2024 at 03:30:19AM +0530, Krishna chaitanya chundru wrote:
>> The current implementation requires iATU for every configuration
>> space access which increases latency & cpu utilization.
>>
>> Configuring iATU in config shift mode enables ECAM feature to access the
>> config space, which avoids iATU configuration for every config access.
>>
>> Add "ctrl2" into struct dw_pcie_ob_atu_cfg  to enable config shift mode.
>>
>> As DBI comes under config space, this avoids remapping of DBI space
>> separately. Instead, it uses the mapped config space address returned from
>> ECAM initialization. Change the order of dw_pcie_get_resources() execution
>> to acheive this.
> 
> s/acheive/achieve/
> 
>> Introduce new ecam_init() function op for the clients to configure after
>> ecam window creation has been done.
>>
>> Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
>> ---
>>   drivers/pci/controller/dwc/pcie-designware-host.c | 114 ++++++++++++++++++----
>>   drivers/pci/controller/dwc/pcie-designware.c      |   2 +-
>>   drivers/pci/controller/dwc/pcie-designware.h      |   6 ++
>>   3 files changed, 102 insertions(+), 20 deletions(-)
>>
>> diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
>> index 3e41865c7290..e98cc841a2a9 100644
>> --- a/drivers/pci/controller/dwc/pcie-designware-host.c
>> +++ b/drivers/pci/controller/dwc/pcie-designware-host.c
>> @@ -418,6 +418,62 @@ static void dw_pcie_host_request_msg_tlp_res(struct dw_pcie_rp *pp)
>>   	}
>>   }
>>   
>> +static int dw_pcie_config_ecam_iatu(struct dw_pcie_rp *pp)
>> +{
>> +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
>> +	struct dw_pcie_ob_atu_cfg atu = {0};
>> +	struct resource_entry *bus;
>> +	int ret, bus_range_max;
>> +
>> +	bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
>> +
>> +	/*
>> +	 * Bus 1 config space needs type 0 atu configuration
>> +	 * Remaining buses need type 1 atu configuration
> 
> s/atu/ATU/ (initialism, looks like "iATU" might be appropriate here?)
> 
> I'm confused about the bus numbering; you refer to "bus 1" and "bus
> 2".  Is bus 1 the root bus, i.e., the primary bus of a Root Port?
> 
> The root bus number would typically be 0, not 1, and is sometimes
> programmable.  I don't know how the DesignWare core works, but since
> you have "bus" here, referring to "bus 1" and "bus 2" here seems
> overly specific.
> 
root bus is bus 0 and we don't need any iATU configuration for it as
its config space is accessible from the system memory, for usp port of
the switch or the direct the endpoint i.e bus 1 we need to send
Configuration Type 0 requests and for other buses we need to send
Configuration Type 1 requests this is as per PCIe spec, I will try to
include PCIe spec details in next patch.
>> +	 */
>> +	atu.index = 0;
>> +	atu.type = PCIE_ATU_TYPE_CFG0;
>> +	atu.cpu_addr = pp->cfg0_base + SZ_1M;
>> +	atu.size = SZ_1M;
>> +	atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE;
>> +	ret = dw_pcie_prog_outbound_atu(pci, &atu);
>> +	if (ret)
>> +		return ret;
>> +
>> +	bus_range_max = bus->res->end - bus->res->start + 1;
>> +
>> +	/* Configure for bus 2 - bus_range_max in type 1 */
>> +	atu.index = 1;
>> +	atu.type = PCIE_ATU_TYPE_CFG1;
>> +	atu.cpu_addr = pp->cfg0_base + SZ_2M;
>> +	atu.size = (SZ_1M * (bus_range_max - 2));
>> +	atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE;
>> +	return dw_pcie_prog_outbound_atu(pci, &atu);
>> +}
>> +
>> +static int dw_pcie_create_ecam_window(struct dw_pcie_rp *pp, struct resource *res)
>> +{
>> +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
>> +	struct device *dev = pci->dev;
>> +	struct resource_entry *bus;
>> +
>> +	bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
>> +	if (!bus)
>> +		return -ENODEV;
>> +
>> +	pp->cfg = pci_ecam_create(dev, res, bus->res, &pci_generic_ecam_ops);
>> +	if (IS_ERR(pp->cfg))
>> +		return PTR_ERR(pp->cfg);
>> +
>> +	pci->dbi_base = pp->cfg->win;
>> +	pci->dbi_phys_addr = res->start;
>> +
>> +	if (pp->ops->ecam_init)
>> +		pp->ops->ecam_init(pci, pp->cfg);
> 
> .ecam_init() is defined to return int, but you ignore the return value.
> If it's practical, I think it would be nicer if you could manage to:
> 
>    - Drop .enable_ecam.
> 
>    - Have .ecam_init() return failure if there's not enough ECAM space
>      or whatever, i.e., move the qcom_pcie_check_ecam_support() code
>      there.
> 
>    - Handle .ecam_init() failure here by doing whatever we did before.
> 
> If there's no useful return value from .ecam_init(), make it void.
> 
In controller driver we need to skip few thing if we want to enable
this feature before calling dw_pcie_host_init, better to have this way
>> +	return 0;
>> +}
> 
>> @@ -454,6 +499,30 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
>>   
>>   	pp->bridge = bridge;
>>   
>> +	pp->cfg0_size = resource_size(res);
>> +	pp->cfg0_base = res->start;
>> +
>> +	if (!pp->enable_ecam) {
> 
> If you can't get rid of .enable_ecam, reverse order so this uses
> positive logic:
> 
>    if (pp->enable_ecam) {
>      ...
>    } else {
>      ...
>    }
> 
ack.

- Krishna Chaitanya.
>> +		pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res);
>> +		if (IS_ERR(pp->va_cfg0_base))
>> +			return PTR_ERR(pp->va_cfg0_base);
>> +
>> +		/* Set default bus ops */
>> +		bridge->ops = &dw_pcie_ops;
>> +		bridge->child_ops = &dw_child_pcie_ops;
>> +		bridge->sysdata = pp;
>> +	} else {
>> +		ret = dw_pcie_create_ecam_window(pp, res);
>> +		if (ret)
>> +			return ret;
>> +		bridge->ops = (struct pci_ops *)&pci_generic_ecam_ops.pci_ops;
>> +		pp->bridge->sysdata = pp->cfg;
>> +	}
>

On Wed, Dec 04, 2024 at 07:45:29AM +0530, Krishna Chaitanya Chundru wrote:
> On 12/4/2024 12:25 AM, Bjorn Helgaas wrote:
> > On Sun, Nov 17, 2024 at 03:30:19AM +0530, Krishna chaitanya chundru wrote:
> > > The current implementation requires iATU for every configuration
> > > space access which increases latency & cpu utilization.
> > > 
> > > Configuring iATU in config shift mode enables ECAM feature to access the
> > > config space, which avoids iATU configuration for every config access.

> > > +static int dw_pcie_config_ecam_iatu(struct dw_pcie_rp *pp)
> > > +{
> > > +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> > > +	struct dw_pcie_ob_atu_cfg atu = {0};
> > > +	struct resource_entry *bus;
> > > +	int ret, bus_range_max;
> > > +
> > > +	bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
> > > +
> > > +	/*
> > > +	 * Bus 1 config space needs type 0 atu configuration
> > > +	 * Remaining buses need type 1 atu configuration
> > 
> > I'm confused about the bus numbering; you refer to "bus 1" and "bus
> > 2".  Is bus 1 the root bus, i.e., the primary bus of a Root Port?
> > 
> > The root bus number would typically be 0, not 1, and is sometimes
> > programmable.  I don't know how the DesignWare core works, but since
> > you have "bus" here, referring to "bus 1" and "bus 2" here seems
> > overly specific.
> > 
> root bus is bus 0 and we don't need any iATU configuration for it as
> its config space is accessible from the system memory, for usp port of
> the switch or the direct the endpoint i.e bus 1 we need to send
> Configuration Type 0 requests and for other buses we need to send
> Configuration Type 1 requests this is as per PCIe spec, I will try to
> include PCIe spec details in next patch.

I understand the Type 0/Type 1 differences.  The question is whether
the root bus number is hard-wired to 0.

I don't think specifying "bus 1" really adds anything.  The point is
that we need Type 0 accesses for anything directly below a Root Port
(regardless of what the RP's secondary bus number is), and Type 1 for
things deeper.

When DWC supports multiple Root Ports in a Root Complex, they will not
all have a secondary bus number of 1.

Bjorn

On 12/5/2024 3:47 AM, Bjorn Helgaas wrote:
> On Wed, Dec 04, 2024 at 07:45:29AM +0530, Krishna Chaitanya Chundru wrote:
>> On 12/4/2024 12:25 AM, Bjorn Helgaas wrote:
>>> On Sun, Nov 17, 2024 at 03:30:19AM +0530, Krishna chaitanya chundru wrote:
>>>> The current implementation requires iATU for every configuration
>>>> space access which increases latency & cpu utilization.
>>>>
>>>> Configuring iATU in config shift mode enables ECAM feature to access the
>>>> config space, which avoids iATU configuration for every config access.
> 
>>>> +static int dw_pcie_config_ecam_iatu(struct dw_pcie_rp *pp)
>>>> +{
>>>> +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
>>>> +	struct dw_pcie_ob_atu_cfg atu = {0};
>>>> +	struct resource_entry *bus;
>>>> +	int ret, bus_range_max;
>>>> +
>>>> +	bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
>>>> +
>>>> +	/*
>>>> +	 * Bus 1 config space needs type 0 atu configuration
>>>> +	 * Remaining buses need type 1 atu configuration
>>>
>>> I'm confused about the bus numbering; you refer to "bus 1" and "bus
>>> 2".  Is bus 1 the root bus, i.e., the primary bus of a Root Port?
>>>
>>> The root bus number would typically be 0, not 1, and is sometimes
>>> programmable.  I don't know how the DesignWare core works, but since
>>> you have "bus" here, referring to "bus 1" and "bus 2" here seems
>>> overly specific.
>>>
>> root bus is bus 0 and we don't need any iATU configuration for it as
>> its config space is accessible from the system memory, for usp port of
>> the switch or the direct the endpoint i.e bus 1 we need to send
>> Configuration Type 0 requests and for other buses we need to send
>> Configuration Type 1 requests this is as per PCIe spec, I will try to
>> include PCIe spec details in next patch.
> 
> I understand the Type 0/Type 1 differences.  The question is whether
> the root bus number is hard-wired to 0.
> 
It is not hard-wired to 0, we can configure it though bus-range property
> I don't think specifying "bus 1" really adds anything.  The point is
> that we need Type 0 accesses for anything directly below a Root Port
> (regardless of what the RP's secondary bus number is), and Type 1 for
> things deeper.
> 
I will update the comment without mentioning the buses as suggested.
> When DWC supports multiple Root Ports in a Root Complex, they will not
> all have a secondary bus number of 1.
mostly they should be in bus number 0 with different device numbers, but
it mostly depends upon the design, currently we don't have any multiple
root ports.

- Krishna Chaitanya.
> 
> Bjorn

On Mon, Dec 09, 2024 at 10:00:06AM +0530, Krishna Chaitanya Chundru wrote:
> On 12/5/2024 3:47 AM, Bjorn Helgaas wrote:
> > On Wed, Dec 04, 2024 at 07:45:29AM +0530, Krishna Chaitanya Chundru wrote:
> > > On 12/4/2024 12:25 AM, Bjorn Helgaas wrote:
> > > > On Sun, Nov 17, 2024 at 03:30:19AM +0530, Krishna chaitanya chundru wrote:
> > > > > The current implementation requires iATU for every configuration
> > > > > space access which increases latency & cpu utilization.
> > > > > 
> > > > > Configuring iATU in config shift mode enables ECAM feature to access the
> > > > > config space, which avoids iATU configuration for every config access.
> > 
> > > > > +static int dw_pcie_config_ecam_iatu(struct dw_pcie_rp *pp)
> > > > > +{
> > > > > +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> > > > > +	struct dw_pcie_ob_atu_cfg atu = {0};
> > > > > +	struct resource_entry *bus;
> > > > > +	int ret, bus_range_max;
> > > > > +
> > > > > +	bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
> > > > > +
> > > > > +	/*
> > > > > +	 * Bus 1 config space needs type 0 atu configuration
> > > > > +	 * Remaining buses need type 1 atu configuration
> > > > 
> > > > I'm confused about the bus numbering; you refer to "bus 1" and "bus
> > > > 2".  Is bus 1 the root bus, i.e., the primary bus of a Root Port?
> > > > 
> > > > The root bus number would typically be 0, not 1, and is sometimes
> > > > programmable.  I don't know how the DesignWare core works, but since
> > > > you have "bus" here, referring to "bus 1" and "bus 2" here seems
> > > > overly specific.
> > > > 
> > > root bus is bus 0 and we don't need any iATU configuration for it as
> > > its config space is accessible from the system memory, for usp port of
> > > the switch or the direct the endpoint i.e bus 1 we need to send
> > > Configuration Type 0 requests and for other buses we need to send
> > > Configuration Type 1 requests this is as per PCIe spec, I will try to
> > > include PCIe spec details in next patch.
> > 
> > I understand the Type 0/Type 1 differences.  The question is whether
> > the root bus number is hard-wired to 0.
> > 
> It is not hard-wired to 0, we can configure it though bus-range property
>
> > I don't think specifying "bus 1" really adds anything.  The point is
> > that we need Type 0 accesses for anything directly below a Root Port
> > (regardless of what the RP's secondary bus number is), and Type 1 for
> > things deeper.
>
> I will update the comment without mentioning the buses as suggested.
>
> > When DWC supports multiple Root Ports in a Root Complex, they will not
> > all have a secondary bus number of 1.
>
> mostly they should be in bus number 0 with different device numbers, but
> it mostly depends upon the design, currently we don't have any multiple
> root ports.

Say "root bus" instead of "bus 0", since you said above that the root
bus number is configurable.

Root Ports should all have a *primary* bus number of the root bus, but
if there are multiple Root Ports, they will all have different
secondary bus numbers.

On Sun, Nov 17, 2024 at 03:30:19AM +0530, Krishna chaitanya chundru wrote:
> The current implementation requires iATU for every configuration
> space access which increases latency & cpu utilization.
> 
> Configuring iATU in config shift mode enables ECAM feature to access the

Can you please elaborate 'config shift mode'? Quote relevant section in DWC
databook for reference.

> config space, which avoids iATU configuration for every config access.
> 
> Add "ctrl2" into struct dw_pcie_ob_atu_cfg  to enable config shift mode.
> 
> As DBI comes under config space, this avoids remapping of DBI space
> separately. Instead, it uses the mapped config space address returned from
> ECAM initialization. Change the order of dw_pcie_get_resources() execution
> to acheive this.
> 
> Introduce new ecam_init() function op for the clients to configure after

We use 'DWC glue drivers' to refer the 'clients' of this driver.

> ecam window creation has been done.
> 

Use 'ECAM' everywhere.

> Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
> ---
>  drivers/pci/controller/dwc/pcie-designware-host.c | 114 ++++++++++++++++++----
>  drivers/pci/controller/dwc/pcie-designware.c      |   2 +-
>  drivers/pci/controller/dwc/pcie-designware.h      |   6 ++
>  3 files changed, 102 insertions(+), 20 deletions(-)
> 
> diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
> index 3e41865c7290..e98cc841a2a9 100644
> --- a/drivers/pci/controller/dwc/pcie-designware-host.c
> +++ b/drivers/pci/controller/dwc/pcie-designware-host.c
> @@ -418,6 +418,62 @@ static void dw_pcie_host_request_msg_tlp_res(struct dw_pcie_rp *pp)
>  	}
>  }
>  
> +static int dw_pcie_config_ecam_iatu(struct dw_pcie_rp *pp)
> +{
> +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> +	struct dw_pcie_ob_atu_cfg atu = {0};
> +	struct resource_entry *bus;
> +	int ret, bus_range_max;
> +
> +	bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
> +
> +	/*
> +	 * Bus 1 config space needs type 0 atu configuration
> +	 * Remaining buses need type 1 atu configuration
> +	 */
> +	atu.index = 0;
> +	atu.type = PCIE_ATU_TYPE_CFG0;
> +	atu.cpu_addr = pp->cfg0_base + SZ_1M;

You didn't mention what occupies the first 1MB.

> +	atu.size = SZ_1M;
> +	atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE;
> +	ret = dw_pcie_prog_outbound_atu(pci, &atu);
> +	if (ret)
> +		return ret;
> +
> +	bus_range_max = bus->res->end - bus->res->start + 1;

resource_size(bus->res)

> +
> +	/* Configure for bus 2 - bus_range_max in type 1 */
> +	atu.index = 1;
> +	atu.type = PCIE_ATU_TYPE_CFG1;
> +	atu.cpu_addr = pp->cfg0_base + SZ_2M;
> +	atu.size = (SZ_1M * (bus_range_max - 2));
> +	atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE;
> +	return dw_pcie_prog_outbound_atu(pci, &atu);
> +}
> +
> +static int dw_pcie_create_ecam_window(struct dw_pcie_rp *pp, struct resource *res)
> +{
> +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> +	struct device *dev = pci->dev;
> +	struct resource_entry *bus;
> +
> +	bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
> +	if (!bus)
> +		return -ENODEV;
> +
> +	pp->cfg = pci_ecam_create(dev, res, bus->res, &pci_generic_ecam_ops);
> +	if (IS_ERR(pp->cfg))
> +		return PTR_ERR(pp->cfg);
> +
> +	pci->dbi_base = pp->cfg->win;
> +	pci->dbi_phys_addr = res->start;
> +
> +	if (pp->ops->ecam_init)
> +		pp->ops->ecam_init(pci, pp->cfg);
> +
> +	return 0;
> +}
> +
>  int dw_pcie_host_init(struct dw_pcie_rp *pp)
>  {
>  	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> @@ -431,19 +487,8 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
>  
>  	raw_spin_lock_init(&pp->lock);
>  
> -	ret = dw_pcie_get_resources(pci);
> -	if (ret)
> -		return ret;
> -
>  	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config");
> -	if (res) {
> -		pp->cfg0_size = resource_size(res);
> -		pp->cfg0_base = res->start;
> -
> -		pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res);
> -		if (IS_ERR(pp->va_cfg0_base))
> -			return PTR_ERR(pp->va_cfg0_base);
> -	} else {
> +	if (!res) {
>  		dev_err(dev, "Missing *config* reg space\n");
>  		return -ENODEV;
>  	}
> @@ -454,6 +499,30 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
>  
>  	pp->bridge = bridge;
>  
> +	pp->cfg0_size = resource_size(res);
> +	pp->cfg0_base = res->start;
> +
> +	if (!pp->enable_ecam) {

Why can't you just use the ECAM mode when there is enough memory defined in DT?
Using this flag slightly defeats the purpose of the ECAM mode.

> +		pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res);
> +		if (IS_ERR(pp->va_cfg0_base))
> +			return PTR_ERR(pp->va_cfg0_base);
> +
> +		/* Set default bus ops */
> +		bridge->ops = &dw_pcie_ops;
> +		bridge->child_ops = &dw_child_pcie_ops;
> +		bridge->sysdata = pp;
> +	} else {
> +		ret = dw_pcie_create_ecam_window(pp, res);
> +		if (ret)
> +			return ret;
> +		bridge->ops = (struct pci_ops *)&pci_generic_ecam_ops.pci_ops;
> +		pp->bridge->sysdata = pp->cfg;

'bridge->sysdata = pp->cfg'?

- Mani

-- 
மணிவண்ணன் சதாசிவம்

On 12/2/2024 10:12 PM, Manivannan Sadhasivam wrote:
> On Sun, Nov 17, 2024 at 03:30:19AM +0530, Krishna chaitanya chundru wrote:
>> The current implementation requires iATU for every configuration
>> space access which increases latency & cpu utilization.
>>
>> Configuring iATU in config shift mode enables ECAM feature to access the
> 
> Can you please elaborate 'config shift mode'? Quote relevant section in DWC
> databook for reference.
> 
>> config space, which avoids iATU configuration for every config access.
>>
>> Add "ctrl2" into struct dw_pcie_ob_atu_cfg  to enable config shift mode.
>>
>> As DBI comes under config space, this avoids remapping of DBI space
>> separately. Instead, it uses the mapped config space address returned from
>> ECAM initialization. Change the order of dw_pcie_get_resources() execution
>> to acheive this.
>>
>> Introduce new ecam_init() function op for the clients to configure after
> 
> We use 'DWC glue drivers' to refer the 'clients' of this driver.
> 
>> ecam window creation has been done.
>>
> 
> Use 'ECAM' everywhere.
> 
>> Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
>> ---
>>   drivers/pci/controller/dwc/pcie-designware-host.c | 114 ++++++++++++++++++----
>>   drivers/pci/controller/dwc/pcie-designware.c      |   2 +-
>>   drivers/pci/controller/dwc/pcie-designware.h      |   6 ++
>>   3 files changed, 102 insertions(+), 20 deletions(-)
>>
>> diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
>> index 3e41865c7290..e98cc841a2a9 100644
>> --- a/drivers/pci/controller/dwc/pcie-designware-host.c
>> +++ b/drivers/pci/controller/dwc/pcie-designware-host.c
>> @@ -418,6 +418,62 @@ static void dw_pcie_host_request_msg_tlp_res(struct dw_pcie_rp *pp)
>>   	}
>>   }
>>   
>> +static int dw_pcie_config_ecam_iatu(struct dw_pcie_rp *pp)
>> +{
>> +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
>> +	struct dw_pcie_ob_atu_cfg atu = {0};
>> +	struct resource_entry *bus;
>> +	int ret, bus_range_max;
>> +
>> +	bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
>> +
>> +	/*
>> +	 * Bus 1 config space needs type 0 atu configuration
>> +	 * Remaining buses need type 1 atu configuration
>> +	 */
>> +	atu.index = 0;
>> +	atu.type = PCIE_ATU_TYPE_CFG0;
>> +	atu.cpu_addr = pp->cfg0_base + SZ_1M;
> 
> You didn't mention what occupies the first 1MB.
> 
>> +	atu.size = SZ_1M;
>> +	atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE;
>> +	ret = dw_pcie_prog_outbound_atu(pci, &atu);
>> +	if (ret)
>> +		return ret;
>> +
>> +	bus_range_max = bus->res->end - bus->res->start + 1;
> 
> resource_size(bus->res)
> 
>> +
>> +	/* Configure for bus 2 - bus_range_max in type 1 */
>> +	atu.index = 1;
>> +	atu.type = PCIE_ATU_TYPE_CFG1;
>> +	atu.cpu_addr = pp->cfg0_base + SZ_2M;
>> +	atu.size = (SZ_1M * (bus_range_max - 2));
>> +	atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE;
>> +	return dw_pcie_prog_outbound_atu(pci, &atu);
>> +}
>> +
>> +static int dw_pcie_create_ecam_window(struct dw_pcie_rp *pp, struct resource *res)
>> +{
>> +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
>> +	struct device *dev = pci->dev;
>> +	struct resource_entry *bus;
>> +
>> +	bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
>> +	if (!bus)
>> +		return -ENODEV;
>> +
>> +	pp->cfg = pci_ecam_create(dev, res, bus->res, &pci_generic_ecam_ops);
>> +	if (IS_ERR(pp->cfg))
>> +		return PTR_ERR(pp->cfg);
>> +
>> +	pci->dbi_base = pp->cfg->win;
>> +	pci->dbi_phys_addr = res->start;
>> +
>> +	if (pp->ops->ecam_init)
>> +		pp->ops->ecam_init(pci, pp->cfg);
>> +
>> +	return 0;
>> +}
>> +
>>   int dw_pcie_host_init(struct dw_pcie_rp *pp)
>>   {
>>   	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
>> @@ -431,19 +487,8 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
>>   
>>   	raw_spin_lock_init(&pp->lock);
>>   
>> -	ret = dw_pcie_get_resources(pci);
>> -	if (ret)
>> -		return ret;
>> -
>>   	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config");
>> -	if (res) {
>> -		pp->cfg0_size = resource_size(res);
>> -		pp->cfg0_base = res->start;
>> -
>> -		pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res);
>> -		if (IS_ERR(pp->va_cfg0_base))
>> -			return PTR_ERR(pp->va_cfg0_base);
>> -	} else {
>> +	if (!res) {
>>   		dev_err(dev, "Missing *config* reg space\n");
>>   		return -ENODEV;
>>   	}
>> @@ -454,6 +499,30 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
>>   
>>   	pp->bridge = bridge;
>>   
>> +	pp->cfg0_size = resource_size(res);
>> +	pp->cfg0_base = res->start;
>> +
>> +	if (!pp->enable_ecam) {
> 
> Why can't you just use the ECAM mode when there is enough memory defined in DT?
> Using this flag slightly defeats the purpose of the ECAM mode.
> 
>> +		pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res);
>> +		if (IS_ERR(pp->va_cfg0_base))
>> +			return PTR_ERR(pp->va_cfg0_base);
>> +
>> +		/* Set default bus ops */
>> +		bridge->ops = &dw_pcie_ops;
>> +		bridge->child_ops = &dw_child_pcie_ops;
>> +		bridge->sysdata = pp;
>> +	} else {
>> +		ret = dw_pcie_create_ecam_window(pp, res);
>> +		if (ret)
>> +			return ret;
>> +		bridge->ops = (struct pci_ops *)&pci_generic_ecam_ops.pci_ops;
>> +		pp->bridge->sysdata = pp->cfg;
> 
> 'bridge->sysdata = pp->cfg'?
>
as we are using pci_generic_ecam_ops.pci_ops for config reads & writes
it expects cfg space as sysdata[1].

[1] https://elixir.bootlin.com/linux/v6.12.1/source/drivers/pci/ecam.c#L170

- Krishna Chaitanya.

> - Mani
>