[v7] PCI: endpoint: add D-state change notifier support

[PATCH v7 2/4] PCI: qcom-ep: Add support for D-state change notification

Posted by Krishna chaitanya chundru 1 year, 5 months ago

Add support to pass D-state change notification to Endpoint
function driver.
Read perst value to determine if the link is in D3Cold/D3hot.

Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
---
 drivers/pci/controller/dwc/pcie-qcom-ep.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c
index 236229f66c80..817fad805c51 100644
--- a/drivers/pci/controller/dwc/pcie-qcom-ep.c
+++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c
@@ -648,6 +648,7 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data)
 	struct device *dev = pci->dev;
 	u32 status = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_STATUS);
 	u32 mask = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_MASK);
+	pci_power_t state;
 	u32 dstate, val;
 
 	writel_relaxed(status, pcie_ep->parf + PARF_INT_ALL_CLEAR);
@@ -671,11 +672,16 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data)
 		dstate = dw_pcie_readl_dbi(pci, DBI_CON_STATUS) &
 					   DBI_CON_STATUS_POWER_STATE_MASK;
 		dev_dbg(dev, "Received D%d state event\n", dstate);
-		if (dstate == 3) {
+		state = dstate;
+		if (dstate == PCI_D3hot) {
 			val = readl_relaxed(pcie_ep->parf + PARF_PM_CTRL);
 			val |= PARF_PM_CTRL_REQ_EXIT_L1;
 			writel_relaxed(val, pcie_ep->parf + PARF_PM_CTRL);
+
+			if (gpiod_get_value(pcie_ep->reset))
+				state = PCI_D3cold;
 		}
+		pci_epc_dstate_notify(pci->ep.epc, state);
 	} else if (FIELD_GET(PARF_INT_ALL_LINK_UP, status)) {
 		dev_dbg(dev, "Received Linkup event. Enumeration complete!\n");
 		dw_pcie_ep_linkup(&pci->ep);

-- 
2.42.0

Re: [PATCH v7 2/4] PCI: qcom-ep: Add support for D-state change notification

Posted by Yogesh Jadav 1 year, 4 months ago


On 7/10/2024 4:38 PM, Krishna chaitanya chundru wrote:
> Add support to pass D-state change notification to Endpoint
> function driver.
> Read perst value to determine if the link is in D3Cold/D3hot.
> 
> Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
> Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
> ---
>   drivers/pci/controller/dwc/pcie-qcom-ep.c | 8 +++++++-
>   1 file changed, 7 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c
> index 236229f66c80..817fad805c51 100644
> --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c
> +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c
> @@ -648,6 +648,7 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data)
>   	struct device *dev = pci->dev;
>   	u32 status = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_STATUS);
>   	u32 mask = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_MASK);
> +	pci_power_t state;
>   	u32 dstate, val;
>   
>   	writel_relaxed(status, pcie_ep->parf + PARF_INT_ALL_CLEAR);
> @@ -671,11 +672,16 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data)
>   		dstate = dw_pcie_readl_dbi(pci, DBI_CON_STATUS) &
>   					   DBI_CON_STATUS_POWER_STATE_MASK;
>   		dev_dbg(dev, "Received D%d state event\n", dstate);
> -		if (dstate == 3) {
> +		state = dstate;
Can we use some meaningful name for variable "state" ? There is 
different purpose of variable "state" and "dstate" which is not getting 
reflected by looking variable names.

- Yogesh
> +		if (dstate == PCI_D3hot) {
>   			val = readl_relaxed(pcie_ep->parf + PARF_PM_CTRL);
>   			val |= PARF_PM_CTRL_REQ_EXIT_L1;
>   			writel_relaxed(val, pcie_ep->parf + PARF_PM_CTRL);
> +
> +			if (gpiod_get_value(pcie_ep->reset))
> +				state = PCI_D3cold;
>   		}
> +		pci_epc_dstate_notify(pci->ep.epc, state);
>   	} else if (FIELD_GET(PARF_INT_ALL_LINK_UP, status)) {
>   		dev_dbg(dev, "Received Linkup event. Enumeration complete!\n");
>   		dw_pcie_ep_linkup(&pci->ep);
>

Re: [PATCH v7 2/4] PCI: qcom-ep: Add support for D-state change notification

Posted by Mayank Rana 1 year, 5 months ago


On 7/10/2024 4:08 AM, Krishna chaitanya chundru wrote:
> Add support to pass D-state change notification to Endpoint
> function driver.
> Read perst value to determine if the link is in D3Cold/D3hot.
> 
> Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
> Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
> ---
>   drivers/pci/controller/dwc/pcie-qcom-ep.c | 8 +++++++-
>   1 file changed, 7 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c
> index 236229f66c80..817fad805c51 100644
> --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c
> +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c
> @@ -648,6 +648,7 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data)
>   	struct device *dev = pci->dev;
>   	u32 status = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_STATUS);
>   	u32 mask = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_MASK);
> +	pci_power_t state;
>   	u32 dstate, val;
>   
>   	writel_relaxed(status, pcie_ep->parf + PARF_INT_ALL_CLEAR);
> @@ -671,11 +672,16 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data)
>   		dstate = dw_pcie_readl_dbi(pci, DBI_CON_STATUS) &
>   					   DBI_CON_STATUS_POWER_STATE_MASK;
>   		dev_dbg(dev, "Received D%d state event\n", dstate);
> -		if (dstate == 3) {
> +		state = dstate;
> +		if (dstate == PCI_D3hot) {
>   			val = readl_relaxed(pcie_ep->parf + PARF_PM_CTRL);
>   			val |= PARF_PM_CTRL_REQ_EXIT_L1;
>   			writel_relaxed(val, pcie_ep->parf + PARF_PM_CTRL);
Can you please also check that do we really need to bring link back out 
of L1/L1SS on receiving D3 hot ?
> +			if (gpiod_get_value(pcie_ep->reset))
> +				state = PCI_D3cold;
>   		}
> +		pci_epc_dstate_notify(pci->ep.epc, state);
>   	} else if (FIELD_GET(PARF_INT_ALL_LINK_UP, status)) {
>   		dev_dbg(dev, "Received Linkup event. Enumeration complete!\n");
>   		dw_pcie_ep_linkup(&pci->ep);
> 

Regards,
Mayank

Re: [PATCH v7 2/4] PCI: qcom-ep: Add support for D-state change notification

Posted by Bjorn Helgaas 1 year, 5 months ago

On Wed, Jul 10, 2024 at 04:38:15PM +0530, Krishna chaitanya chundru wrote:
> Add support to pass D-state change notification to Endpoint
> function driver.

Blank line between paragraphs.

> Read perst value to determine if the link is in D3Cold/D3hot.

I assume this reads the state of the PERST# signal driven by the host.
Style it to match the spec usage ("PERST#") to make that connection
clearer.

D3cold/D3hot is a device state and doesn't apply to a link.  Link
states are L0, L1, L2, L3,etc.  Also in cover letter.

I don't understand the connection between PERST# state and the device
D state.  D3cold is defined to mean main power is absent.  Is the
endpoint firmware still running when main power is absent?

> Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
> Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
> ---
>  drivers/pci/controller/dwc/pcie-qcom-ep.c | 8 +++++++-
>  1 file changed, 7 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c
> index 236229f66c80..817fad805c51 100644
> --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c
> +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c
> @@ -648,6 +648,7 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data)
>  	struct device *dev = pci->dev;
>  	u32 status = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_STATUS);
>  	u32 mask = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_MASK);
> +	pci_power_t state;
>  	u32 dstate, val;
>  
>  	writel_relaxed(status, pcie_ep->parf + PARF_INT_ALL_CLEAR);
> @@ -671,11 +672,16 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data)
>  		dstate = dw_pcie_readl_dbi(pci, DBI_CON_STATUS) &
>  					   DBI_CON_STATUS_POWER_STATE_MASK;
>  		dev_dbg(dev, "Received D%d state event\n", dstate);
> -		if (dstate == 3) {
> +		state = dstate;
> +		if (dstate == PCI_D3hot) {
>  			val = readl_relaxed(pcie_ep->parf + PARF_PM_CTRL);
>  			val |= PARF_PM_CTRL_REQ_EXIT_L1;
>  			writel_relaxed(val, pcie_ep->parf + PARF_PM_CTRL);
> +
> +			if (gpiod_get_value(pcie_ep->reset))
> +				state = PCI_D3cold;
>  		}
> +		pci_epc_dstate_notify(pci->ep.epc, state);
>  	} else if (FIELD_GET(PARF_INT_ALL_LINK_UP, status)) {
>  		dev_dbg(dev, "Received Linkup event. Enumeration complete!\n");
>  		dw_pcie_ep_linkup(&pci->ep);
> 
> -- 
> 2.42.0
>

Re: [PATCH v7 2/4] PCI: qcom-ep: Add support for D-state change notification

Posted by Krishna Chaitanya Chundru 1 year, 5 months ago


On 7/10/2024 5:41 PM, Bjorn Helgaas wrote:
> On Wed, Jul 10, 2024 at 04:38:15PM +0530, Krishna chaitanya chundru wrote:
>> Add support to pass D-state change notification to Endpoint
>> function driver.
> 
> Blank line between paragraphs.
> 
Ack.
>> Read perst value to determine if the link is in D3Cold/D3hot.
> 
> I assume this reads the state of the PERST# signal driven by the host.
> Style it to match the spec usage ("PERST#") to make that connection
> clearer.
> 
Ack.
> D3cold/D3hot is a device state and doesn't apply to a link.  Link
> states are L0, L1, L2, L3,etc.  Also in cover letter.
> 
Ack.
> I don't understand the connection between PERST# state and the device
> D state.  D3cold is defined to mean main power is absent.  Is the
> endpoint firmware still running when main power is absent?
> 
Host as part of its d3cold sequence will assert the perst. so we are
reading perst to know the link the state.

Qcom devices are drawing power from the PCIe, so even when PCIe is in
D3cold endpoint firmware can still run.

- Krishna Chaitanya.
>> Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
>> Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
>> ---
>>   drivers/pci/controller/dwc/pcie-qcom-ep.c | 8 +++++++-
>>   1 file changed, 7 insertions(+), 1 deletion(-)
>>
>> diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c
>> index 236229f66c80..817fad805c51 100644
>> --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c
>> +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c
>> @@ -648,6 +648,7 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data)
>>   	struct device *dev = pci->dev;
>>   	u32 status = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_STATUS);
>>   	u32 mask = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_MASK);
>> +	pci_power_t state;
>>   	u32 dstate, val;
>>   
>>   	writel_relaxed(status, pcie_ep->parf + PARF_INT_ALL_CLEAR);
>> @@ -671,11 +672,16 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data)
>>   		dstate = dw_pcie_readl_dbi(pci, DBI_CON_STATUS) &
>>   					   DBI_CON_STATUS_POWER_STATE_MASK;
>>   		dev_dbg(dev, "Received D%d state event\n", dstate);
>> -		if (dstate == 3) {
>> +		state = dstate;
>> +		if (dstate == PCI_D3hot) {
>>   			val = readl_relaxed(pcie_ep->parf + PARF_PM_CTRL);
>>   			val |= PARF_PM_CTRL_REQ_EXIT_L1;
>>   			writel_relaxed(val, pcie_ep->parf + PARF_PM_CTRL);
>> +
>> +			if (gpiod_get_value(pcie_ep->reset))
>> +				state = PCI_D3cold;
>>   		}
>> +		pci_epc_dstate_notify(pci->ep.epc, state);
>>   	} else if (FIELD_GET(PARF_INT_ALL_LINK_UP, status)) {
>>   		dev_dbg(dev, "Received Linkup event. Enumeration complete!\n");
>>   		dw_pcie_ep_linkup(&pci->ep);
>>
>> -- 
>> 2.42.0
>>

Re: [PATCH v7 2/4] PCI: qcom-ep: Add support for D-state change notification

Posted by Bjorn Helgaas 1 year, 5 months ago

On Thu, Jul 11, 2024 at 11:57:35AM +0530, Krishna Chaitanya Chundru wrote:
> On 7/10/2024 5:41 PM, Bjorn Helgaas wrote:
> > On Wed, Jul 10, 2024 at 04:38:15PM +0530, Krishna chaitanya chundru wrote:
> > > Add support to pass D-state change notification to Endpoint
> > > function driver.
> ...

> > I don't understand the connection between PERST# state and the device
> > D state.  D3cold is defined to mean main power is absent.  Is the
> > endpoint firmware still running when main power is absent?
>
> Host as part of its d3cold sequence will assert the perst. so we are
> reading perst to know the link the state.

I think it's true that when the device is in D3cold, PERST# will be
asserted (PCIe CEM r5.0, sec 2.2.3, fig 2-6).

But I don't think it's necessarily true that when PERST# is asserted,
the device is in D3cold.  For example, PCIe Mini CEM r2.1, sec
3.2.5.3, says "The system may also use PERST# to cause a warm reset of
the add-in card."  In a warm reset, the component remains powered up,
i.e., it is not in D3cold (PCIe r6.0, sec 6.6.1).

I would think the endpoint firmware would be able to directly read the
state of main power or the LTSSM state of the link, without having to
use PERST# to infer it.

I guess the ultimate point of figuring out D3hot vs D3cold is to
figure out whether to use PME or WAKE#?  I'm a little bit dubious
about that being racy, as I mentioned elsewhere.  If there were a way
to attempt PME and fall back to WAKE# if you can determine that PME
failed, maybe that would be safer and obviate the need for the D-state
tracking?

> Qcom devices are drawing power from the PCIe, so even when PCIe is in
> D3cold endpoint firmware can still run.
> 
> - Krishna Chaitanya.
> > > Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
> > > Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
> > > ---
> > >   drivers/pci/controller/dwc/pcie-qcom-ep.c | 8 +++++++-
> > >   1 file changed, 7 insertions(+), 1 deletion(-)
> > > 
> > > diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c
> > > index 236229f66c80..817fad805c51 100644
> > > --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c
> > > +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c
> > > @@ -648,6 +648,7 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data)
> > >   	struct device *dev = pci->dev;
> > >   	u32 status = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_STATUS);
> > >   	u32 mask = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_MASK);
> > > +	pci_power_t state;
> > >   	u32 dstate, val;
> > >   	writel_relaxed(status, pcie_ep->parf + PARF_INT_ALL_CLEAR);
> > > @@ -671,11 +672,16 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data)
> > >   		dstate = dw_pcie_readl_dbi(pci, DBI_CON_STATUS) &
> > >   					   DBI_CON_STATUS_POWER_STATE_MASK;
> > >   		dev_dbg(dev, "Received D%d state event\n", dstate);
> > > -		if (dstate == 3) {
> > > +		state = dstate;
> > > +		if (dstate == PCI_D3hot) {
> > >   			val = readl_relaxed(pcie_ep->parf + PARF_PM_CTRL);
> > >   			val |= PARF_PM_CTRL_REQ_EXIT_L1;
> > >   			writel_relaxed(val, pcie_ep->parf + PARF_PM_CTRL);
> > > +
> > > +			if (gpiod_get_value(pcie_ep->reset))
> > > +				state = PCI_D3cold;
> > >   		}
> > > +		pci_epc_dstate_notify(pci->ep.epc, state);
> > >   	} else if (FIELD_GET(PARF_INT_ALL_LINK_UP, status)) {
> > >   		dev_dbg(dev, "Received Linkup event. Enumeration complete!\n");
> > >   		dw_pcie_ep_linkup(&pci->ep);
> > > 
> > > -- 
> > > 2.42.0
> > >

Re: [PATCH v7 2/4] PCI: qcom-ep: Add support for D-state change notification

Posted by Krishna Chaitanya Chundru 1 year, 4 months ago


On 7/12/2024 12:18 AM, Bjorn Helgaas wrote:
> On Thu, Jul 11, 2024 at 11:57:35AM +0530, Krishna Chaitanya Chundru wrote:
>> On 7/10/2024 5:41 PM, Bjorn Helgaas wrote:
>>> On Wed, Jul 10, 2024 at 04:38:15PM +0530, Krishna chaitanya chundru wrote:
>>>> Add support to pass D-state change notification to Endpoint
>>>> function driver.
>> ...
> 
>>> I don't understand the connection between PERST# state and the device
>>> D state.  D3cold is defined to mean main power is absent.  Is the
>>> endpoint firmware still running when main power is absent?
>>
>> Host as part of its d3cold sequence will assert the perst. so we are
>> reading perst to know the link the state.
> 
> I think it's true that when the device is in D3cold, PERST# will be
> asserted (PCIe CEM r5.0, sec 2.2.3, fig 2-6).
> 
> But I don't think it's necessarily true that when PERST# is asserted,
> the device is in D3cold.  For example, PCIe Mini CEM r2.1, sec
> 3.2.5.3, says "The system may also use PERST# to cause a warm reset of
> the add-in card."  In a warm reset, the component remains powered up,
> i.e., it is not in D3cold (PCIe r6.0, sec 6.6.1).
> 
> I would think the endpoint firmware would be able to directly read the
> state of main power or the LTSSM state of the link, without having to
> use PERST# to infer it.
>
Ack, we will use LTSSM state to know the link state.

> I guess the ultimate point of figuring out D3hot vs D3cold is to
> figure out whether to use PME or WAKE#?  I'm a little bit dubious
> about that being racy, as I mentioned elsewhere.  If there were a way
> to attempt PME and fall back to WAKE# if you can determine that PME
> failed, maybe that would be safer and obviate the need for the D-state
> tracking?
> 
We don't have a way to know that PME is received by the host.

We can add logic to send pme in d3hot and wait for sometime for the
state to move to D0. if it doesn't move to D0 for that period, the we
can check the LTSSM state to see if it in D3cold(L2/L3) then we can use
WAKE# else return with failure.

- Krishna Chaitanya.
>> Qcom devices are drawing power from the PCIe, so even when PCIe is in
>> D3cold endpoint firmware can still run.
>>
>> - Krishna Chaitanya.
>>>> Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
>>>> Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
>>>> ---
>>>>    drivers/pci/controller/dwc/pcie-qcom-ep.c | 8 +++++++-
>>>>    1 file changed, 7 insertions(+), 1 deletion(-)
>>>>
>>>> diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c
>>>> index 236229f66c80..817fad805c51 100644
>>>> --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c
>>>> +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c
>>>> @@ -648,6 +648,7 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data)
>>>>    	struct device *dev = pci->dev;
>>>>    	u32 status = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_STATUS);
>>>>    	u32 mask = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_MASK);
>>>> +	pci_power_t state;
>>>>    	u32 dstate, val;
>>>>    	writel_relaxed(status, pcie_ep->parf + PARF_INT_ALL_CLEAR);
>>>> @@ -671,11 +672,16 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data)
>>>>    		dstate = dw_pcie_readl_dbi(pci, DBI_CON_STATUS) &
>>>>    					   DBI_CON_STATUS_POWER_STATE_MASK;
>>>>    		dev_dbg(dev, "Received D%d state event\n", dstate);
>>>> -		if (dstate == 3) {
>>>> +		state = dstate;
>>>> +		if (dstate == PCI_D3hot) {
>>>>    			val = readl_relaxed(pcie_ep->parf + PARF_PM_CTRL);
>>>>    			val |= PARF_PM_CTRL_REQ_EXIT_L1;
>>>>    			writel_relaxed(val, pcie_ep->parf + PARF_PM_CTRL);
>>>> +
>>>> +			if (gpiod_get_value(pcie_ep->reset))
>>>> +				state = PCI_D3cold;
>>>>    		}
>>>> +		pci_epc_dstate_notify(pci->ep.epc, state);
>>>>    	} else if (FIELD_GET(PARF_INT_ALL_LINK_UP, status)) {
>>>>    		dev_dbg(dev, "Received Linkup event. Enumeration complete!\n");
>>>>    		dw_pcie_ep_linkup(&pci->ep);
>>>>
>>>> -- 
>>>> 2.42.0
>>>>