drivers/pci/controller/pci-hyperv.c | 46 ++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 10 deletions(-)
A guest can wait indefinitely in wait_for_response() for the host to
send either a rescind message or a packet completion. If the
host does not send either, the guest can remain blocked with no
diagnostic indicating a reason.
This was observed during a guest kernel upgrade in which the
host-side application handling the PCI channel faulted, causing the
guest to never receive the completion request.
Add a warning in wait_for_response() when the wait exceeds
a timeout so that such a hang is visible in the guest's kernel log
and can be correlated with host-side state.
Suggested-by: Michael Kelley <mhklinux@outlook.com>
Signed-off-by: Sahil Chandna <sahilchandna@linux.microsoft.com>
---
Changes since v2:
- Add counter based timeout instead of introducing another timer
Link to v2: https://lore.kernel.org/all/20260902115854.2629164-1-sahilchandna@linux.microsoft.com/
Changes since v1:
- Removed periodic warning to one time warning in 2 minutes
- Include vmbus relid and stuck PCI msg.
Link to v1: https://lore.kernel.org/all/20260825051850.2438816-1-sahilchandna@linux.microsoft.com/
drivers/pci/controller/pci-hyperv.c | 46 ++++++++++++++++++++++-------
1 file changed, 36 insertions(+), 10 deletions(-)
diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c
index 89816a2bd7cd..bd07402c47ed 100644
--- a/drivers/pci/controller/pci-hyperv.c
+++ b/drivers/pci/controller/pci-hyperv.c
@@ -1040,19 +1040,38 @@ static void put_pcichild(struct hv_pci_dev *hpdev)
/*
* There is no good way to get notified from vmbus_onoffer_rescind(),
- * so let's use polling here, since this is not a hot path.
+ * so let's use polling here, since this is not a hot path. If
+ * wait_for_response() has been polling for 2 minutes
+ * without either a rescind or completion, add a warning.
*/
+#define PCI_RESPONSE_HANG_TICKS 1200
+
static int wait_for_response(struct hv_device *hdev,
- struct completion *comp)
+ struct completion *comp,
+ const char *msg_type)
{
+ u64 counter = 0;
+
while (true) {
if (hdev->channel->rescind) {
dev_warn_once(&hdev->device, "The device is gone.\n");
return -ENODEV;
}
- if (wait_for_completion_timeout(comp, HZ / 10))
+ counter++;
+
+ if (wait_for_completion_timeout(comp, HZ / 10)) {
+ if (counter > PCI_RESPONSE_HANG_TICKS)
+ dev_warn(&hdev->device,
+ "Late %s completion arrived.\n", msg_type);
break;
+ }
+
+ if (counter == PCI_RESPONSE_HANG_TICKS) {
+ dev_err(&hdev->device,
+ "%s stuck waiting for response, relid = %u\n",
+ msg_type, hdev->channel->offermsg.child_relid);
+ }
}
return 0;
@@ -1518,7 +1537,8 @@ static int hv_read_config_block(struct pci_dev *pdev, void *buf,
if (ret)
return ret;
- ret = wait_for_response(hbus->hdev, &comp_pkt.comp_pkt.host_event);
+ ret = wait_for_response(hbus->hdev, &comp_pkt.comp_pkt.host_event,
+ "PCI_READ_BLOCK");
if (ret)
return ret;
@@ -1607,7 +1627,8 @@ static int hv_write_config_block(struct pci_dev *pdev, void *buf,
if (ret)
return ret;
- ret = wait_for_response(hbus->hdev, &comp_pkt.host_event);
+ ret = wait_for_response(hbus->hdev, &comp_pkt.host_event,
+ "PCI_WRITE_BLOCK");
if (ret)
return ret;
@@ -2624,7 +2645,8 @@ static struct hv_pci_dev *new_pcichild_device(struct hv_pcibus_device *hbus,
if (ret)
goto error;
- if (wait_for_response(hbus->hdev, &comp_pkt.host_event))
+ if (wait_for_response(hbus->hdev, &comp_pkt.host_event,
+ "PCI_QUERY_RESOURCE_REQUIREMENTS"))
goto error;
hpdev->desc = *desc;
@@ -3256,7 +3278,8 @@ static int hv_pci_protocol_negotiation(struct hv_device *hdev,
(unsigned long)pkt, VM_PKT_DATA_INBAND,
VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
if (!ret)
- ret = wait_for_response(hdev, &comp_pkt.host_event);
+ ret = wait_for_response(hdev, &comp_pkt.host_event,
+ "PCI_QUERY_PROTOCOL_VERSION");
if (ret) {
dev_err(&hdev->device,
@@ -3476,7 +3499,8 @@ static int hv_pci_enter_d0(struct hv_device *hdev)
(unsigned long)pkt, VM_PKT_DATA_INBAND,
VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
if (!ret)
- ret = wait_for_response(hdev, &comp_pkt.host_event);
+ ret = wait_for_response(hdev, &comp_pkt.host_event,
+ "PCI_BUS_D0ENTRY");
if (ret)
goto exit;
@@ -3553,7 +3577,8 @@ static int hv_pci_query_relations(struct hv_device *hdev)
ret = vmbus_sendpacket(hdev->channel, &message, sizeof(message),
0, VM_PKT_DATA_INBAND, 0);
if (!ret)
- ret = wait_for_response(hdev, &comp);
+ ret = wait_for_response(hdev, &comp,
+ "PCI_QUERY_BUS_RELATIONS");
/*
* In the case of fast device addition/removal, it's possible that
@@ -3644,7 +3669,8 @@ static int hv_send_resources_allocated(struct hv_device *hdev)
VM_PKT_DATA_INBAND,
VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
if (!ret)
- ret = wait_for_response(hdev, &comp_pkt.host_event);
+ ret = wait_for_response(hdev, &comp_pkt.host_event,
+ "PCI_RESOURCE_ASSIGNED");
if (ret)
break;
--
2.53.0On 9/8/2026 5:00 PM, Sahil Chandna wrote: > A guest can wait indefinitely in wait_for_response() for the host to > send either a rescind message or a packet completion. If the > host does not send either, the guest can remain blocked with no > diagnostic indicating a reason. > This was observed during a guest kernel upgrade in which the > host-side application handling the PCI channel faulted, causing the > guest to never receive the completion request. > Add a warning in wait_for_response() when the wait exceeds > a timeout so that such a hang is visible in the guest's kernel log > and can be correlated with host-side state. > > Suggested-by: Michael Kelley <mhklinux@outlook.com> > Signed-off-by: Sahil Chandna <sahilchandna@linux.microsoft.com> > --- > Changes since v2: > - Add counter based timeout instead of introducing another timer > Link to v2: https://lore.kernel.org/all/20260902115854.2629164-1-sahilchandna@linux.microsoft.com/ > > Changes since v1: > - Removed periodic warning to one time warning in 2 minutes > - Include vmbus relid and stuck PCI msg. > Link to v1: https://lore.kernel.org/all/20260825051850.2438816-1-sahilchandna@linux.microsoft.com/ > drivers/pci/controller/pci-hyperv.c | 46 ++++++++++++++++++++++------- > 1 file changed, 36 insertions(+), 10 deletions(-) > > diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c > index 89816a2bd7cd..bd07402c47ed 100644 > --- a/drivers/pci/controller/pci-hyperv.c > +++ b/drivers/pci/controller/pci-hyperv.c > @@ -1040,19 +1040,38 @@ static void put_pcichild(struct hv_pci_dev *hpdev) > > /* > * There is no good way to get notified from vmbus_onoffer_rescind(), > - * so let's use polling here, since this is not a hot path. > + * so let's use polling here, since this is not a hot path. If > + * wait_for_response() has been polling for 2 minutes > + * without either a rescind or completion, add a warning. > */ > +#define PCI_RESPONSE_HANG_TICKS 1200 > + Nit, I think this variable name could be misleading, as this represents a loop counter for a retry logic, but not exactly *ticks* from kernel terminology. I would have preferred something like: PCI_RESPONSE_WARN_POLL_COUNT. Rest LGTM. Reviewed-by: Naman Jain <namjain@linux.microsoft.com> Regards, Naman Jain
On Wed, Sep 09, 2026 at 03:01:22PM +0530, Naman Jain wrote: > > > On 9/8/2026 5:00 PM, Sahil Chandna wrote: > > A guest can wait indefinitely in wait_for_response() for the host to > > send either a rescind message or a packet completion. If the > > host does not send either, the guest can remain blocked with no > > diagnostic indicating a reason. > > This was observed during a guest kernel upgrade in which the > > host-side application handling the PCI channel faulted, causing the > > guest to never receive the completion request. > > Add a warning in wait_for_response() when the wait exceeds > > a timeout so that such a hang is visible in the guest's kernel log > > and can be correlated with host-side state. > > > > Suggested-by: Michael Kelley <mhklinux@outlook.com> > > Signed-off-by: Sahil Chandna <sahilchandna@linux.microsoft.com> > > --- > > Changes since v2: > > - Add counter based timeout instead of introducing another timer > > Link to v2: https://lore.kernel.org/all/20260902115854.2629164-1-sahilchandna@linux.microsoft.com/ > > > > Changes since v1: > > - Removed periodic warning to one time warning in 2 minutes > > - Include vmbus relid and stuck PCI msg. > > Link to v1: https://lore.kernel.org/all/20260825051850.2438816-1-sahilchandna@linux.microsoft.com/ > > drivers/pci/controller/pci-hyperv.c | 46 ++++++++++++++++++++++------- > > 1 file changed, 36 insertions(+), 10 deletions(-) > > > > diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c > > index 89816a2bd7cd..bd07402c47ed 100644 > > --- a/drivers/pci/controller/pci-hyperv.c > > +++ b/drivers/pci/controller/pci-hyperv.c > > @@ -1040,19 +1040,38 @@ static void put_pcichild(struct hv_pci_dev *hpdev) > > > > /* > > * There is no good way to get notified from vmbus_onoffer_rescind(), > > - * so let's use polling here, since this is not a hot path. > > + * so let's use polling here, since this is not a hot path. If > > + * wait_for_response() has been polling for 2 minutes > > + * without either a rescind or completion, add a warning. > > */ > > +#define PCI_RESPONSE_HANG_TICKS 1200 > > + > > Nit, I think this variable name could be misleading, as this represents a > loop counter for a retry logic, but not exactly *ticks* from kernel > terminology. > > I would have preferred something like: PCI_RESPONSE_WARN_POLL_COUNT. > PCI_RESPONSE_POLL_COUNT? - Mani -- மணிவண்ணன் சதாசிவம்
On Fri, Sep 11, 2026 at 12:29:05PM +0200, Manivannan Sadhasivam wrote: > On Wed, Sep 09, 2026 at 03:01:22PM +0530, Naman Jain wrote: > > > > > > On 9/8/2026 5:00 PM, Sahil Chandna wrote: > > > A guest can wait indefinitely in wait_for_response() for the host to > > > send either a rescind message or a packet completion. If the > > > host does not send either, the guest can remain blocked with no > > > diagnostic indicating a reason. > > > This was observed during a guest kernel upgrade in which the > > > host-side application handling the PCI channel faulted, causing the > > > guest to never receive the completion request. > > > Add a warning in wait_for_response() when the wait exceeds > > > a timeout so that such a hang is visible in the guest's kernel log > > > and can be correlated with host-side state. > > > > > > Suggested-by: Michael Kelley <mhklinux@outlook.com> > > > Signed-off-by: Sahil Chandna <sahilchandna@linux.microsoft.com> > > > --- > > > Changes since v2: > > > - Add counter based timeout instead of introducing another timer > > > Link to v2: https://lore.kernel.org/all/20260902115854.2629164-1-sahilchandna@linux.microsoft.com/ > > > > > > Changes since v1: > > > - Removed periodic warning to one time warning in 2 minutes > > > - Include vmbus relid and stuck PCI msg. > > > Link to v1: https://lore.kernel.org/all/20260825051850.2438816-1-sahilchandna@linux.microsoft.com/ > > > drivers/pci/controller/pci-hyperv.c | 46 ++++++++++++++++++++++------- > > > 1 file changed, 36 insertions(+), 10 deletions(-) > > > > > > diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c > > > index 89816a2bd7cd..bd07402c47ed 100644 > > > --- a/drivers/pci/controller/pci-hyperv.c > > > +++ b/drivers/pci/controller/pci-hyperv.c > > > @@ -1040,19 +1040,38 @@ static void put_pcichild(struct hv_pci_dev *hpdev) > > > > > > /* > > > * There is no good way to get notified from vmbus_onoffer_rescind(), > > > - * so let's use polling here, since this is not a hot path. > > > + * so let's use polling here, since this is not a hot path. If > > > + * wait_for_response() has been polling for 2 minutes > > > + * without either a rescind or completion, add a warning. > > > */ > > > +#define PCI_RESPONSE_HANG_TICKS 1200 > > > + > > > > Nit, I think this variable name could be misleading, as this represents a > > loop counter for a retry logic, but not exactly *ticks* from kernel > > terminology. > > > > I would have preferred something like: PCI_RESPONSE_WARN_POLL_COUNT. > > > > PCI_RESPONSE_POLL_COUNT? Sahil, let me know if you will send out another version. If it is only changing one macro name, I can do that. Wei
On 14-09-2026 04:09, Wei Liu wrote: > On Fri, Sep 11, 2026 at 12:29:05PM +0200, Manivannan Sadhasivam wrote: >> On Wed, Sep 09, 2026 at 03:01:22PM +0530, Naman Jain wrote: >>> >>> >>> On 9/8/2026 5:00 PM, Sahil Chandna wrote: >>>> A guest can wait indefinitely in wait_for_response() for the host to >>>> send either a rescind message or a packet completion. If the >>>> host does not send either, the guest can remain blocked with no >>>> diagnostic indicating a reason. >>>> This was observed during a guest kernel upgrade in which the >>>> host-side application handling the PCI channel faulted, causing the >>>> guest to never receive the completion request. >>>> Add a warning in wait_for_response() when the wait exceeds >>>> a timeout so that such a hang is visible in the guest's kernel log >>>> and can be correlated with host-side state. >>>> >>>> Suggested-by: Michael Kelley <mhklinux@outlook.com> >>>> Signed-off-by: Sahil Chandna <sahilchandna@linux.microsoft.com> >>>> --- >>>> Changes since v2: >>>> - Add counter based timeout instead of introducing another timer >>>> Link to v2: https://lore.kernel.org/all/20260902115854.2629164-1-sahilchandna@linux.microsoft.com/ >>>> >>>> Changes since v1: >>>> - Removed periodic warning to one time warning in 2 minutes >>>> - Include vmbus relid and stuck PCI msg. >>>> Link to v1: https://lore.kernel.org/all/20260825051850.2438816-1-sahilchandna@linux.microsoft.com/ >>>> drivers/pci/controller/pci-hyperv.c | 46 ++++++++++++++++++++++------- >>>> 1 file changed, 36 insertions(+), 10 deletions(-) >>>> >>>> diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c >>>> index 89816a2bd7cd..bd07402c47ed 100644 >>>> --- a/drivers/pci/controller/pci-hyperv.c >>>> +++ b/drivers/pci/controller/pci-hyperv.c >>>> @@ -1040,19 +1040,38 @@ static void put_pcichild(struct hv_pci_dev *hpdev) >>>> >>>> /* >>>> * There is no good way to get notified from vmbus_onoffer_rescind(), >>>> - * so let's use polling here, since this is not a hot path. >>>> + * so let's use polling here, since this is not a hot path. If >>>> + * wait_for_response() has been polling for 2 minutes >>>> + * without either a rescind or completion, add a warning. >>>> */ >>>> +#define PCI_RESPONSE_HANG_TICKS 1200 >>>> + >>> >>> Nit, I think this variable name could be misleading, as this represents a >>> loop counter for a retry logic, but not exactly *ticks* from kernel >>> terminology. >>> >>> I would have preferred something like: PCI_RESPONSE_WARN_POLL_COUNT. Ack, I have added in v4. >>> >> >> PCI_RESPONSE_POLL_COUNT? > Hi Mani, i am inclining towards PCI_RESPONSE_WARN_POLL_COUNT as intention of change is to emit a warning when counter has exceeded a set threshold. > Sahil, let me know if you will send out another version. If it is only > changing one macro name, I can do that. > Hi Wei, I have sent out v4. Regards, Sahil > Wei
© 2016 - 2026 Red Hat, Inc.