During boot, linux would sometimes overwrites control of a powered off
slot before powering it on. Unfortunately QEMU interprets that as a
power off request and ejects the device.

For example:

/x86_64-softmmu/qemu-system-x86_64 -enable-kvm -S -machine q35  \
    -device pcie-root-port,id=pcie_root_port_0,slot=2,chassis=2,addr=0x2,bus=pcie.0 \
    -monitor stdio disk.qcow2
(qemu)device_add virtio-balloon-pci,id=balloon,bus=pcie_root_port_0
(qemu)cont

Balloon is deleted during guest boot.

To fix, save control beforehand and check that power
or led state actually change before ejecting.

Note: this is more a hack than a solution, ideally we'd
find a better way to detect ejects, or move away
from ejects completely and instead monitor whether
it's safe to delete device due to e.g. its power state.

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
 include/hw/pci/pcie.h              |  3 ++-
 hw/pci-bridge/pcie_root_port.c     |  5 ++++-
 hw/pci-bridge/xio3130_downstream.c |  5 ++++-
 hw/pci/pcie.c                      | 14 ++++++++++++--
 4 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/include/hw/pci/pcie.h b/include/hw/pci/pcie.h
index e30334d74d..8d90c0e193 100644
--- a/include/hw/pci/pcie.h
+++ b/include/hw/pci/pcie.h
@@ -107,7 +107,8 @@ void pcie_cap_lnkctl_reset(PCIDevice *dev);
 
 void pcie_cap_slot_init(PCIDevice *dev, uint16_t slot);
 void pcie_cap_slot_reset(PCIDevice *dev);
-void pcie_cap_slot_write_config(PCIDevice *dev,
+void pcie_cap_slot_get(PCIDevice *dev, uint16_t *slot_ctl, uint16_t *slt_sta);
+void pcie_cap_slot_write_config(PCIDevice *dev, uint16_t slot_ctl, uint16_t slt_sta,
                                 uint32_t addr, uint32_t val, int len);
 int pcie_cap_slot_post_load(void *opaque, int version_id);
 void pcie_cap_slot_push_attention_button(PCIDevice *dev);
diff --git a/hw/pci-bridge/pcie_root_port.c b/hw/pci-bridge/pcie_root_port.c
index 92f253c924..09019ca05d 100644
--- a/hw/pci-bridge/pcie_root_port.c
+++ b/hw/pci-bridge/pcie_root_port.c
@@ -31,10 +31,13 @@ static void rp_write_config(PCIDevice *d, uint32_t address,
 {
     uint32_t root_cmd =
         pci_get_long(d->config + d->exp.aer_cap + PCI_ERR_ROOT_COMMAND);
+    uint16_t slt_ctl, slt_sta;
+
+    pcie_cap_slot_get(d, &slt_ctl, &slt_sta);
 
     pci_bridge_write_config(d, address, val, len);
     rp_aer_vector_update(d);
-    pcie_cap_slot_write_config(d, address, val, len);
+    pcie_cap_slot_write_config(d, slt_ctl, slt_sta, address, val, len);
     pcie_aer_write_config(d, address, val, len);
     pcie_aer_root_write_config(d, address, val, len, root_cmd);
 }
diff --git a/hw/pci-bridge/xio3130_downstream.c b/hw/pci-bridge/xio3130_downstream.c
index 264e37d6a6..899b0fd6c9 100644
--- a/hw/pci-bridge/xio3130_downstream.c
+++ b/hw/pci-bridge/xio3130_downstream.c
@@ -41,9 +41,12 @@
 static void xio3130_downstream_write_config(PCIDevice *d, uint32_t address,
                                          uint32_t val, int len)
 {
+    uint16_t slt_ctl, slt_sta;
+
+    pcie_cap_slot_get(d, &slt_sta, &slt_ctl);
     pci_bridge_write_config(d, address, val, len);
     pcie_cap_flr_write_config(d, address, val, len);
-    pcie_cap_slot_write_config(d, address, val, len);
+    pcie_cap_slot_write_config(d, slt_ctl, slt_sta, address, val, len);
     pcie_aer_write_config(d, address, val, len);
 }
 
diff --git a/hw/pci/pcie.c b/hw/pci/pcie.c
index b22527000d..f8490a00de 100644
--- a/hw/pci/pcie.c
+++ b/hw/pci/pcie.c
@@ -594,7 +594,15 @@ void pcie_cap_slot_reset(PCIDevice *dev)
     hotplug_event_update_event_status(dev);
 }
 
-void pcie_cap_slot_write_config(PCIDevice *dev,
+void pcie_cap_slot_get(PCIDevice *dev, uint16_t *slt_ctl, uint16_t *slt_sta)
+{
+    uint32_t pos = dev->exp.exp_cap;
+    uint8_t *exp_cap = dev->config + pos;
+    *slt_ctl = pci_get_word(exp_cap + PCI_EXP_SLTCTL);
+    *slt_sta = pci_get_word(exp_cap + PCI_EXP_SLTSTA);
+}
+
+void pcie_cap_slot_write_config(PCIDevice *dev, uint16_t slt_ctl, uint16_t slt_sta,
                                 uint32_t addr, uint32_t val, int len)
 {
     uint32_t pos = dev->exp.exp_cap;
@@ -623,7 +631,9 @@ void pcie_cap_slot_write_config(PCIDevice *dev,
      * controller is off, it is safe to detach the devices.
      */
     if ((sltsta & PCI_EXP_SLTSTA_PDS) && (val & PCI_EXP_SLTCTL_PCC) &&
-        ((val & PCI_EXP_SLTCTL_PIC_OFF) == PCI_EXP_SLTCTL_PIC_OFF)) {
+        (val & PCI_EXP_SLTCTL_PIC_OFF) == PCI_EXP_SLTCTL_PIC_OFF &&
+        (!(slt_ctl & PCI_EXP_SLTCTL_PCC) ||
+        (slt_ctl & PCI_EXP_SLTCTL_PIC_OFF) != PCI_EXP_SLTCTL_PIC_OFF)) {
         PCIBus *sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(dev));
         pci_for_each_device(sec_bus, pci_bus_num(sec_bus),
                             pcie_unplug_device, NULL);
-- 
MST

On Fri, 21 Jun 2019 02:46:48 -0400
"Michael S. Tsirkin" <mst@redhat.com> wrote:

> During boot, linux would sometimes overwrites control of a powered off
> slot before powering it on. Unfortunately QEMU interprets that as a
> power off request and ejects the device.
> 
> For example:
> 
> /x86_64-softmmu/qemu-system-x86_64 -enable-kvm -S -machine q35  \
>     -device pcie-root-port,id=pcie_root_port_0,slot=2,chassis=2,addr=0x2,bus=pcie.0 \
>     -monitor stdio disk.qcow2
> (qemu)device_add virtio-balloon-pci,id=balloon,bus=pcie_root_port_0
> (qemu)cont
> 
> Balloon is deleted during guest boot.
> 
> To fix, save control beforehand and check that power
> or led state actually change before ejecting.
> 
> Note: this is more a hack than a solution, ideally we'd
> find a better way to detect ejects, or move away
> from ejects completely and instead monitor whether
> it's safe to delete device due to e.g. its power state.
> 
> Signed-off-by: Michael S. Tsirkin <mst@redhat.com>

Reviewed-by: Igor Mammedov <imammedo@redhat.com>
Tested-by: Igor Mammedov <imammedo@redhat.com>

> ---
>  include/hw/pci/pcie.h              |  3 ++-
>  hw/pci-bridge/pcie_root_port.c     |  5 ++++-
>  hw/pci-bridge/xio3130_downstream.c |  5 ++++-
>  hw/pci/pcie.c                      | 14 ++++++++++++--
>  4 files changed, 22 insertions(+), 5 deletions(-)
> 
> diff --git a/include/hw/pci/pcie.h b/include/hw/pci/pcie.h
> index e30334d74d..8d90c0e193 100644
> --- a/include/hw/pci/pcie.h
> +++ b/include/hw/pci/pcie.h
> @@ -107,7 +107,8 @@ void pcie_cap_lnkctl_reset(PCIDevice *dev);
>  
>  void pcie_cap_slot_init(PCIDevice *dev, uint16_t slot);
>  void pcie_cap_slot_reset(PCIDevice *dev);
> -void pcie_cap_slot_write_config(PCIDevice *dev,
> +void pcie_cap_slot_get(PCIDevice *dev, uint16_t *slot_ctl, uint16_t *slt_sta);
> +void pcie_cap_slot_write_config(PCIDevice *dev, uint16_t slot_ctl, uint16_t slt_sta,
>                                  uint32_t addr, uint32_t val, int len);
>  int pcie_cap_slot_post_load(void *opaque, int version_id);
>  void pcie_cap_slot_push_attention_button(PCIDevice *dev);
> diff --git a/hw/pci-bridge/pcie_root_port.c b/hw/pci-bridge/pcie_root_port.c
> index 92f253c924..09019ca05d 100644
> --- a/hw/pci-bridge/pcie_root_port.c
> +++ b/hw/pci-bridge/pcie_root_port.c
> @@ -31,10 +31,13 @@ static void rp_write_config(PCIDevice *d, uint32_t address,
>  {
>      uint32_t root_cmd =
>          pci_get_long(d->config + d->exp.aer_cap + PCI_ERR_ROOT_COMMAND);
> +    uint16_t slt_ctl, slt_sta;
> +
> +    pcie_cap_slot_get(d, &slt_ctl, &slt_sta);
>  
>      pci_bridge_write_config(d, address, val, len);
>      rp_aer_vector_update(d);
> -    pcie_cap_slot_write_config(d, address, val, len);
> +    pcie_cap_slot_write_config(d, slt_ctl, slt_sta, address, val, len);
>      pcie_aer_write_config(d, address, val, len);
>      pcie_aer_root_write_config(d, address, val, len, root_cmd);
>  }
> diff --git a/hw/pci-bridge/xio3130_downstream.c b/hw/pci-bridge/xio3130_downstream.c
> index 264e37d6a6..899b0fd6c9 100644
> --- a/hw/pci-bridge/xio3130_downstream.c
> +++ b/hw/pci-bridge/xio3130_downstream.c
> @@ -41,9 +41,12 @@
>  static void xio3130_downstream_write_config(PCIDevice *d, uint32_t address,
>                                           uint32_t val, int len)
>  {
> +    uint16_t slt_ctl, slt_sta;
> +
> +    pcie_cap_slot_get(d, &slt_sta, &slt_ctl);
>      pci_bridge_write_config(d, address, val, len);
>      pcie_cap_flr_write_config(d, address, val, len);
> -    pcie_cap_slot_write_config(d, address, val, len);
> +    pcie_cap_slot_write_config(d, slt_ctl, slt_sta, address, val, len);
>      pcie_aer_write_config(d, address, val, len);
>  }
>  
> diff --git a/hw/pci/pcie.c b/hw/pci/pcie.c
> index b22527000d..f8490a00de 100644
> --- a/hw/pci/pcie.c
> +++ b/hw/pci/pcie.c
> @@ -594,7 +594,15 @@ void pcie_cap_slot_reset(PCIDevice *dev)
>      hotplug_event_update_event_status(dev);
>  }
>  
> -void pcie_cap_slot_write_config(PCIDevice *dev,
> +void pcie_cap_slot_get(PCIDevice *dev, uint16_t *slt_ctl, uint16_t *slt_sta)
> +{
> +    uint32_t pos = dev->exp.exp_cap;
> +    uint8_t *exp_cap = dev->config + pos;
> +    *slt_ctl = pci_get_word(exp_cap + PCI_EXP_SLTCTL);
> +    *slt_sta = pci_get_word(exp_cap + PCI_EXP_SLTSTA);
> +}
> +
> +void pcie_cap_slot_write_config(PCIDevice *dev, uint16_t slt_ctl, uint16_t slt_sta,
>                                  uint32_t addr, uint32_t val, int len)
>  {
>      uint32_t pos = dev->exp.exp_cap;
> @@ -623,7 +631,9 @@ void pcie_cap_slot_write_config(PCIDevice *dev,
>       * controller is off, it is safe to detach the devices.
>       */
>      if ((sltsta & PCI_EXP_SLTSTA_PDS) && (val & PCI_EXP_SLTCTL_PCC) &&
> -        ((val & PCI_EXP_SLTCTL_PIC_OFF) == PCI_EXP_SLTCTL_PIC_OFF)) {
> +        (val & PCI_EXP_SLTCTL_PIC_OFF) == PCI_EXP_SLTCTL_PIC_OFF &&
> +        (!(slt_ctl & PCI_EXP_SLTCTL_PCC) ||
> +        (slt_ctl & PCI_EXP_SLTCTL_PIC_OFF) != PCI_EXP_SLTCTL_PIC_OFF)) {
>          PCIBus *sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(dev));
>          pci_for_each_device(sec_bus, pci_bus_num(sec_bus),
>                              pcie_unplug_device, NULL);

On 6/21/19 9:46 AM, Michael S. Tsirkin wrote:
> During boot, linux would sometimes overwrites control of a powered off
> slot before powering it on. Unfortunately QEMU interprets that as a
> power off request and ejects the device.
>
> For example:
>
> /x86_64-softmmu/qemu-system-x86_64 -enable-kvm -S -machine q35  \
>      -device pcie-root-port,id=pcie_root_port_0,slot=2,chassis=2,addr=0x2,bus=pcie.0 \
>      -monitor stdio disk.qcow2
> (qemu)device_add virtio-balloon-pci,id=balloon,bus=pcie_root_port_0
> (qemu)cont
>
> Balloon is deleted during guest boot.
>
> To fix, save control beforehand and check that power
> or led state actually change before ejecting.
>
> Note: this is more a hack than a solution, ideally we'd
> find a better way to detect ejects, or move away
> from ejects completely and instead monitor whether
> it's safe to delete device due to e.g. its power state.
>
> Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
> ---
>   include/hw/pci/pcie.h              |  3 ++-
>   hw/pci-bridge/pcie_root_port.c     |  5 ++++-
>   hw/pci-bridge/xio3130_downstream.c |  5 ++++-
>   hw/pci/pcie.c                      | 14 ++++++++++++--
>   4 files changed, 22 insertions(+), 5 deletions(-)
>
> diff --git a/include/hw/pci/pcie.h b/include/hw/pci/pcie.h
> index e30334d74d..8d90c0e193 100644
> --- a/include/hw/pci/pcie.h
> +++ b/include/hw/pci/pcie.h
> @@ -107,7 +107,8 @@ void pcie_cap_lnkctl_reset(PCIDevice *dev);
>   
>   void pcie_cap_slot_init(PCIDevice *dev, uint16_t slot);
>   void pcie_cap_slot_reset(PCIDevice *dev);
> -void pcie_cap_slot_write_config(PCIDevice *dev,
> +void pcie_cap_slot_get(PCIDevice *dev, uint16_t *slot_ctl, uint16_t *slt_sta);
> +void pcie_cap_slot_write_config(PCIDevice *dev, uint16_t slot_ctl, uint16_t slt_sta,
>                                   uint32_t addr, uint32_t val, int len);
>   int pcie_cap_slot_post_load(void *opaque, int version_id);
>   void pcie_cap_slot_push_attention_button(PCIDevice *dev);
> diff --git a/hw/pci-bridge/pcie_root_port.c b/hw/pci-bridge/pcie_root_port.c
> index 92f253c924..09019ca05d 100644
> --- a/hw/pci-bridge/pcie_root_port.c
> +++ b/hw/pci-bridge/pcie_root_port.c
> @@ -31,10 +31,13 @@ static void rp_write_config(PCIDevice *d, uint32_t address,
>   {
>       uint32_t root_cmd =
>           pci_get_long(d->config + d->exp.aer_cap + PCI_ERR_ROOT_COMMAND);
> +    uint16_t slt_ctl, slt_sta;
> +
> +    pcie_cap_slot_get(d, &slt_ctl, &slt_sta);
>   
>       pci_bridge_write_config(d, address, val, len);
>       rp_aer_vector_update(d);
> -    pcie_cap_slot_write_config(d, address, val, len);
> +    pcie_cap_slot_write_config(d, slt_ctl, slt_sta, address, val, len);
>       pcie_aer_write_config(d, address, val, len);
>       pcie_aer_root_write_config(d, address, val, len, root_cmd);
>   }
> diff --git a/hw/pci-bridge/xio3130_downstream.c b/hw/pci-bridge/xio3130_downstream.c
> index 264e37d6a6..899b0fd6c9 100644
> --- a/hw/pci-bridge/xio3130_downstream.c
> +++ b/hw/pci-bridge/xio3130_downstream.c
> @@ -41,9 +41,12 @@
>   static void xio3130_downstream_write_config(PCIDevice *d, uint32_t address,
>                                            uint32_t val, int len)
>   {
> +    uint16_t slt_ctl, slt_sta;
> +
> +    pcie_cap_slot_get(d, &slt_sta, &slt_ctl);
>       pci_bridge_write_config(d, address, val, len);
>       pcie_cap_flr_write_config(d, address, val, len);
> -    pcie_cap_slot_write_config(d, address, val, len);
> +    pcie_cap_slot_write_config(d, slt_ctl, slt_sta, address, val, len);
>       pcie_aer_write_config(d, address, val, len);
>   }
>   
> diff --git a/hw/pci/pcie.c b/hw/pci/pcie.c
> index b22527000d..f8490a00de 100644
> --- a/hw/pci/pcie.c
> +++ b/hw/pci/pcie.c
> @@ -594,7 +594,15 @@ void pcie_cap_slot_reset(PCIDevice *dev)
>       hotplug_event_update_event_status(dev);
>   }
>   
> -void pcie_cap_slot_write_config(PCIDevice *dev,
> +void pcie_cap_slot_get(PCIDevice *dev, uint16_t *slt_ctl, uint16_t *slt_sta)
> +{
> +    uint32_t pos = dev->exp.exp_cap;
> +    uint8_t *exp_cap = dev->config + pos;
> +    *slt_ctl = pci_get_word(exp_cap + PCI_EXP_SLTCTL);
> +    *slt_sta = pci_get_word(exp_cap + PCI_EXP_SLTSTA);
> +}
> +
> +void pcie_cap_slot_write_config(PCIDevice *dev, uint16_t slt_ctl, uint16_t slt_sta,
>                                   uint32_t addr, uint32_t val, int len)
>   {
>       uint32_t pos = dev->exp.exp_cap;
> @@ -623,7 +631,9 @@ void pcie_cap_slot_write_config(PCIDevice *dev,
>        * controller is off, it is safe to detach the devices.
>        */
>       if ((sltsta & PCI_EXP_SLTSTA_PDS) && (val & PCI_EXP_SLTCTL_PCC) &&
> -        ((val & PCI_EXP_SLTCTL_PIC_OFF) == PCI_EXP_SLTCTL_PIC_OFF)) {
> +        (val & PCI_EXP_SLTCTL_PIC_OFF) == PCI_EXP_SLTCTL_PIC_OFF &&
> +        (!(slt_ctl & PCI_EXP_SLTCTL_PCC) ||
> +        (slt_ctl & PCI_EXP_SLTCTL_PIC_OFF) != PCI_EXP_SLTCTL_PIC_OFF)) {
>           PCIBus *sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(dev));
>           pci_for_each_device(sec_bus, pci_bus_num(sec_bus),
>                               pcie_unplug_device, NULL);


Reviewed-by: Marcel Apfelbaum <marcel.apfelbaum@gmail.com>

Thanks,
Marcel

On Fri, 21 Jun 2019 02:46:48 -0400
"Michael S. Tsirkin" <mst@redhat.com> wrote:

> During boot, linux would sometimes overwrites control of a powered off
> slot before powering it on. Unfortunately QEMU interprets that as a
> power off request and ejects the device.
> 
> For example:
> 
> /x86_64-softmmu/qemu-system-x86_64 -enable-kvm -S -machine q35  \
>     -device pcie-root-port,id=pcie_root_port_0,slot=2,chassis=2,addr=0x2,bus=pcie.0 \
>     -monitor stdio disk.qcow2
> (qemu)device_add virtio-balloon-pci,id=balloon,bus=pcie_root_port_0
> (qemu)cont
> 
> Balloon is deleted during guest boot.
> 
> To fix, save control beforehand and check that power
> or led state actually change before ejecting.
> 
> Note: this is more a hack than a solution, ideally we'd
> find a better way to detect ejects, or move away
> from ejects completely and instead monitor whether
> it's safe to delete device due to e.g. its power state.
> 
> Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
> ---
>  include/hw/pci/pcie.h              |  3 ++-
>  hw/pci-bridge/pcie_root_port.c     |  5 ++++-
>  hw/pci-bridge/xio3130_downstream.c |  5 ++++-
>  hw/pci/pcie.c                      | 14 ++++++++++++--
>  4 files changed, 22 insertions(+), 5 deletions(-)
> 
> diff --git a/include/hw/pci/pcie.h b/include/hw/pci/pcie.h
> index e30334d74d..8d90c0e193 100644
> --- a/include/hw/pci/pcie.h
> +++ b/include/hw/pci/pcie.h
> @@ -107,7 +107,8 @@ void pcie_cap_lnkctl_reset(PCIDevice *dev);
>  
>  void pcie_cap_slot_init(PCIDevice *dev, uint16_t slot);
>  void pcie_cap_slot_reset(PCIDevice *dev);
> -void pcie_cap_slot_write_config(PCIDevice *dev,
> +void pcie_cap_slot_get(PCIDevice *dev, uint16_t *slot_ctl, uint16_t *slt_sta);
> +void pcie_cap_slot_write_config(PCIDevice *dev, uint16_t slot_ctl, uint16_t slt_sta,
>                                  uint32_t addr, uint32_t val, int len);
>  int pcie_cap_slot_post_load(void *opaque, int version_id);
>  void pcie_cap_slot_push_attention_button(PCIDevice *dev);
> diff --git a/hw/pci-bridge/pcie_root_port.c b/hw/pci-bridge/pcie_root_port.c
> index 92f253c924..09019ca05d 100644
> --- a/hw/pci-bridge/pcie_root_port.c
> +++ b/hw/pci-bridge/pcie_root_port.c
> @@ -31,10 +31,13 @@ static void rp_write_config(PCIDevice *d, uint32_t address,
>  {
>      uint32_t root_cmd =
>          pci_get_long(d->config + d->exp.aer_cap + PCI_ERR_ROOT_COMMAND);
> +    uint16_t slt_ctl, slt_sta;
> +
> +    pcie_cap_slot_get(d, &slt_ctl, &slt_sta);
>  
>      pci_bridge_write_config(d, address, val, len);
>      rp_aer_vector_update(d);
> -    pcie_cap_slot_write_config(d, address, val, len);
> +    pcie_cap_slot_write_config(d, slt_ctl, slt_sta, address, val, len);
>      pcie_aer_write_config(d, address, val, len);
>      pcie_aer_root_write_config(d, address, val, len, root_cmd);
>  }
> diff --git a/hw/pci-bridge/xio3130_downstream.c b/hw/pci-bridge/xio3130_downstream.c
> index 264e37d6a6..899b0fd6c9 100644
> --- a/hw/pci-bridge/xio3130_downstream.c
> +++ b/hw/pci-bridge/xio3130_downstream.c
> @@ -41,9 +41,12 @@
>  static void xio3130_downstream_write_config(PCIDevice *d, uint32_t address,
>                                           uint32_t val, int len)
>  {
> +    uint16_t slt_ctl, slt_sta;
> +
> +    pcie_cap_slot_get(d, &slt_sta, &slt_ctl);
>      pci_bridge_write_config(d, address, val, len);
>      pcie_cap_flr_write_config(d, address, val, len);
> -    pcie_cap_slot_write_config(d, address, val, len);
> +    pcie_cap_slot_write_config(d, slt_ctl, slt_sta, address, val, len);
>      pcie_aer_write_config(d, address, val, len);
>  }
>  
> diff --git a/hw/pci/pcie.c b/hw/pci/pcie.c
> index b22527000d..f8490a00de 100644
> --- a/hw/pci/pcie.c
> +++ b/hw/pci/pcie.c
> @@ -594,7 +594,15 @@ void pcie_cap_slot_reset(PCIDevice *dev)
>      hotplug_event_update_event_status(dev);
>  }
>  
> -void pcie_cap_slot_write_config(PCIDevice *dev,
> +void pcie_cap_slot_get(PCIDevice *dev, uint16_t *slt_ctl, uint16_t *slt_sta)
> +{
> +    uint32_t pos = dev->exp.exp_cap;
> +    uint8_t *exp_cap = dev->config + pos;
> +    *slt_ctl = pci_get_word(exp_cap + PCI_EXP_SLTCTL);
> +    *slt_sta = pci_get_word(exp_cap + PCI_EXP_SLTSTA);
> +}
> +
> +void pcie_cap_slot_write_config(PCIDevice *dev, uint16_t slt_ctl, uint16_t slt_sta,
>                                  uint32_t addr, uint32_t val, int len)
>  {
>      uint32_t pos = dev->exp.exp_cap;
> @@ -623,7 +631,9 @@ void pcie_cap_slot_write_config(PCIDevice *dev,
>       * controller is off, it is safe to detach the devices.
>       */
comment above probably should be updated to account for new conditions 

>      if ((sltsta & PCI_EXP_SLTSTA_PDS) && (val & PCI_EXP_SLTCTL_PCC) &&
> -        ((val & PCI_EXP_SLTCTL_PIC_OFF) == PCI_EXP_SLTCTL_PIC_OFF)) {
> +        (val & PCI_EXP_SLTCTL_PIC_OFF) == PCI_EXP_SLTCTL_PIC_OFF &&
> +        (!(slt_ctl & PCI_EXP_SLTCTL_PCC) ||
> +        (slt_ctl & PCI_EXP_SLTCTL_PIC_OFF) != PCI_EXP_SLTCTL_PIC_OFF)) {
>          PCIBus *sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(dev));
>          pci_for_each_device(sec_bus, pci_bus_num(sec_bus),
>                              pcie_unplug_device, NULL);

On Tue, Jun 25, 2019 at 02:45:11PM +0200, Igor Mammedov wrote:
> On Fri, 21 Jun 2019 02:46:48 -0400
> "Michael S. Tsirkin" <mst@redhat.com> wrote:
> 
> > During boot, linux would sometimes overwrites control of a powered off
> > slot before powering it on. Unfortunately QEMU interprets that as a
> > power off request and ejects the device.
> > 
> > For example:
> > 
> > /x86_64-softmmu/qemu-system-x86_64 -enable-kvm -S -machine q35  \
> >     -device pcie-root-port,id=pcie_root_port_0,slot=2,chassis=2,addr=0x2,bus=pcie.0 \
> >     -monitor stdio disk.qcow2
> > (qemu)device_add virtio-balloon-pci,id=balloon,bus=pcie_root_port_0
> > (qemu)cont
> > 
> > Balloon is deleted during guest boot.
> > 
> > To fix, save control beforehand and check that power
> > or led state actually change before ejecting.
> > 
> > Note: this is more a hack than a solution, ideally we'd
> > find a better way to detect ejects, or move away
> > from ejects completely and instead monitor whether
> > it's safe to delete device due to e.g. its power state.
> > 
> > Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
> > ---
> >  include/hw/pci/pcie.h              |  3 ++-
> >  hw/pci-bridge/pcie_root_port.c     |  5 ++++-
> >  hw/pci-bridge/xio3130_downstream.c |  5 ++++-
> >  hw/pci/pcie.c                      | 14 ++++++++++++--
> >  4 files changed, 22 insertions(+), 5 deletions(-)
> > 
> > diff --git a/include/hw/pci/pcie.h b/include/hw/pci/pcie.h
> > index e30334d74d..8d90c0e193 100644
> > --- a/include/hw/pci/pcie.h
> > +++ b/include/hw/pci/pcie.h
> > @@ -107,7 +107,8 @@ void pcie_cap_lnkctl_reset(PCIDevice *dev);
> >  
> >  void pcie_cap_slot_init(PCIDevice *dev, uint16_t slot);
> >  void pcie_cap_slot_reset(PCIDevice *dev);
> > -void pcie_cap_slot_write_config(PCIDevice *dev,
> > +void pcie_cap_slot_get(PCIDevice *dev, uint16_t *slot_ctl, uint16_t *slt_sta);
> > +void pcie_cap_slot_write_config(PCIDevice *dev, uint16_t slot_ctl, uint16_t slt_sta,
> >                                  uint32_t addr, uint32_t val, int len);
> >  int pcie_cap_slot_post_load(void *opaque, int version_id);
> >  void pcie_cap_slot_push_attention_button(PCIDevice *dev);
> > diff --git a/hw/pci-bridge/pcie_root_port.c b/hw/pci-bridge/pcie_root_port.c
> > index 92f253c924..09019ca05d 100644
> > --- a/hw/pci-bridge/pcie_root_port.c
> > +++ b/hw/pci-bridge/pcie_root_port.c
> > @@ -31,10 +31,13 @@ static void rp_write_config(PCIDevice *d, uint32_t address,
> >  {
> >      uint32_t root_cmd =
> >          pci_get_long(d->config + d->exp.aer_cap + PCI_ERR_ROOT_COMMAND);
> > +    uint16_t slt_ctl, slt_sta;
> > +
> > +    pcie_cap_slot_get(d, &slt_ctl, &slt_sta);
> >  
> >      pci_bridge_write_config(d, address, val, len);
> >      rp_aer_vector_update(d);
> > -    pcie_cap_slot_write_config(d, address, val, len);
> > +    pcie_cap_slot_write_config(d, slt_ctl, slt_sta, address, val, len);
> >      pcie_aer_write_config(d, address, val, len);
> >      pcie_aer_root_write_config(d, address, val, len, root_cmd);
> >  }
> > diff --git a/hw/pci-bridge/xio3130_downstream.c b/hw/pci-bridge/xio3130_downstream.c
> > index 264e37d6a6..899b0fd6c9 100644
> > --- a/hw/pci-bridge/xio3130_downstream.c
> > +++ b/hw/pci-bridge/xio3130_downstream.c
> > @@ -41,9 +41,12 @@
> >  static void xio3130_downstream_write_config(PCIDevice *d, uint32_t address,
> >                                           uint32_t val, int len)
> >  {
> > +    uint16_t slt_ctl, slt_sta;
> > +
> > +    pcie_cap_slot_get(d, &slt_sta, &slt_ctl);
> >      pci_bridge_write_config(d, address, val, len);
> >      pcie_cap_flr_write_config(d, address, val, len);
> > -    pcie_cap_slot_write_config(d, address, val, len);
> > +    pcie_cap_slot_write_config(d, slt_ctl, slt_sta, address, val, len);
> >      pcie_aer_write_config(d, address, val, len);
> >  }
> >  
> > diff --git a/hw/pci/pcie.c b/hw/pci/pcie.c
> > index b22527000d..f8490a00de 100644
> > --- a/hw/pci/pcie.c
> > +++ b/hw/pci/pcie.c
> > @@ -594,7 +594,15 @@ void pcie_cap_slot_reset(PCIDevice *dev)
> >      hotplug_event_update_event_status(dev);
> >  }
> >  
> > -void pcie_cap_slot_write_config(PCIDevice *dev,
> > +void pcie_cap_slot_get(PCIDevice *dev, uint16_t *slt_ctl, uint16_t *slt_sta)
> > +{
> > +    uint32_t pos = dev->exp.exp_cap;
> > +    uint8_t *exp_cap = dev->config + pos;
> > +    *slt_ctl = pci_get_word(exp_cap + PCI_EXP_SLTCTL);
> > +    *slt_sta = pci_get_word(exp_cap + PCI_EXP_SLTSTA);
> > +}
> > +
> > +void pcie_cap_slot_write_config(PCIDevice *dev, uint16_t slt_ctl, uint16_t slt_sta,
> >                                  uint32_t addr, uint32_t val, int len)
> >  {
> >      uint32_t pos = dev->exp.exp_cap;
> > @@ -623,7 +631,9 @@ void pcie_cap_slot_write_config(PCIDevice *dev,
> >       * controller is off, it is safe to detach the devices.
> >       */
> comment above probably should be updated to account for new conditions 

It's actually the same condition: the change is that
we do not eject if it was already true.

> >      if ((sltsta & PCI_EXP_SLTSTA_PDS) && (val & PCI_EXP_SLTCTL_PCC) &&
> > -        ((val & PCI_EXP_SLTCTL_PIC_OFF) == PCI_EXP_SLTCTL_PIC_OFF)) {
> > +        (val & PCI_EXP_SLTCTL_PIC_OFF) == PCI_EXP_SLTCTL_PIC_OFF &&
> > +        (!(slt_ctl & PCI_EXP_SLTCTL_PCC) ||
> > +        (slt_ctl & PCI_EXP_SLTCTL_PIC_OFF) != PCI_EXP_SLTCTL_PIC_OFF)) {
> >          PCIBus *sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(dev));
> >          pci_for_each_device(sec_bus, pci_bus_num(sec_bus),
> >                              pcie_unplug_device, NULL);