Kernel provides the guidance of dynamic MSI-X allocation support of
passthrough device, by clearing the VFIO_IRQ_INFO_NORESIZE flag to
guide user space.

Fetch the flags from host to determine if dynamic MSI-X allocation is
supported.

Originally-by: Reinette Chatre <reinette.chatre@intel.com>
Signed-off-by: Jing Liu <jing2.liu@intel.com>
---
Changes since RFC v1:
- Filter the dynamic MSI-X allocation flag and store as a bool type.
  (Alex)
- Move the detection to vfio_msix_early_setup(). (Alex)
- Report error of getting irq info and remove the trace of failure
  case. (Alex, Cédric)
---
 hw/vfio/pci.c        | 15 +++++++++++++--
 hw/vfio/pci.h        |  1 +
 hw/vfio/trace-events |  2 +-
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index a205c6b1130f..8a3b34f3c196 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -1493,7 +1493,9 @@ static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
     uint8_t pos;
     uint16_t ctrl;
     uint32_t table, pba;
-    int fd = vdev->vbasedev.fd;
+    int ret, fd = vdev->vbasedev.fd;
+    struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info),
+                                      .index = VFIO_PCI_MSIX_IRQ_INDEX };
     VFIOMSIXInfo *msix;
 
     pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
@@ -1530,6 +1532,14 @@ static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
     msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
     msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
 
+    ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "failed to get MSI-X irq info");
+        return;
+    }
+
+    msix->noresize = !!(irq_info.flags & VFIO_IRQ_INFO_NORESIZE);
+
     /*
      * Test the size of the pba_offset variable and catch if it extends outside
      * of the specified BAR. If it is the case, we need to apply a hardware
@@ -1562,7 +1572,8 @@ static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
     }
 
     trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar,
-                                msix->table_offset, msix->entries);
+                                msix->table_offset, msix->entries,
+                                msix->noresize);
     vdev->msix = msix;
 
     vfio_pci_fixup_msix_region(vdev);
diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h
index a2771b9ff3cc..0717574d79e9 100644
--- a/hw/vfio/pci.h
+++ b/hw/vfio/pci.h
@@ -113,6 +113,7 @@ typedef struct VFIOMSIXInfo {
     uint32_t table_offset;
     uint32_t pba_offset;
     unsigned long *pending;
+    bool noresize;
 } VFIOMSIXInfo;
 
 #define TYPE_VFIO_PCI "vfio-pci"
diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events
index ee7509e68e4f..6de5d9ba8e46 100644
--- a/hw/vfio/trace-events
+++ b/hw/vfio/trace-events
@@ -27,7 +27,7 @@ vfio_vga_read(uint64_t addr, int size, uint64_t data) " (0x%"PRIx64", %d) = 0x%"
 vfio_pci_read_config(const char *name, int addr, int len, int val) " (%s, @0x%x, len=0x%x) 0x%x"
 vfio_pci_write_config(const char *name, int addr, int val, int len) " (%s, @0x%x, 0x%x, len=0x%x)"
 vfio_msi_setup(const char *name, int pos) "%s PCI MSI CAP @0x%x"
-vfio_msix_early_setup(const char *name, int pos, int table_bar, int offset, int entries) "%s PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d"
+vfio_msix_early_setup(const char *name, int pos, int table_bar, int offset, int entries, bool noresize) "%s PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d, noresize %d"
 vfio_check_pcie_flr(const char *name) "%s Supports FLR via PCIe cap"
 vfio_check_pm_reset(const char *name) "%s Supports PM reset"
 vfio_check_af_flr(const char *name) "%s Supports FLR via AF cap"
-- 
2.27.0

Hello Jing,

On 8/22/23 09:29, Jing Liu wrote:
> Kernel provides the guidance of dynamic MSI-X allocation support of
> passthrough device, by clearing the VFIO_IRQ_INFO_NORESIZE flag to
> guide user space.
> 
> Fetch the flags from host to determine if dynamic MSI-X allocation is
> supported.
> 
> Originally-by: Reinette Chatre <reinette.chatre@intel.com>
> Signed-off-by: Jing Liu <jing2.liu@intel.com>
> ---
> Changes since RFC v1:
> - Filter the dynamic MSI-X allocation flag and store as a bool type.
>    (Alex)
> - Move the detection to vfio_msix_early_setup(). (Alex)
> - Report error of getting irq info and remove the trace of failure
>    case. (Alex, Cédric)
> ---
>   hw/vfio/pci.c        | 15 +++++++++++++--
>   hw/vfio/pci.h        |  1 +
>   hw/vfio/trace-events |  2 +-
>   3 files changed, 15 insertions(+), 3 deletions(-)
> 
> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> index a205c6b1130f..8a3b34f3c196 100644
> --- a/hw/vfio/pci.c
> +++ b/hw/vfio/pci.c
> @@ -1493,7 +1493,9 @@ static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
>       uint8_t pos;
>       uint16_t ctrl;
>       uint32_t table, pba;
> -    int fd = vdev->vbasedev.fd;
> +    int ret, fd = vdev->vbasedev.fd;
> +    struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info),
> +                                      .index = VFIO_PCI_MSIX_IRQ_INDEX };
>       VFIOMSIXInfo *msix;
>   
>       pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
> @@ -1530,6 +1532,14 @@ static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
>       msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
>       msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
>   
> +    ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
> +    if (ret < 0) {
> +        error_setg_errno(errp, -ret, "failed to get MSI-X irq info");

Missing :
             g_free(msix);


With this fixed,

Reviewed-by: Cédric Le Goater <clg@redhat.com>

Thanks,

C.


  
> +        return;
> +    }
> +
> +    msix->noresize = !!(irq_info.flags & VFIO_IRQ_INFO_NORESIZE);
> +
>       /*
>        * Test the size of the pba_offset variable and catch if it extends outside
>        * of the specified BAR. If it is the case, we need to apply a hardware
> @@ -1562,7 +1572,8 @@ static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
>       }
>   
>       trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar,
> -                                msix->table_offset, msix->entries);
> +                                msix->table_offset, msix->entries,
> +                                msix->noresize);
>       vdev->msix = msix;
>   
>       vfio_pci_fixup_msix_region(vdev);
> diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h
> index a2771b9ff3cc..0717574d79e9 100644
> --- a/hw/vfio/pci.h
> +++ b/hw/vfio/pci.h
> @@ -113,6 +113,7 @@ typedef struct VFIOMSIXInfo {
>       uint32_t table_offset;
>       uint32_t pba_offset;
>       unsigned long *pending;
> +    bool noresize;
>   } VFIOMSIXInfo;
>   
>   #define TYPE_VFIO_PCI "vfio-pci"
> diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events
> index ee7509e68e4f..6de5d9ba8e46 100644
> --- a/hw/vfio/trace-events
> +++ b/hw/vfio/trace-events
> @@ -27,7 +27,7 @@ vfio_vga_read(uint64_t addr, int size, uint64_t data) " (0x%"PRIx64", %d) = 0x%"
>   vfio_pci_read_config(const char *name, int addr, int len, int val) " (%s, @0x%x, len=0x%x) 0x%x"
>   vfio_pci_write_config(const char *name, int addr, int val, int len) " (%s, @0x%x, 0x%x, len=0x%x)"
>   vfio_msi_setup(const char *name, int pos) "%s PCI MSI CAP @0x%x"
> -vfio_msix_early_setup(const char *name, int pos, int table_bar, int offset, int entries) "%s PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d"
> +vfio_msix_early_setup(const char *name, int pos, int table_bar, int offset, int entries, bool noresize) "%s PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d, noresize %d"
>   vfio_check_pcie_flr(const char *name) "%s Supports FLR via PCIe cap"
>   vfio_check_pm_reset(const char *name) "%s Supports PM reset"
>   vfio_check_af_flr(const char *name) "%s Supports FLR via AF cap"

Hi Cédric,

Thank you for your reviewing.

On 8/29/2023 9:33 PM, Cédric Le Goater wrote:
> Hello Jing,
> 
> On 8/22/23 09:29, Jing Liu wrote:
> > Kernel provides the guidance of dynamic MSI-X allocation support of
> > passthrough device, by clearing the VFIO_IRQ_INFO_NORESIZE flag to
> > guide user space.
> >
> > Fetch the flags from host to determine if dynamic MSI-X allocation is
> > supported.
> >
> > Originally-by: Reinette Chatre <reinette.chatre@intel.com>
> > Signed-off-by: Jing Liu <jing2.liu@intel.com>
> > ---
> > Changes since RFC v1:
> > - Filter the dynamic MSI-X allocation flag and store as a bool type.
> >    (Alex)
> > - Move the detection to vfio_msix_early_setup(). (Alex)
> > - Report error of getting irq info and remove the trace of failure
> >    case. (Alex, Cédric)
> > ---
> >   hw/vfio/pci.c        | 15 +++++++++++++--
> >   hw/vfio/pci.h        |  1 +
> >   hw/vfio/trace-events |  2 +-
> >   3 files changed, 15 insertions(+), 3 deletions(-)
> >
> > diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index
> > a205c6b1130f..8a3b34f3c196 100644
> > --- a/hw/vfio/pci.c
> > +++ b/hw/vfio/pci.c
> > @@ -1493,7 +1493,9 @@ static void vfio_msix_early_setup(VFIOPCIDevice
> *vdev, Error **errp)
> >       uint8_t pos;
> >       uint16_t ctrl;
> >       uint32_t table, pba;
> > -    int fd = vdev->vbasedev.fd;
> > +    int ret, fd = vdev->vbasedev.fd;
> > +    struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info),
> > +                                      .index =
> > + VFIO_PCI_MSIX_IRQ_INDEX };
> >       VFIOMSIXInfo *msix;
> >
> >       pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX); @@
> > -1530,6 +1532,14 @@ static void vfio_msix_early_setup(VFIOPCIDevice *vdev,
> Error **errp)
> >       msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
> >       msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
> >
> > +    ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
> > +    if (ret < 0) {
> > +        error_setg_errno(errp, -ret, "failed to get MSI-X irq info");
> 
> Missing :
>              g_free(msix);
> 
Oh, yes. 

> 
> With this fixed,
> 
> Reviewed-by: Cédric Le Goater <clg@redhat.com>
> 
Thank you. I'll apply it in next version with the fix. 

Jing

> Thanks,
> 
> C.
> 
> 
> 
> > +        return;
> > +    }
> > +
> > +    msix->noresize = !!(irq_info.flags & VFIO_IRQ_INFO_NORESIZE);
> > +
> >       /*
> >        * Test the size of the pba_offset variable and catch if it extends outside
> >        * of the specified BAR. If it is the case, we need to apply a
> > hardware @@ -1562,7 +1572,8 @@ static void
> vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
> >       }
> >
> >       trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar,
> > -                                msix->table_offset, msix->entries);
> > +                                msix->table_offset, msix->entries,
> > +                                msix->noresize);
> >       vdev->msix = msix;
> >
> >       vfio_pci_fixup_msix_region(vdev); diff --git a/hw/vfio/pci.h
> > b/hw/vfio/pci.h index a2771b9ff3cc..0717574d79e9 100644
> > --- a/hw/vfio/pci.h
> > +++ b/hw/vfio/pci.h
> > @@ -113,6 +113,7 @@ typedef struct VFIOMSIXInfo {
> >       uint32_t table_offset;
> >       uint32_t pba_offset;
> >       unsigned long *pending;
> > +    bool noresize;
> >   } VFIOMSIXInfo;
> >
> >   #define TYPE_VFIO_PCI "vfio-pci"
> > diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events index
> > ee7509e68e4f..6de5d9ba8e46 100644
> > --- a/hw/vfio/trace-events
> > +++ b/hw/vfio/trace-events
> > @@ -27,7 +27,7 @@ vfio_vga_read(uint64_t addr, int size, uint64_t data) "
> (0x%"PRIx64", %d) = 0x%"
> >   vfio_pci_read_config(const char *name, int addr, int len, int val) " (%s,
> @0x%x, len=0x%x) 0x%x"
> >   vfio_pci_write_config(const char *name, int addr, int val, int len) " (%s,
> @0x%x, 0x%x, len=0x%x)"
> >   vfio_msi_setup(const char *name, int pos) "%s PCI MSI CAP @0x%x"
> > -vfio_msix_early_setup(const char *name, int pos, int table_bar, int offset, int
> entries) "%s PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d"
> > +vfio_msix_early_setup(const char *name, int pos, int table_bar, int offset, int
> entries, bool noresize) "%s PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x,
> entries %d, noresize %d"
> >   vfio_check_pcie_flr(const char *name) "%s Supports FLR via PCIe cap"
> >   vfio_check_pm_reset(const char *name) "%s Supports PM reset"
> >   vfio_check_af_flr(const char *name) "%s Supports FLR via AF cap"