Series comparison

-[PATCH RFC v3 0/6] virtio-net: add support for SR-IOV emulation
+[PATCH for-10.1 v9 0/9] virtio-net: add support for SR-IOV emulation
-Based-on: <20240228-reuse-v8-0-282660281e60@daynix.com>
+Based-on: <20250104-reuse-v18-0-c349eafd8673@daynix.com>
-("[PATCH v8 00/15] hw/pci: SR-IOV related fixes and improvements")
+("[PATCH v18 00/14] hw/pci: SR-IOV related fixes and improvements")
 Introduction
 ------------
 This series is based on the RFC series submitted by Yui Washizu[1].
 ...
 Summary
 -------
 Patch 1 disables ROM BAR, which virtio-net-pci enables by default, for
 VFs.
-Patch 2 and 3 adds validations.
+Patch 2 makes zero stride valid for 1 VF configuration.
-Patch 4 adds user-created SR-IOV VF infrastructure.
+Patch 3 and 4 adds validations.
-Patch 5 makes virtio-pci work as SR-IOV PF for user-created VFs.
+Patch 5 adds user-created SR-IOV VF infrastructure.
-Patch 6 allows user to create SR-IOV VFs with virtio-net-pci.
+Patch 6 makes virtio-pci work as SR-IOV PF for user-created VFs.
 Patch 7 allows user to create SR-IOV VFs with virtio-net-pci.
 [1] https://patchew.org/QEMU/1689731808-3009-1-git-send-email-yui.washidu@gmail.com/
 [2] https://lore.kernel.org/all/5d46f455-f530-4e5e-9ae7-13a2297d4bc5@daynix.com/
 Co-developed-by: Yui Washizu <yui.washidu@gmail.com>
 Signed-off-by: Akihiko Odaki <akihiko.odaki@daynix.com>
 ---
+Changes in v9:
+- Rebased.
+- Link to v8: https://lore.kernel.org/r/20250104-sriov-v8-0-56144cfdc7d9@daynix.com
+Changes in v8:
+- Rebased.
+- Link to v7: https://lore.kernel.org/r/20240813-sriov-v7-0-8515e3774df7@daynix.com
+Changes in v7:
+- Removed #include <error-report.h>, which is no longer needed.
+- Rebased.
+- Link to v6: https://lore.kernel.org/r/20240802-sriov-v6-0-0c8ff49c4276@daynix.com
+Changes in v6:
+- Added ARI extended capability.
+- Rebased.
+- Link to v5: https://lore.kernel.org/r/20240715-sriov-v5-0-3f5539093ffc@daynix.com
+Changes in v5:
+- Dropped the RFC tag.
+- Fixed device unrealization.
+- Rebased.
+- Link to v4: https://lore.kernel.org/r/20240428-sriov-v4-0-ac8ac6212982@daynix.com
+Changes in v4:
+- Added patch "hw/pci: Fix SR-IOV VF number calculation" to fix division
+  by zero reported by Yui Washizu.
+- Rebased.
+- Link to v3: https://lore.kernel.org/r/20240305-sriov-v3-0-abdb75770372@daynix.com
 Changes in v3:
 - Rebased.
 - Link to v2: https://lore.kernel.org/r/20231210-sriov-v2-0-b959e8a6dfaf@daynix.com
 Changes in v2:
 - Changed to keep VF instances.
 - Link to v1: https://lore.kernel.org/r/20231202-sriov-v1-0-32b3570f7bd6@daynix.com
 ---
-Akihiko Odaki (6):
+Akihiko Odaki (9):
       hw/pci: Do not add ROM BAR for SR-IOV VF
+      hw/pci: Fix SR-IOV VF number calculation
       pcie_sriov: Ensure PF and VF are mutually exclusive
       pcie_sriov: Check PCI Express for SR-IOV PF
       pcie_sriov: Allow user to create SR-IOV device
       virtio-pci: Implement SR-IOV PF
       virtio-net: Implement SR-IOV VF
+      docs: Document composable SR-IOV device
+      pcie_sriov: Make a PCI device with user-created VF ARI-capable
- include/hw/pci/pci_device.h |   6 +-
+ MAINTAINERS                    |   1 +
- include/hw/pci/pcie_sriov.h |  19 +++
+ docs/system/index.rst          |   1 +
- hw/pci/pci.c                |  70 +++++++----
+ docs/system/sriov.rst          |  37 ++++++
- hw/pci/pcie_sriov.c         | 299 +++++++++++++++++++++++++++++++++++---------
+ include/hw/pci/pci_device.h    |   6 +-
- hw/virtio/virtio-net-pci.c  |   1 +
+ include/hw/pci/pcie_sriov.h    |  21 +++
- hw/virtio/virtio-pci.c      |   7 ++
+ include/hw/virtio/virtio-pci.h |   1 +
-files changed, 319 insertions(+), 83 deletions(-)
+ hw/pci/pci.c                   |  76 +++++++----
  hw/pci/pcie_sriov.c            | 294 +++++++++++++++++++++++++++++++++--------
  hw/virtio/virtio-net-pci.c     |   1 +
  hw/virtio/virtio-pci.c         |  24 +++-
 files changed, 378 insertions(+), 84 deletions(-)
 ---
-base-commit: 2c4eb0476e461b8a4b2f745d25f987e831c7f640
+base-commit: 825b96dbcee23d134b691fc75618b59c5f53da32
 change-id: 20231202-sriov-9402fb262be8
 Best regards,
 --
 Akihiko Odaki <akihiko.odaki@daynix.com>

-[PATCH RFC v3 1/6] hw/pci: Do not add ROM BAR for SR-IOV VF
+[PATCH for-10.1 v9 1/9] hw/pci: Do not add ROM BAR for SR-IOV VF
 ...
 @@ -XXX,XX +XXX,XX @@ static void pci_add_option_rom(PCIDevice *pdev, bool is_default_rom,
          return;
      }
 +    if (pci_is_vf(pdev)) {
-+        if (pdev->rom_bar != UINT32_MAX) {
++        if (pdev->rom_bar > 0) {
 +            error_setg(errp, "ROM BAR cannot be enabled for SR-IOV VF");
 +        }
 +
 +        return;
 +    }
 +
      if (load_file || pdev->romsize == UINT32_MAX) {
          path = qemu_find_file(QEMU_FILE_TYPE_BIOS, pdev->romfile);
          if (path == NULL) {
 --
-.44.0
+.48.1

-New patch
+[PATCH for-10.1 v9 2/9] hw/pci: Fix SR-IOV VF number calculation
+pci_config_get_bar_addr() had a division by vf_stride. vf_stride needs
+to be non-zero when there are multiple VFs, but the specification does
+not prohibit to make it zero when there is only one VF.
+Do not perform the division for the first VF to avoid division by zero.
+Signed-off-by: Akihiko Odaki <akihiko.odaki@daynix.com>
+---
+ hw/pci/pci.c | 6 +++++-
+file changed, 5 insertions(+), 1 deletion(-)
+diff --git a/hw/pci/pci.c b/hw/pci/pci.c
+index XXXXXXX..XXXXXXX 100644
+--- a/hw/pci/pci.c
++++ b/hw/pci/pci.c
+@@ -XXX,XX +XXX,XX @@ static pcibus_t pci_config_get_bar_addr(PCIDevice *d, int reg,
+             pci_get_word(pf->config + sriov_cap + PCI_SRIOV_VF_OFFSET);
+         uint16_t vf_stride =
+             pci_get_word(pf->config + sriov_cap + PCI_SRIOV_VF_STRIDE);
+-        uint32_t vf_num = (d->devfn - (pf->devfn + vf_offset)) / vf_stride;
++        uint32_t vf_num = d->devfn - (pf->devfn + vf_offset);
++
++        if (vf_num) {
++            vf_num /= vf_stride;
++        }
+         if (type & PCI_BASE_ADDRESS_MEM_TYPE_64) {
+             new_addr = pci_get_quad(pf->config + bar);
+--
+.48.1

-[PATCH RFC v3 2/6] pcie_sriov: Ensure PF and VF are mutually exclusive
+[PATCH for-10.1 v9 3/9] pcie_sriov: Ensure PF and VF are mutually exclusive
 ...
 +    if (pci_is_vf(dev)) {
 +        error_setg(errp, "a device cannot be a SR-IOV PF and a VF at the same time");
 +        return false;
 +    }
 +
-     pcie_add_capability(dev, PCI_EXT_CAP_ID_SRIOV, 1,
+     if (total_vfs &&
-                         offset, PCI_EXT_CAP_SRIOV_SIZEOF);
+         (uint32_t)devfn + (uint32_t)(total_vfs - 1) * vf_stride >= PCI_DEVFN_MAX) {
-     dev->exp.sriov_cap = offset;
+         error_setg(errp, "VF addr overflows");
 --
-.44.0
+.48.1

-[PATCH RFC v3 3/6] pcie_sriov: Check PCI Express for SR-IOV PF
+[PATCH for-10.1 v9 4/9] pcie_sriov: Check PCI Express for SR-IOV PF
 ...
      if (pci_is_vf(dev)) {
          error_setg(errp, "a device cannot be a SR-IOV PF and a VF at the same time");
          return false;
 --
-.44.0
+.48.1

-[PATCH RFC v3 4/6] pcie_sriov: Allow user to create SR-IOV device
+[PATCH for-10.1 v9 5/9] pcie_sriov: Allow user to create SR-IOV device
 ...
 pcie_sriov_pf_exit() when exiting.
 Signed-off-by: Akihiko Odaki <akihiko.odaki@daynix.com>
 ---
  include/hw/pci/pci_device.h |   6 +-
- include/hw/pci/pcie_sriov.h |  19 +++
+ include/hw/pci/pcie_sriov.h |  18 +++
  hw/pci/pci.c                |  62 ++++++----
- hw/pci/pcie_sriov.c         | 289 +++++++++++++++++++++++++++++++++++---------
+ hw/pci/pcie_sriov.c         | 278 +++++++++++++++++++++++++++++++++++---------
-files changed, 293 insertions(+), 83 deletions(-)
+files changed, 286 insertions(+), 78 deletions(-)
 diff --git a/include/hw/pci/pci_device.h b/include/hw/pci/pci_device.h
 index XXXXXXX..XXXXXXX 100644
 --- a/include/hw/pci/pci_device.h
 +++ b/include/hw/pci/pci_device.h
 ...
 +    bool sriov_vf_user_creatable;
  };
  enum PCIReqIDType {
 @@ -XXX,XX +XXX,XX @@ struct PCIDevice {
-     /* ID of standby device in net_failover pair */
+      * realizing the device.
-     char *failover_pair_id;
+      */
-     uint32_t acpi_index;
+     uint32_t max_bounce_buffer_size;
 +
 +    char *sriov_pf;
  };
  static inline int pci_intx(PCIDevice *pci_dev)
 ...
 diff --git a/include/hw/pci/pcie_sriov.h b/include/hw/pci/pcie_sriov.h
 index XXXXXXX..XXXXXXX 100644
 --- a/include/hw/pci/pcie_sriov.h
 +++ b/include/hw/pci/pcie_sriov.h
 @@ -XXX,XX +XXX,XX @@
- struct PCIESriovPF {
+ typedef struct PCIESriovPF {
      uint8_t vf_bar_type[PCI_NUM_REGIONS];   /* Store type for each VF bar */
      PCIDevice **vf;     /* Pointer to an array of num_vfs VF devices */
 +    bool vf_user_created; /* If VFs are created by user */
- };
+ } PCIESriovPF;
- struct PCIESriovVF {
+ typedef struct PCIESriovVF {
 @@ -XXX,XX +XXX,XX @@ void pcie_sriov_pf_init_vf_bar(PCIDevice *dev, int region_num,
  void pcie_sriov_vf_register_bar(PCIDevice *dev, int region_num,
                                  MemoryRegion *memory);
 +/**
 + * pcie_sriov_pf_init_from_user_created_vfs() - Initialize PF with user-created
 + *                                              VFs.
 + * @dev: A PCIe device being realized.
 + * @offset: The offset of the SR-IOV capability.
 + * @errp: pointer to Error*, to store an error if it happens.
 + *
-+ * Return:
++ * Return: The size of added capability. 0 if the user did not create VFs.
-+ * * true - @dev is initialized as a PCIe SR-IOV PF.
++ *         -1 if failed.
 + * * false - @dev is not initialized because there is no SR-IOV VFs or an error
 + *           occurred.
 + */
-+bool pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev, uint16_t offset,
++int16_t pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev,
-+                                              Error **errp);
++                                                 uint16_t offset,
 +                                                 Error **errp);
 +
 +bool pcie_sriov_register_device(PCIDevice *dev, Error **errp);
 +void pcie_sriov_unregister_device(PCIDevice *dev);
 +
  /*
   * Default (minimal) page size support values
   * as required by the SR/IOV standard:
 diff --git a/hw/pci/pci.c b/hw/pci/pci.c
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/pci/pci.c
 +++ b/hw/pci/pci.c
-@@ -XXX,XX +XXX,XX @@ static Property pci_props[] = {
+@@ -XXX,XX +XXX,XX @@ static const Property pci_props[] = {
                      QEMU_PCIE_ERR_UNC_MASK_BITNR, true),
      DEFINE_PROP_BIT("x-pcie-ari-nextfn-1", PCIDevice, cap_present,
                      QEMU_PCIE_ARI_NEXTFN_1_BITNR, false),
+     DEFINE_PROP_SIZE32("x-max-bounce-buffer-size", PCIDevice,
+                      max_bounce_buffer_size, DEFAULT_MAX_BOUNCE_BUFFER_SIZE),
 +    DEFINE_PROP_STRING("sriov-pf", PCIDevice, sriov_pf),
-     DEFINE_PROP_END_OF_LIST()
+     DEFINE_PROP_BIT("x-pcie-ext-tag", PCIDevice, cap_present,
- };
+                     QEMU_PCIE_EXT_TAG_BITNR, true),
+     { .name = "busnr", .info = &prop_pci_busnr },
 @@ -XXX,XX +XXX,XX @@ static void pci_init_multifunction(PCIBus *bus, PCIDevice *dev, Error **errp)
          dev->config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION;
      }
 -    /*
 ...
 -    assert(!pci_is_vf(pci_dev)); /* VFs must use pcie_sriov_vf_register_bar */
      assert(region_num >= 0);
      assert(region_num < PCI_NUM_REGIONS);
      assert(is_power_of_2(size));
 @@ -XXX,XX +XXX,XX @@ void pci_register_bar(PCIDevice *pci_dev, int region_num,
-     assert(hdr_type != PCI_HEADER_TYPE_BRIDGE || region_num < 2);
      r = &pci_dev->io_regions[region_num];
+     assert(!r->size);
 -    r->addr = PCI_BAR_UNMAPPED;
      r->size = size;
      r->type = type;
      r->memory = memory;
 @@ -XXX,XX +XXX,XX @@ void pci_register_bar(PCIDevice *pci_dev, int region_num,
 ...
 diff --git a/hw/pci/pcie_sriov.c b/hw/pci/pcie_sriov.c
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/pci/pcie_sriov.c
 +++ b/hw/pci/pcie_sriov.c
 @@ -XXX,XX +XXX,XX @@
+ #include "hw/pci/pcie.h"
+ #include "hw/pci/pci_bus.h"
  #include "hw/qdev-properties.h"
- #include "qemu/error-report.h"
+-#include "qemu/error-report.h"
  #include "qemu/range.h"
-+#include "qapi/error.h"
+ #include "qapi/error.h"
  #include "trace.h"
 +static GHashTable *pfs;
 +
  static void unparent_vfs(PCIDevice *dev, uint16_t total_vfs)
 ...
 -bool pcie_sriov_pf_init(PCIDevice *dev, uint16_t offset,
 -                        const char *vfname, uint16_t vf_dev_id,
 -                        uint16_t init_vfs, uint16_t total_vfs,
 -                        uint16_t vf_offset, uint16_t vf_stride,
 -                        Error **errp)
-+static void clear_ctrl_vfe(PCIDevice *dev)
-+{
-+    uint8_t *ctrl = dev->config + dev->exp.sriov_cap + PCI_SRIOV_CTRL;
-+    pci_set_word(ctrl, pci_get_word(ctrl) & ~PCI_SRIOV_CTRL_VFE);
-+}
-+
 +static void register_vfs(PCIDevice *dev)
 +{
 +    uint16_t num_vfs;
 +    uint16_t i;
 +    uint16_t sriov_cap = dev->exp.sriov_cap;
 +
 +    assert(sriov_cap > 0);
 +    num_vfs = pci_get_word(dev->config + sriov_cap + PCI_SRIOV_NUM_VF);
-+    if (num_vfs > pci_get_word(dev->config + sriov_cap + PCI_SRIOV_TOTAL_VF)) {
-+        clear_ctrl_vfe(dev);
-+        return;
-+    }
 +
 +    trace_sriov_register_vfs(dev->name, PCI_SLOT(dev->devfn),
 +                             PCI_FUNC(dev->devfn), num_vfs);
 +    for (i = 0; i < num_vfs; i++) {
 +        pci_set_enabled(dev->exp.sriov_pf.vf[i], true);
 +    }
++
++    pci_set_word(dev->wmask + sriov_cap + PCI_SRIOV_NUM_VF, 0);
 +}
 +
 +static void unregister_vfs(PCIDevice *dev)
 +{
++    uint8_t *cfg = dev->config + dev->exp.sriov_cap;
 +    uint16_t i;
-+    uint8_t *cfg = dev->config + dev->exp.sriov_cap;
 +
 +    trace_sriov_unregister_vfs(dev->name, PCI_SLOT(dev->devfn),
 +                               PCI_FUNC(dev->devfn));
 +    for (i = 0; i < pci_get_word(cfg + PCI_SRIOV_TOTAL_VF); i++) {
 +        pci_set_enabled(dev->exp.sriov_pf.vf[i], false);
 +    }
++
++    pci_set_word(dev->wmask + dev->exp.sriov_cap + PCI_SRIOV_NUM_VF, 0xffff);
 +}
 +
 +static bool pcie_sriov_pf_init_common(PCIDevice *dev, uint16_t offset,
 +                                      uint16_t vf_dev_id, uint16_t init_vfs,
 +                                      uint16_t total_vfs, uint16_t vf_offset,
 +                                      uint16_t vf_stride, Error **errp)
  {
 -    BusState *bus = qdev_get_parent_bus(&dev->qdev);
--    int32_t devfn = dev->devfn + vf_offset;
+     int32_t devfn = dev->devfn + vf_offset;
      uint8_t *cfg = dev->config + offset;
      uint8_t *wmask;
 @@ -XXX,XX +XXX,XX @@ bool pcie_sriov_pf_init(PCIDevice *dev, uint16_t offset,
      qdev_prop_set_bit(&dev->qdev, "multifunction", true);
 +    return true;
 ...
 -        error_report("%s: PCI region size must be a power"
 -                     " of two - type=0x%x, size=0x%"FMT_PCIBUS,
 -                     __func__, type, size);
 -        exit(1);
 -    }
--
++    return pci_register_bar(dev, region_num, type, memory);
 +}
 -    r = &dev->io_regions[region_num];
 -    r->memory = memory;
 -    r->address_space =
 -        type & PCI_BASE_ADDRESS_SPACE_IO
 -        ? bus->address_space_io
 ...
 -    r->addr = pci_bar_address(dev, region_num, r->type, r->size);
 -    if (r->addr != PCI_BAR_UNMAPPED) {
 -        memory_region_add_subregion_overlap(r->address_space,
 -                                            r->addr, r->memory, 1);
 -    }
-+    return pci_register_bar(dev, region_num, type, memory);
- }
--static void clear_ctrl_vfe(PCIDevice *dev)
 +static gint compare_vf_devfns(gconstpointer a, gconstpointer b)
- {
++{
 -    uint8_t *ctrl = dev->config + dev->exp.sriov_cap + PCI_SRIOV_CTRL;
 -    pci_set_word(ctrl, pci_get_word(ctrl) & ~PCI_SRIOV_CTRL_VFE);
 +    return (*(PCIDevice **)a)->devfn - (*(PCIDevice **)b)->devfn;
  }
 -static void register_vfs(PCIDevice *dev)
-+bool pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev, uint16_t offset,
++int16_t pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev,
-+                                              Error **errp)
++                                                 uint16_t offset,
 +                                                 Error **errp)
  {
 -    uint16_t num_vfs;
 +    GPtrArray *pf;
 +    PCIDevice **vfs;
 +    BusState *bus = qdev_get_parent_bus(DEVICE(dev));
 ...
      uint16_t i;
 -    uint16_t sriov_cap = dev->exp.sriov_cap;
 -    assert(sriov_cap > 0);
 -    num_vfs = pci_get_word(dev->config + sriov_cap + PCI_SRIOV_NUM_VF);
--    if (num_vfs > pci_get_word(dev->config + sriov_cap + PCI_SRIOV_TOTAL_VF)) {
--        clear_ctrl_vfe(dev);
--        return;
 +    if (!pfs || !dev->qdev.id) {
-+        return false;
++        return 0;
-     }
++    }
 -    trace_sriov_register_vfs(dev->name, PCI_SLOT(dev->devfn),
 -                             PCI_FUNC(dev->devfn), num_vfs);
 -    for (i = 0; i < num_vfs; i++) {
 -        pci_set_enabled(dev->exp.sriov_pf.vf[i], true);
 +    pf = g_hash_table_lookup(pfs, dev->qdev.id);
 +    if (!pf) {
-+        return false;
++        return 0;
-+    }
+     }
-+
 -    pci_set_word(dev->wmask + sriov_cap + PCI_SRIOV_NUM_VF, 0);
 +    if (pf->len > UINT16_MAX) {
 +        error_setg(errp, "too many VFs");
-+        return false;
++        return -1;
 +    }
 +
 +    g_ptr_array_sort(pf, compare_vf_devfns);
 +    vfs = (void *)pf->pdata;
 +
 +    if (vfs[0]->devfn <= dev->devfn) {
 +        error_setg(errp, "a VF function number is less than the PF function number");
-+        return false;
++        return -1;
-     }
++    }
 +
 +    vf_dev_id = pci_get_word(vfs[0]->config + PCI_DEVICE_ID);
 +    vf_offset = vfs[0]->devfn - dev->devfn;
 +    vf_stride = pf->len < 2 ? 0 : vfs[1]->devfn - vfs[0]->devfn;
 +
 +    for (i = 0; i < pf->len; i++) {
 +        if (bus != qdev_get_parent_bus(&vfs[i]->qdev)) {
 +            error_setg(errp, "SR-IOV VF parent bus mismatches with PF");
-+            return false;
++            return -1;
 +        }
 +
 +        if (ven_id != pci_get_word(vfs[i]->config + PCI_VENDOR_ID)) {
 +            error_setg(errp, "SR-IOV VF vendor ID mismatches with PF");
-+            return false;
++            return -1;
 +        }
 +
 +        if (vf_dev_id != pci_get_word(vfs[i]->config + PCI_DEVICE_ID)) {
 +            error_setg(errp, "inconsistent SR-IOV VF device IDs");
-+            return false;
++            return -1;
 +        }
 +
 +        for (size_t j = 0; j < PCI_NUM_REGIONS; j++) {
 +            if (vfs[i]->io_regions[j].size != vfs[0]->io_regions[j].size ||
 +                vfs[i]->io_regions[j].type != vfs[0]->io_regions[j].type) {
 +                error_setg(errp, "inconsistent SR-IOV BARs");
-+                return false;
++                return -1;
 +            }
 +        }
 +
 +        if (vfs[i]->devfn - vfs[0]->devfn != vf_stride * i) {
 +            error_setg(errp, "inconsistent SR-IOV stride");
-+            return false;
++            return -1;
 +        }
 +    }
 +
 +    if (!pcie_sriov_pf_init_common(dev, offset, vf_dev_id, pf->len,
 +                                   pf->len, vf_offset, vf_stride, errp)) {
-+        return false;
++        return -1;
 +    }
 +
 +    for (i = 0; i < pf->len; i++) {
 +        vfs[i]->exp.sriov_vf.pf = dev;
 +        vfs[i]->exp.sriov_vf.vf_number = i;
 ...
 +
 +    dev->exp.sriov_pf.vf = vfs;
 +    dev->exp.sriov_pf.vf_user_created = true;
 +
 +    for (i = 0; i < PCI_NUM_REGIONS; i++) {
-+        uint8_t type = vfs[0]->io_regions[i].type;
++        PCIIORegion *region = &vfs[0]->io_regions[i];
-+        pcibus_t size = vfs[0]->io_regions[i].size;
++
-+
++        if (region->size) {
-+        if (size) {
++            pcie_sriov_pf_init_vf_bar(dev, i, region->type, region->size);
-+            pcie_sriov_pf_init_vf_bar(dev, i, type, size);
++        }
-+        }
++    }
-+    }
++
-+
++    return PCI_EXT_CAP_SRIOV_SIZEOF;
-+    return true;
+ }
  }
 -static void unregister_vfs(PCIDevice *dev)
 +bool pcie_sriov_register_device(PCIDevice *dev, Error **errp)
  {
+-    uint8_t *cfg = dev->config + dev->exp.sriov_cap;
 -    uint16_t i;
--    uint8_t *cfg = dev->config + dev->exp.sriov_cap;
 +    if (!dev->exp.sriov_pf.vf && dev->qdev.id &&
 +        pfs && g_hash_table_contains(pfs, dev->qdev.id)) {
 +        error_setg(errp, "attaching user-created SR-IOV VF unsupported");
 +        return false;
 +    }
 ...
 +            pf = g_ptr_array_new();
 +            g_hash_table_insert(pfs, g_strdup(dev->sriov_pf), pf);
 +        }
 +
 +        g_ptr_array_add(pf, dev);
-+    }
+     }
-+
 -    pci_set_word(dev->wmask + dev->exp.sriov_cap + PCI_SRIOV_NUM_VF, 0xffff);
 +    return true;
 +}
 +
 +void pcie_sriov_unregister_device(PCIDevice *dev)
 +{
 +    if (dev->sriov_pf && pfs) {
-+        GPtrArray *pf = g_hash_table_lookup(pfs, dev->qdev.id);
++        GPtrArray *pf = g_hash_table_lookup(pfs, dev->sriov_pf);
 +
 +        if (pf) {
 +            g_ptr_array_remove_fast(pf, dev);
 +
 +            if (!pf->len) {
-+                g_hash_table_remove(pfs, dev->qdev.id);
++                g_hash_table_remove(pfs, dev->sriov_pf);
 +                g_ptr_array_free(pf, FALSE);
 +            }
 +        }
-     }
++    }
  }
  void pcie_sriov_config_write(PCIDevice *dev, uint32_t address,
 @@ -XXX,XX +XXX,XX @@ void pcie_sriov_pf_add_sup_pgsize(PCIDevice *dev, uint16_t opt_sup_pgsize)
  uint16_t pcie_sriov_vf_number(PCIDevice *dev)
  {
 -    assert(pci_is_vf(dev));
 +    assert(dev->exp.sriov_vf.pf);
      return dev->exp.sriov_vf.vf_number;
  }
 --
-.44.0
+.48.1

-[PATCH RFC v3 5/6] virtio-pci: Implement SR-IOV PF
+[PATCH for-10.1 v9 6/9] virtio-pci: Implement SR-IOV PF
 Allow user to attach SR-IOV VF to a virtio-pci PF.
 Signed-off-by: Akihiko Odaki <akihiko.odaki@daynix.com>
 ---
- hw/virtio/virtio-pci.c | 7 +++++++
+ include/hw/virtio/virtio-pci.h |  1 +
-file changed, 7 insertions(+)
+ hw/virtio/virtio-pci.c         | 20 +++++++++++++++-----
 files changed, 16 insertions(+), 5 deletions(-)
+diff --git a/include/hw/virtio/virtio-pci.h b/include/hw/virtio/virtio-pci.h
+index XXXXXXX..XXXXXXX 100644
+--- a/include/hw/virtio/virtio-pci.h
++++ b/include/hw/virtio/virtio-pci.h
+@@ -XXX,XX +XXX,XX @@ struct VirtIOPCIProxy {
+     uint32_t modern_io_bar_idx;
+     uint32_t modern_mem_bar_idx;
+     int config_cap;
++    uint16_t last_pcie_cap_offset;
+     uint32_t flags;
+     bool disable_modern;
+     bool ignore_backend_features;
 diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/virtio/virtio-pci.c
 +++ b/hw/virtio/virtio-pci.c
 @@ -XXX,XX +XXX,XX @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
+     uint8_t *config;
+     uint32_t size;
+     VirtIODevice *vdev = virtio_bus_get_device(bus);
++    int16_t res;
+     /*
+      * Virtio capabilities present without
+@@ -XXX,XX +XXX,XX @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
          pci_register_bar(&proxy->pci_dev, proxy->legacy_io_bar_idx,
                           PCI_BASE_ADDRESS_SPACE_IO, &proxy->bar);
      }
 +
-+    if (pcie_sriov_pf_init_from_user_created_vfs(&proxy->pci_dev,
++    res = pcie_sriov_pf_init_from_user_created_vfs(&proxy->pci_dev,
-+                                                 PCI_CONFIG_SPACE_SIZE,
++                                                   proxy->last_pcie_cap_offset,
-+                                                 errp)) {
++                                                   errp);
 +    if (res > 0) {
 +        proxy->last_pcie_cap_offset += res;
 +        virtio_add_feature(&vdev->host_features, VIRTIO_F_SR_IOV);
 +    }
  }
  static void virtio_pci_device_unplugged(DeviceState *d)
-@@ -XXX,XX +XXX,XX @@ static void virtio_pci_device_unplugged(DeviceState *d)
+@@ -XXX,XX +XXX,XX @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
-     bool modern = virtio_pci_modern(proxy);
      if (pcie_port && pci_is_express(pci_dev)) {
          int pos;
 -        uint16_t last_pcie_cap_offset = PCI_CONFIG_SPACE_SIZE;
 +        proxy->last_pcie_cap_offset = PCI_CONFIG_SPACE_SIZE;
          pos = pcie_endpoint_cap_init(pci_dev, 0);
          assert(pos > 0);
@@ -XXX,XX +XXX,XX @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
          pci_set_word(pci_dev->config + pos + PCI_PM_PMC, 0x3);
          if (proxy->flags & VIRTIO_PCI_FLAG_AER) {
 -            pcie_aer_init(pci_dev, PCI_ERR_VER, last_pcie_cap_offset,
 +            pcie_aer_init(pci_dev, PCI_ERR_VER, proxy->last_pcie_cap_offset,
                            PCI_ERR_SIZEOF, NULL);
 -            last_pcie_cap_offset += PCI_ERR_SIZEOF;
 +            proxy->last_pcie_cap_offset += PCI_ERR_SIZEOF;
          }
          if (proxy->flags & VIRTIO_PCI_FLAG_INIT_DEVERR) {
@@ -XXX,XX +XXX,XX @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
          }
          if (proxy->flags & VIRTIO_PCI_FLAG_ATS) {
 -            pcie_ats_init(pci_dev, last_pcie_cap_offset,
 +            pcie_ats_init(pci_dev, proxy->last_pcie_cap_offset,
                            proxy->flags & VIRTIO_PCI_FLAG_ATS_PAGE_ALIGNED);
 -            last_pcie_cap_offset += PCI_EXT_CAP_ATS_SIZEOF;
 +            proxy->last_pcie_cap_offset += PCI_EXT_CAP_ATS_SIZEOF;
          }
          if (proxy->flags & VIRTIO_PCI_FLAG_INIT_FLR) {
@@ -XXX,XX +XXX,XX @@ static void virtio_pci_exit(PCIDevice *pci_dev)
                       !pci_bus_is_root(pci_get_bus(pci_dev));
      bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
 +    pcie_sriov_pf_exit(&proxy->pci_dev);
-     virtio_pci_stop_ioeventfd(proxy);
+     msix_uninit_exclusive_bar(pci_dev);
+     if (proxy->flags & VIRTIO_PCI_FLAG_AER && pcie_port &&
-     if (modern) {
+         pci_is_express(pci_dev)) {
 --
-.44.0
+.48.1

-[PATCH RFC v3 6/6] virtio-net: Implement SR-IOV VF
+[PATCH for-10.1 v9 7/9] virtio-net: Implement SR-IOV VF
 ...
      set_bit(DEVICE_CATEGORY_NETWORK, dc->categories);
      device_class_set_props(dc, virtio_net_properties);
      vpciklass->realize = virtio_net_pci_realize;
 --
-.44.0
+.48.1

-New patch
+[PATCH for-10.1 v9 8/9] docs: Document composable SR-IOV device
+Signed-off-by: Akihiko Odaki <akihiko.odaki@daynix.com>
+---
+ MAINTAINERS           |  1 +
+ docs/system/index.rst |  1 +
+ docs/system/sriov.rst | 36 ++++++++++++++++++++++++++++++++++++
+files changed, 38 insertions(+)
+diff --git a/MAINTAINERS b/MAINTAINERS
+index XXXXXXX..XXXXXXX 100644
+--- a/MAINTAINERS
++++ b/MAINTAINERS
+@@ -XXX,XX +XXX,XX @@ F: hw/pci-bridge/*
+ F: qapi/pci.json
+ F: docs/pci*
+ F: docs/specs/*pci*
++F: docs/system/sriov.rst
+ PCIE DOE
+ M: Huai-Cheng Kuo <hchkuo@avery-design.com.tw>
+diff --git a/docs/system/index.rst b/docs/system/index.rst
+index XXXXXXX..XXXXXXX 100644
+--- a/docs/system/index.rst
++++ b/docs/system/index.rst
+@@ -XXX,XX +XXX,XX @@ or Hypervisor.Framework.
+    multi-process
+    confidential-guest-support
+    vm-templating
++   sriov
+diff --git a/docs/system/sriov.rst b/docs/system/sriov.rst
+new file mode 100644
+index XXXXXXX..XXXXXXX
+--- /dev/null
++++ b/docs/system/sriov.rst
+@@ -XXX,XX +XXX,XX @@
++.. SPDX-License-Identifier: GPL-2.0-or-later
++
++Compsable SR-IOV device
++=======================
++
++SR-IOV (Single Root I/O Virtualization) is an optional extended capability of a
++PCI Express device. It allows a single physical function (PF) to appear as
++multiple virtual functions (VFs) for the main purpose of eliminating software
++overhead in I/O from virtual machines.
++
++There are devices with predefined SR-IOV configurations, but it is also possible
++to compose an SR-IOV device yourself. Composing an SR-IOV device is currently
++only supported by virtio-net-pci.
++
++Users can configure an SR-IOV-capable virtio-net device by adding
++virtio-net-pci functions to a bus. Below is a command line example:
++
++.. code-block:: shell
++
++    -netdev user,id=n -netdev user,id=o
++    -netdev user,id=p -netdev user,id=q
++    -device pcie-root-port,id=b
++    -device virtio-net-pci,bus=b,addr=0x0.0x3,netdev=q,sriov-pf=f
++    -device virtio-net-pci,bus=b,addr=0x0.0x2,netdev=p,sriov-pf=f
++    -device virtio-net-pci,bus=b,addr=0x0.0x1,netdev=o,sriov-pf=f
++    -device virtio-net-pci,bus=b,addr=0x0.0x0,netdev=n,id=f
++
++The VFs specify the paired PF with ``sriov-pf`` property. The PF must be
++added after all VFs. It is the user's responsibility to ensure that VFs have
++function numbers larger than one of the PF, and that the function numbers
++have a consistent stride.
++
++You may also need to perform additional steps to activate the SR-IOV feature on
++your guest. For Linux, refer to [1]_.
++
++.. [1] https://docs.kernel.org/PCI/pci-iov-howto.html
+--
+.48.1

-New patch
+[PATCH for-10.1 v9 9/9] pcie_sriov: Make a PCI device with user-created VF ARI-capable
+Signed-off-by: Akihiko Odaki <akihiko.odaki@daynix.com>
+---
+ docs/system/sriov.rst       |  3 ++-
+ include/hw/pci/pcie_sriov.h |  7 +++++--
+ hw/pci/pcie_sriov.c         |  8 +++++++-
+ hw/virtio/virtio-pci.c      | 16 ++++++++++------
+files changed, 24 insertions(+), 10 deletions(-)
+diff --git a/docs/system/sriov.rst b/docs/system/sriov.rst
+index XXXXXXX..XXXXXXX 100644
+--- a/docs/system/sriov.rst
++++ b/docs/system/sriov.rst
+@@ -XXX,XX +XXX,XX @@ virtio-net-pci functions to a bus. Below is a command line example:
+ The VFs specify the paired PF with ``sriov-pf`` property. The PF must be
+ added after all VFs. It is the user's responsibility to ensure that VFs have
+ function numbers larger than one of the PF, and that the function numbers
+-have a consistent stride.
++have a consistent stride. Both the PF and VFs are ARI-capable so you can have
++255 VFs at maximum.
+ You may also need to perform additional steps to activate the SR-IOV feature on
+ your guest. For Linux, refer to [1]_.
+diff --git a/include/hw/pci/pcie_sriov.h b/include/hw/pci/pcie_sriov.h
+index XXXXXXX..XXXXXXX 100644
+--- a/include/hw/pci/pcie_sriov.h
++++ b/include/hw/pci/pcie_sriov.h
+@@ -XXX,XX +XXX,XX @@ void pcie_sriov_vf_register_bar(PCIDevice *dev, int region_num,
+ /**
+  * pcie_sriov_pf_init_from_user_created_vfs() - Initialize PF with user-created
+- *                                              VFs.
++ *                                              VFs, adding ARI to PF
+  * @dev: A PCIe device being realized.
+  * @offset: The offset of the SR-IOV capability.
+  * @errp: pointer to Error*, to store an error if it happens.
+  *
+- * Return: The size of added capability. 0 if the user did not create VFs.
++ * Initializes a PF with user-created VFs, adding the ARI extended capability to
++ * the PF. The VFs should call pcie_ari_init() to form an ARI device.
++ *
++ * Return: The size of added capabilities. 0 if the user did not create VFs.
+  *         -1 if failed.
+  */
+ int16_t pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev,
+diff --git a/hw/pci/pcie_sriov.c b/hw/pci/pcie_sriov.c
+index XXXXXXX..XXXXXXX 100644
+--- a/hw/pci/pcie_sriov.c
++++ b/hw/pci/pcie_sriov.c
+@@ -XXX,XX +XXX,XX @@ int16_t pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev,
+     PCIDevice **vfs;
+     BusState *bus = qdev_get_parent_bus(DEVICE(dev));
+     uint16_t ven_id = pci_get_word(dev->config + PCI_VENDOR_ID);
++    uint16_t size = PCI_EXT_CAP_SRIOV_SIZEOF;
+     uint16_t vf_dev_id;
+     uint16_t vf_offset;
+     uint16_t vf_stride;
+@@ -XXX,XX +XXX,XX @@ int16_t pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev,
+         return -1;
+     }
++    if (!pcie_find_capability(dev, PCI_EXT_CAP_ID_ARI)) {
++        pcie_ari_init(dev, offset + size);
++        size += PCI_ARI_SIZEOF;
++    }
++
+     for (i = 0; i < pf->len; i++) {
+         vfs[i]->exp.sriov_vf.pf = dev;
+         vfs[i]->exp.sriov_vf.vf_number = i;
+@@ -XXX,XX +XXX,XX @@ int16_t pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev,
+         }
+     }
+-    return PCI_EXT_CAP_SRIOV_SIZEOF;
++    return size;
+ }
+ bool pcie_sriov_register_device(PCIDevice *dev, Error **errp)
+diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
+index XXXXXXX..XXXXXXX 100644
+--- a/hw/virtio/virtio-pci.c
++++ b/hw/virtio/virtio-pci.c
+@@ -XXX,XX +XXX,XX @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
+                          PCI_BASE_ADDRESS_SPACE_IO, &proxy->bar);
+     }
+-    res = pcie_sriov_pf_init_from_user_created_vfs(&proxy->pci_dev,
+-                                                   proxy->last_pcie_cap_offset,
+-                                                   errp);
+-    if (res > 0) {
+-        proxy->last_pcie_cap_offset += res;
+-        virtio_add_feature(&vdev->host_features, VIRTIO_F_SR_IOV);
++    if (pci_is_vf(&proxy->pci_dev)) {
++        pcie_ari_init(&proxy->pci_dev, proxy->last_pcie_cap_offset);
++        proxy->last_pcie_cap_offset += PCI_ARI_SIZEOF;
++    } else {
++        res = pcie_sriov_pf_init_from_user_created_vfs(
++            &proxy->pci_dev, proxy->last_pcie_cap_offset, errp);
++        if (res > 0) {
++            proxy->last_pcie_cap_offset += res;
++            virtio_add_feature(&vdev->host_features, VIRTIO_F_SR_IOV);
++        }
+     }
+ }
+--
+.48.1

Based-on: <20240228-reuse-v8-0-282660281e60@daynix.com>
("[PATCH v8 00/15] hw/pci: SR-IOV related fixes and improvements")

Introduction
------------

This series is based on the RFC series submitted by Yui Washizu[1].
See also [2] for the context.

This series enables SR-IOV emulation for virtio-net. It is useful
to test SR-IOV support on the guest, or to expose several vDPA devices
in a VM. vDPA devices can also provide L2 switching feature for
offloading though it is out of scope to allow the guest to configure
such a feature.

The PF side code resides in virtio-pci. The VF side code resides in
the PCI common infrastructure, but it is restricted to work only for
virtio-net-pci because of lack of validation.

User Interface
--------------

A user can configure a SR-IOV capable virtio-net device by adding
virtio-net-pci functions to a bus. Below is a command line example:
  -netdev user,id=n -netdev user,id=o
  -netdev user,id=p -netdev user,id=q
  -device pcie-root-port,id=b
  -device virtio-net-pci,bus=b,addr=0x0.0x3,netdev=q,sriov-pf=f
  -device virtio-net-pci,bus=b,addr=0x0.0x2,netdev=p,sriov-pf=f
  -device virtio-net-pci,bus=b,addr=0x0.0x1,netdev=o,sriov-pf=f
  -device virtio-net-pci,bus=b,addr=0x0.0x0,netdev=n,id=f

The VFs specify the paired PF with "sriov-pf" property. The PF must be
added after all VFs. It is user's responsibility to ensure that VFs have
function numbers larger than one of the PF, and the function numbers
have a consistent stride.

Keeping VF instances
--------------------

A problem with SR-IOV emulation is that it needs to hotplug the VFs as
the guest requests. Previously, this behavior was implemented by
realizing and unrealizing VFs at runtime. However, this strategy does
not work well for the proposed virtio-net emulation; in this proposal,
device options passed in the command line must be maintained as VFs
are hotplugged, but they are consumed when the machine starts and not
available after that, which makes realizing VFs at runtime impossible.

As an strategy alternative to runtime realization/unrealization, this
series proposes to reuse the code to power down PCI Express devices.
When a PCI Express device is powered down, it will be hidden from the
guest but will be kept realized. This effectively implements the
behavior we need for the SR-IOV emulation.

Summary
-------

Patch 1 disables ROM BAR, which virtio-net-pci enables by default, for
VFs.
Patch 2 and 3 adds validations.
Patch 4 adds user-created SR-IOV VF infrastructure.
Patch 5 makes virtio-pci work as SR-IOV PF for user-created VFs.
Patch 6 allows user to create SR-IOV VFs with virtio-net-pci.

[1] https://patchew.org/QEMU/1689731808-3009-1-git-send-email-yui.washidu@gmail.com/
[2] https://lore.kernel.org/all/5d46f455-f530-4e5e-9ae7-13a2297d4bc5@daynix.com/

Co-developed-by: Yui Washizu <yui.washidu@gmail.com>
Signed-off-by: Akihiko Odaki <akihiko.odaki@daynix.com>
---
Changes in v3:
- Rebased.
- Link to v2: https://lore.kernel.org/r/20231210-sriov-v2-0-b959e8a6dfaf@daynix.com

Changes in v2:
- Changed to keep VF instances.
- Link to v1: https://lore.kernel.org/r/20231202-sriov-v1-0-32b3570f7bd6@daynix.com

---
Akihiko Odaki (6):
      hw/pci: Do not add ROM BAR for SR-IOV VF
      pcie_sriov: Ensure PF and VF are mutually exclusive
      pcie_sriov: Check PCI Express for SR-IOV PF
      pcie_sriov: Allow user to create SR-IOV device
      virtio-pci: Implement SR-IOV PF
      virtio-net: Implement SR-IOV VF

include/hw/pci/pci_device.h |   6 +-
 include/hw/pci/pcie_sriov.h |  19 +++
 hw/pci/pci.c                |  70 +++++++----
 hw/pci/pcie_sriov.c         | 299 +++++++++++++++++++++++++++++++++++---------
 hw/virtio/virtio-net-pci.c  |   1 +
 hw/virtio/virtio-pci.c      |   7 ++
 6 files changed, 319 insertions(+), 83 deletions(-)
---
base-commit: 2c4eb0476e461b8a4b2f745d25f987e831c7f640
change-id: 20231202-sriov-9402fb262be8

Best regards,
-- 
Akihiko Odaki <akihiko.odaki@daynix.com>

A user can create a SR-IOV device by specifying the PF with the
sriov-pf property of the VFs. The VFs must be added before the PF.

A user-creatable VF must have PCIDeviceClass::sriov_vf_user_creatable
set. Such a VF cannot refer to the PF because it is created before the
PF.

A PF that user-creatable VFs can be attached calls
pcie_sriov_pf_init_from_user_created_vfs() during realization and
pcie_sriov_pf_exit() when exiting.

Signed-off-by: Akihiko Odaki <akihiko.odaki@daynix.com>
---
 include/hw/pci/pci_device.h |   6 +-
 include/hw/pci/pcie_sriov.h |  19 +++
 hw/pci/pci.c                |  62 ++++++----
 hw/pci/pcie_sriov.c         | 289 +++++++++++++++++++++++++++++++++++---------
 4 files changed, 293 insertions(+), 83 deletions(-)

diff --git a/include/hw/pci/pci_device.h b/include/hw/pci/pci_device.h
index XXXXXXX..XXXXXXX 100644
--- a/include/hw/pci/pci_device.h
+++ b/include/hw/pci/pci_device.h
@@ -XXX,XX +XXX,XX @@ struct PCIDeviceClass {
     uint16_t subsystem_id;              /* only for header type = 0 */
 
     const char *romfile;                /* rom bar */
+
+    bool sriov_vf_user_creatable;
 };
 
 enum PCIReqIDType {
@@ -XXX,XX +XXX,XX @@ struct PCIDevice {
     /* ID of standby device in net_failover pair */
     char *failover_pair_id;
     uint32_t acpi_index;
+
+    char *sriov_pf;
 };
 
 static inline int pci_intx(PCIDevice *pci_dev)
@@ -XXX,XX +XXX,XX @@ static inline int pci_is_express_downstream_port(const PCIDevice *d)
 
 static inline int pci_is_vf(const PCIDevice *d)
 {
-    return d->exp.sriov_vf.pf != NULL;
+    return d->sriov_pf || d->exp.sriov_vf.pf != NULL;
 }
 
 static inline uint32_t pci_config_size(const PCIDevice *d)
diff --git a/include/hw/pci/pcie_sriov.h b/include/hw/pci/pcie_sriov.h
index XXXXXXX..XXXXXXX 100644
--- a/include/hw/pci/pcie_sriov.h
+++ b/include/hw/pci/pcie_sriov.h
@@ -XXX,XX +XXX,XX @@
 struct PCIESriovPF {
     uint8_t vf_bar_type[PCI_NUM_REGIONS];   /* Store type for each VF bar */
     PCIDevice **vf;     /* Pointer to an array of num_vfs VF devices */
+    bool vf_user_created; /* If VFs are created by user */
 };
 
 struct PCIESriovVF {
@@ -XXX,XX +XXX,XX @@ void pcie_sriov_pf_init_vf_bar(PCIDevice *dev, int region_num,
 void pcie_sriov_vf_register_bar(PCIDevice *dev, int region_num,
                                 MemoryRegion *memory);
 
+/**
+ * pcie_sriov_pf_init_from_user_created_vfs() - Initialize PF with user-created
+ *                                              VFs.
+ * @dev: A PCIe device being realized.
+ * @offset: The offset of the SR-IOV capability.
+ * @errp: pointer to Error*, to store an error if it happens.
+ *
+ * Return:
+ * * true - @dev is initialized as a PCIe SR-IOV PF.
+ * * false - @dev is not initialized because there is no SR-IOV VFs or an error
+ *           occurred.
+ */
+bool pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev, uint16_t offset,
+                                              Error **errp);
+
+bool pcie_sriov_register_device(PCIDevice *dev, Error **errp);
+void pcie_sriov_unregister_device(PCIDevice *dev);
+
 /*
  * Default (minimal) page size support values
  * as required by the SR/IOV standard:
diff --git a/hw/pci/pci.c b/hw/pci/pci.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/pci/pci.c
+++ b/hw/pci/pci.c
@@ -XXX,XX +XXX,XX @@ static Property pci_props[] = {
                     QEMU_PCIE_ERR_UNC_MASK_BITNR, true),
     DEFINE_PROP_BIT("x-pcie-ari-nextfn-1", PCIDevice, cap_present,
                     QEMU_PCIE_ARI_NEXTFN_1_BITNR, false),
+    DEFINE_PROP_STRING("sriov-pf", PCIDevice, sriov_pf),
     DEFINE_PROP_END_OF_LIST()
 };
 
@@ -XXX,XX +XXX,XX @@ static void pci_init_multifunction(PCIBus *bus, PCIDevice *dev, Error **errp)
         dev->config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION;
     }
 
-    /*
-     * With SR/IOV and ARI, a device at function 0 need not be a multifunction
-     * device, as it may just be a VF that ended up with function 0 in
-     * the legacy PCI interpretation. Avoid failing in such cases:
-     */
-    if (pci_is_vf(dev) &&
-        dev->exp.sriov_vf.pf->cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
+    /* SR/IOV is not handled here. */
+    if (pci_is_vf(dev)) {
         return;
     }
 
@@ -XXX,XX +XXX,XX @@ static void pci_init_multifunction(PCIBus *bus, PCIDevice *dev, Error **errp)
     }
     /* function 0 indicates single function, so function > 0 must be NULL */
     for (func = 1; func < PCI_FUNC_MAX; ++func) {
-        if (bus->devices[PCI_DEVFN(slot, func)]) {
+        PCIDevice *device = bus->devices[PCI_DEVFN(slot, func)];
+        if (device && !pci_is_vf(device)) {
             error_setg(errp, "PCI: %x.0 indicates single function, "
                        "but %x.%x is already populated.",
                        slot, slot, func);
@@ -XXX,XX +XXX,XX @@ static void pci_qdev_unrealize(DeviceState *dev)
 
     pci_unregister_io_regions(pci_dev);
     pci_del_option_rom(pci_dev);
+    pcie_sriov_unregister_device(pci_dev);
 
     if (pc->exit) {
         pc->exit(pci_dev);
@@ -XXX,XX +XXX,XX @@ void pci_register_bar(PCIDevice *pci_dev, int region_num,
     pcibus_t size = memory_region_size(memory);
     uint8_t hdr_type;
 
-    assert(!pci_is_vf(pci_dev)); /* VFs must use pcie_sriov_vf_register_bar */
     assert(region_num >= 0);
     assert(region_num < PCI_NUM_REGIONS);
     assert(is_power_of_2(size));
@@ -XXX,XX +XXX,XX @@ void pci_register_bar(PCIDevice *pci_dev, int region_num,
     assert(hdr_type != PCI_HEADER_TYPE_BRIDGE || region_num < 2);
 
     r = &pci_dev->io_regions[region_num];
-    r->addr = PCI_BAR_UNMAPPED;
     r->size = size;
     r->type = type;
     r->memory = memory;
@@ -XXX,XX +XXX,XX @@ void pci_register_bar(PCIDevice *pci_dev, int region_num,
                         ? pci_get_bus(pci_dev)->address_space_io
                         : pci_get_bus(pci_dev)->address_space_mem;
 
-    wmask = ~(size - 1);
-    if (region_num == PCI_ROM_SLOT) {
-        /* ROM enable bit is writable */
-        wmask |= PCI_ROM_ADDRESS_ENABLE;
-    }
-
-    addr = pci_bar(pci_dev, region_num);
-    pci_set_long(pci_dev->config + addr, type);
+    if (pci_is_vf(pci_dev)) {
+        PCIDevice *pf = pci_dev->exp.sriov_vf.pf;
+        assert(!pf || type == pf->exp.sriov_pf.vf_bar_type[region_num]);
 
-    if (!(r->type & PCI_BASE_ADDRESS_SPACE_IO) &&
-        r->type & PCI_BASE_ADDRESS_MEM_TYPE_64) {
-        pci_set_quad(pci_dev->wmask + addr, wmask);
-        pci_set_quad(pci_dev->cmask + addr, ~0ULL);
+        r->addr = pci_bar_address(pci_dev, region_num, r->type, r->size);
+        if (r->addr != PCI_BAR_UNMAPPED) {
+            memory_region_add_subregion_overlap(r->address_space,
+                                                r->addr, r->memory, 1);
+        }
     } else {
-        pci_set_long(pci_dev->wmask + addr, wmask & 0xffffffff);
-        pci_set_long(pci_dev->cmask + addr, 0xffffffff);
+        r->addr = PCI_BAR_UNMAPPED;
+
+        wmask = ~(size - 1);
+        if (region_num == PCI_ROM_SLOT) {
+            /* ROM enable bit is writable */
+            wmask |= PCI_ROM_ADDRESS_ENABLE;
+        }
+
+        addr = pci_bar(pci_dev, region_num);
+        pci_set_long(pci_dev->config + addr, type);
+
+        if (!(r->type & PCI_BASE_ADDRESS_SPACE_IO) &&
+            r->type & PCI_BASE_ADDRESS_MEM_TYPE_64) {
+            pci_set_quad(pci_dev->wmask + addr, wmask);
+            pci_set_quad(pci_dev->cmask + addr, ~0ULL);
+        } else {
+            pci_set_long(pci_dev->wmask + addr, wmask & 0xffffffff);
+            pci_set_long(pci_dev->cmask + addr, 0xffffffff);
+        }
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static void pci_qdev_realize(DeviceState *qdev, Error **errp)
         }
     }
 
+    if (!pcie_sriov_register_device(pci_dev, errp)) {
+        pci_qdev_unrealize(DEVICE(pci_dev));
+        return;
+    }
+
     /*
      * A PCIe Downstream Port that do not have ARI Forwarding enabled must
      * associate only Device 0 with the device attached to the bus
diff --git a/hw/pci/pcie_sriov.c b/hw/pci/pcie_sriov.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/pci/pcie_sriov.c
+++ b/hw/pci/pcie_sriov.c
@@ -XXX,XX +XXX,XX @@
 #include "hw/qdev-properties.h"
 #include "qemu/error-report.h"
 #include "qemu/range.h"
+#include "qapi/error.h"
 #include "trace.h"
 
+static GHashTable *pfs;
+
 static void unparent_vfs(PCIDevice *dev, uint16_t total_vfs)
 {
     for (uint16_t i = 0; i < total_vfs; i++) {
@@ -XXX,XX +XXX,XX @@ static void unparent_vfs(PCIDevice *dev, uint16_t total_vfs)
     dev->exp.sriov_pf.vf = NULL;
 }
 
-bool pcie_sriov_pf_init(PCIDevice *dev, uint16_t offset,
-                        const char *vfname, uint16_t vf_dev_id,
-                        uint16_t init_vfs, uint16_t total_vfs,
-                        uint16_t vf_offset, uint16_t vf_stride,
-                        Error **errp)
+static void clear_ctrl_vfe(PCIDevice *dev)
+{
+    uint8_t *ctrl = dev->config + dev->exp.sriov_cap + PCI_SRIOV_CTRL;
+    pci_set_word(ctrl, pci_get_word(ctrl) & ~PCI_SRIOV_CTRL_VFE);
+}
+
+static void register_vfs(PCIDevice *dev)
+{
+    uint16_t num_vfs;
+    uint16_t i;
+    uint16_t sriov_cap = dev->exp.sriov_cap;
+
+    assert(sriov_cap > 0);
+    num_vfs = pci_get_word(dev->config + sriov_cap + PCI_SRIOV_NUM_VF);
+    if (num_vfs > pci_get_word(dev->config + sriov_cap + PCI_SRIOV_TOTAL_VF)) {
+        clear_ctrl_vfe(dev);
+        return;
+    }
+
+    trace_sriov_register_vfs(dev->name, PCI_SLOT(dev->devfn),
+                             PCI_FUNC(dev->devfn), num_vfs);
+    for (i = 0; i < num_vfs; i++) {
+        pci_set_enabled(dev->exp.sriov_pf.vf[i], true);
+    }
+}
+
+static void unregister_vfs(PCIDevice *dev)
+{
+    uint16_t i;
+    uint8_t *cfg = dev->config + dev->exp.sriov_cap;
+
+    trace_sriov_unregister_vfs(dev->name, PCI_SLOT(dev->devfn),
+                               PCI_FUNC(dev->devfn));
+    for (i = 0; i < pci_get_word(cfg + PCI_SRIOV_TOTAL_VF); i++) {
+        pci_set_enabled(dev->exp.sriov_pf.vf[i], false);
+    }
+}
+
+static bool pcie_sriov_pf_init_common(PCIDevice *dev, uint16_t offset,
+                                      uint16_t vf_dev_id, uint16_t init_vfs,
+                                      uint16_t total_vfs, uint16_t vf_offset,
+                                      uint16_t vf_stride, Error **errp)
 {
-    BusState *bus = qdev_get_parent_bus(&dev->qdev);
-    int32_t devfn = dev->devfn + vf_offset;
     uint8_t *cfg = dev->config + offset;
     uint8_t *wmask;
 
@@ -XXX,XX +XXX,XX @@ bool pcie_sriov_pf_init(PCIDevice *dev, uint16_t offset,
 
     qdev_prop_set_bit(&dev->qdev, "multifunction", true);
 
+    return true;
+}
+
+bool pcie_sriov_pf_init(PCIDevice *dev, uint16_t offset,
+                        const char *vfname, uint16_t vf_dev_id,
+                        uint16_t init_vfs, uint16_t total_vfs,
+                        uint16_t vf_offset, uint16_t vf_stride,
+                        Error **errp)
+{
+    BusState *bus = qdev_get_parent_bus(&dev->qdev);
+    int32_t devfn = dev->devfn + vf_offset;
+
+    if (pfs && g_hash_table_contains(pfs, dev->qdev.id)) {
+        error_setg(errp, "attaching user-created SR-IOV VF unsupported");
+        return false;
+    }
+
+    if (!pcie_sriov_pf_init_common(dev, offset, vf_dev_id, init_vfs,
+                                   total_vfs, vf_offset, vf_stride, errp)) {
+        return false;
+    }
+
     dev->exp.sriov_pf.vf = g_new(PCIDevice *, total_vfs);
 
     for (uint16_t i = 0; i < total_vfs; i++) {
@@ -XXX,XX +XXX,XX @@ void pcie_sriov_pf_exit(PCIDevice *dev)
 {
     uint8_t *cfg = dev->config + dev->exp.sriov_cap;
 
-    unparent_vfs(dev, pci_get_word(cfg + PCI_SRIOV_TOTAL_VF));
+    if (dev->exp.sriov_pf.vf_user_created) {
+        uint16_t ven_id = pci_get_word(dev->config + PCI_VENDOR_ID);
+        uint16_t total_vfs = pci_get_word(dev->config + PCI_SRIOV_TOTAL_VF);
+        uint16_t vf_dev_id = pci_get_word(dev->config + PCI_SRIOV_VF_DID);
+
+        unregister_vfs(dev);
+
+        for (uint16_t i = 0; i < total_vfs; i++) {
+            dev->exp.sriov_pf.vf[i]->exp.sriov_vf.pf = NULL;
+
+            pci_config_set_vendor_id(dev->exp.sriov_pf.vf[i]->config, ven_id);
+            pci_config_set_device_id(dev->exp.sriov_pf.vf[i]->config, vf_dev_id);
+        }
+    } else {
+        unparent_vfs(dev, pci_get_word(cfg + PCI_SRIOV_TOTAL_VF));
+    }
 }
 
 void pcie_sriov_pf_init_vf_bar(PCIDevice *dev, int region_num,
@@ -XXX,XX +XXX,XX @@ void pcie_sriov_pf_init_vf_bar(PCIDevice *dev, int region_num,
 void pcie_sriov_vf_register_bar(PCIDevice *dev, int region_num,
                                 MemoryRegion *memory)
 {
-    PCIIORegion *r;
-    PCIBus *bus = pci_get_bus(dev);
     uint8_t type;
-    pcibus_t size = memory_region_size(memory);
 
-    assert(pci_is_vf(dev)); /* PFs must use pci_register_bar */
-    assert(region_num >= 0);
-    assert(region_num < PCI_NUM_REGIONS);
+    assert(dev->exp.sriov_vf.pf);
     type = dev->exp.sriov_vf.pf->exp.sriov_pf.vf_bar_type[region_num];
 
-    if (!is_power_of_2(size)) {
-        error_report("%s: PCI region size must be a power"
-                     " of two - type=0x%x, size=0x%"FMT_PCIBUS,
-                     __func__, type, size);
-        exit(1);
-    }
-
-    r = &dev->io_regions[region_num];
-    r->memory = memory;
-    r->address_space =
-        type & PCI_BASE_ADDRESS_SPACE_IO
-        ? bus->address_space_io
-        : bus->address_space_mem;
-    r->size = size;
-    r->type = type;
-
-    r->addr = pci_bar_address(dev, region_num, r->type, r->size);
-    if (r->addr != PCI_BAR_UNMAPPED) {
-        memory_region_add_subregion_overlap(r->address_space,
-                                            r->addr, r->memory, 1);
-    }
+    return pci_register_bar(dev, region_num, type, memory);
 }
 
-static void clear_ctrl_vfe(PCIDevice *dev)
+static gint compare_vf_devfns(gconstpointer a, gconstpointer b)
 {
-    uint8_t *ctrl = dev->config + dev->exp.sriov_cap + PCI_SRIOV_CTRL;
-    pci_set_word(ctrl, pci_get_word(ctrl) & ~PCI_SRIOV_CTRL_VFE);
+    return (*(PCIDevice **)a)->devfn - (*(PCIDevice **)b)->devfn;
 }
 
-static void register_vfs(PCIDevice *dev)
+bool pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev, uint16_t offset,
+                                              Error **errp)
 {
-    uint16_t num_vfs;
+    GPtrArray *pf;
+    PCIDevice **vfs;
+    BusState *bus = qdev_get_parent_bus(DEVICE(dev));
+    uint16_t ven_id = pci_get_word(dev->config + PCI_VENDOR_ID);
+    uint16_t vf_dev_id;
+    uint16_t vf_offset;
+    uint16_t vf_stride;
     uint16_t i;
-    uint16_t sriov_cap = dev->exp.sriov_cap;
 
-    assert(sriov_cap > 0);
-    num_vfs = pci_get_word(dev->config + sriov_cap + PCI_SRIOV_NUM_VF);
-    if (num_vfs > pci_get_word(dev->config + sriov_cap + PCI_SRIOV_TOTAL_VF)) {
-        clear_ctrl_vfe(dev);
-        return;
+    if (!pfs || !dev->qdev.id) {
+        return false;
     }
 
-    trace_sriov_register_vfs(dev->name, PCI_SLOT(dev->devfn),
-                             PCI_FUNC(dev->devfn), num_vfs);
-    for (i = 0; i < num_vfs; i++) {
-        pci_set_enabled(dev->exp.sriov_pf.vf[i], true);
+    pf = g_hash_table_lookup(pfs, dev->qdev.id);
+    if (!pf) {
+        return false;
+    }
+
+    if (pf->len > UINT16_MAX) {
+        error_setg(errp, "too many VFs");
+        return false;
+    }
+
+    g_ptr_array_sort(pf, compare_vf_devfns);
+    vfs = (void *)pf->pdata;
+
+    if (vfs[0]->devfn <= dev->devfn) {
+        error_setg(errp, "a VF function number is less than the PF function number");
+        return false;
     }
+
+    vf_dev_id = pci_get_word(vfs[0]->config + PCI_DEVICE_ID);
+    vf_offset = vfs[0]->devfn - dev->devfn;
+    vf_stride = pf->len < 2 ? 0 : vfs[1]->devfn - vfs[0]->devfn;
+
+    for (i = 0; i < pf->len; i++) {
+        if (bus != qdev_get_parent_bus(&vfs[i]->qdev)) {
+            error_setg(errp, "SR-IOV VF parent bus mismatches with PF");
+            return false;
+        }
+
+        if (ven_id != pci_get_word(vfs[i]->config + PCI_VENDOR_ID)) {
+            error_setg(errp, "SR-IOV VF vendor ID mismatches with PF");
+            return false;
+        }
+
+        if (vf_dev_id != pci_get_word(vfs[i]->config + PCI_DEVICE_ID)) {
+            error_setg(errp, "inconsistent SR-IOV VF device IDs");
+            return false;
+        }
+
+        for (size_t j = 0; j < PCI_NUM_REGIONS; j++) {
+            if (vfs[i]->io_regions[j].size != vfs[0]->io_regions[j].size ||
+                vfs[i]->io_regions[j].type != vfs[0]->io_regions[j].type) {
+                error_setg(errp, "inconsistent SR-IOV BARs");
+                return false;
+            }
+        }
+
+        if (vfs[i]->devfn - vfs[0]->devfn != vf_stride * i) {
+            error_setg(errp, "inconsistent SR-IOV stride");
+            return false;
+        }
+    }
+
+    if (!pcie_sriov_pf_init_common(dev, offset, vf_dev_id, pf->len,
+                                   pf->len, vf_offset, vf_stride, errp)) {
+        return false;
+    }
+
+    for (i = 0; i < pf->len; i++) {
+        vfs[i]->exp.sriov_vf.pf = dev;
+        vfs[i]->exp.sriov_vf.vf_number = i;
+
+        /* set vid/did according to sr/iov spec - they are not used */
+        pci_config_set_vendor_id(vfs[i]->config, 0xffff);
+        pci_config_set_device_id(vfs[i]->config, 0xffff);
+    }
+
+    dev->exp.sriov_pf.vf = vfs;
+    dev->exp.sriov_pf.vf_user_created = true;
+
+    for (i = 0; i < PCI_NUM_REGIONS; i++) {
+        uint8_t type = vfs[0]->io_regions[i].type;
+        pcibus_t size = vfs[0]->io_regions[i].size;
+
+        if (size) {
+            pcie_sriov_pf_init_vf_bar(dev, i, type, size);
+        }
+    }
+
+    return true;
 }
 
-static void unregister_vfs(PCIDevice *dev)
+bool pcie_sriov_register_device(PCIDevice *dev, Error **errp)
 {
-    uint16_t i;
-    uint8_t *cfg = dev->config + dev->exp.sriov_cap;
+    if (!dev->exp.sriov_pf.vf && dev->qdev.id &&
+        pfs && g_hash_table_contains(pfs, dev->qdev.id)) {
+        error_setg(errp, "attaching user-created SR-IOV VF unsupported");
+        return false;
+    }
 
-    trace_sriov_unregister_vfs(dev->name, PCI_SLOT(dev->devfn),
-                               PCI_FUNC(dev->devfn));
-    for (i = 0; i < pci_get_word(cfg + PCI_SRIOV_TOTAL_VF); i++) {
-        pci_set_enabled(dev->exp.sriov_pf.vf[i], false);
+    if (dev->sriov_pf) {
+        PCIDevice *pci_pf;
+        GPtrArray *pf;
+
+        if (!PCI_DEVICE_GET_CLASS(dev)->sriov_vf_user_creatable) {
+            error_setg(errp, "user cannot create SR-IOV VF with this device type");
+            return false;
+        }
+
+        if (!pci_is_express(dev)) {
+            error_setg(errp, "PCI Express is required for SR-IOV VF");
+            return false;
+        }
+
+        if (!pci_qdev_find_device(dev->sriov_pf, &pci_pf)) {
+            error_setg(errp, "PCI device specified as SR-IOV PF already exists");
+            return false;
+        }
+
+        if (!pfs) {
+            pfs = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
+        }
+
+        pf = g_hash_table_lookup(pfs, dev->sriov_pf);
+        if (!pf) {
+            pf = g_ptr_array_new();
+            g_hash_table_insert(pfs, g_strdup(dev->sriov_pf), pf);
+        }
+
+        g_ptr_array_add(pf, dev);
+    }
+
+    return true;
+}
+
+void pcie_sriov_unregister_device(PCIDevice *dev)
+{
+    if (dev->sriov_pf && pfs) {
+        GPtrArray *pf = g_hash_table_lookup(pfs, dev->qdev.id);
+
+        if (pf) {
+            g_ptr_array_remove_fast(pf, dev);
+
+            if (!pf->len) {
+                g_hash_table_remove(pfs, dev->qdev.id);
+                g_ptr_array_free(pf, FALSE);
+            }
+        }
     }
 }
 
@@ -XXX,XX +XXX,XX @@ void pcie_sriov_pf_add_sup_pgsize(PCIDevice *dev, uint16_t opt_sup_pgsize)
 
 uint16_t pcie_sriov_vf_number(PCIDevice *dev)
 {
-    assert(pci_is_vf(dev));
+    assert(dev->exp.sriov_vf.pf);
     return dev->exp.sriov_vf.vf_number;
 }

-- 
2.44.0

Allow user to attach SR-IOV VF to a virtio-pci PF.

Signed-off-by: Akihiko Odaki <akihiko.odaki@daynix.com>
---
 hw/virtio/virtio-pci.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/virtio-pci.c
+++ b/hw/virtio/virtio-pci.c
@@ -XXX,XX +XXX,XX @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
         pci_register_bar(&proxy->pci_dev, proxy->legacy_io_bar_idx,
                          PCI_BASE_ADDRESS_SPACE_IO, &proxy->bar);
     }
+
+    if (pcie_sriov_pf_init_from_user_created_vfs(&proxy->pci_dev,
+                                                 PCI_CONFIG_SPACE_SIZE,
+                                                 errp)) {
+        virtio_add_feature(&vdev->host_features, VIRTIO_F_SR_IOV);
+    }
 }
 
 static void virtio_pci_device_unplugged(DeviceState *d)
@@ -XXX,XX +XXX,XX @@ static void virtio_pci_device_unplugged(DeviceState *d)
     bool modern = virtio_pci_modern(proxy);
     bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
 
+    pcie_sriov_pf_exit(&proxy->pci_dev);
     virtio_pci_stop_ioeventfd(proxy);
 
     if (modern) {

-- 
2.44.0

Based-on: <20250104-reuse-v18-0-c349eafd8673@daynix.com>
("[PATCH v18 00/14] hw/pci: SR-IOV related fixes and improvements")

Introduction
------------

This series is based on the RFC series submitted by Yui Washizu[1].
See also [2] for the context.

The PF side code resides in virtio-pci. The VF side code resides in
the PCI common infrastructure, but it is restricted to work only for
virtio-net-pci because of lack of validation.

User Interface
--------------

Keeping VF instances
--------------------

Summary
-------

Patch 1 disables ROM BAR, which virtio-net-pci enables by default, for
VFs.
Patch 2 makes zero stride valid for 1 VF configuration.
Patch 3 and 4 adds validations.
Patch 5 adds user-created SR-IOV VF infrastructure.
Patch 6 makes virtio-pci work as SR-IOV PF for user-created VFs.
Patch 7 allows user to create SR-IOV VFs with virtio-net-pci.

[1] https://patchew.org/QEMU/1689731808-3009-1-git-send-email-yui.washidu@gmail.com/
[2] https://lore.kernel.org/all/5d46f455-f530-4e5e-9ae7-13a2297d4bc5@daynix.com/

Co-developed-by: Yui Washizu <yui.washidu@gmail.com>
Signed-off-by: Akihiko Odaki <akihiko.odaki@daynix.com>
---
Changes in v9:
- Rebased.
- Link to v8: https://lore.kernel.org/r/20250104-sriov-v8-0-56144cfdc7d9@daynix.com

Changes in v8:
- Rebased.
- Link to v7: https://lore.kernel.org/r/20240813-sriov-v7-0-8515e3774df7@daynix.com

Changes in v7:
- Removed #include <error-report.h>, which is no longer needed.
- Rebased.
- Link to v6: https://lore.kernel.org/r/20240802-sriov-v6-0-0c8ff49c4276@daynix.com

Changes in v6:
- Added ARI extended capability.
- Rebased.
- Link to v5: https://lore.kernel.org/r/20240715-sriov-v5-0-3f5539093ffc@daynix.com

Changes in v5:
- Dropped the RFC tag.
- Fixed device unrealization.
- Rebased.
- Link to v4: https://lore.kernel.org/r/20240428-sriov-v4-0-ac8ac6212982@daynix.com

Changes in v4:
- Added patch "hw/pci: Fix SR-IOV VF number calculation" to fix division
  by zero reported by Yui Washizu.
- Rebased.
- Link to v3: https://lore.kernel.org/r/20240305-sriov-v3-0-abdb75770372@daynix.com

Changes in v3:
- Rebased.
- Link to v2: https://lore.kernel.org/r/20231210-sriov-v2-0-b959e8a6dfaf@daynix.com

Changes in v2:
- Changed to keep VF instances.
- Link to v1: https://lore.kernel.org/r/20231202-sriov-v1-0-32b3570f7bd6@daynix.com

---
Akihiko Odaki (9):
      hw/pci: Do not add ROM BAR for SR-IOV VF
      hw/pci: Fix SR-IOV VF number calculation
      pcie_sriov: Ensure PF and VF are mutually exclusive
      pcie_sriov: Check PCI Express for SR-IOV PF
      pcie_sriov: Allow user to create SR-IOV device
      virtio-pci: Implement SR-IOV PF
      virtio-net: Implement SR-IOV VF
      docs: Document composable SR-IOV device
      pcie_sriov: Make a PCI device with user-created VF ARI-capable

MAINTAINERS                    |   1 +
 docs/system/index.rst          |   1 +
 docs/system/sriov.rst          |  37 ++++++
 include/hw/pci/pci_device.h    |   6 +-
 include/hw/pci/pcie_sriov.h    |  21 +++
 include/hw/virtio/virtio-pci.h |   1 +
 hw/pci/pci.c                   |  76 +++++++----
 hw/pci/pcie_sriov.c            | 294 +++++++++++++++++++++++++++++++++--------
 hw/virtio/virtio-net-pci.c     |   1 +
 hw/virtio/virtio-pci.c         |  24 +++-
 10 files changed, 378 insertions(+), 84 deletions(-)
---
base-commit: 825b96dbcee23d134b691fc75618b59c5f53da32
change-id: 20231202-sriov-9402fb262be8

Best regards,
-- 
Akihiko Odaki <akihiko.odaki@daynix.com>

pci_config_get_bar_addr() had a division by vf_stride. vf_stride needs
to be non-zero when there are multiple VFs, but the specification does
not prohibit to make it zero when there is only one VF.

Do not perform the division for the first VF to avoid division by zero.

Signed-off-by: Akihiko Odaki <akihiko.odaki@daynix.com>
---
 hw/pci/pci.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/hw/pci/pci.c b/hw/pci/pci.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/pci/pci.c
+++ b/hw/pci/pci.c
@@ -XXX,XX +XXX,XX @@ static pcibus_t pci_config_get_bar_addr(PCIDevice *d, int reg,
             pci_get_word(pf->config + sriov_cap + PCI_SRIOV_VF_OFFSET);
         uint16_t vf_stride =
             pci_get_word(pf->config + sriov_cap + PCI_SRIOV_VF_STRIDE);
-        uint32_t vf_num = (d->devfn - (pf->devfn + vf_offset)) / vf_stride;
+        uint32_t vf_num = d->devfn - (pf->devfn + vf_offset);
+
+        if (vf_num) {
+            vf_num /= vf_stride;
+        }
 
         if (type & PCI_BASE_ADDRESS_MEM_TYPE_64) {
             new_addr = pci_get_quad(pf->config + bar);

-- 
2.48.1

A user can create a SR-IOV device by specifying the PF with the
sriov-pf property of the VFs. The VFs must be added before the PF.

A user-creatable VF must have PCIDeviceClass::sriov_vf_user_creatable
set. Such a VF cannot refer to the PF because it is created before the
PF.

A PF that user-creatable VFs can be attached calls
pcie_sriov_pf_init_from_user_created_vfs() during realization and
pcie_sriov_pf_exit() when exiting.

Signed-off-by: Akihiko Odaki <akihiko.odaki@daynix.com>
---
 include/hw/pci/pci_device.h |   6 +-
 include/hw/pci/pcie_sriov.h |  18 +++
 hw/pci/pci.c                |  62 ++++++----
 hw/pci/pcie_sriov.c         | 278 +++++++++++++++++++++++++++++++++++---------
 4 files changed, 286 insertions(+), 78 deletions(-)

diff --git a/include/hw/pci/pci_device.h b/include/hw/pci/pci_device.h
index XXXXXXX..XXXXXXX 100644
--- a/include/hw/pci/pci_device.h
+++ b/include/hw/pci/pci_device.h
@@ -XXX,XX +XXX,XX @@ struct PCIDeviceClass {
     uint16_t subsystem_id;              /* only for header type = 0 */
 
     const char *romfile;                /* rom bar */
+
+    bool sriov_vf_user_creatable;
 };
 
 enum PCIReqIDType {
@@ -XXX,XX +XXX,XX @@ struct PCIDevice {
      * realizing the device.
      */
     uint32_t max_bounce_buffer_size;
+
+    char *sriov_pf;
 };
 
 static inline int pci_intx(PCIDevice *pci_dev)
@@ -XXX,XX +XXX,XX @@ static inline int pci_is_express_downstream_port(const PCIDevice *d)
 
 static inline int pci_is_vf(const PCIDevice *d)
 {
-    return d->exp.sriov_vf.pf != NULL;
+    return d->sriov_pf || d->exp.sriov_vf.pf != NULL;
 }
 
 static inline uint32_t pci_config_size(const PCIDevice *d)
diff --git a/include/hw/pci/pcie_sriov.h b/include/hw/pci/pcie_sriov.h
index XXXXXXX..XXXXXXX 100644
--- a/include/hw/pci/pcie_sriov.h
+++ b/include/hw/pci/pcie_sriov.h
@@ -XXX,XX +XXX,XX @@
 typedef struct PCIESriovPF {
     uint8_t vf_bar_type[PCI_NUM_REGIONS];   /* Store type for each VF bar */
     PCIDevice **vf;     /* Pointer to an array of num_vfs VF devices */
+    bool vf_user_created; /* If VFs are created by user */
 } PCIESriovPF;
 
 typedef struct PCIESriovVF {
@@ -XXX,XX +XXX,XX @@ void pcie_sriov_pf_init_vf_bar(PCIDevice *dev, int region_num,
 void pcie_sriov_vf_register_bar(PCIDevice *dev, int region_num,
                                 MemoryRegion *memory);
 
+/**
+ * pcie_sriov_pf_init_from_user_created_vfs() - Initialize PF with user-created
+ *                                              VFs.
+ * @dev: A PCIe device being realized.
+ * @offset: The offset of the SR-IOV capability.
+ * @errp: pointer to Error*, to store an error if it happens.
+ *
+ * Return: The size of added capability. 0 if the user did not create VFs.
+ *         -1 if failed.
+ */
+int16_t pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev,
+                                                 uint16_t offset,
+                                                 Error **errp);
+
+bool pcie_sriov_register_device(PCIDevice *dev, Error **errp);
+void pcie_sriov_unregister_device(PCIDevice *dev);
+
 /*
  * Default (minimal) page size support values
  * as required by the SR/IOV standard:
diff --git a/hw/pci/pci.c b/hw/pci/pci.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/pci/pci.c
+++ b/hw/pci/pci.c
@@ -XXX,XX +XXX,XX @@ static const Property pci_props[] = {
                     QEMU_PCIE_ARI_NEXTFN_1_BITNR, false),
     DEFINE_PROP_SIZE32("x-max-bounce-buffer-size", PCIDevice,
                      max_bounce_buffer_size, DEFAULT_MAX_BOUNCE_BUFFER_SIZE),
+    DEFINE_PROP_STRING("sriov-pf", PCIDevice, sriov_pf),
     DEFINE_PROP_BIT("x-pcie-ext-tag", PCIDevice, cap_present,
                     QEMU_PCIE_EXT_TAG_BITNR, true),
     { .name = "busnr", .info = &prop_pci_busnr },
@@ -XXX,XX +XXX,XX @@ static void pci_init_multifunction(PCIBus *bus, PCIDevice *dev, Error **errp)
         dev->config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION;
     }
 
-    /*
-     * With SR/IOV and ARI, a device at function 0 need not be a multifunction
-     * device, as it may just be a VF that ended up with function 0 in
-     * the legacy PCI interpretation. Avoid failing in such cases:
-     */
-    if (pci_is_vf(dev) &&
-        dev->exp.sriov_vf.pf->cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
+    /* SR/IOV is not handled here. */
+    if (pci_is_vf(dev)) {
         return;
     }
 
@@ -XXX,XX +XXX,XX @@ static void pci_init_multifunction(PCIBus *bus, PCIDevice *dev, Error **errp)
     }
     /* function 0 indicates single function, so function > 0 must be NULL */
     for (func = 1; func < PCI_FUNC_MAX; ++func) {
-        if (bus->devices[PCI_DEVFN(slot, func)]) {
+        PCIDevice *device = bus->devices[PCI_DEVFN(slot, func)];
+        if (device && !pci_is_vf(device)) {
             error_setg(errp, "PCI: %x.0 indicates single function, "
                        "but %x.%x is already populated.",
                        slot, slot, func);
@@ -XXX,XX +XXX,XX @@ static void pci_qdev_unrealize(DeviceState *dev)
 
     pci_unregister_io_regions(pci_dev);
     pci_del_option_rom(pci_dev);
+    pcie_sriov_unregister_device(pci_dev);
 
     if (pc->exit) {
         pc->exit(pci_dev);
@@ -XXX,XX +XXX,XX @@ void pci_register_bar(PCIDevice *pci_dev, int region_num,
     pcibus_t size = memory_region_size(memory);
     uint8_t hdr_type;
 
-    assert(!pci_is_vf(pci_dev)); /* VFs must use pcie_sriov_vf_register_bar */
     assert(region_num >= 0);
     assert(region_num < PCI_NUM_REGIONS);
     assert(is_power_of_2(size));
@@ -XXX,XX +XXX,XX @@ void pci_register_bar(PCIDevice *pci_dev, int region_num,
 
     r = &pci_dev->io_regions[region_num];
     assert(!r->size);
-    r->addr = PCI_BAR_UNMAPPED;
     r->size = size;
     r->type = type;
     r->memory = memory;
@@ -XXX,XX +XXX,XX @@ void pci_register_bar(PCIDevice *pci_dev, int region_num,
                         ? pci_get_bus(pci_dev)->address_space_io
                         : pci_get_bus(pci_dev)->address_space_mem;
 
-    wmask = ~(size - 1);
-    if (region_num == PCI_ROM_SLOT) {
-        /* ROM enable bit is writable */
-        wmask |= PCI_ROM_ADDRESS_ENABLE;
-    }
-
-    addr = pci_bar(pci_dev, region_num);
-    pci_set_long(pci_dev->config + addr, type);
+    if (pci_is_vf(pci_dev)) {
+        PCIDevice *pf = pci_dev->exp.sriov_vf.pf;
+        assert(!pf || type == pf->exp.sriov_pf.vf_bar_type[region_num]);
 
-    if (!(r->type & PCI_BASE_ADDRESS_SPACE_IO) &&
-        r->type & PCI_BASE_ADDRESS_MEM_TYPE_64) {
-        pci_set_quad(pci_dev->wmask + addr, wmask);
-        pci_set_quad(pci_dev->cmask + addr, ~0ULL);
+        r->addr = pci_bar_address(pci_dev, region_num, r->type, r->size);
+        if (r->addr != PCI_BAR_UNMAPPED) {
+            memory_region_add_subregion_overlap(r->address_space,
+                                                r->addr, r->memory, 1);
+        }
     } else {
-        pci_set_long(pci_dev->wmask + addr, wmask & 0xffffffff);
-        pci_set_long(pci_dev->cmask + addr, 0xffffffff);
+        r->addr = PCI_BAR_UNMAPPED;
+
+        wmask = ~(size - 1);
+        if (region_num == PCI_ROM_SLOT) {
+            /* ROM enable bit is writable */
+            wmask |= PCI_ROM_ADDRESS_ENABLE;
+        }
+
+        addr = pci_bar(pci_dev, region_num);
+        pci_set_long(pci_dev->config + addr, type);
+
+        if (!(r->type & PCI_BASE_ADDRESS_SPACE_IO) &&
+            r->type & PCI_BASE_ADDRESS_MEM_TYPE_64) {
+            pci_set_quad(pci_dev->wmask + addr, wmask);
+            pci_set_quad(pci_dev->cmask + addr, ~0ULL);
+        } else {
+            pci_set_long(pci_dev->wmask + addr, wmask & 0xffffffff);
+            pci_set_long(pci_dev->cmask + addr, 0xffffffff);
+        }
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static void pci_qdev_realize(DeviceState *qdev, Error **errp)
         }
     }
 
+    if (!pcie_sriov_register_device(pci_dev, errp)) {
+        pci_qdev_unrealize(DEVICE(pci_dev));
+        return;
+    }
+
     /*
      * A PCIe Downstream Port that do not have ARI Forwarding enabled must
      * associate only Device 0 with the device attached to the bus
diff --git a/hw/pci/pcie_sriov.c b/hw/pci/pcie_sriov.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/pci/pcie_sriov.c
+++ b/hw/pci/pcie_sriov.c
@@ -XXX,XX +XXX,XX @@
 #include "hw/pci/pcie.h"
 #include "hw/pci/pci_bus.h"
 #include "hw/qdev-properties.h"
-#include "qemu/error-report.h"
 #include "qemu/range.h"
 #include "qapi/error.h"
 #include "trace.h"
 
+static GHashTable *pfs;
+
 static void unparent_vfs(PCIDevice *dev, uint16_t total_vfs)
 {
     for (uint16_t i = 0; i < total_vfs; i++) {
@@ -XXX,XX +XXX,XX @@ static void unparent_vfs(PCIDevice *dev, uint16_t total_vfs)
     dev->exp.sriov_pf.vf = NULL;
 }
 
-bool pcie_sriov_pf_init(PCIDevice *dev, uint16_t offset,
-                        const char *vfname, uint16_t vf_dev_id,
-                        uint16_t init_vfs, uint16_t total_vfs,
-                        uint16_t vf_offset, uint16_t vf_stride,
-                        Error **errp)
+static void register_vfs(PCIDevice *dev)
+{
+    uint16_t num_vfs;
+    uint16_t i;
+    uint16_t sriov_cap = dev->exp.sriov_cap;
+
+    assert(sriov_cap > 0);
+    num_vfs = pci_get_word(dev->config + sriov_cap + PCI_SRIOV_NUM_VF);
+
+    trace_sriov_register_vfs(dev->name, PCI_SLOT(dev->devfn),
+                             PCI_FUNC(dev->devfn), num_vfs);
+    for (i = 0; i < num_vfs; i++) {
+        pci_set_enabled(dev->exp.sriov_pf.vf[i], true);
+    }
+
+    pci_set_word(dev->wmask + sriov_cap + PCI_SRIOV_NUM_VF, 0);
+}
+
+static void unregister_vfs(PCIDevice *dev)
+{
+    uint8_t *cfg = dev->config + dev->exp.sriov_cap;
+    uint16_t i;
+
+    trace_sriov_unregister_vfs(dev->name, PCI_SLOT(dev->devfn),
+                               PCI_FUNC(dev->devfn));
+    for (i = 0; i < pci_get_word(cfg + PCI_SRIOV_TOTAL_VF); i++) {
+        pci_set_enabled(dev->exp.sriov_pf.vf[i], false);
+    }
+
+    pci_set_word(dev->wmask + dev->exp.sriov_cap + PCI_SRIOV_NUM_VF, 0xffff);
+}
+
+static bool pcie_sriov_pf_init_common(PCIDevice *dev, uint16_t offset,
+                                      uint16_t vf_dev_id, uint16_t init_vfs,
+                                      uint16_t total_vfs, uint16_t vf_offset,
+                                      uint16_t vf_stride, Error **errp)
 {
-    BusState *bus = qdev_get_parent_bus(&dev->qdev);
     int32_t devfn = dev->devfn + vf_offset;
     uint8_t *cfg = dev->config + offset;
     uint8_t *wmask;
@@ -XXX,XX +XXX,XX @@ bool pcie_sriov_pf_init(PCIDevice *dev, uint16_t offset,
 
     qdev_prop_set_bit(&dev->qdev, "multifunction", true);
 
+    return true;
+}
+
+bool pcie_sriov_pf_init(PCIDevice *dev, uint16_t offset,
+                        const char *vfname, uint16_t vf_dev_id,
+                        uint16_t init_vfs, uint16_t total_vfs,
+                        uint16_t vf_offset, uint16_t vf_stride,
+                        Error **errp)
+{
+    BusState *bus = qdev_get_parent_bus(&dev->qdev);
+    int32_t devfn = dev->devfn + vf_offset;
+
+    if (pfs && g_hash_table_contains(pfs, dev->qdev.id)) {
+        error_setg(errp, "attaching user-created SR-IOV VF unsupported");
+        return false;
+    }
+
+    if (!pcie_sriov_pf_init_common(dev, offset, vf_dev_id, init_vfs,
+                                   total_vfs, vf_offset, vf_stride, errp)) {
+        return false;
+    }
+
     dev->exp.sriov_pf.vf = g_new(PCIDevice *, total_vfs);
 
     for (uint16_t i = 0; i < total_vfs; i++) {
@@ -XXX,XX +XXX,XX @@ void pcie_sriov_pf_exit(PCIDevice *dev)
 {
     uint8_t *cfg = dev->config + dev->exp.sriov_cap;
 
-    unparent_vfs(dev, pci_get_word(cfg + PCI_SRIOV_TOTAL_VF));
+    if (dev->exp.sriov_pf.vf_user_created) {
+        uint16_t ven_id = pci_get_word(dev->config + PCI_VENDOR_ID);
+        uint16_t total_vfs = pci_get_word(dev->config + PCI_SRIOV_TOTAL_VF);
+        uint16_t vf_dev_id = pci_get_word(dev->config + PCI_SRIOV_VF_DID);
+
+        unregister_vfs(dev);
+
+        for (uint16_t i = 0; i < total_vfs; i++) {
+            dev->exp.sriov_pf.vf[i]->exp.sriov_vf.pf = NULL;
+
+            pci_config_set_vendor_id(dev->exp.sriov_pf.vf[i]->config, ven_id);
+            pci_config_set_device_id(dev->exp.sriov_pf.vf[i]->config, vf_dev_id);
+        }
+    } else {
+        unparent_vfs(dev, pci_get_word(cfg + PCI_SRIOV_TOTAL_VF));
+    }
 }
 
 void pcie_sriov_pf_init_vf_bar(PCIDevice *dev, int region_num,
@@ -XXX,XX +XXX,XX @@ void pcie_sriov_pf_init_vf_bar(PCIDevice *dev, int region_num,
 void pcie_sriov_vf_register_bar(PCIDevice *dev, int region_num,
                                 MemoryRegion *memory)
 {
-    PCIIORegion *r;
-    PCIBus *bus = pci_get_bus(dev);
     uint8_t type;
-    pcibus_t size = memory_region_size(memory);
 
-    assert(pci_is_vf(dev)); /* PFs must use pci_register_bar */
-    assert(region_num >= 0);
-    assert(region_num < PCI_NUM_REGIONS);
+    assert(dev->exp.sriov_vf.pf);
     type = dev->exp.sriov_vf.pf->exp.sriov_pf.vf_bar_type[region_num];
 
-    if (!is_power_of_2(size)) {
-        error_report("%s: PCI region size must be a power"
-                     " of two - type=0x%x, size=0x%"FMT_PCIBUS,
-                     __func__, type, size);
-        exit(1);
-    }
+    return pci_register_bar(dev, region_num, type, memory);
+}
 
-    r = &dev->io_regions[region_num];
-    r->memory = memory;
-    r->address_space =
-        type & PCI_BASE_ADDRESS_SPACE_IO
-        ? bus->address_space_io
-        : bus->address_space_mem;
-    r->size = size;
-    r->type = type;
-
-    r->addr = pci_bar_address(dev, region_num, r->type, r->size);
-    if (r->addr != PCI_BAR_UNMAPPED) {
-        memory_region_add_subregion_overlap(r->address_space,
-                                            r->addr, r->memory, 1);
-    }
+static gint compare_vf_devfns(gconstpointer a, gconstpointer b)
+{
+    return (*(PCIDevice **)a)->devfn - (*(PCIDevice **)b)->devfn;
 }
 
-static void register_vfs(PCIDevice *dev)
+int16_t pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev,
+                                                 uint16_t offset,
+                                                 Error **errp)
 {
-    uint16_t num_vfs;
+    GPtrArray *pf;
+    PCIDevice **vfs;
+    BusState *bus = qdev_get_parent_bus(DEVICE(dev));
+    uint16_t ven_id = pci_get_word(dev->config + PCI_VENDOR_ID);
+    uint16_t vf_dev_id;
+    uint16_t vf_offset;
+    uint16_t vf_stride;
     uint16_t i;
-    uint16_t sriov_cap = dev->exp.sriov_cap;
 
-    assert(sriov_cap > 0);
-    num_vfs = pci_get_word(dev->config + sriov_cap + PCI_SRIOV_NUM_VF);
+    if (!pfs || !dev->qdev.id) {
+        return 0;
+    }
 
-    trace_sriov_register_vfs(dev->name, PCI_SLOT(dev->devfn),
-                             PCI_FUNC(dev->devfn), num_vfs);
-    for (i = 0; i < num_vfs; i++) {
-        pci_set_enabled(dev->exp.sriov_pf.vf[i], true);
+    pf = g_hash_table_lookup(pfs, dev->qdev.id);
+    if (!pf) {
+        return 0;
     }
 
-    pci_set_word(dev->wmask + sriov_cap + PCI_SRIOV_NUM_VF, 0);
+    if (pf->len > UINT16_MAX) {
+        error_setg(errp, "too many VFs");
+        return -1;
+    }
+
+    g_ptr_array_sort(pf, compare_vf_devfns);
+    vfs = (void *)pf->pdata;
+
+    if (vfs[0]->devfn <= dev->devfn) {
+        error_setg(errp, "a VF function number is less than the PF function number");
+        return -1;
+    }
+
+    vf_dev_id = pci_get_word(vfs[0]->config + PCI_DEVICE_ID);
+    vf_offset = vfs[0]->devfn - dev->devfn;
+    vf_stride = pf->len < 2 ? 0 : vfs[1]->devfn - vfs[0]->devfn;
+
+    for (i = 0; i < pf->len; i++) {
+        if (bus != qdev_get_parent_bus(&vfs[i]->qdev)) {
+            error_setg(errp, "SR-IOV VF parent bus mismatches with PF");
+            return -1;
+        }
+
+        if (ven_id != pci_get_word(vfs[i]->config + PCI_VENDOR_ID)) {
+            error_setg(errp, "SR-IOV VF vendor ID mismatches with PF");
+            return -1;
+        }
+
+        if (vf_dev_id != pci_get_word(vfs[i]->config + PCI_DEVICE_ID)) {
+            error_setg(errp, "inconsistent SR-IOV VF device IDs");
+            return -1;
+        }
+
+        for (size_t j = 0; j < PCI_NUM_REGIONS; j++) {
+            if (vfs[i]->io_regions[j].size != vfs[0]->io_regions[j].size ||
+                vfs[i]->io_regions[j].type != vfs[0]->io_regions[j].type) {
+                error_setg(errp, "inconsistent SR-IOV BARs");
+                return -1;
+            }
+        }
+
+        if (vfs[i]->devfn - vfs[0]->devfn != vf_stride * i) {
+            error_setg(errp, "inconsistent SR-IOV stride");
+            return -1;
+        }
+    }
+
+    if (!pcie_sriov_pf_init_common(dev, offset, vf_dev_id, pf->len,
+                                   pf->len, vf_offset, vf_stride, errp)) {
+        return -1;
+    }
+
+    for (i = 0; i < pf->len; i++) {
+        vfs[i]->exp.sriov_vf.pf = dev;
+        vfs[i]->exp.sriov_vf.vf_number = i;
+
+        /* set vid/did according to sr/iov spec - they are not used */
+        pci_config_set_vendor_id(vfs[i]->config, 0xffff);
+        pci_config_set_device_id(vfs[i]->config, 0xffff);
+    }
+
+    dev->exp.sriov_pf.vf = vfs;
+    dev->exp.sriov_pf.vf_user_created = true;
+
+    for (i = 0; i < PCI_NUM_REGIONS; i++) {
+        PCIIORegion *region = &vfs[0]->io_regions[i];
+
+        if (region->size) {
+            pcie_sriov_pf_init_vf_bar(dev, i, region->type, region->size);
+        }
+    }
+
+    return PCI_EXT_CAP_SRIOV_SIZEOF;
 }
 
-static void unregister_vfs(PCIDevice *dev)
+bool pcie_sriov_register_device(PCIDevice *dev, Error **errp)
 {
-    uint8_t *cfg = dev->config + dev->exp.sriov_cap;
-    uint16_t i;
+    if (!dev->exp.sriov_pf.vf && dev->qdev.id &&
+        pfs && g_hash_table_contains(pfs, dev->qdev.id)) {
+        error_setg(errp, "attaching user-created SR-IOV VF unsupported");
+        return false;
+    }
 
-    trace_sriov_unregister_vfs(dev->name, PCI_SLOT(dev->devfn),
-                               PCI_FUNC(dev->devfn));
-    for (i = 0; i < pci_get_word(cfg + PCI_SRIOV_TOTAL_VF); i++) {
-        pci_set_enabled(dev->exp.sriov_pf.vf[i], false);
+    if (dev->sriov_pf) {
+        PCIDevice *pci_pf;
+        GPtrArray *pf;
+
+        if (!PCI_DEVICE_GET_CLASS(dev)->sriov_vf_user_creatable) {
+            error_setg(errp, "user cannot create SR-IOV VF with this device type");
+            return false;
+        }
+
+        if (!pci_is_express(dev)) {
+            error_setg(errp, "PCI Express is required for SR-IOV VF");
+            return false;
+        }
+
+        if (!pci_qdev_find_device(dev->sriov_pf, &pci_pf)) {
+            error_setg(errp, "PCI device specified as SR-IOV PF already exists");
+            return false;
+        }
+
+        if (!pfs) {
+            pfs = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
+        }
+
+        pf = g_hash_table_lookup(pfs, dev->sriov_pf);
+        if (!pf) {
+            pf = g_ptr_array_new();
+            g_hash_table_insert(pfs, g_strdup(dev->sriov_pf), pf);
+        }
+
+        g_ptr_array_add(pf, dev);
     }
 
-    pci_set_word(dev->wmask + dev->exp.sriov_cap + PCI_SRIOV_NUM_VF, 0xffff);
+    return true;
+}
+
+void pcie_sriov_unregister_device(PCIDevice *dev)
+{
+    if (dev->sriov_pf && pfs) {
+        GPtrArray *pf = g_hash_table_lookup(pfs, dev->sriov_pf);
+
+        if (pf) {
+            g_ptr_array_remove_fast(pf, dev);
+
+            if (!pf->len) {
+                g_hash_table_remove(pfs, dev->sriov_pf);
+                g_ptr_array_free(pf, FALSE);
+            }
+        }
+    }
 }
 
 void pcie_sriov_config_write(PCIDevice *dev, uint32_t address,
@@ -XXX,XX +XXX,XX @@ void pcie_sriov_pf_add_sup_pgsize(PCIDevice *dev, uint16_t opt_sup_pgsize)
 
 uint16_t pcie_sriov_vf_number(PCIDevice *dev)
 {
-    assert(pci_is_vf(dev));
+    assert(dev->exp.sriov_vf.pf);
     return dev->exp.sriov_vf.vf_number;
 }

-- 
2.48.1

Allow user to attach SR-IOV VF to a virtio-pci PF.

Signed-off-by: Akihiko Odaki <akihiko.odaki@daynix.com>
---
 include/hw/virtio/virtio-pci.h |  1 +
 hw/virtio/virtio-pci.c         | 20 +++++++++++++++-----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/include/hw/virtio/virtio-pci.h b/include/hw/virtio/virtio-pci.h
index XXXXXXX..XXXXXXX 100644
--- a/include/hw/virtio/virtio-pci.h
+++ b/include/hw/virtio/virtio-pci.h
@@ -XXX,XX +XXX,XX @@ struct VirtIOPCIProxy {
     uint32_t modern_io_bar_idx;
     uint32_t modern_mem_bar_idx;
     int config_cap;
+    uint16_t last_pcie_cap_offset;
     uint32_t flags;
     bool disable_modern;
     bool ignore_backend_features;
diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/virtio-pci.c
+++ b/hw/virtio/virtio-pci.c
@@ -XXX,XX +XXX,XX @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
     uint8_t *config;
     uint32_t size;
     VirtIODevice *vdev = virtio_bus_get_device(bus);
+    int16_t res;
 
     /*
      * Virtio capabilities present without
@@ -XXX,XX +XXX,XX @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
         pci_register_bar(&proxy->pci_dev, proxy->legacy_io_bar_idx,
                          PCI_BASE_ADDRESS_SPACE_IO, &proxy->bar);
     }
+
+    res = pcie_sriov_pf_init_from_user_created_vfs(&proxy->pci_dev,
+                                                   proxy->last_pcie_cap_offset,
+                                                   errp);
+    if (res > 0) {
+        proxy->last_pcie_cap_offset += res;
+        virtio_add_feature(&vdev->host_features, VIRTIO_F_SR_IOV);
+    }
 }
 
 static void virtio_pci_device_unplugged(DeviceState *d)
@@ -XXX,XX +XXX,XX @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
 
     if (pcie_port && pci_is_express(pci_dev)) {
         int pos;
-        uint16_t last_pcie_cap_offset = PCI_CONFIG_SPACE_SIZE;
+        proxy->last_pcie_cap_offset = PCI_CONFIG_SPACE_SIZE;
 
         pos = pcie_endpoint_cap_init(pci_dev, 0);
         assert(pos > 0);
@@ -XXX,XX +XXX,XX @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
         pci_set_word(pci_dev->config + pos + PCI_PM_PMC, 0x3);
 
         if (proxy->flags & VIRTIO_PCI_FLAG_AER) {
-            pcie_aer_init(pci_dev, PCI_ERR_VER, last_pcie_cap_offset,
+            pcie_aer_init(pci_dev, PCI_ERR_VER, proxy->last_pcie_cap_offset,
                           PCI_ERR_SIZEOF, NULL);
-            last_pcie_cap_offset += PCI_ERR_SIZEOF;
+            proxy->last_pcie_cap_offset += PCI_ERR_SIZEOF;
         }
 
         if (proxy->flags & VIRTIO_PCI_FLAG_INIT_DEVERR) {
@@ -XXX,XX +XXX,XX @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
         }
 
         if (proxy->flags & VIRTIO_PCI_FLAG_ATS) {
-            pcie_ats_init(pci_dev, last_pcie_cap_offset,
+            pcie_ats_init(pci_dev, proxy->last_pcie_cap_offset,
                           proxy->flags & VIRTIO_PCI_FLAG_ATS_PAGE_ALIGNED);
-            last_pcie_cap_offset += PCI_EXT_CAP_ATS_SIZEOF;
+            proxy->last_pcie_cap_offset += PCI_EXT_CAP_ATS_SIZEOF;
         }
 
         if (proxy->flags & VIRTIO_PCI_FLAG_INIT_FLR) {
@@ -XXX,XX +XXX,XX @@ static void virtio_pci_exit(PCIDevice *pci_dev)
                      !pci_bus_is_root(pci_get_bus(pci_dev));
     bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
 
+    pcie_sriov_pf_exit(&proxy->pci_dev);
     msix_uninit_exclusive_bar(pci_dev);
     if (proxy->flags & VIRTIO_PCI_FLAG_AER && pcie_port &&
         pci_is_express(pci_dev)) {

-- 
2.48.1

Signed-off-by: Akihiko Odaki <akihiko.odaki@daynix.com>
---
 MAINTAINERS           |  1 +
 docs/system/index.rst |  1 +
 docs/system/sriov.rst | 36 ++++++++++++++++++++++++++++++++++++
 3 files changed, 38 insertions(+)

diff --git a/MAINTAINERS b/MAINTAINERS
index XXXXXXX..XXXXXXX 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -XXX,XX +XXX,XX @@ F: hw/pci-bridge/*
 F: qapi/pci.json
 F: docs/pci*
 F: docs/specs/*pci*
+F: docs/system/sriov.rst
 
 PCIE DOE
 M: Huai-Cheng Kuo <hchkuo@avery-design.com.tw>
diff --git a/docs/system/index.rst b/docs/system/index.rst
index XXXXXXX..XXXXXXX 100644
--- a/docs/system/index.rst
+++ b/docs/system/index.rst
@@ -XXX,XX +XXX,XX @@ or Hypervisor.Framework.
    multi-process
    confidential-guest-support
    vm-templating
+   sriov
diff --git a/docs/system/sriov.rst b/docs/system/sriov.rst
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/docs/system/sriov.rst
@@ -XXX,XX +XXX,XX @@
+.. SPDX-License-Identifier: GPL-2.0-or-later
+
+Compsable SR-IOV device
+=======================
+
+SR-IOV (Single Root I/O Virtualization) is an optional extended capability of a
+PCI Express device. It allows a single physical function (PF) to appear as
+multiple virtual functions (VFs) for the main purpose of eliminating software
+overhead in I/O from virtual machines.
+
+There are devices with predefined SR-IOV configurations, but it is also possible
+to compose an SR-IOV device yourself. Composing an SR-IOV device is currently
+only supported by virtio-net-pci.
+
+Users can configure an SR-IOV-capable virtio-net device by adding
+virtio-net-pci functions to a bus. Below is a command line example:
+
+.. code-block:: shell
+
+    -netdev user,id=n -netdev user,id=o
+    -netdev user,id=p -netdev user,id=q
+    -device pcie-root-port,id=b
+    -device virtio-net-pci,bus=b,addr=0x0.0x3,netdev=q,sriov-pf=f
+    -device virtio-net-pci,bus=b,addr=0x0.0x2,netdev=p,sriov-pf=f
+    -device virtio-net-pci,bus=b,addr=0x0.0x1,netdev=o,sriov-pf=f
+    -device virtio-net-pci,bus=b,addr=0x0.0x0,netdev=n,id=f
+
+The VFs specify the paired PF with ``sriov-pf`` property. The PF must be
+added after all VFs. It is the user's responsibility to ensure that VFs have
+function numbers larger than one of the PF, and that the function numbers
+have a consistent stride.
+
+You may also need to perform additional steps to activate the SR-IOV feature on
+your guest. For Linux, refer to [1]_.
+
+.. [1] https://docs.kernel.org/PCI/pci-iov-howto.html

-- 
2.48.1

Signed-off-by: Akihiko Odaki <akihiko.odaki@daynix.com>
---
 docs/system/sriov.rst       |  3 ++-
 include/hw/pci/pcie_sriov.h |  7 +++++--
 hw/pci/pcie_sriov.c         |  8 +++++++-
 hw/virtio/virtio-pci.c      | 16 ++++++++++------
 4 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/docs/system/sriov.rst b/docs/system/sriov.rst
index XXXXXXX..XXXXXXX 100644
--- a/docs/system/sriov.rst
+++ b/docs/system/sriov.rst
@@ -XXX,XX +XXX,XX @@ virtio-net-pci functions to a bus. Below is a command line example:
 The VFs specify the paired PF with ``sriov-pf`` property. The PF must be
 added after all VFs. It is the user's responsibility to ensure that VFs have
 function numbers larger than one of the PF, and that the function numbers
-have a consistent stride.
+have a consistent stride. Both the PF and VFs are ARI-capable so you can have
+255 VFs at maximum.
 
 You may also need to perform additional steps to activate the SR-IOV feature on
 your guest. For Linux, refer to [1]_.
diff --git a/include/hw/pci/pcie_sriov.h b/include/hw/pci/pcie_sriov.h
index XXXXXXX..XXXXXXX 100644
--- a/include/hw/pci/pcie_sriov.h
+++ b/include/hw/pci/pcie_sriov.h
@@ -XXX,XX +XXX,XX @@ void pcie_sriov_vf_register_bar(PCIDevice *dev, int region_num,
 
 /**
  * pcie_sriov_pf_init_from_user_created_vfs() - Initialize PF with user-created
- *                                              VFs.
+ *                                              VFs, adding ARI to PF
  * @dev: A PCIe device being realized.
  * @offset: The offset of the SR-IOV capability.
  * @errp: pointer to Error*, to store an error if it happens.
  *
- * Return: The size of added capability. 0 if the user did not create VFs.
+ * Initializes a PF with user-created VFs, adding the ARI extended capability to
+ * the PF. The VFs should call pcie_ari_init() to form an ARI device.
+ *
+ * Return: The size of added capabilities. 0 if the user did not create VFs.
  *         -1 if failed.
  */
 int16_t pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev,
diff --git a/hw/pci/pcie_sriov.c b/hw/pci/pcie_sriov.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/pci/pcie_sriov.c
+++ b/hw/pci/pcie_sriov.c
@@ -XXX,XX +XXX,XX @@ int16_t pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev,
     PCIDevice **vfs;
     BusState *bus = qdev_get_parent_bus(DEVICE(dev));
     uint16_t ven_id = pci_get_word(dev->config + PCI_VENDOR_ID);
+    uint16_t size = PCI_EXT_CAP_SRIOV_SIZEOF;
     uint16_t vf_dev_id;
     uint16_t vf_offset;
     uint16_t vf_stride;
@@ -XXX,XX +XXX,XX @@ int16_t pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev,
         return -1;
     }
 
+    if (!pcie_find_capability(dev, PCI_EXT_CAP_ID_ARI)) {
+        pcie_ari_init(dev, offset + size);
+        size += PCI_ARI_SIZEOF;
+    }
+
     for (i = 0; i < pf->len; i++) {
         vfs[i]->exp.sriov_vf.pf = dev;
         vfs[i]->exp.sriov_vf.vf_number = i;
@@ -XXX,XX +XXX,XX @@ int16_t pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev,
         }
     }
 
-    return PCI_EXT_CAP_SRIOV_SIZEOF;
+    return size;
 }
 
 bool pcie_sriov_register_device(PCIDevice *dev, Error **errp)
diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/virtio-pci.c
+++ b/hw/virtio/virtio-pci.c
@@ -XXX,XX +XXX,XX @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
                          PCI_BASE_ADDRESS_SPACE_IO, &proxy->bar);
     }
 
-    res = pcie_sriov_pf_init_from_user_created_vfs(&proxy->pci_dev,
-                                                   proxy->last_pcie_cap_offset,
-                                                   errp);
-    if (res > 0) {
-        proxy->last_pcie_cap_offset += res;
-        virtio_add_feature(&vdev->host_features, VIRTIO_F_SR_IOV);
+    if (pci_is_vf(&proxy->pci_dev)) {
+        pcie_ari_init(&proxy->pci_dev, proxy->last_pcie_cap_offset);
+        proxy->last_pcie_cap_offset += PCI_ARI_SIZEOF;
+    } else {
+        res = pcie_sriov_pf_init_from_user_created_vfs(
+            &proxy->pci_dev, proxy->last_pcie_cap_offset, errp);
+        if (res > 0) {
+            proxy->last_pcie_cap_offset += res;
+            virtio_add_feature(&vdev->host_features, VIRTIO_F_SR_IOV);
+        }
     }
 }

-- 
2.48.1