Patch [1, 5] refactors the PCI infrastructure code.

Patch [6, 10] adds user-created SR-IOV VF infrastructure.

Patch 11 makes virtio-pci work as SR-IOV PF for user-created VFs.

Patch 12 allows user to create SR-IOV VFs with virtio-net-pci.

Akihiko Odaki (12):

hw/pci: Initialize PCI multifunction after realization

hw/pci: Determine if rombar is explicitly enabled

vfio: Avoid inspecting option QDict for rombar

hw/qdev: Remove opts member

pcie_sriov: Reuse SR-IOV VF device instances

pcie_sriov: Release VFs failed to realize

docs/pcie_sriov.txt | 8 +-

include/hw/pci/pci.h | 2 +-

include/hw/pci/pci_device.h | 13 +-

include/hw/pci/pcie_sriov.h | 25 ++-

include/hw/qdev-core.h | 4 -

hw/core/qdev.c | 1 -

hw/net/igb.c | 3 +-

hw/nvme/ctrl.c | 3 +-

hw/pci/pci.c | 98 +++++++-----

hw/pci/pci_host.c | 4 +-

hw/pci/pcie.c | 4 +-

hw/pci/pcie_sriov.c | 360 +++++++++++++++++++++++++++++++++-----------

hw/vfio/pci.c | 3 +-

hw/virtio/virtio-net-pci.c | 1 +

hw/virtio/virtio-pci.c | 7 +

system/qdev-monitor.c | 12 +-

16 files changed, 395 insertions(+), 153 deletions(-)

base-commit: 4705fc0c8511d073bee4751c3c974aab2b10a970

+ if (pdev->rom_bar && pdev->rom_bar != -1) {

if (load_file || pdev->romsize == -1) {

2.43.0

The device realization code may enable PCI multifunction for SR-IOV.

hw/pci/pci.c | 17 +++++++++--------

1 file changed, 9 insertions(+), 8 deletions(-)

@@ -XXX,XX +XXX,XX @@ static void pci_init_mask_bridge(PCIDevice *d)

PCI_PREF_RANGE_TYPE_MASK);

}

-static void pci_init_multifunction(PCIBus *bus, PCIDevice *dev, Error **errp)

+static void pci_init_multifunction(PCIDevice *dev, Error **errp)

{

+ PCIBus *bus = pci_get_bus(dev);

uint8_t slot = PCI_SLOT(dev->devfn);

uint8_t func;

@@ -XXX,XX +XXX,XX @@ static PCIDevice *do_pci_register_device(PCIDevice *pci_dev,

PCIDeviceClass *pc = PCI_DEVICE_GET_CLASS(pci_dev);

PCIConfigReadFunc *config_read = pc->config_read;

PCIConfigWriteFunc *config_write = pc->config_write;

- Error *local_err = NULL;

DeviceState *dev = DEVICE(pci_dev);

PCIBus *bus = pci_get_bus(pci_dev);

bool is_bridge = IS_PCI_BRIDGE(pci_dev);

@@ -XXX,XX +XXX,XX @@ static PCIDevice *do_pci_register_device(PCIDevice *pci_dev,

if (is_bridge) {

pci_init_mask_bridge(pci_dev);

}

- pci_init_multifunction(bus, pci_dev, &local_err);

- if (local_err) {

- error_propagate(errp, local_err);

- do_pci_unregister_device(pci_dev);

- return NULL;

- }

if (!config_read)

config_read = pci_default_read_config;

@@ -XXX,XX +XXX,XX @@ static void pci_qdev_realize(DeviceState *qdev, Error **errp)

}

}

+ pci_init_multifunction(pci_dev, &local_err);

+ if (local_err) {

+ error_propagate(errp, local_err);

+ pci_qdev_unrealize(DEVICE(pci_dev));

+ return;

+ }

/*

* A PCIe Downstream Port that do not have ARI Forwarding enabled must

* associate only Device 0 with the device attached to the bus

2.43.0

pcie_add_capability(dev, PCI_EXT_CAP_ID_SRIOV, 1,

offset, PCI_EXT_CAP_SRIOV_SIZEOF);

dev->exp.sriov_cap = offset;

2.43.0

2.43.0

include/hw/pci/pcie_sriov.h | 19 +++

hw/pci/pci.c | 53 ++++++---

hw/pci/pcie_sriov.c | 279 +++++++++++++++++++++++++++++++++++---------

4 files changed, 283 insertions(+), 74 deletions(-)

/* ID of standby device in net_failover pair */

char *failover_pair_id;

uint32_t acpi_index;

@@ -XXX,XX +XXX,XX @@ struct PCIESriovPF {

uint16_t num_vfs; /* Number of virtual functions created */

};

struct PCIESriovVF {

+ * Return:

+ * * true - @dev is initialized as a PCIe SR-IOV PF.

+ * * false - @dev is not initialized because there is no SR-IOV VFs or an error

+ * occurred.

+bool pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev, uint16_t offset,

+ Error **errp);

@@ -XXX,XX +XXX,XX @@ static Property pci_props[] = {

QEMU_PCIE_ERR_UNC_MASK_BITNR, true),

DEFINE_PROP_BIT("x-pcie-ari-nextfn-1", PCIDevice, cap_present,

DEFINE_PROP_END_OF_LIST()

};

@@ -XXX,XX +XXX,XX @@ static void pci_init_multifunction(PCIDevice *dev, Error **errp)

* device, as it may just be a VF that ended up with function 0 in

* the legacy PCI interpretation. Avoid failing in such cases:

*/

pci_init_multifunction(pci_dev, &local_err);

if (local_err) {

error_propagate(errp, local_err);

static void unrealize_vfs(PCIDevice *dev, uint16_t total_vfs)

@@ -XXX,XX +XXX,XX @@ static void unrealize_vfs(PCIDevice *dev, uint16_t total_vfs)

+ dev->exp.sriov_pf.num_vfs = num_vfs;

+ uint16_t num_vfs = dev->exp.sriov_pf.num_vfs;

+ PCI_FUNC(dev->devfn), num_vfs);

+ for (i = 0; i < num_vfs; i++) {

+ dev->exp.sriov_pf.num_vfs = 0;

+ pci_set_word(dev->config + dev->exp.sriov_cap + PCI_SRIOV_NUM_VF, 0);

- int32_t devfn = dev->devfn + vf_offset;

assert(dev->exp.sriov_pf.vf);

- unrealize_vfs(dev, pci_get_word(cfg + PCI_SRIOV_TOTAL_VF));

+ unrealize_vfs(dev, pci_get_word(cfg + PCI_SRIOV_TOTAL_VF));

+ }

}

+bool pcie_sriov_pf_init_from_user_created_vfs(PCIDevice *dev, uint16_t offset,

+ Error **errp)

+ return false;

}

+ return false;

}

- dev->exp.sriov_pf.num_vfs = num_vfs;

+

+ return false;

+ return false;

+ return false;

+ return false;

+ return false;

+ return false;

+ return false;

+ return false;

+ uint8_t type = vfs[0]->io_regions[i].type;

+ pcibus_t size = vfs[0]->io_regions[i].size;

+

+ if (size) {

+ pcie_sriov_pf_init_vf_bar(dev, i, type, size);

+ }

+ }

+

+ return true;

}

- uint16_t num_vfs = dev->exp.sriov_pf.num_vfs;

- PCI_FUNC(dev->devfn), num_vfs);

- for (i = 0; i < num_vfs; i++) {

+ }

+

+ GPtrArray *pf = g_hash_table_lookup(pfs, dev->qdev.id);

+ g_hash_table_remove(pfs, dev->qdev.id);

}

- dev->exp.sriov_pf.num_vfs = 0;

- pci_set_word(dev->config + dev->exp.sriov_cap + PCI_SRIOV_NUM_VF, 0);

}

2.43.0

hw/virtio/virtio-pci.c | 7 +++++++

1 file changed, 7 insertions(+)

+ if (pcie_sriov_pf_init_from_user_created_vfs(&proxy->pci_dev,

+ PCI_CONFIG_SPACE_SIZE,

+ errp)) {

@@ -XXX,XX +XXX,XX @@ static void virtio_pci_device_unplugged(DeviceState *d)

bool modern = virtio_pci_modern(proxy);

virtio_pci_stop_ioeventfd(proxy);

if (modern) {

2.43.0

2.43.0

hw/pci/pcie_sriov.c | 2 ++

1 file changed, 2 insertions(+)

diff --git a/hw/pci/pcie_sriov.c b/hw/pci/pcie_sriov.c

--- a/hw/pci/pcie_sriov.c

+++ b/hw/pci/pcie_sriov.c

@@ -XXX,XX +XXX,XX @@ bool pcie_sriov_pf_init(PCIDevice *dev, uint16_t offset,

vf->exp.sriov_vf.vf_number = i;

if (!qdev_realize(&vf->qdev, bus, errp)) {

+ object_unparent(OBJECT(vf));

+ object_unref(vf);

unrealize_vfs(dev, i);

return false;

}

2.43.0

docs/pcie_sriov.txt | 8 ++--

include/hw/pci/pci.h | 2 +-

include/hw/pci/pci_device.h | 2 +-

include/hw/pci/pcie_sriov.h | 6 +--

hw/net/igb.c | 3 +-

hw/nvme/ctrl.c | 3 +-

hw/pci/pci.c | 18 ++++----

hw/pci/pci_host.c | 4 +-

hw/pci/pcie.c | 4 +-

hw/pci/pcie_sriov.c | 105 +++++++++++++++++++++-----------------------

10 files changed, 79 insertions(+), 76 deletions(-)

diff --git a/docs/pcie_sriov.txt b/docs/pcie_sriov.txt

--- a/docs/pcie_sriov.txt

+++ b/docs/pcie_sriov.txt

@@ -XXX,XX +XXX,XX @@ setting up a BAR for a VF.

...

/* Add and initialize the SR/IOV capability */

- pcie_sriov_pf_init(d, 0x200, "your_virtual_dev",

- vf_devid, initial_vfs, total_vfs,

- fun_offset, stride);

+ if (!pcie_sriov_pf_init(d, 0x200, "your_virtual_dev",

+ vf_devid, initial_vfs, total_vfs,

+ fun_offset, stride, errp)) {

+ return;

+ }

/* Set up individual VF BARs (parameters as for normal BARs) */

pcie_sriov_pf_init_vf_bar( ... )

diff --git a/include/hw/pci/pci.h b/include/hw/pci/pci.h

index XXXXXXX..XXXXXXX 100644

--- a/include/hw/pci/pci.h

+++ b/include/hw/pci/pci.h

@@ -XXX,XX +XXX,XX @@ static inline void pci_irq_pulse(PCIDevice *pci_dev)

}

MSIMessage pci_get_msi_message(PCIDevice *dev, int vector);

-void pci_set_power(PCIDevice *pci_dev, bool state);

+void pci_set_enabled(PCIDevice *pci_dev, bool state);

#endif

diff --git a/include/hw/pci/pci_device.h b/include/hw/pci/pci_device.h

index XXXXXXX..XXXXXXX 100644

--- a/include/hw/pci/pci_device.h

+++ b/include/hw/pci/pci_device.h

@@ -XXX,XX +XXX,XX @@ typedef struct PCIReqIDCache PCIReqIDCache;

struct PCIDevice {

DeviceState qdev;

bool partially_hotplugged;

- bool has_power;

+ bool is_enabled;

/* PCI config space */

uint8_t *config;

@@ -XXX,XX +XXX,XX @@

struct PCIESriovPF {

uint16_t num_vfs; /* Number of virtual functions created */

uint8_t vf_bar_type[PCI_NUM_REGIONS]; /* Store type for each VF bar */

- const char *vfname; /* Reference to the device type used for the VFs */

PCIDevice **vf; /* Pointer to an array of num_vfs VF devices */

};

@@ -XXX,XX +XXX,XX @@ struct PCIESriovVF {

uint16_t vf_number; /* Logical VF number of this function */

};

-void pcie_sriov_pf_init(PCIDevice *dev, uint16_t offset,

+bool pcie_sriov_pf_init(PCIDevice *dev, uint16_t offset,

const char *vfname, uint16_t vf_dev_id,

uint16_t init_vfs, uint16_t total_vfs,

- uint16_t vf_offset, uint16_t vf_stride);

+ uint16_t vf_offset, uint16_t vf_stride,

+ Error **errp);

void pcie_sriov_pf_exit(PCIDevice *dev);

/* Set up a VF bar in the SR/IOV bar area */

diff --git a/hw/net/igb.c b/hw/net/igb.c

index XXXXXXX..XXXXXXX 100644

--- a/hw/net/igb.c

+++ b/hw/net/igb.c

@@ -XXX,XX +XXX,XX @@ static void igb_pci_realize(PCIDevice *pci_dev, Error **errp)

pcie_sriov_pf_init(pci_dev, IGB_CAP_SRIOV_OFFSET, TYPE_IGBVF,

IGB_82576_VF_DEV_ID, IGB_MAX_VF_FUNCTIONS, IGB_MAX_VF_FUNCTIONS,

- IGB_VF_OFFSET, IGB_VF_STRIDE);

+ IGB_VF_OFFSET, IGB_VF_STRIDE,

+ &error_abort);

pcie_sriov_pf_init_vf_bar(pci_dev, IGBVF_MMIO_BAR_IDX,

PCI_BASE_ADDRESS_MEM_TYPE_64 | PCI_BASE_ADDRESS_MEM_PREFETCH,

diff --git a/hw/nvme/ctrl.c b/hw/nvme/ctrl.c

index XXXXXXX..XXXXXXX 100644

--- a/hw/nvme/ctrl.c

+++ b/hw/nvme/ctrl.c

@@ -XXX,XX +XXX,XX @@ static void nvme_init_sriov(NvmeCtrl *n, PCIDevice *pci_dev, uint16_t offset)

pcie_sriov_pf_init(pci_dev, offset, "nvme", vf_dev_id,

n->params.sriov_max_vfs, n->params.sriov_max_vfs,

- NVME_VF_OFFSET, NVME_VF_STRIDE);

+ NVME_VF_OFFSET, NVME_VF_STRIDE,

+ &error_abort);

pcie_sriov_pf_init_vf_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |

PCI_BASE_ADDRESS_MEM_TYPE_64, bar_size);

diff --git a/hw/pci/pci.c b/hw/pci/pci.c

index XXXXXXX..XXXXXXX 100644

--- a/hw/pci/pci.c

+++ b/hw/pci/pci.c

@@ -XXX,XX +XXX,XX @@ static void pci_update_mappings(PCIDevice *d)

continue;

new_addr = pci_bar_address(d, i, r->type, r->size);

- if (!d->has_power) {

+ if (!d->is_enabled) {

new_addr = PCI_BAR_UNMAPPED;

}

@@ -XXX,XX +XXX,XX @@ void pci_default_write_config(PCIDevice *d, uint32_t addr, uint32_t val_in, int

pci_update_irq_disabled(d, was_irq_disabled);

memory_region_set_enabled(&d->bus_master_enable_region,

(pci_get_word(d->config + PCI_COMMAND)

- & PCI_COMMAND_MASTER) && d->has_power);

+ & PCI_COMMAND_MASTER) && d->is_enabled);

}

msi_write_config(d, addr, val_in, l);

@@ -XXX,XX +XXX,XX @@ static void pci_qdev_realize(DeviceState *qdev, Error **errp)

return;

}

- pci_set_power(pci_dev, true);

+ if (!pci_is_vf(pci_dev)) {

+ pci_set_enabled(pci_dev, true);

+ }

pci_dev->msi_trigger = pci_msi_trigger;

}

@@ -XXX,XX +XXX,XX @@ MSIMessage pci_get_msi_message(PCIDevice *dev, int vector)

return msg;

}

-void pci_set_power(PCIDevice *d, bool state)

+void pci_set_enabled(PCIDevice *d, bool state)

{

- if (d->has_power == state) {

+ if (d->is_enabled == state) {

return;

}

- d->has_power = state;

+ d->is_enabled = state;

pci_update_mappings(d);

memory_region_set_enabled(&d->bus_master_enable_region,

(pci_get_word(d->config + PCI_COMMAND)

- & PCI_COMMAND_MASTER) && d->has_power);

- if (!d->has_power) {

+ & PCI_COMMAND_MASTER) && d->is_enabled);

+ if (!d->is_enabled) {

pci_device_reset(d);

}

}

diff --git a/hw/pci/pci_host.c b/hw/pci/pci_host.c

index XXXXXXX..XXXXXXX 100644

--- a/hw/pci/pci_host.c

+++ b/hw/pci/pci_host.c

@@ -XXX,XX +XXX,XX @@ void pci_host_config_write_common(PCIDevice *pci_dev, uint32_t addr,

* allowing direct removal of unexposed functions.

*/

if ((pci_dev->qdev.hotplugged && !pci_get_function_0(pci_dev)) ||

- !pci_dev->has_power || is_pci_dev_ejected(pci_dev)) {

+ !pci_dev->is_enabled || is_pci_dev_ejected(pci_dev)) {

return;

}

@@ -XXX,XX +XXX,XX @@ uint32_t pci_host_config_read_common(PCIDevice *pci_dev, uint32_t addr,

* allowing direct removal of unexposed functions.

*/

if ((pci_dev->qdev.hotplugged && !pci_get_function_0(pci_dev)) ||

- !pci_dev->has_power || is_pci_dev_ejected(pci_dev)) {

+ !pci_dev->is_enabled || is_pci_dev_ejected(pci_dev)) {

return ~0x0;

}

diff --git a/hw/pci/pcie.c b/hw/pci/pcie.c

index XXXXXXX..XXXXXXX 100644

--- a/hw/pci/pcie.c

+++ b/hw/pci/pcie.c

@@ -XXX,XX +XXX,XX @@ static void pcie_set_power_device(PCIBus *bus, PCIDevice *dev, void *opaque)

{

bool *power = opaque;

- pci_set_power(dev, *power);

+ if (!pci_is_vf(dev)) {

+ pci_set_enabled(dev, *power);

+ }

}

static void pcie_cap_update_power(PCIDevice *hotplug_dev)

@@ -XXX,XX +XXX,XX @@

#include "qapi/error.h"

#include "trace.h"

-static PCIDevice *register_vf(PCIDevice *pf, int devfn,

- const char *name, uint16_t vf_num);

-static void unregister_vfs(PCIDevice *dev);

+static void unrealize_vfs(PCIDevice *dev, uint16_t total_vfs)

+{

+ for (uint16_t i = 0; i < total_vfs; i++) {

+ Error *err = NULL;

+ PCIDevice *vf = dev->exp.sriov_pf.vf[i];

+ if (!object_property_set_bool(OBJECT(vf), "realized", false, &err)) {

+ error_reportf_err(err, "Failed to unplug: ");

+ }

+ object_unparent(OBJECT(vf));

+ object_unref(OBJECT(vf));

+ }

+ g_free(dev->exp.sriov_pf.vf);

+ dev->exp.sriov_pf.vf = NULL;

+}

-void pcie_sriov_pf_init(PCIDevice *dev, uint16_t offset,

+bool pcie_sriov_pf_init(PCIDevice *dev, uint16_t offset,

const char *vfname, uint16_t vf_dev_id,

uint16_t init_vfs, uint16_t total_vfs,

- uint16_t vf_offset, uint16_t vf_stride)

+ uint16_t vf_offset, uint16_t vf_stride,