Finish cpr for vfio-pci MSI/MSI-X devices by preserving eventfd's and
vector state.
Signed-off-by: Steve Sistare <steven.sistare@oracle.com>
---
hw/vfio/pci.c | 122 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 121 insertions(+), 1 deletion(-)
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 237231b..2fd7121 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -53,17 +53,53 @@ static void vfio_disable_interrupts(VFIOPCIDevice *vdev);
static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled);
static void vfio_msi_disable_common(VFIOPCIDevice *vdev);
+#define EVENT_FD_NAME(vdev, name) \
+ g_strdup_printf("%s_%s", (vdev)->vbasedev.name, (name))
+
+static int save_event_fd(VFIOPCIDevice *vdev, const char *name, int nr,
+ EventNotifier *ev)
+{
+ int fd = event_notifier_get_fd(ev);
+
+ if (fd >= 0) {
+ Error *err;
+ g_autofree char *fdname = EVENT_FD_NAME(vdev, name);
+
+ if (cpr_resave_fd(fdname, nr, fd, &err)) {
+ error_report_err(err);
+ return 1;
+ }
+ }
+ return 0;
+}
+
+static int load_event_fd(VFIOPCIDevice *vdev, const char *name, int nr)
+{
+ g_autofree char *fdname = EVENT_FD_NAME(vdev, name);
+ int fd = cpr_find_fd(fdname, nr);
+ return fd;
+}
+
+static void delete_event_fd(VFIOPCIDevice *vdev, const char *name, int nr)
+{
+ g_autofree char *fdname = EVENT_FD_NAME(vdev, name);
+ cpr_delete_fd(fdname, nr);
+}
+
/* Create new or reuse existing eventfd */
static int vfio_notifier_init(VFIOPCIDevice *vdev, EventNotifier *e,
const char *name, int nr)
{
- int fd = -1; /* placeholder until a subsequent patch */
int ret = 0;
+ int fd = load_event_fd(vdev, name, nr);
if (fd >= 0) {
event_notifier_init_fd(e, fd);
} else {
ret = event_notifier_init(e, 0);
+ if (!ret) {
+ save_event_fd(vdev, name, nr, e);
+ }
}
return ret;
}
@@ -71,6 +107,7 @@ static int vfio_notifier_init(VFIOPCIDevice *vdev, EventNotifier *e,
static void vfio_notifier_cleanup(VFIOPCIDevice *vdev, EventNotifier *e,
const char *name, int nr)
{
+ delete_event_fd(vdev, name, nr);
event_notifier_cleanup(e);
}
@@ -511,6 +548,15 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
VFIOMSIVector *vector;
int ret;
+ /*
+ * Ignore the callback from msix_set_vector_notifiers during resume.
+ * The necessary subset of these actions is called from vfio_claim_vectors
+ * during post load.
+ */
+ if (vdev->vbasedev.reused) {
+ return 0;
+ }
+
trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr);
vector = &vdev->msi_vectors[nr];
@@ -2784,6 +2830,11 @@ static void vfio_register_err_notifier(VFIOPCIDevice *vdev)
fd = event_notifier_get_fd(&vdev->err_notifier);
qemu_set_fd_handler(fd, vfio_err_notifier_handler, NULL, vdev);
+ /* Do not alter irq_signaling during vfio_realize for cpr */
+ if (vdev->vbasedev.reused) {
+ return;
+ }
+
if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0,
VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
@@ -2849,6 +2900,12 @@ static void vfio_register_req_notifier(VFIOPCIDevice *vdev)
fd = event_notifier_get_fd(&vdev->req_notifier);
qemu_set_fd_handler(fd, vfio_req_notifier_handler, NULL, vdev);
+ /* Do not alter irq_signaling during vfio_realize for cpr */
+ if (vdev->vbasedev.reused) {
+ vdev->req_enabled = true;
+ return;
+ }
+
if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0,
VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
@@ -3357,6 +3414,43 @@ static Property vfio_pci_dev_properties[] = {
DEFINE_PROP_END_OF_LIST(),
};
+static void vfio_claim_vectors(VFIOPCIDevice *vdev, int nr_vectors, bool msix)
+{
+ int i, fd;
+ bool pending = false;
+ PCIDevice *pdev = &vdev->pdev;
+
+ vdev->nr_vectors = nr_vectors;
+ vdev->msi_vectors = g_new0(VFIOMSIVector, nr_vectors);
+ vdev->interrupt = msix ? VFIO_INT_MSIX : VFIO_INT_MSI;
+
+ for (i = 0; i < nr_vectors; i++) {
+ VFIOMSIVector *vector = &vdev->msi_vectors[i];
+
+ fd = load_event_fd(vdev, "interrupt", i);
+ if (fd >= 0) {
+ vfio_vector_init(vdev, i);
+ qemu_set_fd_handler(fd, vfio_msi_interrupt, NULL, vector);
+ }
+
+ if (load_event_fd(vdev, "kvm_interrupt", i) >= 0) {
+ vfio_route_change = kvm_irqchip_begin_route_changes(kvm_state);
+ vfio_add_kvm_msi_virq(vdev, vector, i, msix);
+ kvm_irqchip_commit_route_changes(&vfio_route_change);
+ vfio_connect_kvm_msi_virq(vector, i);
+ }
+
+ if (msix && msix_is_pending(pdev, i) && msix_is_masked(pdev, i)) {
+ set_bit(i, vdev->msix->pending);
+ pending = true;
+ }
+ }
+
+ if (msix) {
+ memory_region_set_enabled(&pdev->msix_pba_mmio, pending);
+ }
+}
+
/*
* The kernel may change non-emulated config bits. Exclude them from the
* changed-bits check in get_pci_config_device.
@@ -3375,6 +3469,29 @@ static int vfio_pci_pre_load(void *opaque)
return 0;
}
+static int vfio_pci_post_load(void *opaque, int version_id)
+{
+ VFIOPCIDevice *vdev = opaque;
+ PCIDevice *pdev = &vdev->pdev;
+ int nr_vectors;
+
+ if (msix_enabled(pdev)) {
+ msix_set_vector_notifiers(pdev, vfio_msix_vector_use,
+ vfio_msix_vector_release, NULL);
+ nr_vectors = vdev->msix->entries;
+ vfio_claim_vectors(vdev, nr_vectors, true);
+
+ } else if (msi_enabled(pdev)) {
+ nr_vectors = msi_nr_vectors_allocated(pdev);
+ vfio_claim_vectors(vdev, nr_vectors, false);
+
+ } else if (vfio_pci_read_config(pdev, PCI_INTERRUPT_PIN, 1)) {
+ assert(0); /* completed in a subsequent patch */
+ }
+
+ return 0;
+}
+
static bool vfio_pci_needed(void *opaque)
{
return cpr_get_mode() == CPR_MODE_RESTART;
@@ -3387,8 +3504,11 @@ static const VMStateDescription vfio_pci_vmstate = {
.minimum_version_id = 0,
.priority = MIG_PRI_VFIO_PCI, /* must load before container */
.pre_load = vfio_pci_pre_load,
+ .post_load = vfio_pci_post_load,
.needed = vfio_pci_needed,
.fields = (VMStateField[]) {
+ VMSTATE_PCI_DEVICE(pdev, VFIOPCIDevice),
+ VMSTATE_MSIX_TEST(pdev, VFIOPCIDevice, vfio_msix_present),
VMSTATE_END_OF_LIST()
}
};
--
1.8.3.1On Wed, 15 Jun 2022 07:52:15 -0700
Steve Sistare <steven.sistare@oracle.com> wrote:
> Finish cpr for vfio-pci MSI/MSI-X devices by preserving eventfd's and
> vector state.
>
> Signed-off-by: Steve Sistare <steven.sistare@oracle.com>
> ---
> hw/vfio/pci.c | 122 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
> 1 file changed, 121 insertions(+), 1 deletion(-)
>
> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> index 237231b..2fd7121 100644
> --- a/hw/vfio/pci.c
> +++ b/hw/vfio/pci.c
> @@ -53,17 +53,53 @@ static void vfio_disable_interrupts(VFIOPCIDevice *vdev);
> static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled);
> static void vfio_msi_disable_common(VFIOPCIDevice *vdev);
>
> +#define EVENT_FD_NAME(vdev, name) \
> + g_strdup_printf("%s_%s", (vdev)->vbasedev.name, (name))
> +
> +static int save_event_fd(VFIOPCIDevice *vdev, const char *name, int nr,
> + EventNotifier *ev)
> +{
> + int fd = event_notifier_get_fd(ev);
> +
> + if (fd >= 0) {
> + Error *err;
> + g_autofree char *fdname = EVENT_FD_NAME(vdev, name);
> +
> + if (cpr_resave_fd(fdname, nr, fd, &err)) {
> + error_report_err(err);
> + return 1;
Preferably -1, but the caller doesn't actually test the return value
anyway :-\
> + }
> + }
> + return 0;
> +}
> +
> +static int load_event_fd(VFIOPCIDevice *vdev, const char *name, int nr)
> +{
> + g_autofree char *fdname = EVENT_FD_NAME(vdev, name);
> + int fd = cpr_find_fd(fdname, nr);
> + return fd;
return cpr_find_fd(EVENT_FD_NAME(vdev, name), nr);
> +}
> +
> +static void delete_event_fd(VFIOPCIDevice *vdev, const char *name, int nr)
> +{
> + g_autofree char *fdname = EVENT_FD_NAME(vdev, name);
> + cpr_delete_fd(fdname, nr);
cpr_delete_fd(EVENT_FD_NAME(vdev, name), nr);
> +}
> +
> /* Create new or reuse existing eventfd */
> static int vfio_notifier_init(VFIOPCIDevice *vdev, EventNotifier *e,
> const char *name, int nr)
> {
> - int fd = -1; /* placeholder until a subsequent patch */
> int ret = 0;
> + int fd = load_event_fd(vdev, name, nr);
>
> if (fd >= 0) {
> event_notifier_init_fd(e, fd);
> } else {
> ret = event_notifier_init(e, 0);
> + if (!ret) {
> + save_event_fd(vdev, name, nr, e);
Return value not tested. The function generates an error report if it
fails, but it doesn't seem that actually blocks a cpr attempt. Do we
just wind up with that error report as a breadcrumb to why cpr breaks
with a missing fd down the road?
> + }
> }
> return ret;
> }
> @@ -71,6 +107,7 @@ static int vfio_notifier_init(VFIOPCIDevice *vdev, EventNotifier *e,
> static void vfio_notifier_cleanup(VFIOPCIDevice *vdev, EventNotifier *e,
> const char *name, int nr)
> {
> + delete_event_fd(vdev, name, nr);
> event_notifier_cleanup(e);
> }
>
> @@ -511,6 +548,15 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
> VFIOMSIVector *vector;
> int ret;
>
> + /*
> + * Ignore the callback from msix_set_vector_notifiers during resume.
> + * The necessary subset of these actions is called from vfio_claim_vectors
> + * during post load.
> + */
> + if (vdev->vbasedev.reused) {
> + return 0;
> + }
> +
> trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr);
>
> vector = &vdev->msi_vectors[nr];
> @@ -2784,6 +2830,11 @@ static void vfio_register_err_notifier(VFIOPCIDevice *vdev)
> fd = event_notifier_get_fd(&vdev->err_notifier);
> qemu_set_fd_handler(fd, vfio_err_notifier_handler, NULL, vdev);
>
> + /* Do not alter irq_signaling during vfio_realize for cpr */
> + if (vdev->vbasedev.reused) {
> + return;
> + }
> +
> if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0,
> VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
> error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
> @@ -2849,6 +2900,12 @@ static void vfio_register_req_notifier(VFIOPCIDevice *vdev)
> fd = event_notifier_get_fd(&vdev->req_notifier);
> qemu_set_fd_handler(fd, vfio_req_notifier_handler, NULL, vdev);
>
> + /* Do not alter irq_signaling during vfio_realize for cpr */
> + if (vdev->vbasedev.reused) {
> + vdev->req_enabled = true;
> + return;
> + }
vfio_notifier_init() transparently gets the old fd or creates a new
one, how do we know which has occurred to know that this eventfd is
already configured?
Don't we also have the same issue relative to vdev->pci_aer for the
error handler?
> +
> if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0,
> VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
> error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
> @@ -3357,6 +3414,43 @@ static Property vfio_pci_dev_properties[] = {
> DEFINE_PROP_END_OF_LIST(),
> };
>
> +static void vfio_claim_vectors(VFIOPCIDevice *vdev, int nr_vectors, bool msix)
> +{
> + int i, fd;
> + bool pending = false;
> + PCIDevice *pdev = &vdev->pdev;
> +
> + vdev->nr_vectors = nr_vectors;
> + vdev->msi_vectors = g_new0(VFIOMSIVector, nr_vectors);
> + vdev->interrupt = msix ? VFIO_INT_MSIX : VFIO_INT_MSI;
> +
> + for (i = 0; i < nr_vectors; i++) {
> + VFIOMSIVector *vector = &vdev->msi_vectors[i];
> +
> + fd = load_event_fd(vdev, "interrupt", i);
> + if (fd >= 0) {
> + vfio_vector_init(vdev, i);
> + qemu_set_fd_handler(fd, vfio_msi_interrupt, NULL, vector);
> + }
> +
> + if (load_event_fd(vdev, "kvm_interrupt", i) >= 0) {
> + vfio_route_change = kvm_irqchip_begin_route_changes(kvm_state);
> + vfio_add_kvm_msi_virq(vdev, vector, i, msix);
> + kvm_irqchip_commit_route_changes(&vfio_route_change);
> + vfio_connect_kvm_msi_virq(vector, i);
Shouldn't we take advantage of the batching support here?
> + }
How do we debug if one of the above fails that shouldn't have failed?
Should we have an assert or change this to a non-void return if we
cannot setup an interrupt that we think is configured?
> +
> + if (msix && msix_is_pending(pdev, i) && msix_is_masked(pdev, i)) {
> + set_bit(i, vdev->msix->pending);
> + pending = true;
> + }
> + }
> +
> + if (msix) {
> + memory_region_set_enabled(&pdev->msix_pba_mmio, pending);
> + }
> +}
> +
> /*
> * The kernel may change non-emulated config bits. Exclude them from the
> * changed-bits check in get_pci_config_device.
> @@ -3375,6 +3469,29 @@ static int vfio_pci_pre_load(void *opaque)
> return 0;
> }
>
> +static int vfio_pci_post_load(void *opaque, int version_id)
> +{
> + VFIOPCIDevice *vdev = opaque;
> + PCIDevice *pdev = &vdev->pdev;
> + int nr_vectors;
> +
> + if (msix_enabled(pdev)) {
> + msix_set_vector_notifiers(pdev, vfio_msix_vector_use,
> + vfio_msix_vector_release, NULL);
> + nr_vectors = vdev->msix->entries;
Maybe this is why we're not generating an error above, we don't know
which vectors are configured other than if they have a saved eventfd,
where we don't test whether we were able to actually save the fd.
Thanks,
Alex
> + vfio_claim_vectors(vdev, nr_vectors, true);
> +
> + } else if (msi_enabled(pdev)) {
> + nr_vectors = msi_nr_vectors_allocated(pdev);
> + vfio_claim_vectors(vdev, nr_vectors, false);
> +
> + } else if (vfio_pci_read_config(pdev, PCI_INTERRUPT_PIN, 1)) {
> + assert(0); /* completed in a subsequent patch */
> + }
> +
> + return 0;
> +}
> +
> static bool vfio_pci_needed(void *opaque)
> {
> return cpr_get_mode() == CPR_MODE_RESTART;
> @@ -3387,8 +3504,11 @@ static const VMStateDescription vfio_pci_vmstate = {
> .minimum_version_id = 0,
> .priority = MIG_PRI_VFIO_PCI, /* must load before container */
> .pre_load = vfio_pci_pre_load,
> + .post_load = vfio_pci_post_load,
> .needed = vfio_pci_needed,
> .fields = (VMStateField[]) {
> + VMSTATE_PCI_DEVICE(pdev, VFIOPCIDevice),
> + VMSTATE_MSIX_TEST(pdev, VFIOPCIDevice, vfio_msix_present),
> VMSTATE_END_OF_LIST()
> }
> };
On 6/29/2022 4:19 PM, Alex Williamson wrote:
> On Wed, 15 Jun 2022 07:52:15 -0700
> Steve Sistare <steven.sistare@oracle.com> wrote:
>
>> Finish cpr for vfio-pci MSI/MSI-X devices by preserving eventfd's and
>> vector state.
>>
>> Signed-off-by: Steve Sistare <steven.sistare@oracle.com>
>> ---
>> hw/vfio/pci.c | 122 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
>> 1 file changed, 121 insertions(+), 1 deletion(-)
>>
>> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
>> index 237231b..2fd7121 100644
>> --- a/hw/vfio/pci.c
>> +++ b/hw/vfio/pci.c
>> @@ -53,17 +53,53 @@ static void vfio_disable_interrupts(VFIOPCIDevice *vdev);
>> static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled);
>> static void vfio_msi_disable_common(VFIOPCIDevice *vdev);
>>
>> +#define EVENT_FD_NAME(vdev, name) \
>> + g_strdup_printf("%s_%s", (vdev)->vbasedev.name, (name))
>> +
>> +static int save_event_fd(VFIOPCIDevice *vdev, const char *name, int nr,
>> + EventNotifier *ev)
>> +{
>> + int fd = event_notifier_get_fd(ev);
>> +
>> + if (fd >= 0) {
>> + Error *err;
>> + g_autofree char *fdname = EVENT_FD_NAME(vdev, name);
>> +
>> + if (cpr_resave_fd(fdname, nr, fd, &err)) {
>> + error_report_err(err);
>> + return 1;
>
>
> Preferably -1, but the caller doesn't actually test the return value
> anyway :-\
Per my previous email, I suggest that cpr_resave_fd return void, and hence
save_event_fd becomes void as well.
>> + }
>> + }
>> + return 0;
>> +}
>> +
>> +static int load_event_fd(VFIOPCIDevice *vdev, const char *name, int nr)
>> +{
>> + g_autofree char *fdname = EVENT_FD_NAME(vdev, name);
>> + int fd = cpr_find_fd(fdname, nr);
>> + return fd;
>
>
> return cpr_find_fd(EVENT_FD_NAME(vdev, name), nr);
That leaks EVENT_FD_NAME, produced by g_strdup_printf, but I can reduce it to:
g_autofree char *fdname = EVENT_FD_NAME(vdev, name);
return cpr_find_fd(fdname, nr);
>> +}
>> +
>> +static void delete_event_fd(VFIOPCIDevice *vdev, const char *name, int nr)
>> +{
>> + g_autofree char *fdname = EVENT_FD_NAME(vdev, name);
>> + cpr_delete_fd(fdname, nr);
>
>
> cpr_delete_fd(EVENT_FD_NAME(vdev, name), nr);
Ditto.
>> +}
>> +
>> /* Create new or reuse existing eventfd */
>> static int vfio_notifier_init(VFIOPCIDevice *vdev, EventNotifier *e,
>> const char *name, int nr)
>> {
>> - int fd = -1; /* placeholder until a subsequent patch */
>> int ret = 0;
>> + int fd = load_event_fd(vdev, name, nr);
>>
>> if (fd >= 0) {
>> event_notifier_init_fd(e, fd);
>> } else {
>> ret = event_notifier_init(e, 0);
>> + if (!ret) {
>> + save_event_fd(vdev, name, nr, e);
>
>
> Return value not tested. The function generates an error report if it
> fails, but it doesn't seem that actually blocks a cpr attempt. Do we
> just wind up with that error report as a breadcrumb to why cpr breaks
> with a missing fd down the road?
Thanks, that is a bug, it should have been:
ret = save_event_fd(vdev, name, nr, e)
... but per the previous comment save_event_fd becomes void.
>> + }
>> }
>> return ret;
>> }
>> @@ -71,6 +107,7 @@ static int vfio_notifier_init(VFIOPCIDevice *vdev, EventNotifier *e,
>> static void vfio_notifier_cleanup(VFIOPCIDevice *vdev, EventNotifier *e,
>> const char *name, int nr)
>> {
>> + delete_event_fd(vdev, name, nr);
>> event_notifier_cleanup(e);
>> }
>>
>> @@ -511,6 +548,15 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
>> VFIOMSIVector *vector;
>> int ret;
>>
>> + /*
>> + * Ignore the callback from msix_set_vector_notifiers during resume.
>> + * The necessary subset of these actions is called from vfio_claim_vectors
>> + * during post load.
>> + */
>> + if (vdev->vbasedev.reused) {
>> + return 0;
>> + }
>> +
>> trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr);
>>
>> vector = &vdev->msi_vectors[nr];
>> @@ -2784,6 +2830,11 @@ static void vfio_register_err_notifier(VFIOPCIDevice *vdev)
>> fd = event_notifier_get_fd(&vdev->err_notifier);
>> qemu_set_fd_handler(fd, vfio_err_notifier_handler, NULL, vdev);
>>
>> + /* Do not alter irq_signaling during vfio_realize for cpr */
>> + if (vdev->vbasedev.reused) {
>> + return;
>> + }
>> +
>> if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0,
>> VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
>> error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
>> @@ -2849,6 +2900,12 @@ static void vfio_register_req_notifier(VFIOPCIDevice *vdev)
>> fd = event_notifier_get_fd(&vdev->req_notifier);
>> qemu_set_fd_handler(fd, vfio_req_notifier_handler, NULL, vdev);
>>
>> + /* Do not alter irq_signaling during vfio_realize for cpr */
>> + if (vdev->vbasedev.reused) {
>> + vdev->req_enabled = true;
>> + return;
>> + }
>
> vfio_notifier_init() transparently gets the old fd or creates a new
> one, how do we know which has occurred to know that this eventfd is
> already configured?
The caller can check the reused flag, which is set iff an old fd exists.
I could pass reused to vfio_notifier_init to assert that, but in some cases
I would need to pass a reused flag down through several functions to reach
vfio_notifier_init, which just seems ugly.
> Don't we also have the same issue relative to vdev->pci_aer for the
> error handler?
Same answer:
vfio_register_err_notifier()
vfio_notifier_init();
if (vdev->vbasedev.reused)
return;
vfio_set_irq_signaling() ...
>> +
>> if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0,
>> VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
>> error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
>> @@ -3357,6 +3414,43 @@ static Property vfio_pci_dev_properties[] = {
>> DEFINE_PROP_END_OF_LIST(),
>> };
>>
>> +static void vfio_claim_vectors(VFIOPCIDevice *vdev, int nr_vectors, bool msix)
>> +{
>> + int i, fd;
>> + bool pending = false;
>> + PCIDevice *pdev = &vdev->pdev;
>> +
>> + vdev->nr_vectors = nr_vectors;
>> + vdev->msi_vectors = g_new0(VFIOMSIVector, nr_vectors);
>> + vdev->interrupt = msix ? VFIO_INT_MSIX : VFIO_INT_MSI;
>> +
>> + for (i = 0; i < nr_vectors; i++) {
>> + VFIOMSIVector *vector = &vdev->msi_vectors[i];
>> +
>> + fd = load_event_fd(vdev, "interrupt", i);
>> + if (fd >= 0) {
>> + vfio_vector_init(vdev, i);
>> + qemu_set_fd_handler(fd, vfio_msi_interrupt, NULL, vector);
>> + }
>> +
>> + if (load_event_fd(vdev, "kvm_interrupt", i) >= 0) {
>> + vfio_route_change = kvm_irqchip_begin_route_changes(kvm_state);
>> + vfio_add_kvm_msi_virq(vdev, vector, i, msix);
>> + kvm_irqchip_commit_route_changes(&vfio_route_change);
>> + vfio_connect_kvm_msi_virq(vector, i);
>
>
> Shouldn't we take advantage of the batching support here?
OK, will do.
>> + }
>
> How do we debug if one of the above fails that shouldn't have failed?
> Should we have an assert or change this to a non-void return if we
> cannot setup an interrupt that we think is configured?
The path above ending with qemu_set_fd_handler always succeeds, because:
fd = load_event_fd(vdev, "interrupt", i);
if (fd >= 0) {
vfio_vector_init(vdev, i)
vfio_notifier_init(..., "interrupt", i)
int fd = load_event_fd(vdev, name, i);
if (fd >= 0) {
event_notifier_init_fd(e, fd); <-- void, never fails
In the kvm_interrupt clause, only vfio_connect_kvm_msi_virq() can fail. But, it
returns void, and other callers also assume it succeeds. Good enough, or do
you want to do better here?
>> +
>> + if (msix && msix_is_pending(pdev, i) && msix_is_masked(pdev, i)) {
>> + set_bit(i, vdev->msix->pending);
>> + pending = true;
>> + }
>> + }
>> +
>> + if (msix) {
>> + memory_region_set_enabled(&pdev->msix_pba_mmio, pending);
>> + }
>> +}
>> +
>> /*
>> * The kernel may change non-emulated config bits. Exclude them from the
>> * changed-bits check in get_pci_config_device.
>> @@ -3375,6 +3469,29 @@ static int vfio_pci_pre_load(void *opaque)
>> return 0;
>> }
>>
>> +static int vfio_pci_post_load(void *opaque, int version_id)
>> +{
>> + VFIOPCIDevice *vdev = opaque;
>> + PCIDevice *pdev = &vdev->pdev;
>> + int nr_vectors;
>> +
>> + if (msix_enabled(pdev)) {
>> + msix_set_vector_notifiers(pdev, vfio_msix_vector_use,
>> + vfio_msix_vector_release, NULL);
>> + nr_vectors = vdev->msix->entries;
>
> Maybe this is why we're not generating an error above, we don't know
> which vectors are configured other than if they have a saved eventfd,
> where we don't test whether we were able to actually save the fd.
> Thanks,
>
> Alex
>
>
>> + vfio_claim_vectors(vdev, nr_vectors, true);
>> +
>> + } else if (msi_enabled(pdev)) {
>> + nr_vectors = msi_nr_vectors_allocated(pdev);
>> + vfio_claim_vectors(vdev, nr_vectors, false);
>> +
>> + } else if (vfio_pci_read_config(pdev, PCI_INTERRUPT_PIN, 1)) {
>> + assert(0); /* completed in a subsequent patch */
>> + }
>> +
>> + return 0;
>> +}
>> +
>> static bool vfio_pci_needed(void *opaque)
>> {
>> return cpr_get_mode() == CPR_MODE_RESTART;
>> @@ -3387,8 +3504,11 @@ static const VMStateDescription vfio_pci_vmstate = {
>> .minimum_version_id = 0,
>> .priority = MIG_PRI_VFIO_PCI, /* must load before container */
>> .pre_load = vfio_pci_pre_load,
>> + .post_load = vfio_pci_post_load,
>> .needed = vfio_pci_needed,
>> .fields = (VMStateField[]) {
>> + VMSTATE_PCI_DEVICE(pdev, VFIOPCIDevice),
>> + VMSTATE_MSIX_TEST(pdev, VFIOPCIDevice, vfio_msix_present),
>> VMSTATE_END_OF_LIST()
>> }
>> };
>
© 2016 - 2026 Red Hat, Inc.