In VFIO migration resume phase and some guest startups, there are
already unmasked vectors in the vector table when calling
vfio_msix_enable(). So in order to avoid inefficiently disabling
and enabling vectors repeatedly, let's allocate all needed vectors
first and then enable these unmasked vectors one by one without
disabling.
Signed-off-by: Shenming Lu <lushenming@huawei.com>
---
hw/pci/msix.c | 2 +-
hw/vfio/pci.c | 20 +++++++++++++++++---
include/hw/pci/msix.h | 1 +
3 files changed, 19 insertions(+), 4 deletions(-)
diff --git a/hw/pci/msix.c b/hw/pci/msix.c
index ae9331cd0b..e057958fcd 100644
--- a/hw/pci/msix.c
+++ b/hw/pci/msix.c
@@ -131,7 +131,7 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked)
}
}
-static bool msix_masked(PCIDevice *dev)
+bool msix_masked(PCIDevice *dev)
{
return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK;
}
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index f74be78209..088fd41926 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -569,6 +569,9 @@ static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
static void vfio_msix_enable(VFIOPCIDevice *vdev)
{
+ PCIDevice *pdev = &vdev->pdev;
+ unsigned int nr, max_vec = 0;
+
vfio_disable_interrupts(vdev);
vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
@@ -587,11 +590,22 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev)
* triggering to userspace, then immediately release the vector, leaving
* the physical device with no vectors enabled, but MSI-X enabled, just
* like the guest view.
+ * If there are already unmasked vectors (in migration resume phase and
+ * some guest startups) which will be enabled soon, we can allocate all
+ * of them here to avoid inefficiently disabling and enabling vectors
+ * repeatedly later.
*/
- vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
- vfio_msix_vector_release(&vdev->pdev, 0);
+ if (!msix_masked(pdev)) {
+ for (nr = 0; nr < msix_nr_vectors_allocated(pdev); nr++) {
+ if (!msix_is_masked(pdev, nr)) {
+ max_vec = nr;
+ }
+ }
+ }
+ vfio_msix_vector_do_use(pdev, max_vec, NULL, NULL);
+ vfio_msix_vector_release(pdev, max_vec);
- if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
+ if (msix_set_vector_notifiers(pdev, vfio_msix_vector_use,
vfio_msix_vector_release, NULL)) {
error_report("vfio: msix_set_vector_notifiers failed");
}
diff --git a/include/hw/pci/msix.h b/include/hw/pci/msix.h
index 4c4a60c739..b3cd88e262 100644
--- a/include/hw/pci/msix.h
+++ b/include/hw/pci/msix.h
@@ -28,6 +28,7 @@ void msix_load(PCIDevice *dev, QEMUFile *f);
int msix_enabled(PCIDevice *dev);
int msix_present(PCIDevice *dev);
+bool msix_masked(PCIDevice *dev);
bool msix_is_masked(PCIDevice *dev, unsigned vector);
void msix_set_pending(PCIDevice *dev, unsigned vector);
--
2.19.1
MST/Marcel,
Do you have an Ack or objection to exporting msix_masked() as below?
Thanks,
Alex
On Tue, 23 Feb 2021 10:22:25 +0800
Shenming Lu <lushenming@huawei.com> wrote:
> In VFIO migration resume phase and some guest startups, there are
> already unmasked vectors in the vector table when calling
> vfio_msix_enable(). So in order to avoid inefficiently disabling
> and enabling vectors repeatedly, let's allocate all needed vectors
> first and then enable these unmasked vectors one by one without
> disabling.
>
> Signed-off-by: Shenming Lu <lushenming@huawei.com>
> ---
> hw/pci/msix.c | 2 +-
> hw/vfio/pci.c | 20 +++++++++++++++++---
> include/hw/pci/msix.h | 1 +
> 3 files changed, 19 insertions(+), 4 deletions(-)
>
> diff --git a/hw/pci/msix.c b/hw/pci/msix.c
> index ae9331cd0b..e057958fcd 100644
> --- a/hw/pci/msix.c
> +++ b/hw/pci/msix.c
> @@ -131,7 +131,7 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked)
> }
> }
>
> -static bool msix_masked(PCIDevice *dev)
> +bool msix_masked(PCIDevice *dev)
> {
> return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK;
> }
> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> index f74be78209..088fd41926 100644
> --- a/hw/vfio/pci.c
> +++ b/hw/vfio/pci.c
> @@ -569,6 +569,9 @@ static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
>
> static void vfio_msix_enable(VFIOPCIDevice *vdev)
> {
> + PCIDevice *pdev = &vdev->pdev;
> + unsigned int nr, max_vec = 0;
> +
> vfio_disable_interrupts(vdev);
>
> vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
> @@ -587,11 +590,22 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev)
> * triggering to userspace, then immediately release the vector, leaving
> * the physical device with no vectors enabled, but MSI-X enabled, just
> * like the guest view.
> + * If there are already unmasked vectors (in migration resume phase and
> + * some guest startups) which will be enabled soon, we can allocate all
> + * of them here to avoid inefficiently disabling and enabling vectors
> + * repeatedly later.
> */
> - vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
> - vfio_msix_vector_release(&vdev->pdev, 0);
> + if (!msix_masked(pdev)) {
> + for (nr = 0; nr < msix_nr_vectors_allocated(pdev); nr++) {
> + if (!msix_is_masked(pdev, nr)) {
> + max_vec = nr;
> + }
> + }
> + }
> + vfio_msix_vector_do_use(pdev, max_vec, NULL, NULL);
> + vfio_msix_vector_release(pdev, max_vec);
>
> - if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
> + if (msix_set_vector_notifiers(pdev, vfio_msix_vector_use,
> vfio_msix_vector_release, NULL)) {
> error_report("vfio: msix_set_vector_notifiers failed");
> }
> diff --git a/include/hw/pci/msix.h b/include/hw/pci/msix.h
> index 4c4a60c739..b3cd88e262 100644
> --- a/include/hw/pci/msix.h
> +++ b/include/hw/pci/msix.h
> @@ -28,6 +28,7 @@ void msix_load(PCIDevice *dev, QEMUFile *f);
>
> int msix_enabled(PCIDevice *dev);
> int msix_present(PCIDevice *dev);
> +bool msix_masked(PCIDevice *dev);
>
> bool msix_is_masked(PCIDevice *dev, unsigned vector);
> void msix_set_pending(PCIDevice *dev, unsigned vector);
On 2021/3/3 5:26, Alex Williamson wrote:
>
> MST/Marcel,
>
> Do you have an Ack or objection to exporting msix_masked() as below?
Or could we use msix_function_masked instead as below? :-)
if (!pdev->msix_function_masked) {
for (nr = 0; nr < msix_nr_vectors_allocated(pdev); nr++) {
if (!msix_is_masked(pdev, nr)) {
max_vec = nr;
}
}
}
Thanks,
Shenming
> Thanks,
>
> Alex
>
> On Tue, 23 Feb 2021 10:22:25 +0800
> Shenming Lu <lushenming@huawei.com> wrote:
>
>> In VFIO migration resume phase and some guest startups, there are
>> already unmasked vectors in the vector table when calling
>> vfio_msix_enable(). So in order to avoid inefficiently disabling
>> and enabling vectors repeatedly, let's allocate all needed vectors
>> first and then enable these unmasked vectors one by one without
>> disabling.
>>
>> Signed-off-by: Shenming Lu <lushenming@huawei.com>
>> ---
>> hw/pci/msix.c | 2 +-
>> hw/vfio/pci.c | 20 +++++++++++++++++---
>> include/hw/pci/msix.h | 1 +
>> 3 files changed, 19 insertions(+), 4 deletions(-)
>>
>> diff --git a/hw/pci/msix.c b/hw/pci/msix.c
>> index ae9331cd0b..e057958fcd 100644
>> --- a/hw/pci/msix.c
>> +++ b/hw/pci/msix.c
>> @@ -131,7 +131,7 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked)
>> }
>> }
>>
>> -static bool msix_masked(PCIDevice *dev)
>> +bool msix_masked(PCIDevice *dev)
>> {
>> return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK;
>> }
>> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
>> index f74be78209..088fd41926 100644
>> --- a/hw/vfio/pci.c
>> +++ b/hw/vfio/pci.c
>> @@ -569,6 +569,9 @@ static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
>>
>> static void vfio_msix_enable(VFIOPCIDevice *vdev)
>> {
>> + PCIDevice *pdev = &vdev->pdev;
>> + unsigned int nr, max_vec = 0;
>> +
>> vfio_disable_interrupts(vdev);
>>
>> vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
>> @@ -587,11 +590,22 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev)
>> * triggering to userspace, then immediately release the vector, leaving
>> * the physical device with no vectors enabled, but MSI-X enabled, just
>> * like the guest view.
>> + * If there are already unmasked vectors (in migration resume phase and
>> + * some guest startups) which will be enabled soon, we can allocate all
>> + * of them here to avoid inefficiently disabling and enabling vectors
>> + * repeatedly later.
>> */
>> - vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
>> - vfio_msix_vector_release(&vdev->pdev, 0);
>> + if (!msix_masked(pdev)) {
>> + for (nr = 0; nr < msix_nr_vectors_allocated(pdev); nr++) {
>> + if (!msix_is_masked(pdev, nr)) {
>> + max_vec = nr;
>> + }
>> + }
>> + }
>> + vfio_msix_vector_do_use(pdev, max_vec, NULL, NULL);
>> + vfio_msix_vector_release(pdev, max_vec);
>>
>> - if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
>> + if (msix_set_vector_notifiers(pdev, vfio_msix_vector_use,
>> vfio_msix_vector_release, NULL)) {
>> error_report("vfio: msix_set_vector_notifiers failed");
>> }
>> diff --git a/include/hw/pci/msix.h b/include/hw/pci/msix.h
>> index 4c4a60c739..b3cd88e262 100644
>> --- a/include/hw/pci/msix.h
>> +++ b/include/hw/pci/msix.h
>> @@ -28,6 +28,7 @@ void msix_load(PCIDevice *dev, QEMUFile *f);
>>
>> int msix_enabled(PCIDevice *dev);
>> int msix_present(PCIDevice *dev);
>> +bool msix_masked(PCIDevice *dev);
>>
>> bool msix_is_masked(PCIDevice *dev, unsigned vector);
>> void msix_set_pending(PCIDevice *dev, unsigned vector);
>
> .
>
On Tue, 9 Mar 2021 15:25:52 +0800
Shenming Lu <lushenming@huawei.com> wrote:
> On 2021/3/3 5:26, Alex Williamson wrote:
> >
> > MST/Marcel,
> >
> > Do you have an Ack or objection to exporting msix_masked() as below?
>
> Or could we use msix_function_masked instead as below? :-)
>
> if (!pdev->msix_function_masked) {
> for (nr = 0; nr < msix_nr_vectors_allocated(pdev); nr++) {
> if (!msix_is_masked(pdev, nr)) {
> max_vec = nr;
> }
> }
> }
Yeah, it looks like that should work. I'll look for a v4 with that
change. Thanks,
Alex
> > On Tue, 23 Feb 2021 10:22:25 +0800
> > Shenming Lu <lushenming@huawei.com> wrote:
> >
> >> In VFIO migration resume phase and some guest startups, there are
> >> already unmasked vectors in the vector table when calling
> >> vfio_msix_enable(). So in order to avoid inefficiently disabling
> >> and enabling vectors repeatedly, let's allocate all needed vectors
> >> first and then enable these unmasked vectors one by one without
> >> disabling.
> >>
> >> Signed-off-by: Shenming Lu <lushenming@huawei.com>
> >> ---
> >> hw/pci/msix.c | 2 +-
> >> hw/vfio/pci.c | 20 +++++++++++++++++---
> >> include/hw/pci/msix.h | 1 +
> >> 3 files changed, 19 insertions(+), 4 deletions(-)
> >>
> >> diff --git a/hw/pci/msix.c b/hw/pci/msix.c
> >> index ae9331cd0b..e057958fcd 100644
> >> --- a/hw/pci/msix.c
> >> +++ b/hw/pci/msix.c
> >> @@ -131,7 +131,7 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked)
> >> }
> >> }
> >>
> >> -static bool msix_masked(PCIDevice *dev)
> >> +bool msix_masked(PCIDevice *dev)
> >> {
> >> return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK;
> >> }
> >> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> >> index f74be78209..088fd41926 100644
> >> --- a/hw/vfio/pci.c
> >> +++ b/hw/vfio/pci.c
> >> @@ -569,6 +569,9 @@ static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
> >>
> >> static void vfio_msix_enable(VFIOPCIDevice *vdev)
> >> {
> >> + PCIDevice *pdev = &vdev->pdev;
> >> + unsigned int nr, max_vec = 0;
> >> +
> >> vfio_disable_interrupts(vdev);
> >>
> >> vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
> >> @@ -587,11 +590,22 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev)
> >> * triggering to userspace, then immediately release the vector, leaving
> >> * the physical device with no vectors enabled, but MSI-X enabled, just
> >> * like the guest view.
> >> + * If there are already unmasked vectors (in migration resume phase and
> >> + * some guest startups) which will be enabled soon, we can allocate all
> >> + * of them here to avoid inefficiently disabling and enabling vectors
> >> + * repeatedly later.
> >> */
> >> - vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
> >> - vfio_msix_vector_release(&vdev->pdev, 0);
> >> + if (!msix_masked(pdev)) {
> >> + for (nr = 0; nr < msix_nr_vectors_allocated(pdev); nr++) {
> >> + if (!msix_is_masked(pdev, nr)) {
> >> + max_vec = nr;
> >> + }
> >> + }
> >> + }
> >> + vfio_msix_vector_do_use(pdev, max_vec, NULL, NULL);
> >> + vfio_msix_vector_release(pdev, max_vec);
> >>
> >> - if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
> >> + if (msix_set_vector_notifiers(pdev, vfio_msix_vector_use,
> >> vfio_msix_vector_release, NULL)) {
> >> error_report("vfio: msix_set_vector_notifiers failed");
> >> }
> >> diff --git a/include/hw/pci/msix.h b/include/hw/pci/msix.h
> >> index 4c4a60c739..b3cd88e262 100644
> >> --- a/include/hw/pci/msix.h
> >> +++ b/include/hw/pci/msix.h
> >> @@ -28,6 +28,7 @@ void msix_load(PCIDevice *dev, QEMUFile *f);
> >>
> >> int msix_enabled(PCIDevice *dev);
> >> int msix_present(PCIDevice *dev);
> >> +bool msix_masked(PCIDevice *dev);
> >>
> >> bool msix_is_masked(PCIDevice *dev, unsigned vector);
> >> void msix_set_pending(PCIDevice *dev, unsigned vector);
> >
> > .
> >
>
© 2016 - 2026 Red Hat, Inc.