Add support for configure interrupt, use kvm_irqfd_assign and set the
gsi to kernel. When the configure notifier was eventfd_signal by host
kernel, this will finally inject an msix interrupt to guest

Signed-off-by: Cindy Lu <lulu@redhat.com>
---
 hw/virtio/virtio-pci.c         | 93 ++++++++++++++++++++++++++++++++++
 include/hw/virtio/virtio-bus.h |  2 +
 2 files changed, 95 insertions(+)

diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
index 36524a5728..f8053e1fab 100644
--- a/hw/virtio/virtio-pci.c
+++ b/hw/virtio/virtio-pci.c
@@ -762,6 +762,98 @@ undo:
     return ret;
 }
 
+ static int kvm_virtio_pci_config_irqfd_use(VirtIOPCIProxy *proxy,
+                                 unsigned int vector)
+{
+    VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+    EventNotifier *n = virtio_queue_get_config_notifier(vdev);
+    assert(irqfd);
+    return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, irqfd->virq);
+}
+
+static void kvm_virtio_pci_config_irqfd_release(VirtIOPCIProxy *proxy,
+                                      unsigned int vector)
+{
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+    EventNotifier *n = virtio_queue_get_config_notifier(vdev);
+    VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
+    assert(irqfd);
+    kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, irqfd->virq);
+    return;
+}
+static int kvm_virtio_pci_config_vector_use(VirtIOPCIProxy *proxy,
+                                        unsigned int vector)
+{
+    VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
+    int ret;
+
+    if (irqfd->users == 0) {
+        ret = kvm_irqchip_add_msi_route(kvm_state, vector, &proxy->pci_dev);
+        if (ret < 0) {
+            return ret;
+        }
+        irqfd->virq = ret;
+    }
+    irqfd->users++;
+
+    return 0;
+}
+static int kvm_virtio_pci_vector_config_use(VirtIOPCIProxy *proxy)
+{
+
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+    unsigned int vector;
+    int ret;
+    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
+
+    vector = vdev->config_vector ;
+    ret = kvm_virtio_pci_config_vector_use(proxy, vector);
+    if (ret < 0) {
+        goto undo;
+    }
+    ret = kvm_virtio_pci_config_irqfd_use(proxy,  vector);
+    if (ret < 0) {
+        goto undo;
+    }
+    return 0;
+undo:
+    kvm_virtio_pci_config_irqfd_release(proxy, vector);
+    return ret;
+}
+static void kvm_virtio_pci_vector_config_release(VirtIOPCIProxy *proxy)
+{
+    PCIDevice *dev = &proxy->pci_dev;
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+    unsigned int vector;
+    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
+    vector = vdev->config_vector ;
+    if (vector >= msix_nr_vectors_allocated(dev)) {
+        return;
+    }
+    kvm_virtio_pci_config_irqfd_release(proxy, vector);
+    kvm_virtio_pci_vq_vector_release(proxy, vector);
+}
+
+static int virtio_pci_set_guest_config_notifier(DeviceState *d,  bool assign,
+                                         bool with_irqfd)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+    EventNotifier *notifier = virtio_queue_get_config_notifier(vdev);
+    int r = 0;
+    if (assign) {
+        r = event_notifier_init(notifier, 1);
+        virtio_set_config_notifier_fd_handler(vdev, true, with_irqfd);
+        kvm_virtio_pci_vector_config_use(proxy);
+    } else {
+        virtio_set_config_notifier_fd_handler(vdev, false, with_irqfd);
+        kvm_virtio_pci_vector_config_release(proxy);
+        event_notifier_cleanup(notifier);
+    }
+    return r;
+}
+
 static void kvm_virtio_pci_vector_release(VirtIOPCIProxy *proxy, int nvqs)
 {
     PCIDevice *dev = &proxy->pci_dev;
@@ -2137,6 +2229,7 @@ static void virtio_pci_bus_class_init(ObjectClass *klass, void *data)
     k->ioeventfd_assign = virtio_pci_ioeventfd_assign;
     k->get_dma_as = virtio_pci_get_dma_as;
     k->queue_enabled = virtio_pci_queue_enabled;
+    k->set_config_notifiers = virtio_pci_set_guest_config_notifier;
 }
 
 static const TypeInfo virtio_pci_bus_info = {
diff --git a/include/hw/virtio/virtio-bus.h b/include/hw/virtio/virtio-bus.h
index ef8abe49c5..dae81ee414 100644
--- a/include/hw/virtio/virtio-bus.h
+++ b/include/hw/virtio/virtio-bus.h
@@ -93,6 +93,8 @@ struct VirtioBusClass {
      */
     bool has_variable_vring_alignment;
     AddressSpace *(*get_dma_as)(DeviceState *d);
+    int (*set_config_notifiers)(DeviceState *d, bool assign, bool with_irqfd);
+
 };
 
 struct VirtioBusState {
-- 
2.21.3

On 2021/1/13 下午11:45, Cindy Lu wrote:
> Add support for configure interrupt, use kvm_irqfd_assign and set the
> gsi to kernel. When the configure notifier was eventfd_signal by host
> kernel, this will finally inject an msix interrupt to guest
>
> Signed-off-by: Cindy Lu <lulu@redhat.com>
> ---
>   hw/virtio/virtio-pci.c         | 93 ++++++++++++++++++++++++++++++++++
>   include/hw/virtio/virtio-bus.h |  2 +
>   2 files changed, 95 insertions(+)
>
> diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
> index 36524a5728..f8053e1fab 100644
> --- a/hw/virtio/virtio-pci.c
> +++ b/hw/virtio/virtio-pci.c
> @@ -762,6 +762,98 @@ undo:
>       return ret;
>   }
>   
> + static int kvm_virtio_pci_config_irqfd_use(VirtIOPCIProxy *proxy,
> +                                 unsigned int vector)
> +{
> +    VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
> +    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
> +    EventNotifier *n = virtio_queue_get_config_notifier(vdev);
> +    assert(irqfd);
> +    return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, irqfd->virq);
> +}
> +
> +static void kvm_virtio_pci_config_irqfd_release(VirtIOPCIProxy *proxy,
> +                                      unsigned int vector)
> +{
> +    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
> +    EventNotifier *n = virtio_queue_get_config_notifier(vdev);
> +    VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
> +    assert(irqfd);
> +    kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, irqfd->virq);
> +    return;
> +}
> +static int kvm_virtio_pci_config_vector_use(VirtIOPCIProxy *proxy,
> +                                        unsigned int vector)
> +{
> +    VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
> +    int ret;
> +
> +    if (irqfd->users == 0) {
> +        ret = kvm_irqchip_add_msi_route(kvm_state, vector, &proxy->pci_dev);
> +        if (ret < 0) {
> +            return ret;
> +        }
> +        irqfd->virq = ret;
> +    }
> +    irqfd->users++;
> +
> +    return 0;
> +}


Any reason that we can't resue kvm_virtio_pci_vq_vector_use()? Note that 
at MSIX level there's no difference if it belongs to vq or config.


> +static int kvm_virtio_pci_vector_config_use(VirtIOPCIProxy *proxy)
> +{
> +
> +    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
> +    unsigned int vector;
> +    int ret;
> +    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
> +
> +    vector = vdev->config_vector ;
> +    ret = kvm_virtio_pci_config_vector_use(proxy, vector);
> +    if (ret < 0) {
> +        goto undo;
> +    }
> +    ret = kvm_virtio_pci_config_irqfd_use(proxy,  vector);
> +    if (ret < 0) {
> +        goto undo;
> +    }
> +    return 0;
> +undo:
> +    kvm_virtio_pci_config_irqfd_release(proxy, vector);
> +    return ret;
> +}
> +static void kvm_virtio_pci_vector_config_release(VirtIOPCIProxy *proxy)
> +{
> +    PCIDevice *dev = &proxy->pci_dev;
> +    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
> +    unsigned int vector;
> +    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
> +    vector = vdev->config_vector ;
> +    if (vector >= msix_nr_vectors_allocated(dev)) {
> +        return;
> +    }
> +    kvm_virtio_pci_config_irqfd_release(proxy, vector);
> +    kvm_virtio_pci_vq_vector_release(proxy, vector);
> +}
> +
> +static int virtio_pci_set_guest_config_notifier(DeviceState *d,  bool assign,
> +                                         bool with_irqfd)
> +{
> +    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
> +    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
> +    EventNotifier *notifier = virtio_queue_get_config_notifier(vdev);
> +    int r = 0;
> +    if (assign) {
> +        r = event_notifier_init(notifier, 1);
> +        virtio_set_config_notifier_fd_handler(vdev, true, with_irqfd);
> +        kvm_virtio_pci_vector_config_use(proxy);
> +    } else {
> +        virtio_set_config_notifier_fd_handler(vdev, false, with_irqfd);
> +        kvm_virtio_pci_vector_config_release(proxy);
> +        event_notifier_cleanup(notifier);
> +    }
> +    return r;
> +}


Any way to re-use virtio_pci_set_guest_notifier() here?

Thanks


> +
>   static void kvm_virtio_pci_vector_release(VirtIOPCIProxy *proxy, int nvqs)
>   {
>       PCIDevice *dev = &proxy->pci_dev;
> @@ -2137,6 +2229,7 @@ static void virtio_pci_bus_class_init(ObjectClass *klass, void *data)
>       k->ioeventfd_assign = virtio_pci_ioeventfd_assign;
>       k->get_dma_as = virtio_pci_get_dma_as;
>       k->queue_enabled = virtio_pci_queue_enabled;
> +    k->set_config_notifiers = virtio_pci_set_guest_config_notifier;
>   }
>   
>   static const TypeInfo virtio_pci_bus_info = {
> diff --git a/include/hw/virtio/virtio-bus.h b/include/hw/virtio/virtio-bus.h
> index ef8abe49c5..dae81ee414 100644
> --- a/include/hw/virtio/virtio-bus.h
> +++ b/include/hw/virtio/virtio-bus.h
> @@ -93,6 +93,8 @@ struct VirtioBusClass {
>        */
>       bool has_variable_vring_alignment;
>       AddressSpace *(*get_dma_as)(DeviceState *d);
> +    int (*set_config_notifiers)(DeviceState *d, bool assign, bool with_irqfd);
> +
>   };
>   
>   struct VirtioBusState {

On Thu, Jan 14, 2021 at 12:43 PM Jason Wang <jasowang@redhat.com> wrote:
>
>
> On 2021/1/13 下午11:45, Cindy Lu wrote:
> > Add support for configure interrupt, use kvm_irqfd_assign and set the
> > gsi to kernel. When the configure notifier was eventfd_signal by host
> > kernel, this will finally inject an msix interrupt to guest
> >
> > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > ---
> >   hw/virtio/virtio-pci.c         | 93 ++++++++++++++++++++++++++++++++++
> >   include/hw/virtio/virtio-bus.h |  2 +
> >   2 files changed, 95 insertions(+)
> >
> > diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
> > index 36524a5728..f8053e1fab 100644
> > --- a/hw/virtio/virtio-pci.c
> > +++ b/hw/virtio/virtio-pci.c
> > @@ -762,6 +762,98 @@ undo:
> >       return ret;
> >   }
> >
> > + static int kvm_virtio_pci_config_irqfd_use(VirtIOPCIProxy *proxy,
> > +                                 unsigned int vector)
> > +{
> > +    VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
> > +    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
> > +    EventNotifier *n = virtio_queue_get_config_notifier(vdev);
> > +    assert(irqfd);
> > +    return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, irqfd->virq);
> > +}
> > +
> > +static void kvm_virtio_pci_config_irqfd_release(VirtIOPCIProxy *proxy,
> > +                                      unsigned int vector)
> > +{
> > +    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
> > +    EventNotifier *n = virtio_queue_get_config_notifier(vdev);
> > +    VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
> > +    assert(irqfd);
> > +    kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, irqfd->virq);
> > +    return;
> > +}
> > +static int kvm_virtio_pci_config_vector_use(VirtIOPCIProxy *proxy,
> > +                                        unsigned int vector)
> > +{
> > +    VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
> > +    int ret;
> > +
> > +    if (irqfd->users == 0) {
> > +        ret = kvm_irqchip_add_msi_route(kvm_state, vector, &proxy->pci_dev);
> > +        if (ret < 0) {
> > +            return ret;
> > +        }
> > +        irqfd->virq = ret;
> > +    }
> > +    irqfd->users++;
> > +
> > +    return 0;
> > +}
>
>
> Any reason that we can't resue kvm_virtio_pci_vq_vector_use()? Note that
> at MSIX level there's no difference if it belongs to vq or config.
>
>
> > +static int kvm_virtio_pci_vector_config_use(VirtIOPCIProxy *proxy)
> > +{
> > +
> > +    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
> > +    unsigned int vector;
> > +    int ret;
> > +    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
> > +
> > +    vector = vdev->config_vector ;
> > +    ret = kvm_virtio_pci_config_vector_use(proxy, vector);
> > +    if (ret < 0) {
> > +        goto undo;
> > +    }
> > +    ret = kvm_virtio_pci_config_irqfd_use(proxy,  vector);
> > +    if (ret < 0) {
> > +        goto undo;
> > +    }
> > +    return 0;
> > +undo:
> > +    kvm_virtio_pci_config_irqfd_release(proxy, vector);
> > +    return ret;
> > +}
> > +static void kvm_virtio_pci_vector_config_release(VirtIOPCIProxy *proxy)
> > +{
> > +    PCIDevice *dev = &proxy->pci_dev;
> > +    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
> > +    unsigned int vector;
> > +    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
> > +    vector = vdev->config_vector ;
> > +    if (vector >= msix_nr_vectors_allocated(dev)) {
> > +        return;
> > +    }
> > +    kvm_virtio_pci_config_irqfd_release(proxy, vector);
> > +    kvm_virtio_pci_vq_vector_release(proxy, vector);
> > +}
> > +
> > +static int virtio_pci_set_guest_config_notifier(DeviceState *d,  bool assign,
> > +                                         bool with_irqfd)
> > +{
> > +    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
> > +    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
> > +    EventNotifier *notifier = virtio_queue_get_config_notifier(vdev);
> > +    int r = 0;
> > +    if (assign) {
> > +        r = event_notifier_init(notifier, 1);
> > +        virtio_set_config_notifier_fd_handler(vdev, true, with_irqfd);
> > +        kvm_virtio_pci_vector_config_use(proxy);
> > +    } else {
> > +        virtio_set_config_notifier_fd_handler(vdev, false, with_irqfd);
> > +        kvm_virtio_pci_vector_config_release(proxy);
> > +        event_notifier_cleanup(notifier);
> > +    }
> > +    return r;
> > +}
>
>
> Any way to re-use virtio_pci_set_guest_notifier() here?
>
> Thanks
>
These  two functions you mentioned all have something needed such as
queue no/guest_notifier. so I created a new one.
but it's a good idea to re-organize it,  I will post a new version
>
> > +
> >   static void kvm_virtio_pci_vector_release(VirtIOPCIProxy *proxy, int nvqs)
> >   {
> >       PCIDevice *dev = &proxy->pci_dev;
> > @@ -2137,6 +2229,7 @@ static void virtio_pci_bus_class_init(ObjectClass *klass, void *data)
> >       k->ioeventfd_assign = virtio_pci_ioeventfd_assign;
> >       k->get_dma_as = virtio_pci_get_dma_as;
> >       k->queue_enabled = virtio_pci_queue_enabled;
> > +    k->set_config_notifiers = virtio_pci_set_guest_config_notifier;
> >   }
> >
> >   static const TypeInfo virtio_pci_bus_info = {
> > diff --git a/include/hw/virtio/virtio-bus.h b/include/hw/virtio/virtio-bus.h
> > index ef8abe49c5..dae81ee414 100644
> > --- a/include/hw/virtio/virtio-bus.h
> > +++ b/include/hw/virtio/virtio-bus.h
> > @@ -93,6 +93,8 @@ struct VirtioBusClass {
> >        */
> >       bool has_variable_vring_alignment;
> >       AddressSpace *(*get_dma_as)(DeviceState *d);
> > +    int (*set_config_notifiers)(DeviceState *d, bool assign, bool with_irqfd);
> > +
> >   };
> >
> >   struct VirtioBusState {
>