From nobody Sun May 19 09:08:36 2024 Delivered-To: importer@patchew.org Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=pass(p=none dis=none) header.from=nongnu.org ARC-Seal: i=1; a=rsa-sha256; t=1677577242; cv=none; d=zohomail.com; s=zohoarc; b=d+f0/demwnJ9YaeSNrM4tJS8YY3T3hkKY11wPxrKGGXb9ZJTZG2JvMEifPMYLyErrRE+xGlqiL6+tMlD+t9ha14wqOZmNxtpKbaZMDvA2imr4kaEEop2mZxWenxD3nTPMVWoc5kpws2zkmvHqRTm9L1WKpIt7YSYLs4upSRHQmc= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1677577242; h=Content-Type:Content-Transfer-Encoding:Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Archive:List-Help:List-Unsubscribe:MIME-Version:Message-ID:Reply-To:References:Sender:Subject:To; bh=bo6Vg60eQ6GecgKmQlI82TfylZAqPfN+6thoYnIiClg=; b=FvTIpKmj8Ni9JzIWY1lSotm7jhArOGHfx+9uUfikO2+qo5dZh3M5CzhHUjM/SVyDqaJFU66gtd7MgjQRKsJBpo6vEh3bUtqYCuavG3xMch2hQvMqAqPtGjMSwEoV+JfSC+/6/gT/zelypLuYyY9eg/CCN+S6cOm3BwOZjFU8oxw= ARC-Authentication-Results: i=1; mx.zohomail.com; spf=pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=pass header.from= (p=none dis=none) Return-Path: Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) by mx.zohomail.com with SMTPS id 167757724236234.183252306710756; Tue, 28 Feb 2023 01:40:42 -0800 (PST) Received: from localhost ([::1] helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1pWwSo-0002N3-Dx; Tue, 28 Feb 2023 04:40:02 -0500 Received: from eggs.gnu.org ([2001:470:142:3::10]) by lists.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1pWwSm-0002Kn-HO for qemu-devel@nongnu.org; Tue, 28 Feb 2023 04:40:00 -0500 Received: from szxga02-in.huawei.com ([45.249.212.188]) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1pWwSi-0003Be-CK for qemu-devel@nongnu.org; Tue, 28 Feb 2023 04:40:00 -0500 Received: from kwepemi100025.china.huawei.com (unknown [172.30.72.54]) by szxga02-in.huawei.com (SkyGuard) with ESMTP id 4PQsjM4WG4z9tD0; Tue, 28 Feb 2023 17:37:43 +0800 (CST) Received: from DESKTOP-27KDQMV.china.huawei.com (10.174.148.223) by kwepemi100025.china.huawei.com (7.221.188.158) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2507.21; Tue, 28 Feb 2023 17:39:41 +0800 To: , , CC: , , , , , , , Longpeng Subject: [PATCH v1 1/3] virtio-pci: submit msi route changes in batch Date: Tue, 28 Feb 2023 17:39:35 +0800 Message-ID: <20230228093937.2515-2-longpeng2@huawei.com> X-Mailer: git-send-email 2.25.0.windows.1 In-Reply-To: <20230228093937.2515-1-longpeng2@huawei.com> References: <20230228093937.2515-1-longpeng2@huawei.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Originating-IP: [10.174.148.223] X-ClientProxiedBy: dggems705-chm.china.huawei.com (10.3.19.182) To kwepemi100025.china.huawei.com (7.221.188.158) X-CFilter-Loop: Reflected Received-SPF: pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) client-ip=209.51.188.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Received-SPF: pass client-ip=45.249.212.188; envelope-from=longpeng2@huawei.com; helo=szxga02-in.huawei.com X-Spam_score_int: -41 X-Spam_score: -4.2 X-Spam_bar: ---- X-Spam_report: (-4.2 / 5.0 requ) BAYES_00=-1.9, RCVD_IN_DNSWL_MED=-2.3, SPF_HELO_NONE=0.001, SPF_PASS=-0.001 autolearn=ham autolearn_force=no X-Spam_action: no action X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-to: "Longpeng(Mike)" From: "Longpeng(Mike)" via Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: qemu-devel-bounces+importer=patchew.org@nongnu.org X-ZM-MESSAGEID: 1677577243145100001 Content-Type: text/plain; charset="utf-8" From: Longpeng The kvm_irqchip_commit_routes() is a time-intensive operation, it needs scan and update all irqfds that are already assigned during each invocation, so more vectors means need more time to process them. For virtio-pci, we can just submit once when enabling vectors of a virtio-pci device. This can reduce the downtime when migrating a VM with vhost-vdpa devices. Signed-off-by: Longpeng --- hw/virtio/virtio-pci.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c index 247325c193..22e76e3902 100644 --- a/hw/virtio/virtio-pci.c +++ b/hw/virtio/virtio-pci.c @@ -49,6 +49,19 @@ * configuration space */ #define VIRTIO_PCI_CONFIG_SIZE(dev) VIRTIO_PCI_CONFIG_OFF(msix_enabled= (dev)) =20 +/* Protected by the BQL */ +static KVMRouteChange virtio_pci_route_change; + +static inline void virtio_pci_begin_route_changes(void) +{ + virtio_pci_route_change =3D kvm_irqchip_begin_route_changes(kvm_state); +} + +static inline void virtio_pci_commit_route_changes(void) +{ + kvm_irqchip_commit_route_changes(&virtio_pci_route_change); +} + static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size, VirtIOPCIProxy *dev); static void virtio_pci_reset(DeviceState *qdev); @@ -790,12 +803,11 @@ static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProx= y *proxy, int ret; =20 if (irqfd->users =3D=3D 0) { - KVMRouteChange c =3D kvm_irqchip_begin_route_changes(kvm_state); - ret =3D kvm_irqchip_add_msi_route(&c, vector, &proxy->pci_dev); + ret =3D kvm_irqchip_add_msi_route(&virtio_pci_route_change, vector, + &proxy->pci_dev); if (ret < 0) { return ret; } - kvm_irqchip_commit_route_changes(&c); irqfd->virq =3D ret; } irqfd->users++; @@ -903,12 +915,18 @@ static int kvm_virtio_pci_vector_vq_use(VirtIOPCIProx= y *proxy, int nvqs) int ret =3D 0; VirtIODevice *vdev =3D virtio_bus_get_device(&proxy->bus); =20 + virtio_pci_begin_route_changes(); + for (queue_no =3D 0; queue_no < nvqs; queue_no++) { if (!virtio_queue_get_num(vdev, queue_no)) { + virtio_pci_commit_route_changes(); return -1; } ret =3D kvm_virtio_pci_vector_use_one(proxy, queue_no); } + + virtio_pci_commit_route_changes(); + return ret; } =20 --=20 2.23.0 From nobody Sun May 19 09:08:36 2024 Delivered-To: importer@patchew.org Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=pass(p=none dis=none) header.from=nongnu.org ARC-Seal: i=1; a=rsa-sha256; t=1677577252; cv=none; d=zohomail.com; s=zohoarc; b=Kd7lsAA65tFoYpm70TCET5jIQMSrveF2VsO1Rav+F31gCQaiCrmwKqYCxLIwj8XxHZC8ykCSzGk8BGg+n7Cthys17Lys0KcQyGZz0QPZ907OpBo0x1zJEIS8GGo2VQJSwWYyD3w6nCshlglsLPLaMA02uMWp/qx5IrLQ25aiAcQ= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1677577252; h=Content-Type:Content-Transfer-Encoding:Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Archive:List-Help:List-Unsubscribe:MIME-Version:Message-ID:Reply-To:References:Sender:Subject:To; bh=RTz8iPoO+hzXvy7QAVTGmUJ+Nrzu9RKtdCz1Yjt/GD8=; b=HcMUrZYRsTpj1TJma19uNTop3mNlQvx0W8WKcvtOti4kTkR/KXJBaraA11jAihL/EUkrFI4buCSPhSLu46A15sAcJGR6b5Xm2S01WfySr3agbyh0vj5G+M2Wy/G7j2gattGh+bHMKuyP7Tw4tDTZizILorlLX467EoJzo8EOk1g= ARC-Authentication-Results: i=1; mx.zohomail.com; spf=pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=pass header.from= (p=none dis=none) Return-Path: Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) by mx.zohomail.com with SMTPS id 1677577252137138.33211694955185; Tue, 28 Feb 2023 01:40:52 -0800 (PST) Received: from localhost ([::1] helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1pWwSp-0002Qe-DK; Tue, 28 Feb 2023 04:40:03 -0500 Received: from eggs.gnu.org ([2001:470:142:3::10]) by lists.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1pWwSn-0002LF-3U for qemu-devel@nongnu.org; Tue, 28 Feb 2023 04:40:01 -0500 Received: from szxga08-in.huawei.com ([45.249.212.255]) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1pWwSi-0003Bf-Bb for qemu-devel@nongnu.org; Tue, 28 Feb 2023 04:40:00 -0500 Received: from kwepemi100025.china.huawei.com (unknown [172.30.72.54]) by szxga08-in.huawei.com (SkyGuard) with ESMTP id 4PQshd3tBjz16NxQ; Tue, 28 Feb 2023 17:37:05 +0800 (CST) Received: from DESKTOP-27KDQMV.china.huawei.com (10.174.148.223) by kwepemi100025.china.huawei.com (7.221.188.158) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2507.21; Tue, 28 Feb 2023 17:39:42 +0800 To: , , CC: , , , , , , , Longpeng Subject: [PATCH v1 2/3] kvm-irqchip: use KVMRouteChange API to update msi route Date: Tue, 28 Feb 2023 17:39:36 +0800 Message-ID: <20230228093937.2515-3-longpeng2@huawei.com> X-Mailer: git-send-email 2.25.0.windows.1 In-Reply-To: <20230228093937.2515-1-longpeng2@huawei.com> References: <20230228093937.2515-1-longpeng2@huawei.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Originating-IP: [10.174.148.223] X-ClientProxiedBy: dggems705-chm.china.huawei.com (10.3.19.182) To kwepemi100025.china.huawei.com (7.221.188.158) X-CFilter-Loop: Reflected Received-SPF: pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) client-ip=209.51.188.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Received-SPF: pass client-ip=45.249.212.255; envelope-from=longpeng2@huawei.com; helo=szxga08-in.huawei.com X-Spam_score_int: -41 X-Spam_score: -4.2 X-Spam_bar: ---- X-Spam_report: (-4.2 / 5.0 requ) BAYES_00=-1.9, RCVD_IN_DNSWL_MED=-2.3, RCVD_IN_MSPIKE_H2=-0.001, SPF_HELO_NONE=0.001, SPF_PASS=-0.001 autolearn=ham autolearn_force=no X-Spam_action: no action X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-to: "Longpeng(Mike)" From: "Longpeng(Mike)" via Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: qemu-devel-bounces+importer=patchew.org@nongnu.org X-ZM-MESSAGEID: 1677577252995100001 Content-Type: text/plain; charset="utf-8" From: Longpeng The KVMRouteChange API is added by commit 9568690868e ("kvm-irqchip: introduce new API to support route change"). We can also apply it on kvm_irqchip_update_msi_route(), there are no functional changes and we can optimize the virtio-pci core base on this change in the next patch. Signed-off-by: Longpeng --- accel/kvm/kvm-all.c | 10 ++++++---- accel/stubs/kvm-stub.c | 2 +- hw/intc/ioapic.c | 5 +++-- hw/misc/ivshmem.c | 6 ++++-- hw/vfio/pci.c | 5 +++-- hw/virtio/virtio-pci.c | 7 +++++-- include/sysemu/kvm.h | 2 +- target/i386/kvm/kvm.c | 6 ++++-- 8 files changed, 27 insertions(+), 16 deletions(-) diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 9b26582655..1ed0dc4c9d 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -1820,10 +1820,11 @@ static void kvm_add_routing_entry(KVMState *s, set_gsi(s, entry->gsi); } =20 -static int kvm_update_routing_entry(KVMState *s, +static int kvm_update_routing_entry(KVMRouteChange *c, struct kvm_irq_routing_entry *new_entr= y) { struct kvm_irq_routing_entry *entry; + KVMState *s =3D c->s; int n; =20 for (n =3D 0; n < s->irq_routes->nr; n++) { @@ -1837,6 +1838,7 @@ static int kvm_update_routing_entry(KVMState *s, } =20 *entry =3D *new_entry; + c->changes++; =20 return 0; } @@ -2046,7 +2048,7 @@ int kvm_irqchip_add_msi_route(KVMRouteChange *c, int = vector, PCIDevice *dev) return virq; } =20 -int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg, +int kvm_irqchip_update_msi_route(KVMRouteChange *c, int virq, MSIMessage m= sg, PCIDevice *dev) { struct kvm_irq_routing_entry kroute =3D {}; @@ -2075,7 +2077,7 @@ int kvm_irqchip_update_msi_route(KVMState *s, int vir= q, MSIMessage msg, =20 trace_kvm_irqchip_update_msi_route(virq); =20 - return kvm_update_routing_entry(s, &kroute); + return kvm_update_routing_entry(c, &kroute); } =20 static int kvm_irqchip_assign_irqfd(KVMState *s, EventNotifier *event, @@ -2221,7 +2223,7 @@ static int kvm_irqchip_assign_irqfd(KVMState *s, Even= tNotifier *event, abort(); } =20 -int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg) +int kvm_irqchip_update_msi_route(KVMRouteChange *c, int virq, MSIMessage m= sg) { return -ENOSYS; } diff --git a/accel/stubs/kvm-stub.c b/accel/stubs/kvm-stub.c index 5d2dd8f351..5bcf98b9ab 100644 --- a/accel/stubs/kvm-stub.c +++ b/accel/stubs/kvm-stub.c @@ -69,7 +69,7 @@ void kvm_irqchip_release_virq(KVMState *s, int virq) { } =20 -int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg, +int kvm_irqchip_update_msi_route(KVMRouteChange *c, int virq, MSIMessage m= sg, PCIDevice *dev) { return -ENOSYS; diff --git a/hw/intc/ioapic.c b/hw/intc/ioapic.c index 264262959d..07b9cf7705 100644 --- a/hw/intc/ioapic.c +++ b/hw/intc/ioapic.c @@ -195,6 +195,7 @@ static void ioapic_update_kvm_routes(IOAPICCommonState = *s) int i; =20 if (kvm_irqchip_is_split()) { + KVMRouteChange c =3D kvm_irqchip_begin_route_changes(kvm_state); for (i =3D 0; i < IOAPIC_NUM_PINS; i++) { MSIMessage msg; struct ioapic_entry_info info; @@ -202,10 +203,10 @@ static void ioapic_update_kvm_routes(IOAPICCommonStat= e *s) if (!info.masked) { msg.address =3D info.addr; msg.data =3D info.data; - kvm_irqchip_update_msi_route(kvm_state, i, msg, NULL); + kvm_irqchip_update_msi_route(&c, i, msg, NULL); } } - kvm_irqchip_commit_routes(kvm_state); + kvm_irqchip_commit_route_changes(&c); } #endif } diff --git a/hw/misc/ivshmem.c b/hw/misc/ivshmem.c index d66d912172..0e9427be42 100644 --- a/hw/misc/ivshmem.c +++ b/hw/misc/ivshmem.c @@ -278,6 +278,7 @@ static int ivshmem_vector_unmask(PCIDevice *dev, unsign= ed vector, IVShmemState *s =3D IVSHMEM_COMMON(dev); EventNotifier *n =3D &s->peers[s->vm_id].eventfds[vector]; MSIVector *v =3D &s->msi_vectors[vector]; + KVMRouteChange c; int ret; =20 IVSHMEM_DPRINTF("vector unmask %p %d\n", dev, vector); @@ -287,11 +288,12 @@ static int ivshmem_vector_unmask(PCIDevice *dev, unsi= gned vector, } assert(!v->unmasked); =20 - ret =3D kvm_irqchip_update_msi_route(kvm_state, v->virq, msg, dev); + c =3D kvm_irqchip_begin_route_changes(kvm_state); + ret =3D kvm_irqchip_update_msi_route(&c, v->virq, msg, dev); if (ret < 0) { return ret; } - kvm_irqchip_commit_routes(kvm_state); + kvm_irqchip_commit_route_changes(&c); =20 ret =3D kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, v->virq= ); if (ret < 0) { diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index 939dcc3d4a..fb69cc9965 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -460,8 +460,9 @@ static void vfio_remove_kvm_msi_virq(VFIOMSIVector *vec= tor) static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg, PCIDevice *pdev) { - kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg, pdev); - kvm_irqchip_commit_routes(kvm_state); + KVMRouteChange c =3D kvm_irqchip_begin_route_changes(kvm_state); + kvm_irqchip_update_msi_route(&c, vector->virq, msg, pdev); + kvm_irqchip_commit_route_changes(&c); } =20 static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr, diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c index 22e76e3902..5fd02b7cb8 100644 --- a/hw/virtio/virtio-pci.c +++ b/hw/virtio/virtio-pci.c @@ -990,12 +990,15 @@ static int virtio_pci_one_vector_unmask(VirtIOPCIProx= y *proxy, if (proxy->vector_irqfd) { irqfd =3D &proxy->vector_irqfd[vector]; if (irqfd->msg.data !=3D msg.data || irqfd->msg.address !=3D msg.a= ddress) { - ret =3D kvm_irqchip_update_msi_route(kvm_state, irqfd->virq, m= sg, + virtio_pci_begin_route_changes(); + ret =3D kvm_irqchip_update_msi_route(&virtio_pci_route_change, + irqfd->virq, msg, &proxy->pci_dev); if (ret < 0) { + virtio_pci_commit_route_changes(); return ret; } - kvm_irqchip_commit_routes(kvm_state); + virtio_pci_commit_route_changes(); } } =20 diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h index e9a97eda8c..4c52a39efa 100644 --- a/include/sysemu/kvm.h +++ b/include/sysemu/kvm.h @@ -494,7 +494,7 @@ void kvm_init_cpu_signals(CPUState *cpu); * @return: virq (>=3D0) when success, errno (<0) when failed. */ int kvm_irqchip_add_msi_route(KVMRouteChange *c, int vector, PCIDevice *de= v); -int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg, +int kvm_irqchip_update_msi_route(KVMRouteChange *c, int virq, MSIMessage m= sg, PCIDevice *dev); void kvm_irqchip_commit_routes(KVMState *s); =20 diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index 5870301991..ceb61badb4 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -5532,9 +5532,11 @@ static void kvm_update_msi_routes_all(void *private,= bool global, { int cnt =3D 0, vector; MSIRouteEntry *entry; + KVMRouteChange c; MSIMessage msg; PCIDevice *dev; =20 + c =3D kvm_irqchip_begin_route_changes(kvm_state); /* TODO: explicit route update */ QLIST_FOREACH(entry, &msi_route_list, list) { cnt++; @@ -5551,9 +5553,9 @@ static void kvm_update_msi_routes_all(void *private, = bool global, */ continue; } - kvm_irqchip_update_msi_route(kvm_state, entry->virq, msg, dev); + kvm_irqchip_update_msi_route(&c, entry->virq, msg, dev); } - kvm_irqchip_commit_routes(kvm_state); + kvm_irqchip_commit_route_changes(&c); trace_kvm_x86_update_msi_routes(cnt); } =20 --=20 2.23.0 From nobody Sun May 19 09:08:36 2024 Delivered-To: importer@patchew.org Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=pass(p=none dis=none) header.from=nongnu.org ARC-Seal: i=1; a=rsa-sha256; t=1677577242; cv=none; d=zohomail.com; s=zohoarc; b=E5CfRSHQgauhcfTnKXppOtioXRprxvCfk3XeQm5mOLsBorTWYh1sJhddxZRNLXemgJ3aXBaHBUnmzH+3UwO1jSZF3DvEl4RVmaeaIyIpboZIGmg7zkbcVIRJ7y5dIY4dtr61B+UNVuLAQgPfFwF+L6PpZd6FQ0Q0SK2Y2CNHQEs= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1677577242; h=Content-Type:Content-Transfer-Encoding:Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Archive:List-Help:List-Unsubscribe:MIME-Version:Message-ID:Reply-To:References:Sender:Subject:To; bh=EX4budolvhauRLH6sTkFKFcjPZePxunPFwIgI3KsUwg=; b=PdVOkOVpX1A1Npv6QUV4OwQhZ+6ZNbYFsyz4PGLSrPcMnRycVKQUEEZtRFd7wSEylErCJIutwCl8KU/x3Nxy3IjcTQ9bc8+udiiZ+o9LU0Er/KY+ygzLsX7Cl8THpFc4N2HN/Bha6WDAhGZY1/0Zf9fLQA5KGtAJetTphP/xhPI= ARC-Authentication-Results: i=1; mx.zohomail.com; spf=pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=pass header.from= (p=none dis=none) Return-Path: Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) by mx.zohomail.com with SMTPS id 1677577242855916.4208756466401; Tue, 28 Feb 2023 01:40:42 -0800 (PST) Received: from localhost ([::1] helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1pWwSo-0002Lu-4K; Tue, 28 Feb 2023 04:40:02 -0500 Received: from eggs.gnu.org ([2001:470:142:3::10]) by lists.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1pWwSm-0002Kr-La for qemu-devel@nongnu.org; Tue, 28 Feb 2023 04:40:00 -0500 Received: from szxga03-in.huawei.com ([45.249.212.189]) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1pWwSi-0003Bq-CF for qemu-devel@nongnu.org; Tue, 28 Feb 2023 04:40:00 -0500 Received: from kwepemi100025.china.huawei.com (unknown [172.30.72.54]) by szxga03-in.huawei.com (SkyGuard) with ESMTP id 4PQsdy27S8zKmKW; Tue, 28 Feb 2023 17:34:46 +0800 (CST) Received: from DESKTOP-27KDQMV.china.huawei.com (10.174.148.223) by kwepemi100025.china.huawei.com (7.221.188.158) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2507.21; Tue, 28 Feb 2023 17:39:42 +0800 To: , , CC: , , , , , , , Longpeng Subject: [PATCH v1 3/3] virtio-pci: defer to commit kvm irq routing when enable msi/msix Date: Tue, 28 Feb 2023 17:39:37 +0800 Message-ID: <20230228093937.2515-4-longpeng2@huawei.com> X-Mailer: git-send-email 2.25.0.windows.1 In-Reply-To: <20230228093937.2515-1-longpeng2@huawei.com> References: <20230228093937.2515-1-longpeng2@huawei.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Originating-IP: [10.174.148.223] X-ClientProxiedBy: dggems705-chm.china.huawei.com (10.3.19.182) To kwepemi100025.china.huawei.com (7.221.188.158) X-CFilter-Loop: Reflected Received-SPF: pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) client-ip=209.51.188.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Received-SPF: pass client-ip=45.249.212.189; envelope-from=longpeng2@huawei.com; helo=szxga03-in.huawei.com X-Spam_score_int: -41 X-Spam_score: -4.2 X-Spam_bar: ---- X-Spam_report: (-4.2 / 5.0 requ) BAYES_00=-1.9, RCVD_IN_DNSWL_MED=-2.3, SPF_HELO_NONE=0.001, SPF_PASS=-0.001 autolearn=ham autolearn_force=no X-Spam_action: no action X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-to: "Longpeng(Mike)" From: "Longpeng(Mike)" via Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: qemu-devel-bounces+importer=patchew.org@nongnu.org X-ZM-MESSAGEID: 1677577244960100007 Content-Type: text/plain; charset="utf-8" From: Longpeng All unmasked vectors will be setup in msix_set_vector_notifiers(), which is a time-consuming operation because each vector need to be submit to KVM once. It's even worse if the VM has several devices and each devices has dozens of vectors. We can defer and commit the vectors in batch, just like the commit dc580d51= f7 ("vfio: defer to commit kvm irq routing when enable msi/msix"), The can reduce 80% of the time spending on virtio_pci_set_guest_notifiers(). Signed-off-by: Longpeng --- hw/virtio/virtio-pci.c | 113 ++++++++++++++++++++++++++++++++----- include/hw/virtio/virtio.h | 1 + 2 files changed, 99 insertions(+), 15 deletions(-) diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c index 5fd02b7cb8..13f9c31009 100644 --- a/hw/virtio/virtio-pci.c +++ b/hw/virtio/virtio-pci.c @@ -51,15 +51,22 @@ =20 /* Protected by the BQL */ static KVMRouteChange virtio_pci_route_change; +static unsigned virtio_pci_route_change_depth; =20 static inline void virtio_pci_begin_route_changes(void) { - virtio_pci_route_change =3D kvm_irqchip_begin_route_changes(kvm_state); + if (!virtio_pci_route_change_depth) { + virtio_pci_route_change =3D kvm_irqchip_begin_route_changes(kvm_st= ate); + } + virtio_pci_route_change_depth++; } =20 static inline void virtio_pci_commit_route_changes(void) { - kvm_irqchip_commit_route_changes(&virtio_pci_route_change); + virtio_pci_route_change_depth--; + if (!virtio_pci_route_change_depth) { + kvm_irqchip_commit_route_changes(&virtio_pci_route_change); + } } =20 static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size, @@ -976,6 +983,88 @@ static void kvm_virtio_pci_vector_config_release(VirtI= OPCIProxy *proxy) kvm_virtio_pci_vector_release_one(proxy, VIRTIO_CONFIG_IRQ_IDX); } =20 +static int virtio_pci_vector_do_unmask(VirtIOPCIProxy *proxy, + unsigned int queue_no, + unsigned int vector, + EventNotifier *n) +{ + VirtIODevice *vdev =3D virtio_bus_get_device(&proxy->bus); + VirtioDeviceClass *k =3D VIRTIO_DEVICE_GET_CLASS(vdev); + int ret =3D 0; + + /* + * If guest supports masking, irqfd is already setup, unmask it. + * Otherwise, set it up now. + */ + if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) { + k->guest_notifier_mask(vdev, queue_no, false); + /* Test after unmasking to avoid losing events. */ + if (k->guest_notifier_pending && + k->guest_notifier_pending(vdev, queue_no)) { + event_notifier_set(n); + } + } else { + ret =3D kvm_virtio_pci_irqfd_use(proxy, n, vector); + } + + return ret; +} + +static void virtio_pci_prepare_kvm_msi_virq_batch(VirtIOPCIProxy *proxy) +{ + VirtIODevice *vdev =3D virtio_bus_get_device(&proxy->bus); + + assert(!vdev->defer_kvm_irq_routing); + vdev->defer_kvm_irq_routing =3D true; + virtio_pci_begin_route_changes(); +} + +static void virtio_pci_commit_kvm_msi_virq_batch(VirtIOPCIProxy *proxy) +{ + VirtIODevice *vdev =3D virtio_bus_get_device(&proxy->bus); + PCIDevice *dev =3D &proxy->pci_dev; + VirtQueue *vq; + EventNotifier *n; + int vector, index; + int ret; + + assert(vdev->defer_kvm_irq_routing); + virtio_pci_commit_route_changes(); + vdev->defer_kvm_irq_routing =3D false; + + if (!msix_enabled(dev)) { + return; + } + + /* Unmask all unmasked vectors */ + for (vector =3D 0; vector < dev->msix_entries_nr; vector++) { + if (msix_is_masked(dev, vector)) { + continue; + } + + vq =3D virtio_vector_first_queue(vdev, vector); + while (vq) { + index =3D virtio_get_queue_index(vq); + if (!virtio_queue_get_num(vdev, index)) { + break; + } + if (index < proxy->nvqs_with_notifiers) { + n =3D virtio_queue_get_guest_notifier(vq); + ret =3D virtio_pci_vector_do_unmask(proxy, index, vector, = n); + assert(ret >=3D 0); + } + vq =3D virtio_vector_next_queue(vq); + } + + if (vector =3D=3D vdev->config_vector) { + n =3D virtio_config_get_guest_notifier(vdev); + ret =3D virtio_pci_vector_do_unmask(proxy, VIRTIO_CONFIG_IRQ_I= DX, + vector, n); + assert(ret >=3D 0); + } + } +} + static int virtio_pci_one_vector_unmask(VirtIOPCIProxy *proxy, unsigned int queue_no, unsigned int vector, @@ -983,7 +1072,6 @@ static int virtio_pci_one_vector_unmask(VirtIOPCIProxy= *proxy, EventNotifier *n) { VirtIODevice *vdev =3D virtio_bus_get_device(&proxy->bus); - VirtioDeviceClass *k =3D VIRTIO_DEVICE_GET_CLASS(vdev); VirtIOIRQFD *irqfd; int ret =3D 0; =20 @@ -1002,19 +1090,10 @@ static int virtio_pci_one_vector_unmask(VirtIOPCIPr= oxy *proxy, } } =20 - /* If guest supports masking, irqfd is already setup, unmask it. - * Otherwise, set it up now. - */ - if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) { - k->guest_notifier_mask(vdev, queue_no, false); - /* Test after unmasking to avoid losing events. */ - if (k->guest_notifier_pending && - k->guest_notifier_pending(vdev, queue_no)) { - event_notifier_set(n); - } - } else { - ret =3D kvm_virtio_pci_irqfd_use(proxy, n, vector); + if (!vdev->defer_kvm_irq_routing) { + ret =3D virtio_pci_vector_do_unmask(proxy, queue_no, vector, n); } + return ret; } =20 @@ -1284,12 +1363,16 @@ static int virtio_pci_set_guest_notifiers(DeviceSta= te *d, int nvqs, bool assign) } } =20 + virtio_pci_prepare_kvm_msi_virq_batch(proxy); + r =3D msix_set_vector_notifiers(&proxy->pci_dev, virtio_pci_vector= _unmask, virtio_pci_vector_mask, virtio_pci_vector_poll); if (r < 0) { goto notifiers_error; } + + virtio_pci_commit_kvm_msi_virq_batch(proxy); } =20 return 0; diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h index 77c6c55929..9d82831350 100644 --- a/include/hw/virtio/virtio.h +++ b/include/hw/virtio/virtio.h @@ -147,6 +147,7 @@ struct VirtIODevice bool start_on_kick; /* when virtio 1.0 feature has not been negotiated= */ bool disable_legacy_check; bool vhost_started; + bool defer_kvm_irq_routing; VMChangeStateEntry *vmstate; char *bus_name; uint8_t device_endian; --=20 2.23.0