From nobody Sun Apr 28 13:46:19 2024 Delivered-To: importer@patchew.org Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=huawei.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) by mx.zohomail.com with SMTPS id 1629878301049338.50238237527026; Wed, 25 Aug 2021 00:58:21 -0700 (PDT) Received: from localhost ([::1]:35174 helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1mInnb-0001lb-CY for importer@patchew.org; Wed, 25 Aug 2021 03:58:15 -0400 Received: from eggs.gnu.org ([2001:470:142:3::10]:44290) by lists.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1mInmK-0007XU-Kd for qemu-devel@nongnu.org; Wed, 25 Aug 2021 03:56:56 -0400 Received: from szxga03-in.huawei.com ([45.249.212.189]:2225) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1mInmG-0005cr-2z for qemu-devel@nongnu.org; Wed, 25 Aug 2021 03:56:56 -0400 Received: from dggemv704-chm.china.huawei.com (unknown [172.30.72.57]) by szxga03-in.huawei.com (SkyGuard) with ESMTP id 4GvdbD43GZz89Bv; Wed, 25 Aug 2021 15:56:24 +0800 (CST) Received: from dggpeml500016.china.huawei.com (7.185.36.70) by dggemv704-chm.china.huawei.com (10.3.19.47) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2176.2; Wed, 25 Aug 2021 15:56:39 +0800 Received: from DESKTOP-27KDQMV.china.huawei.com (10.174.148.223) by dggpeml500016.china.huawei.com (7.185.36.70) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2176.2; Wed, 25 Aug 2021 15:56:35 +0800 From: "Longpeng(Mike)" To: , , , Subject: [PATCH 1/5] vfio: use helper to simplfy the failure path in vfio_msi_enable Date: Wed, 25 Aug 2021 15:56:16 +0800 Message-ID: <20210825075620.2607-2-longpeng2@huawei.com> X-Mailer: git-send-email 2.25.0.windows.1 In-Reply-To: <20210825075620.2607-1-longpeng2@huawei.com> References: <20210825075620.2607-1-longpeng2@huawei.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Originating-IP: [10.174.148.223] X-ClientProxiedBy: dggems702-chm.china.huawei.com (10.3.19.179) To dggpeml500016.china.huawei.com (7.185.36.70) X-CFilter-Loop: Reflected Received-SPF: pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) client-ip=209.51.188.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Received-SPF: pass client-ip=45.249.212.189; envelope-from=longpeng2@huawei.com; helo=szxga03-in.huawei.com X-Spam_score_int: -41 X-Spam_score: -4.2 X-Spam_bar: ---- X-Spam_report: (-4.2 / 5.0 requ) BAYES_00=-1.9, RCVD_IN_DNSWL_MED=-2.3, RCVD_IN_MSPIKE_H2=-0.001, SPF_HELO_NONE=0.001, SPF_PASS=-0.001 autolearn=ham autolearn_force=no X-Spam_action: no action X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: "Longpeng\(Mike\)" , arei.gonglei@huawei.com, huangzhichao@huawei.com, qemu-devel@nongnu.org Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZM-MESSAGEID: 1629878301756100002 Content-Type: text/plain; charset="utf-8" The main difference of the failure path in vfio_msi_enable and vfio_msi_disable_common is enable INTX or not. Extend the vfio_msi_disable_common to provide a arg to decide whether need to fallback, and then we can use this helper to instead the redundant code in vfio_msi_enable. Signed-off-by: Longpeng(Mike) --- hw/vfio/pci.c | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index e1ea1d8..7cc43fe 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -47,6 +47,7 @@ =20 static void vfio_disable_interrupts(VFIOPCIDevice *vdev); static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled); +static void vfio_msi_disable_common(VFIOPCIDevice *vdev, bool enable_intx); =20 /* * Disabling BAR mmaping can be slow, but toggling it around INTx can @@ -650,29 +651,17 @@ retry: if (ret) { if (ret < 0) { error_report("vfio: Error: Failed to setup MSI fds: %m"); - } else if (ret !=3D vdev->nr_vectors) { + } else { error_report("vfio: Error: Failed to enable %d " "MSI vectors, retry with %d", vdev->nr_vectors, r= et); } =20 - for (i =3D 0; i < vdev->nr_vectors; i++) { - VFIOMSIVector *vector =3D &vdev->msi_vectors[i]; - if (vector->virq >=3D 0) { - vfio_remove_kvm_msi_virq(vector); - } - qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt), - NULL, NULL, NULL); - event_notifier_cleanup(&vector->interrupt); - } - - g_free(vdev->msi_vectors); - vdev->msi_vectors =3D NULL; + vfio_msi_disable_common(vdev, false); =20 - if (ret > 0 && ret !=3D vdev->nr_vectors) { + if (ret > 0) { vdev->nr_vectors =3D ret; goto retry; } - vdev->nr_vectors =3D 0; =20 /* * Failing to setup MSI doesn't really fall within any specificati= on. @@ -680,7 +669,6 @@ retry: * out to fall back to INTx for this device. */ error_report("vfio: Error: Failed to enable MSI"); - vdev->interrupt =3D VFIO_INT_NONE; =20 return; } @@ -688,7 +676,7 @@ retry: trace_vfio_msi_enable(vdev->vbasedev.name, vdev->nr_vectors); } =20 -static void vfio_msi_disable_common(VFIOPCIDevice *vdev) +static void vfio_msi_disable_common(VFIOPCIDevice *vdev, bool enable_intx) { Error *err =3D NULL; int i; @@ -710,9 +698,11 @@ static void vfio_msi_disable_common(VFIOPCIDevice *vde= v) vdev->nr_vectors =3D 0; vdev->interrupt =3D VFIO_INT_NONE; =20 - vfio_intx_enable(vdev, &err); - if (err) { - error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); + if (enable_intx) { + vfio_intx_enable(vdev, &err); + if (err) { + error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); + } } } =20 @@ -737,7 +727,7 @@ static void vfio_msix_disable(VFIOPCIDevice *vdev) vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX); } =20 - vfio_msi_disable_common(vdev); + vfio_msi_disable_common(vdev, true); =20 memset(vdev->msix->pending, 0, BITS_TO_LONGS(vdev->msix->entries) * sizeof(unsigned long)); @@ -748,7 +738,7 @@ static void vfio_msix_disable(VFIOPCIDevice *vdev) static void vfio_msi_disable(VFIOPCIDevice *vdev) { vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSI_IRQ_INDEX); - vfio_msi_disable_common(vdev); + vfio_msi_disable_common(vdev, true); =20 trace_vfio_msi_disable(vdev->vbasedev.name); } --=20 1.8.3.1 From nobody Sun Apr 28 13:46:19 2024 Delivered-To: importer@patchew.org Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=huawei.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) by mx.zohomail.com with SMTPS id 1629878295890777.9417179445685; Wed, 25 Aug 2021 00:58:15 -0700 (PDT) Received: from localhost ([::1]:35154 helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1mInna-0001kj-H2 for importer@patchew.org; Wed, 25 Aug 2021 03:58:14 -0400 Received: from eggs.gnu.org ([2001:470:142:3::10]:44258) by lists.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1mInmJ-0007XL-Uk for qemu-devel@nongnu.org; Wed, 25 Aug 2021 03:56:55 -0400 Received: from szxga03-in.huawei.com ([45.249.212.189]:2224) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1mInmG-0005cq-1A for qemu-devel@nongnu.org; Wed, 25 Aug 2021 03:56:55 -0400 Received: from dggemv704-chm.china.huawei.com (unknown [172.30.72.57]) by szxga03-in.huawei.com (SkyGuard) with ESMTP id 4GvdbD4mVFz89xc; Wed, 25 Aug 2021 15:56:24 +0800 (CST) Received: from dggpeml500016.china.huawei.com (7.185.36.70) by dggemv704-chm.china.huawei.com (10.3.19.47) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2176.2; Wed, 25 Aug 2021 15:56:39 +0800 Received: from DESKTOP-27KDQMV.china.huawei.com (10.174.148.223) by dggpeml500016.china.huawei.com (7.185.36.70) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2176.2; Wed, 25 Aug 2021 15:56:36 +0800 From: "Longpeng(Mike)" To: , , , Subject: [PATCH 2/5] msix: simplfy the conditional in msix_set/unset_vector_notifiers Date: Wed, 25 Aug 2021 15:56:17 +0800 Message-ID: <20210825075620.2607-3-longpeng2@huawei.com> X-Mailer: git-send-email 2.25.0.windows.1 In-Reply-To: <20210825075620.2607-1-longpeng2@huawei.com> References: <20210825075620.2607-1-longpeng2@huawei.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Originating-IP: [10.174.148.223] X-ClientProxiedBy: dggems702-chm.china.huawei.com (10.3.19.179) To dggpeml500016.china.huawei.com (7.185.36.70) X-CFilter-Loop: Reflected Received-SPF: pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) client-ip=209.51.188.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Received-SPF: pass client-ip=45.249.212.189; envelope-from=longpeng2@huawei.com; helo=szxga03-in.huawei.com X-Spam_score_int: -41 X-Spam_score: -4.2 X-Spam_bar: ---- X-Spam_report: (-4.2 / 5.0 requ) BAYES_00=-1.9, RCVD_IN_DNSWL_MED=-2.3, RCVD_IN_MSPIKE_H2=-0.001, SPF_HELO_NONE=0.001, SPF_PASS=-0.001 autolearn=ham autolearn_force=no X-Spam_action: no action X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: "Longpeng\(Mike\)" , arei.gonglei@huawei.com, huangzhichao@huawei.com, qemu-devel@nongnu.org Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZM-MESSAGEID: 1629878297289100001 Content-Type: text/plain; charset="utf-8" 'msix_function_masked' is kept pace with the device's config, we can use it to replace the complex conditional in msix_set/unset_vector_notifiers. poll_notifier should be reset to NULL in the error path in msix_set_vector_notifiers, fix it incidentally. Signed-off-by: Longpeng(Mike) --- hw/pci/msix.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/hw/pci/msix.c b/hw/pci/msix.c index ae9331c..8057709 100644 --- a/hw/pci/msix.c +++ b/hw/pci/msix.c @@ -592,8 +592,7 @@ int msix_set_vector_notifiers(PCIDevice *dev, dev->msix_vector_release_notifier =3D release_notifier; dev->msix_vector_poll_notifier =3D poll_notifier; =20 - if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & - (MSIX_ENABLE_MASK | MSIX_MASKALL_MASK)) =3D=3D MSIX_ENABLE_MASK) { + if (!dev->msix_function_masked) { for (vector =3D 0; vector < dev->msix_entries_nr; vector++) { ret =3D msix_set_notifier_for_vector(dev, vector); if (ret < 0) { @@ -612,6 +611,7 @@ undo: } dev->msix_vector_use_notifier =3D NULL; dev->msix_vector_release_notifier =3D NULL; + dev->msix_vector_poll_notifier =3D NULL; return ret; } =20 @@ -622,8 +622,7 @@ void msix_unset_vector_notifiers(PCIDevice *dev) assert(dev->msix_vector_use_notifier && dev->msix_vector_release_notifier); =20 - if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & - (MSIX_ENABLE_MASK | MSIX_MASKALL_MASK)) =3D=3D MSIX_ENABLE_MASK) { + if (!dev->msix_function_masked) { for (vector =3D 0; vector < dev->msix_entries_nr; vector++) { msix_unset_notifier_for_vector(dev, vector); } --=20 1.8.3.1 From nobody Sun Apr 28 13:46:19 2024 Delivered-To: importer@patchew.org Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=huawei.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) by mx.zohomail.com with SMTPS id 1629878300855735.6949177494197; Wed, 25 Aug 2021 00:58:20 -0700 (PDT) Received: from localhost ([::1]:35184 helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1mInna-0001lt-S4 for importer@patchew.org; Wed, 25 Aug 2021 03:58:14 -0400 Received: from eggs.gnu.org ([2001:470:142:3::10]:44294) by lists.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1mInmL-0007Xc-25 for qemu-devel@nongnu.org; Wed, 25 Aug 2021 03:56:57 -0400 Received: from szxga02-in.huawei.com ([45.249.212.188]:2102) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1mInmG-0005cu-NR for qemu-devel@nongnu.org; Wed, 25 Aug 2021 03:56:56 -0400 Received: from dggemv704-chm.china.huawei.com (unknown [172.30.72.53]) by szxga02-in.huawei.com (SkyGuard) with ESMTP id 4GvdW54g2dzbdXk; Wed, 25 Aug 2021 15:52:49 +0800 (CST) Received: from dggpeml500016.china.huawei.com (7.185.36.70) by dggemv704-chm.china.huawei.com (10.3.19.47) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2176.2; Wed, 25 Aug 2021 15:56:39 +0800 Received: from DESKTOP-27KDQMV.china.huawei.com (10.174.148.223) by dggpeml500016.china.huawei.com (7.185.36.70) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2176.2; Wed, 25 Aug 2021 15:56:38 +0800 From: "Longpeng(Mike)" To: , , , Subject: [PATCH 3/5] vfio: defer to enable msix in migration resume phase Date: Wed, 25 Aug 2021 15:56:18 +0800 Message-ID: <20210825075620.2607-4-longpeng2@huawei.com> X-Mailer: git-send-email 2.25.0.windows.1 In-Reply-To: <20210825075620.2607-1-longpeng2@huawei.com> References: <20210825075620.2607-1-longpeng2@huawei.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Originating-IP: [10.174.148.223] X-ClientProxiedBy: dggems702-chm.china.huawei.com (10.3.19.179) To dggpeml500016.china.huawei.com (7.185.36.70) X-CFilter-Loop: Reflected Received-SPF: pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) client-ip=209.51.188.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Received-SPF: pass client-ip=45.249.212.188; envelope-from=longpeng2@huawei.com; helo=szxga02-in.huawei.com X-Spam_score_int: -41 X-Spam_score: -4.2 X-Spam_bar: ---- X-Spam_report: (-4.2 / 5.0 requ) BAYES_00=-1.9, RCVD_IN_DNSWL_MED=-2.3, RCVD_IN_MSPIKE_H2=-0.001, SPF_HELO_NONE=0.001, SPF_PASS=-0.001 autolearn=ham autolearn_force=no X-Spam_action: no action X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: "Longpeng\(Mike\)" , arei.gonglei@huawei.com, huangzhichao@huawei.com, qemu-devel@nongnu.org Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZM-MESSAGEID: 1629878301754100001 Content-Type: text/plain; charset="utf-8" The vf's unmasked msix vectors will be enable one by one in migraiton resume phase, VFIO_DEVICE_SET_IRQS will be called for each vector, it's a bit expensive if the vf has more vectors. We can call VFIO_DEVICE_SET_IRQS once outside the loop of set vector notifiers to reduce the cost. The test VM has 128 vcpus and 8 VF (with 65 vectors enabled), we mesure the cost of the vfio_msix_enable for each one, and we can see 10% costs can be reduced. Origin Apply this patch 1st 8 4 2nd 15 11 3rd 22 18 4th 24 25 5th 36 33 6th 44 40 7th 51 47 8th 58 54 Total 258ms 232ms Signed-off-by: Longpeng(Mike) --- hw/vfio/pci.c | 22 ++++++++++++++++++++++ hw/vfio/pci.h | 1 + 2 files changed, 23 insertions(+) diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index 7cc43fe..ca37fb7 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -372,6 +372,10 @@ static int vfio_enable_vectors(VFIOPCIDevice *vdev, bo= ol msix) int ret =3D 0, i, argsz; int32_t *fds; =20 + if (!vdev->nr_vectors) { + return 0; + } + argsz =3D sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds)); =20 irq_set =3D g_malloc0(argsz); @@ -495,6 +499,11 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, un= signed int nr, } } =20 + if (vdev->defer_add_virq) { + vdev->nr_vectors =3D MAX(vdev->nr_vectors, nr + 1); + goto clear_pending; + } + /* * We don't want to have the host allocate all possible MSI vectors * for a device if they're not in use, so we shutdown and incrementally @@ -524,6 +533,7 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, uns= igned int nr, } } =20 +clear_pending: /* Disable PBA emulation when nothing more is pending. */ clear_bit(nr, vdev->msix->pending); if (find_first_bit(vdev->msix->pending, @@ -608,6 +618,16 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev) if (msix_set_vector_notifiers(pdev, vfio_msix_vector_use, vfio_msix_vector_release, NULL)) { error_report("vfio: msix_set_vector_notifiers failed"); + return; + } + + if (!pdev->msix_function_masked && vdev->defer_add_virq) { + int ret; + vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX); + ret =3D vfio_enable_vectors(vdev, true); + if (ret) { + error_report("vfio: failed to enable vectors, %d", ret); + } } =20 trace_vfio_msix_enable(vdev->vbasedev.name); @@ -2456,7 +2476,9 @@ static int vfio_pci_load_config(VFIODevice *vbasedev,= QEMUFile *f) if (msi_enabled(pdev)) { vfio_msi_enable(vdev); } else if (msix_enabled(pdev)) { + vdev->defer_add_virq =3D true; vfio_msix_enable(vdev); + vdev->defer_add_virq =3D false; } =20 return ret; diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h index 6477751..4235c83 100644 --- a/hw/vfio/pci.h +++ b/hw/vfio/pci.h @@ -171,6 +171,7 @@ struct VFIOPCIDevice { bool no_kvm_ioeventfd; bool no_vfio_ioeventfd; bool enable_ramfb; + bool defer_add_virq; VFIODisplay *dpy; Notifier irqchip_change_notifier; }; --=20 1.8.3.1 From nobody Sun Apr 28 13:46:19 2024 Delivered-To: importer@patchew.org Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=huawei.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) by mx.zohomail.com with SMTPS id 1629878297874195.07147977072577; Wed, 25 Aug 2021 00:58:17 -0700 (PDT) Received: from localhost ([::1]:35404 helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1mInnc-0001uG-L7 for importer@patchew.org; Wed, 25 Aug 2021 03:58:16 -0400 Received: from eggs.gnu.org ([2001:470:142:3::10]:44314) by lists.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1mInmM-0007YK-5I for qemu-devel@nongnu.org; Wed, 25 Aug 2021 03:56:58 -0400 Received: from szxga08-in.huawei.com ([45.249.212.255]:2854) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1mInmI-0005ct-Ji for qemu-devel@nongnu.org; Wed, 25 Aug 2021 03:56:57 -0400 Received: from dggemv703-chm.china.huawei.com (unknown [172.30.72.57]) by szxga08-in.huawei.com (SkyGuard) with ESMTP id 4GvdZt6b8bz1DDHL; Wed, 25 Aug 2021 15:56:06 +0800 (CST) Received: from dggpeml500016.china.huawei.com (7.185.36.70) by dggemv703-chm.china.huawei.com (10.3.19.46) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2176.2; Wed, 25 Aug 2021 15:56:39 +0800 Received: from DESKTOP-27KDQMV.china.huawei.com (10.174.148.223) by dggpeml500016.china.huawei.com (7.185.36.70) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2176.2; Wed, 25 Aug 2021 15:56:39 +0800 From: "Longpeng(Mike)" To: , , , Subject: [PATCH 4/5] kvm: irqchip: support defer to commit the route Date: Wed, 25 Aug 2021 15:56:19 +0800 Message-ID: <20210825075620.2607-5-longpeng2@huawei.com> X-Mailer: git-send-email 2.25.0.windows.1 In-Reply-To: <20210825075620.2607-1-longpeng2@huawei.com> References: <20210825075620.2607-1-longpeng2@huawei.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Originating-IP: [10.174.148.223] X-ClientProxiedBy: dggems702-chm.china.huawei.com (10.3.19.179) To dggpeml500016.china.huawei.com (7.185.36.70) X-CFilter-Loop: Reflected Received-SPF: pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) client-ip=209.51.188.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Received-SPF: pass client-ip=45.249.212.255; envelope-from=longpeng2@huawei.com; helo=szxga08-in.huawei.com X-Spam_score_int: -22 X-Spam_score: -2.3 X-Spam_bar: -- X-Spam_report: (-2.3 / 5.0 requ) RCVD_IN_DNSWL_MED=-2.3, SPF_HELO_NONE=0.001, SPF_PASS=-0.001 autolearn=ham autolearn_force=no X-Spam_action: no action X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: "Longpeng\(Mike\)" , arei.gonglei@huawei.com, huangzhichao@huawei.com, qemu-devel@nongnu.org Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZM-MESSAGEID: 1629878299317100003 Content-Type: text/plain; charset="utf-8" The kvm_irqchip_commit_routes() is relatively expensive, so provide the users a choice to commit the route immediately or not when they add msi/msix route. Signed-off-by: Longpeng(Mike) --- accel/kvm/kvm-all.c | 10 +++++++--- accel/stubs/kvm-stub.c | 3 ++- hw/misc/ivshmem.c | 2 +- hw/vfio/pci.c | 2 +- hw/virtio/virtio-pci.c | 2 +- include/sysemu/kvm.h | 4 +++- target/i386/kvm/kvm.c | 2 +- 7 files changed, 16 insertions(+), 9 deletions(-) diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 0125c17..1f788a2 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -1950,7 +1950,8 @@ int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg) return kvm_set_irq(s, route->kroute.gsi, 1); } =20 -int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev) +int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev, + bool defer_commit) { struct kvm_irq_routing_entry kroute =3D {}; int virq; @@ -1993,7 +1994,9 @@ int kvm_irqchip_add_msi_route(KVMState *s, int vector= , PCIDevice *dev) =20 kvm_add_routing_entry(s, &kroute); kvm_arch_add_msi_route_post(&kroute, vector, dev); - kvm_irqchip_commit_routes(s); + if (!defer_commit) { + kvm_irqchip_commit_routes(s); + } =20 return virq; } @@ -2151,7 +2154,8 @@ int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg) abort(); } =20 -int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev) +int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev, + bool defer_commit) { return -ENOSYS; } diff --git a/accel/stubs/kvm-stub.c b/accel/stubs/kvm-stub.c index 5b1d00a..d5caaca 100644 --- a/accel/stubs/kvm-stub.c +++ b/accel/stubs/kvm-stub.c @@ -81,7 +81,8 @@ int kvm_on_sigbus(int code, void *addr) } =20 #ifndef CONFIG_USER_ONLY -int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev) +int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev, + bool defer_commit) { return -ENOSYS; } diff --git a/hw/misc/ivshmem.c b/hw/misc/ivshmem.c index 1ba4a98..98b14cc 100644 --- a/hw/misc/ivshmem.c +++ b/hw/misc/ivshmem.c @@ -429,7 +429,7 @@ static void ivshmem_add_kvm_msi_virq(IVShmemState *s, i= nt vector, IVSHMEM_DPRINTF("ivshmem_add_kvm_msi_virq vector:%d\n", vector); assert(!s->msi_vectors[vector].pdev); =20 - ret =3D kvm_irqchip_add_msi_route(kvm_state, vector, pdev); + ret =3D kvm_irqchip_add_msi_route(kvm_state, vector, pdev, false); if (ret < 0) { error_setg(errp, "kvm_irqchip_add_msi_route failed"); return; diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index ca37fb7..3ab67d6 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -427,7 +427,7 @@ static void vfio_add_kvm_msi_virq(VFIOPCIDevice *vdev, = VFIOMSIVector *vector, return; } =20 - virq =3D kvm_irqchip_add_msi_route(kvm_state, vector_n, &vdev->pdev); + virq =3D kvm_irqchip_add_msi_route(kvm_state, vector_n, &vdev->pdev, f= alse); if (virq < 0) { event_notifier_cleanup(&vector->kvm_interrupt); return; diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c index 433060a..7e2d021 100644 --- a/hw/virtio/virtio-pci.c +++ b/hw/virtio/virtio-pci.c @@ -684,7 +684,7 @@ static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy = *proxy, int ret; =20 if (irqfd->users =3D=3D 0) { - ret =3D kvm_irqchip_add_msi_route(kvm_state, vector, &proxy->pci_d= ev); + ret =3D kvm_irqchip_add_msi_route(kvm_state, vector, &proxy->pci_d= ev, false); if (ret < 0) { return ret; } diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h index a1ab1ee..1932dc0 100644 --- a/include/sysemu/kvm.h +++ b/include/sysemu/kvm.h @@ -473,9 +473,11 @@ void kvm_init_cpu_signals(CPUState *cpu); * message. * @dev: Owner PCI device to add the route. If @dev is specified * as @NULL, an empty MSI message will be inited. + * @defer_commit: Defer to commit new route to the KVM core. * @return: virq (>=3D0) when success, errno (<0) when failed. */ -int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev); +int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev, + bool defer_commit); int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg, PCIDevice *dev); void kvm_irqchip_commit_routes(KVMState *s); diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index e69abe4..896406b 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -4724,7 +4724,7 @@ void kvm_arch_init_irq_routing(KVMState *s) /* If the ioapic is in QEMU and the lapics are in KVM, reserve MSI routes for signaling interrupts to the local apics. */ for (i =3D 0; i < IOAPIC_NUM_PINS; i++) { - if (kvm_irqchip_add_msi_route(s, 0, NULL) < 0) { + if (kvm_irqchip_add_msi_route(s, 0, NULL, false) < 0) { error_report("Could not enable split IRQ mode."); exit(1); } --=20 1.8.3.1 From nobody Sun Apr 28 13:46:19 2024 Delivered-To: importer@patchew.org Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=huawei.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) by mx.zohomail.com with SMTPS id 1629878365624372.70908402190446; Wed, 25 Aug 2021 00:59:25 -0700 (PDT) Received: from localhost ([::1]:41848 helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1mInoi-0006Bs-K3 for importer@patchew.org; Wed, 25 Aug 2021 03:59:24 -0400 Received: from eggs.gnu.org ([2001:470:142:3::10]:44316) by lists.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1mInmO-0007hJ-L4 for qemu-devel@nongnu.org; Wed, 25 Aug 2021 03:57:00 -0400 Received: from szxga01-in.huawei.com ([45.249.212.187]:2474) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1mInmI-0005eK-7J for qemu-devel@nongnu.org; Wed, 25 Aug 2021 03:57:00 -0400 Received: from dggemv711-chm.china.huawei.com (unknown [172.30.72.56]) by szxga01-in.huawei.com (SkyGuard) with ESMTP id 4GvdW73xNtzbj4g; Wed, 25 Aug 2021 15:52:51 +0800 (CST) Received: from dggpeml500016.china.huawei.com (7.185.36.70) by dggemv711-chm.china.huawei.com (10.1.198.66) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2176.2; Wed, 25 Aug 2021 15:56:41 +0800 Received: from DESKTOP-27KDQMV.china.huawei.com (10.174.148.223) by dggpeml500016.china.huawei.com (7.185.36.70) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2176.2; Wed, 25 Aug 2021 15:56:40 +0800 From: "Longpeng(Mike)" To: , , , Subject: [PATCH 5/5] vfio: defer to commit kvm route in migraiton resume phase Date: Wed, 25 Aug 2021 15:56:20 +0800 Message-ID: <20210825075620.2607-6-longpeng2@huawei.com> X-Mailer: git-send-email 2.25.0.windows.1 In-Reply-To: <20210825075620.2607-1-longpeng2@huawei.com> References: <20210825075620.2607-1-longpeng2@huawei.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Originating-IP: [10.174.148.223] X-ClientProxiedBy: dggems702-chm.china.huawei.com (10.3.19.179) To dggpeml500016.china.huawei.com (7.185.36.70) X-CFilter-Loop: Reflected Received-SPF: pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) client-ip=209.51.188.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Received-SPF: pass client-ip=45.249.212.187; envelope-from=longpeng2@huawei.com; helo=szxga01-in.huawei.com X-Spam_score_int: -41 X-Spam_score: -4.2 X-Spam_bar: ---- X-Spam_report: (-4.2 / 5.0 requ) BAYES_00=-1.9, RCVD_IN_DNSWL_MED=-2.3, RCVD_IN_MSPIKE_H2=-0.001, SPF_HELO_NONE=0.001, SPF_PASS=-0.001 autolearn=ham autolearn_force=no X-Spam_action: no action X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: "Longpeng\(Mike\)" , arei.gonglei@huawei.com, huangzhichao@huawei.com, qemu-devel@nongnu.org Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZM-MESSAGEID: 1629878366057100003 Content-Type: text/plain; charset="utf-8" In migration resume phase, all unmasked msix vectors need to be setup when load the VF state. However, the setup operation would takes longer if the VF has more unmasked vectors. In our case, the VF has 65 vectors and each one spend at most 0.8ms on setup operation the total cost of the VF is about 8-58ms. For a VM that has 8 VFs of this type, the total cost is more than 250ms. vfio_pci_load_config vfio_msix_enable msix_set_vector_notifiers for (vector =3D 0; vector < dev->msix_entries_nr; vector++) { vfio_msix_vector_do_use vfio_add_kvm_msi_virq kvm_irqchip_commit_routes <-- expensive } We can reduce the cost by only commit once outside the loop. The routes is cached in kvm_state, we commit them first and then bind irqfd for each vector. The test VM has 128 vcpus and 8 VF (with 65 vectors enabled), we mesure the cost of the vfio_msix_enable for each one, and we can see 90+% costs can be reduce. Origin Apply this patch and vfio enable optimization 1st 8 2 2nd 15 2 3rd 22 2 4th 24 3 5th 36 2 6th 44 3 7th 51 3 8th 58 4 Total 258ms 21ms The optimition can be also applied to msi type. Signed-off-by: Longpeng(Mike) --- hw/vfio/pci.c | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index 3ab67d6..50e7ec7 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -427,12 +427,17 @@ static void vfio_add_kvm_msi_virq(VFIOPCIDevice *vdev= , VFIOMSIVector *vector, return; } =20 - virq =3D kvm_irqchip_add_msi_route(kvm_state, vector_n, &vdev->pdev, f= alse); + virq =3D kvm_irqchip_add_msi_route(kvm_state, vector_n, &vdev->pdev, + vdev->defer_add_virq); if (virq < 0) { event_notifier_cleanup(&vector->kvm_interrupt); return; } =20 + if (vdev->defer_add_virq) { + goto out; + } + if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, &vector->kvm_interru= pt, NULL, virq) < 0) { kvm_irqchip_release_virq(kvm_state, virq); @@ -440,6 +445,7 @@ static void vfio_add_kvm_msi_virq(VFIOPCIDevice *vdev, = VFIOMSIVector *vector, return; } =20 +out: vector->virq =3D virq; } =20 @@ -577,6 +583,36 @@ static void vfio_msix_vector_release(PCIDevice *pdev, = unsigned int nr) } } =20 +static void vfio_commit_kvm_msi_virq(VFIOPCIDevice *vdev) +{ + int i; + VFIOMSIVector *vector; + bool commited =3D false; + + for (i =3D 0; i < vdev->nr_vectors; i++) { + vector =3D &vdev->msi_vectors[i]; + + if (vector->virq < 0) { + continue; + } + + /* Commit cached route entries to KVM core first if not yet */ + if (!commited) { + kvm_irqchip_commit_routes(kvm_state); + commited =3D true; + } + + if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, + &vector->kvm_interrupt, + NULL, vector->virq) < 0) { + kvm_irqchip_release_virq(kvm_state, vector->virq); + event_notifier_cleanup(&vector->kvm_interrupt); + vector->virq =3D -1; + return; + } + } +} + static void vfio_msix_enable(VFIOPCIDevice *vdev) { PCIDevice *pdev =3D &vdev->pdev; @@ -624,6 +660,7 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev) if (!pdev->msix_function_masked && vdev->defer_add_virq) { int ret; vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX); + vfio_commit_kvm_msi_virq(vdev); ret =3D vfio_enable_vectors(vdev, true); if (ret) { error_report("vfio: failed to enable vectors, %d", ret); @@ -664,6 +701,10 @@ retry: vfio_add_kvm_msi_virq(vdev, vector, i, false); } =20 + if (vdev->defer_add_virq){ + vfio_commit_kvm_msi_virq(vdev); + } + /* Set interrupt type prior to possible interrupts */ vdev->interrupt =3D VFIO_INT_MSI; =20 @@ -2473,13 +2514,13 @@ static int vfio_pci_load_config(VFIODevice *vbasede= v, QEMUFile *f) vfio_pci_write_config(pdev, PCI_COMMAND, pci_get_word(pdev->config + PCI_COMMAND), 2); =20 + vdev->defer_add_virq =3D true; if (msi_enabled(pdev)) { vfio_msi_enable(vdev); } else if (msix_enabled(pdev)) { - vdev->defer_add_virq =3D true; vfio_msix_enable(vdev); - vdev->defer_add_virq =3D false; } + vdev->defer_add_virq =3D false; =20 return ret; } --=20 1.8.3.1