From nobody Tue Oct 28 01:56:35 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (208.118.235.17 [208.118.235.17]) by mx.zohomail.com with SMTPS id 1514871025957418.2512500300202; Mon, 1 Jan 2018 21:30:25 -0800 (PST) Received: from localhost ([::1]:57807 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1eWF9T-0007LX-Ui for importer@patchew.org; Tue, 02 Jan 2018 00:30:15 -0500 Received: from eggs.gnu.org ([2001:4830:134:3::10]:50199) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1eWF7Z-0006EK-5e for qemu-devel@nongnu.org; Tue, 02 Jan 2018 00:28:18 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1eWF7W-000076-QO for qemu-devel@nongnu.org; Tue, 02 Jan 2018 00:28:17 -0500 Received: from ozlabs.ru ([107.173.13.209]:37032) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1eWF7W-00006p-Ka; Tue, 02 Jan 2018 00:28:14 -0500 Received: from vpl1.ozlabs.ibm.com (localhost [IPv6:::1]) by ozlabs.ru (Postfix) with ESMTP id 421893A60024; Tue, 2 Jan 2018 00:27:49 -0500 (EST) From: Alexey Kardashevskiy To: qemu-devel@nongnu.org Date: Tue, 2 Jan 2018 16:28:03 +1100 Message-Id: <20180102052805.20498-2-aik@ozlabs.ru> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20180102052805.20498-1-aik@ozlabs.ru> References: <20180102052805.20498-1-aik@ozlabs.ru> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 107.173.13.209 Subject: [Qemu-devel] [PATCH qemu 1/3] memory/iommu: Add get_attr() X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Alexey Kardashevskiy , Paolo Bonzini , Alex Williamson , qemu-ppc@nongnu.org, David Gibson Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" This adds get_attr() to IOMMUMemoryRegionClass, like iommu_ops::domain_get_attr in the Linux kernel. This defines the first attribute - IOMMU_ATTR_SPAPR_TCE_FD - which will be used between the pSeries machine and VFIO-PCI. Signed-off-by: Alexey Kardashevskiy Acked-by: David Gibson Acked-by: Paolo Bonzini --- include/exec/memory.h | 22 ++++++++++++++++++++++ memory.c | 13 +++++++++++++ 2 files changed, 35 insertions(+) diff --git a/include/exec/memory.h b/include/exec/memory.h index a4cabdf..e1e0a8e 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -190,6 +190,10 @@ struct MemoryRegionOps { const MemoryRegionMmio old_mmio; }; =20 +enum IOMMUMemoryRegionAttr { + IOMMU_ATTR_SPAPR_TCE_FD +}; + typedef struct IOMMUMemoryRegionClass { /* private */ struct DeviceClass parent_class; @@ -210,6 +214,10 @@ typedef struct IOMMUMemoryRegionClass { IOMMUNotifierFlag new_flags); /* Set this up to provide customized IOMMU replay function */ void (*replay)(IOMMUMemoryRegion *iommu, IOMMUNotifier *notifier); + + /* Get IOMMU misc attributes */ + int (*get_attr)(IOMMUMemoryRegion *iommu, enum IOMMUMemoryRegionAttr, + void *data); } IOMMUMemoryRegionClass; =20 typedef struct CoalescedMemoryRange CoalescedMemoryRange; @@ -924,6 +932,20 @@ void memory_region_unregister_iommu_notifier(MemoryReg= ion *mr, IOMMUNotifier *n); =20 /** + * memory_region_iommu_get_attr: return an IOMMU attr if get_attr() is + * defined on the IOMMU. + * + * Returns 0 if succeded, error code otherwise. + * + * @iommu_mr: the memory region + * @attr: the requested attribute + * @data: a pointer to the requested attribute data + */ +int memory_region_iommu_get_attr(IOMMUMemoryRegion *iommu_mr, + enum IOMMUMemoryRegionAttr attr, + void *data); + +/** * memory_region_name: get a memory region's name * * Returns the string that was used to initialize the memory region. diff --git a/memory.c b/memory.c index 4b41fb8..39ab25a 100644 --- a/memory.c +++ b/memory.c @@ -1920,6 +1920,19 @@ void memory_region_notify_iommu(IOMMUMemoryRegion *i= ommu_mr, } } =20 +int memory_region_iommu_get_attr(IOMMUMemoryRegion *iommu_mr, + enum IOMMUMemoryRegionAttr attr, + void *data) +{ + IOMMUMemoryRegionClass *imrc =3D IOMMU_MEMORY_REGION_GET_CLASS(iommu_m= r); + + if (!imrc->get_attr) { + return -EINVAL; + } + + return imrc->get_attr(iommu_mr, attr, data); +} + void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client) { uint8_t mask =3D 1 << client; --=20 2.11.0 From nobody Tue Oct 28 01:56:35 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (208.118.235.17 [208.118.235.17]) by mx.zohomail.com with SMTPS id 1514871025954851.7928302059655; Mon, 1 Jan 2018 21:30:25 -0800 (PST) Received: from localhost ([::1]:57808 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1eWF9V-0007MT-5l for importer@patchew.org; Tue, 02 Jan 2018 00:30:17 -0500 Received: from eggs.gnu.org ([2001:4830:134:3::10]:50216) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1eWF7Z-0006EO-NT for qemu-devel@nongnu.org; Tue, 02 Jan 2018 00:28:18 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1eWF7Y-00007t-OL for qemu-devel@nongnu.org; Tue, 02 Jan 2018 00:28:17 -0500 Received: from ozlabs.ru ([107.173.13.209]:37038) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1eWF7Y-00007i-IE; Tue, 02 Jan 2018 00:28:16 -0500 Received: from vpl1.ozlabs.ibm.com (localhost [IPv6:::1]) by ozlabs.ru (Postfix) with ESMTP id 210E73A6004C; Tue, 2 Jan 2018 00:27:50 -0500 (EST) From: Alexey Kardashevskiy To: qemu-devel@nongnu.org Date: Tue, 2 Jan 2018 16:28:04 +1100 Message-Id: <20180102052805.20498-3-aik@ozlabs.ru> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20180102052805.20498-1-aik@ozlabs.ru> References: <20180102052805.20498-1-aik@ozlabs.ru> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 107.173.13.209 Subject: [Qemu-devel] [PATCH qemu 2/3] vfio/spapr: Use iommu memory region's get_attr() X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Alexey Kardashevskiy , Paolo Bonzini , Alex Williamson , qemu-ppc@nongnu.org, David Gibson Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" In order to enable TCE operations support in KVM, we have to inform the KVM about VFIO groups being attached to specific LIOBNs. The KVM already knows about VFIO groups, the only bit missing is which in-kernel TCE table (the one with user visible TCEs) should update the attached broups. There is an KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE attribute of the VFIO KVM device which receives a groupfd/tablefd couple. This uses a new memory_region_iommu_get_attr() helper to get the IOMMU fd and calls KVM to establish the link. As get_attr() is not implemented yet, this should cause no behavioural change. Signed-off-by: Alexey Kardashevskiy Acked-by: David Gibson Acked-by: Paolo Bonzini --- hw/vfio/common.c | 26 ++++++++++++++++++++++++++ hw/vfio/trace-events | 1 + 2 files changed, 27 insertions(+) diff --git a/hw/vfio/common.c b/hw/vfio/common.c index b77be3a..ebe28fe 100644 --- a/hw/vfio/common.c +++ b/hw/vfio/common.c @@ -457,6 +457,32 @@ static void vfio_listener_region_add(MemoryListener *l= istener, vfio_host_win_add(container, section->offset_within_address_space, section->offset_within_address_space + int128_get64(section->size) - 1, pgsize); +#ifdef CONFIG_KVM + if (kvm_enabled()) { + VFIOGroup *group; + IOMMUMemoryRegion *iommu_mr =3D IOMMU_MEMORY_REGION(section->m= r); + struct kvm_vfio_spapr_tce param; + struct kvm_device_attr attr =3D { + .group =3D KVM_DEV_VFIO_GROUP, + .attr =3D KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE, + .addr =3D (uint64_t)(unsigned long)¶m, + }; + + if (!memory_region_iommu_get_attr(iommu_mr, IOMMU_ATTR_SPAPR_T= CE_FD, + ¶m.tablefd)) { + QLIST_FOREACH(group, &container->group_list, container_nex= t) { + param.groupfd =3D group->fd; + if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &at= tr)) { + error_report("vfio: failed to setup fd %d for a gr= oup with fd %d: %s", + param.tablefd, param.groupfd, + strerror(errno)); + return; + } + trace_vfio_spapr_group_attach(param.groupfd, param.tab= lefd); + } + } + } +#endif } =20 hostwin_found =3D false; diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events index fae096c..3d34fe8 100644 --- a/hw/vfio/trace-events +++ b/hw/vfio/trace-events @@ -123,3 +123,4 @@ vfio_prereg_register(uint64_t va, uint64_t size, int re= t) "va=3D0x%"PRIx64" size=3D0 vfio_prereg_unregister(uint64_t va, uint64_t size, int ret) "va=3D0x%"PRIx= 64" size=3D0x%"PRIx64" ret=3D%d" vfio_spapr_create_window(int ps, uint64_t ws, uint64_t off) "pageshift=3D0= x%x winsize=3D0x%"PRIx64" offset=3D0x%"PRIx64 vfio_spapr_remove_window(uint64_t off) "offset=3D0x%"PRIx64 +vfio_spapr_group_attach(int groupfd, int tablefd) "Attached groupfd %d to = liobn fd %d" --=20 2.11.0 From nobody Tue Oct 28 01:56:35 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1514871058163845.0836006353953; Mon, 1 Jan 2018 21:30:58 -0800 (PST) Received: from localhost ([::1]:57811 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1eWFA9-0007rD-5W for importer@patchew.org; Tue, 02 Jan 2018 00:30:57 -0500 Received: from eggs.gnu.org ([2001:4830:134:3::10]:50477) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1eWF87-0006dl-MA for qemu-devel@nongnu.org; Tue, 02 Jan 2018 00:28:52 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1eWF84-0000Ph-J5 for qemu-devel@nongnu.org; Tue, 02 Jan 2018 00:28:51 -0500 Received: from ozlabs.ru ([107.173.13.209]:37078) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1eWF84-0000PL-9b; Tue, 02 Jan 2018 00:28:48 -0500 Received: from vpl1.ozlabs.ibm.com (localhost [IPv6:::1]) by ozlabs.ru (Postfix) with ESMTP id 0534A3A6005E; Tue, 2 Jan 2018 00:27:52 -0500 (EST) From: Alexey Kardashevskiy To: qemu-devel@nongnu.org Date: Tue, 2 Jan 2018 16:28:05 +1100 Message-Id: <20180102052805.20498-4-aik@ozlabs.ru> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20180102052805.20498-1-aik@ozlabs.ru> References: <20180102052805.20498-1-aik@ozlabs.ru> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 107.173.13.209 Subject: [Qemu-devel] [PATCH qemu 3/3] spapr/iommu: Enable in-kernel TCE acceleration via VFIO KVM device X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Alexey Kardashevskiy , Paolo Bonzini , Alex Williamson , qemu-ppc@nongnu.org, David Gibson Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" In order to enable TCE operations support in KVM, we have to inform the KVM about VFIO groups being attached to specific LIOBNs; the necessary bits are implemented already by IOMMU MR and VFIO. This defines get_attr() for the SPAPR TCE IOMMU MR which makes VFIO call the KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE ioctl and establish LIOBN-to-IOMMU link. This changes spapr_tce_set_need_vfio() to avoid TCE table reallocation if the kernel supports the TCE acceleration. Signed-off-by: Alexey Kardashevskiy Acked-by: David Gibson Acked-by: Paolo Bonzini --- target/ppc/kvm_ppc.h | 6 ++++++ hw/ppc/spapr_iommu.c | 19 +++++++++++++++++++ target/ppc/kvm.c | 7 ++++++- 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/target/ppc/kvm_ppc.h b/target/ppc/kvm_ppc.h index ecb5549..c55bb67 100644 --- a/target/ppc/kvm_ppc.h +++ b/target/ppc/kvm_ppc.h @@ -46,6 +46,7 @@ void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t pa= ge_shift, int kvmppc_remove_spapr_tce(void *table, int pfd, uint32_t window_size); int kvmppc_reset_htab(int shift_hint); uint64_t kvmppc_rma_size(uint64_t current_size, unsigned int hash_shift); +bool kvmppc_has_cap_spapr_vfio(void); #endif /* !CONFIG_USER_ONLY */ bool kvmppc_has_cap_epr(void); int kvmppc_define_rtas_kernel_token(uint32_t token, const char *function); @@ -229,6 +230,11 @@ static inline bool kvmppc_is_mem_backend_page_size_ok(= const char *obj_path) return true; } =20 +static inline bool kvmppc_has_cap_spapr_vfio(void) +{ + return false; +} + #endif /* !CONFIG_USER_ONLY */ =20 static inline bool kvmppc_has_cap_epr(void) diff --git a/hw/ppc/spapr_iommu.c b/hw/ppc/spapr_iommu.c index 5ccd785..f45538c 100644 --- a/hw/ppc/spapr_iommu.c +++ b/hw/ppc/spapr_iommu.c @@ -17,6 +17,7 @@ * License along with this library; if not, see . */ #include "qemu/osdep.h" +#include #include "qemu/error-report.h" #include "hw/hw.h" #include "qemu/log.h" @@ -160,6 +161,19 @@ static uint64_t spapr_tce_get_min_page_size(IOMMUMemor= yRegion *iommu) return 1ULL << tcet->page_shift; } =20 +static int spapr_tce_get_attr(IOMMUMemoryRegion *iommu, + enum IOMMUMemoryRegionAttr attr, void *data) +{ + sPAPRTCETable *tcet =3D container_of(iommu, sPAPRTCETable, iommu); + + if (attr =3D=3D IOMMU_ATTR_SPAPR_TCE_FD && kvmppc_has_cap_spapr_vfio()= ) { + *(int *) data =3D tcet->fd; + return 0; + } + + return -EINVAL; +} + static void spapr_tce_notify_flag_changed(IOMMUMemoryRegion *iommu, IOMMUNotifierFlag old, IOMMUNotifierFlag new) @@ -284,6 +298,10 @@ void spapr_tce_set_need_vfio(sPAPRTCETable *tcet, bool= need_vfio) =20 tcet->need_vfio =3D need_vfio; =20 + if (!need_vfio || (tcet->fd !=3D -1 && kvmppc_has_cap_spapr_vfio())) { + return; + } + oldtable =3D tcet->table; =20 tcet->table =3D spapr_tce_alloc_table(tcet->liobn, @@ -643,6 +661,7 @@ static void spapr_iommu_memory_region_class_init(Object= Class *klass, void *data) imrc->translate =3D spapr_tce_translate_iommu; imrc->get_min_page_size =3D spapr_tce_get_min_page_size; imrc->notify_flag_changed =3D spapr_tce_notify_flag_changed; + imrc->get_attr =3D spapr_tce_get_attr; } =20 static const TypeInfo spapr_iommu_memory_region_info =3D { diff --git a/target/ppc/kvm.c b/target/ppc/kvm.c index 518dd06..6faeb25 100644 --- a/target/ppc/kvm.c +++ b/target/ppc/kvm.c @@ -133,7 +133,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s) cap_spapr_tce =3D kvm_check_extension(s, KVM_CAP_SPAPR_TCE); cap_spapr_tce_64 =3D kvm_check_extension(s, KVM_CAP_SPAPR_TCE_64); cap_spapr_multitce =3D kvm_check_extension(s, KVM_CAP_SPAPR_MULTITCE); - cap_spapr_vfio =3D false; + cap_spapr_vfio =3D kvm_vm_check_extension(s, KVM_CAP_SPAPR_TCE_VFIO); cap_one_reg =3D kvm_check_extension(s, KVM_CAP_ONE_REG); cap_hior =3D kvm_check_extension(s, KVM_CAP_PPC_HIOR); cap_epr =3D kvm_check_extension(s, KVM_CAP_PPC_EPR); @@ -2456,6 +2456,11 @@ bool kvmppc_has_cap_mmu_hash_v3(void) return cap_mmu_hash_v3; } =20 +bool kvmppc_has_cap_spapr_vfio(void) +{ + return cap_spapr_vfio; +} + PowerPCCPUClass *kvm_ppc_get_host_cpu_class(void) { uint32_t host_pvr =3D mfpvr(); --=20 2.11.0