From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 622F45FDA7; Fri, 22 May 2026 04:28:50 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424131; cv=none; b=PGL+F1Ti3+Zq0+vyAJIz1DzsiBQJxu/f02UWbTVBqGfP/rwlRxWr8Mf2Ep/29uzMPgFJX1ivFnHjDoxKPD9azF3teiA2CsZQFE5tthmu95+xd7IfDPxQAYUyzqMwr+AJfAnZWimeGfg20FhTVmYFTmJh+zSoALmn6hUOyh2BDYQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424131; c=relaxed/simple; bh=K3DN8+vdSo15J0oWGBTO6XyJKH+l3IUXmlWKiJTGMdA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=KVacUEZKjd7nUYPV1/g2rHBSfBEEER9vjEWpnKfix6BcZFgRtgoHJcu9xfmncRmr5enP0k7mrHVz017iT4W4L/iTShUzdN8tLIHp6LsQD2HtJ2+S1/bsj+62/0emS/sVhordr4M3lLZw70GCl9YISaPwmHhC/mI4q+xLZ5dw7aw= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=oofhJ+Bn; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="oofhJ+Bn" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 78D9E1F00A3D; Fri, 22 May 2026 04:28:39 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424130; bh=EN0IQUuPXitnl1iltsyZkrJ7mdFplVIW+Qac7dXtXac=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=oofhJ+BnD8hM/LCVdJ/7G55+n1MfvMzsj1mvKIs9w+pDJYq+ccbHNjqkIxMBcMsOI IYRjlhqi87N4ou0jrg8o6TYwheH8tqcQW+W6RvZNEy5/RiBBWG59HTEH7imcMyRY78 5/FXpn57pnIkN9R+y0xvddQGPGhc1Fihzc94yaxS4G9cm94A92wehVQmntTf0ceGoP EqaRcHfsrsIFCyQeY7lGyxVXnADKgk4z1iHqzSRL+6PrxS/DIveIY6JZWQDQmDwNUf ne3/dijtNZY+hUzTFcJggFSsezzF436KsKMclkD8FbeXxxhRKLI+pnP1/+1D51tgO7 0D9yvrU1JTw9w== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org Subject: [PATCH v5 01/20] [DO NOT MERGE] arm64/coco: Add pKVM as a CC platform Date: Fri, 22 May 2026 09:57:56 +0530 Message-ID: <20260522042815.370873-2-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" pKVM does support memory encryption, expose that to the rest of the kernel through cc_platform_has() At the moment, all devices inside the guest are emulated which requires its memory to be shared back to the host (decrypted), so set force_dma_unencrypted() to always return true. Signed-off-by: Mostafa Saleh Signed-off-by: Aneesh Kumar K.V (Arm) --- arch/arm64/include/asm/hypervisor.h | 6 ++++++ arch/arm64/include/asm/mem_encrypt.h | 3 ++- arch/arm64/kernel/rsi.c | 12 ------------ arch/arm64/mm/init.c | 13 +++++++++++++ drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c | 5 +++++ 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/arch/arm64/include/asm/hypervisor.h b/arch/arm64/include/asm/h= ypervisor.h index a12fd897c877..1b0e15f290be 100644 --- a/arch/arm64/include/asm/hypervisor.h +++ b/arch/arm64/include/asm/hypervisor.h @@ -10,8 +10,14 @@ void kvm_arm_target_impl_cpu_init(void); =20 #ifdef CONFIG_ARM_PKVM_GUEST void pkvm_init_hyp_services(void); +bool is_protected_kvm_guest(void); #else static inline void pkvm_init_hyp_services(void) { }; + +static inline bool is_protected_kvm_guest(void) +{ + return false; +} #endif =20 static inline void kvm_arch_init_hyp_services(void) diff --git a/arch/arm64/include/asm/mem_encrypt.h b/arch/arm64/include/asm/= mem_encrypt.h index 314b2b52025f..636f45b4d8af 100644 --- a/arch/arm64/include/asm/mem_encrypt.h +++ b/arch/arm64/include/asm/mem_encrypt.h @@ -2,6 +2,7 @@ #ifndef __ASM_MEM_ENCRYPT_H #define __ASM_MEM_ENCRYPT_H =20 +#include #include =20 struct device; @@ -20,7 +21,7 @@ int realm_register_memory_enc_ops(void); =20 static inline bool force_dma_unencrypted(struct device *dev) { - return is_realm_world(); + return is_realm_world() || is_protected_kvm_guest(); } =20 /* diff --git a/arch/arm64/kernel/rsi.c b/arch/arm64/kernel/rsi.c index 92160f2e57ff..25ca75ce1a4d 100644 --- a/arch/arm64/kernel/rsi.c +++ b/arch/arm64/kernel/rsi.c @@ -7,7 +7,6 @@ #include #include #include -#include #include =20 #include @@ -23,17 +22,6 @@ EXPORT_SYMBOL(prot_ns_shared); DEFINE_STATIC_KEY_FALSE_RO(rsi_present); EXPORT_SYMBOL(rsi_present); =20 -bool cc_platform_has(enum cc_attr attr) -{ - switch (attr) { - case CC_ATTR_MEM_ENCRYPT: - return is_realm_world(); - default: - return false; - } -} -EXPORT_SYMBOL_GPL(cc_platform_has); - static bool rsi_version_matches(void) { unsigned long ver_lower, ver_higher; diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 97987f850a33..c1b223e7cc8e 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +37,7 @@ =20 #include #include +#include #include #include #include @@ -416,6 +418,17 @@ void dump_mem_limit(void) } } =20 +bool cc_platform_has(enum cc_attr attr) +{ + switch (attr) { + case CC_ATTR_MEM_ENCRYPT: + return is_realm_world() || is_protected_kvm_guest(); + default: + return false; + } +} +EXPORT_SYMBOL_GPL(cc_platform_has); + #ifdef CONFIG_EXECMEM static u64 module_direct_base __ro_after_init =3D 0; static u64 module_plt_base __ro_after_init =3D 0; diff --git a/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c b/drivers/virt/c= oco/pkvm-guest/arm-pkvm-guest.c index 4230b817a80b..297e6d6019b8 100644 --- a/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c +++ b/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c @@ -95,6 +95,11 @@ static int mmio_guard_ioremap_hook(phys_addr_t phys, siz= e_t size, return 0; } =20 +bool is_protected_kvm_guest(void) +{ + return !!pkvm_granule; +} + void pkvm_init_hyp_services(void) { int i; --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4B6F025B0A6; Fri, 22 May 2026 04:29:02 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424143; cv=none; b=ON2mWV0RwWKMgKICMSgh+hxJd0beDedBCvjxwV1fY2ulvVgowZ1yQHO4jXCEt2g7fr+2eEYqAS2VU9nccL5bEEgdSSrV88ZiFEoOcI/NskOUHwRMjvx8mRv2ftvMKo04E80il/W5UeYfG6l57orRGpwxQxkTrx8PB/YuJzf0uMw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424143; c=relaxed/simple; bh=iylfyXBNO6S14YbSeAPeNtbQXHgcnAOZOy6z6gxK0Ig=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=bcQBUp4INNRj/3Vi88fPbwoZx6ug0fpcebVB09MxSqPqlQ4cpJLV0Q/2NRA7RPtvur/9JSBT4WW2mDMH9coQoi64cuwVxxXFHPiRJz6mdQaJZZuz70Z45FRHEpYuZ7GReS/AGtP6W8Gf/6eKc5n6/UV9/wmZCfE49j8IFvxXfNM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=oQoyawK8; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="oQoyawK8" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 9D13A1F000E9; Fri, 22 May 2026 04:28:50 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424141; bh=4aF9+d8jwgDAX56PY9Oecf4wOZ9YOQdZwhZO6X/dmDA=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=oQoyawK8y+HW7VDzUabZEzIrZMxRHXdpf/BDYPznoAE+Q6F0scH3UnMFqICzd6qWv 4LbbljSgBLQjqtntkeO/+IVq4sO6eqL3fn3WTONJfIDhngF9Ngl7YYO9x4U4fdJwcr 7hThxT/Jz/LlAG05H970ULFr26ngAdhF3QzgfIFW5MKIZA1M4jQ1W5XJyM7zc8EVoG KB1BH9PWN7nrgb79LWOpW0nTN03ONx6qqd0lRzmgZnfJ874m5C3f6fGj2Kqqa7TcBy DLNynTYhdlh9Cey8CGdd/mrmFS+4vFir/UlmRMRvjCphkLeHJFBObp2YnnyhkwIu+j LsbLqIQqmxb0Q== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org, Halil Pasic , Matthew Rosato , Jaehoon Kim Subject: [PATCH v5 02/20] [DO NOT MERGE] s390: Expose protected virtualization through cc_platform_has() Date: Fri, 22 May 2026 09:57:57 +0530 Message-ID: <20260522042815.370873-3-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Protected virtualization guests use memory encryption, so advertise that to the rest of the kernel through cc_platform_has(CC_ATTR_MEM_ENCRYPT). s390 already forces DMA mappings to be unencrypted for protected virtualization guests through force_dma_unencrypted(). Add ARCH_HAS_CC_PLATFORM and provide the matching cc_platform_has() implementation Signed-off-by: Aneesh Kumar K.V (Arm) --- Cc: Halil Pasic Cc: Matthew Rosato Cc: Jaehoon Kim --- arch/s390/Kconfig | 1 + arch/s390/mm/init.c | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index ecbcbb781e40..9b5e6029e043 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -87,6 +87,7 @@ config S390 select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE select ARCH_HAS_CC_CAN_LINK + select ARCH_HAS_CC_PLATFORM select ARCH_HAS_CPU_FINALIZE_INIT select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VIRTUAL diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 1f72efc2a579..ad3c6d92b801 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -50,6 +50,7 @@ #include #include #include +#include =20 pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir"); pgd_t invalid_pg_dir[PTRS_PER_PGD] __section(".bss..invalid_pg_dir"); @@ -140,6 +141,19 @@ bool force_dma_unencrypted(struct device *dev) return is_prot_virt_guest(); } =20 + +bool cc_platform_has(enum cc_attr attr) +{ + switch (attr) { + case CC_ATTR_MEM_ENCRYPT: + return is_prot_virt_guest(); + + default: + return false; + } +} +EXPORT_SYMBOL_GPL(cc_platform_has); + /* protected virtualization */ static void __init pv_init(void) { --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C02EA2798EA; Fri, 22 May 2026 04:29:13 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424155; cv=none; b=OdbhWyApLuC41onb/VxKwrg2lrK7YnKY+oHngtfFHjmBMeCeoWuA0/cUcCxp214Y4Qzh+6DNMspH6kt1PYu+OgFm6ASrzDT8TuiVCoSwzI+ciprQjnXwNV9+dYAmDEBEVOCKYlkNQVfDOaW/TkjNXzd7Wb5K9ose3loxKh5FLFM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424155; c=relaxed/simple; bh=SkjdkOrEG6S2/NwI67FiZittwap4DupuSeNR5t/x19A=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=QjfAjou8wBLBjGMkZ0hk+b91EYZBKfXh2OpkcJTL3pFsXaafIRi801nV9NfdLZOLPZGf+xhOq8QMGvaoGzccMrWXwsXYyMKUg208fxrdS6U4tblXda9opV45vpl5GlFfJ44F1ueCYOY/GMT9m8RyZPRnB+Yy3r3uw7ASJD10tSw= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=igaRhG3F; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="igaRhG3F" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 9966F1F00A3D; Fri, 22 May 2026 04:29:02 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424153; bh=XzNUfuSsVIAYZs66SuzTocA+H2G8cNxm1JUvnlC50Gw=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=igaRhG3Fk+QrZB5QIMIHo9UEo10rvWO62VQbeVrlG+1DKAlRtl7YvtV/YorKdk/SR QjdGSX4OLX4hNF2YRB0PfJqJzkI3jcZDR76lUz9I+EAQxWWSc3ykJmTVf4+1AXMjlz IVDuroWAsA4x+wlSsDe29o4KhwOHghD3wC8Etc90wcyVVPX8kEeokLxzXOMVt95pGG sW2oH4ydn6pS5UlUa0gza6XErSSZBCjGnO1IGX7EGBFDgPHUQHAIprsdCbSfCEW6lx 9S3RUM0vSUPddF7CBrslgvxePZvSZNcotzES+bYz0szlIIUFIGVfLZBwQl5UZTcEis +6KmxV0iynWXA== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org, Jiri Pirko Subject: [PATCH v5 03/20] dma-direct: swiotlb: handle swiotlb alloc/free outside __dma_direct_alloc_pages Date: Fri, 22 May 2026 09:57:58 +0530 Message-ID: <20260522042815.370873-4-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Move swiotlb allocation out of __dma_direct_alloc_pages() and handle it in dma_direct_alloc() / dma_direct_alloc_pages(). This is needed for follow-up changes that simplify the handling of memory encryption/decryption based on the DMA attribute flags. swiotlb backing pages are already mapped decrypted by swiotlb_update_mem_attributes() and rmem_swiotlb_device_init(), so dma-direct should not call dma_set_decrypted() on allocation nor dma_set_encrypted() on free for swiotlb-backed memory. Update alloc/free paths to detect swiotlb-backed pages and skip encrypt/decrypt transitions for those paths. Keep the existing highmem rejection in dma_direct_alloc_pages() for swiotlb allocations. Only for "restricted-dma-pool", we currently set `for_alloc =3D true`, while rmem_swiotlb_device_init() decrypts the whole pool up front. This pool is typically used together with "shared-dma-pool", where the shared region is accessed after remap/ioremap and the returned address is suitable for decrypted memory access. So existing code paths remain valid. Tested-by: Jiri Pirko Signed-off-by: Aneesh Kumar K.V (Arm) --- include/linux/swiotlb.h | 6 ++++ kernel/dma/direct.c | 71 ++++++++++++++++++++++++++++++----------- kernel/dma/swiotlb.c | 6 ++++ 3 files changed, 65 insertions(+), 18 deletions(-) diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 3dae0f592063..133bb8ca9032 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -284,6 +284,8 @@ extern void swiotlb_print_info(void); #ifdef CONFIG_DMA_RESTRICTED_POOL struct page *swiotlb_alloc(struct device *dev, size_t size); bool swiotlb_free(struct device *dev, struct page *page, size_t size); +void swiotlb_free_from_pool(struct device *dev, phys_addr_t tlb_addr, + size_t size, struct io_tlb_pool *pool); =20 static inline bool is_swiotlb_for_alloc(struct device *dev) { @@ -299,6 +301,10 @@ static inline bool swiotlb_free(struct device *dev, st= ruct page *page, { return false; } +static inline void swiotlb_free_from_pool(struct device *dev, phys_addr_t = tlb_addr, + size_t size, struct io_tlb_pool *pool) +{ +} static inline bool is_swiotlb_for_alloc(struct device *dev) { return false; diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index ec887f443741..fe8e83a36058 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -96,14 +96,6 @@ static int dma_set_encrypted(struct device *dev, void *v= addr, size_t size) return ret; } =20 -static void __dma_direct_free_pages(struct device *dev, struct page *page, - size_t size) -{ - if (swiotlb_free(dev, page, size)) - return; - dma_free_contiguous(dev, page, size); -} - static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t si= ze) { struct page *page =3D swiotlb_alloc(dev, size); @@ -125,9 +117,6 @@ static struct page *__dma_direct_alloc_pages(struct dev= ice *dev, size_t size, =20 WARN_ON_ONCE(!PAGE_ALIGNED(size)); =20 - if (is_swiotlb_for_alloc(dev)) - return dma_direct_alloc_swiotlb(dev, size); - gfp |=3D dma_direct_optimal_gfp_mask(dev, &phys_limit); page =3D dma_alloc_contiguous(dev, size, gfp); if (page) { @@ -204,6 +193,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { bool remap =3D false, set_uncached =3D false; + bool mark_mem_decrypt =3D true; struct page *page; void *ret; =20 @@ -250,11 +240,21 @@ void *dma_direct_alloc(struct device *dev, size_t siz= e, dma_direct_use_pool(dev, gfp)) return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); =20 + if (is_swiotlb_for_alloc(dev)) { + page =3D dma_direct_alloc_swiotlb(dev, size); + if (page) { + mark_mem_decrypt =3D false; + goto setup_page; + } + return NULL; + } + /* we always manually zero the memory once we are done */ page =3D __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true); if (!page) return NULL; =20 +setup_page: /* * dma_alloc_contiguous can return highmem pages depending on a * combination the cma=3D arguments and per-arch setup. These need to be @@ -281,7 +281,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, goto out_free_pages; } else { ret =3D page_address(page); - if (dma_set_decrypted(dev, ret, size)) + if (mark_mem_decrypt && dma_set_decrypted(dev, ret, size)) goto out_leak_pages; } =20 @@ -298,10 +298,11 @@ void *dma_direct_alloc(struct device *dev, size_t siz= e, return ret; =20 out_encrypt_pages: - if (dma_set_encrypted(dev, page_address(page), size)) + if (mark_mem_decrypt && dma_set_encrypted(dev, page_address(page), size)) return NULL; out_free_pages: - __dma_direct_free_pages(dev, page, size); + if (!swiotlb_free(dev, page, size)) + dma_free_contiguous(dev, page, size); return NULL; out_leak_pages: return NULL; @@ -310,6 +311,9 @@ void *dma_direct_alloc(struct device *dev, size_t size, void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) { + phys_addr_t phys; + bool mark_mem_encrypted =3D true; + struct io_tlb_pool *swiotlb_pool; unsigned int page_order =3D get_order(size); =20 if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) && @@ -338,16 +342,25 @@ void dma_direct_free(struct device *dev, size_t size, dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size))) return; =20 + phys =3D dma_to_phys(dev, dma_addr); + swiotlb_pool =3D swiotlb_find_pool(dev, phys); + if (swiotlb_pool) + /* Swiotlb doesn't need a page attribute update on free */ + mark_mem_encrypted =3D false; + if (is_vmalloc_addr(cpu_addr)) { vunmap(cpu_addr); } else { if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED)) arch_dma_clear_uncached(cpu_addr, size); - if (dma_set_encrypted(dev, cpu_addr, size)) + if (mark_mem_encrypted && dma_set_encrypted(dev, cpu_addr, size)) return; } =20 - __dma_direct_free_pages(dev, dma_direct_to_page(dev, dma_addr), size); + if (swiotlb_pool) + swiotlb_free_from_pool(dev, phys, size, swiotlb_pool); + else + dma_free_contiguous(dev, dma_direct_to_page(dev, dma_addr), size); } =20 struct page *dma_direct_alloc_pages(struct device *dev, size_t size, @@ -359,6 +372,15 @@ struct page *dma_direct_alloc_pages(struct device *dev= , size_t size, if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp)) return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); =20 + if (is_swiotlb_for_alloc(dev)) { + page =3D dma_direct_alloc_swiotlb(dev, size); + if (!page) + return NULL; + + ret =3D page_address(page); + goto setup_page; + } + page =3D __dma_direct_alloc_pages(dev, size, gfp, false); if (!page) return NULL; @@ -366,6 +388,7 @@ struct page *dma_direct_alloc_pages(struct device *dev,= size_t size, ret =3D page_address(page); if (dma_set_decrypted(dev, ret, size)) goto out_leak_pages; +setup_page: memset(ret, 0, size); *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page)); return page; @@ -377,16 +400,28 @@ void dma_direct_free_pages(struct device *dev, size_t= size, struct page *page, dma_addr_t dma_addr, enum dma_data_direction dir) { + phys_addr_t phys; void *vaddr =3D page_address(page); + struct io_tlb_pool *swiotlb_pool; + bool mark_mem_encrypted =3D true; =20 /* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */ if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) && dma_free_from_pool(dev, vaddr, size)) return; =20 - if (dma_set_encrypted(dev, vaddr, size)) + phys =3D page_to_phys(page); + swiotlb_pool =3D swiotlb_find_pool(dev, phys); + if (swiotlb_pool) + mark_mem_encrypted =3D false; + + if (mark_mem_encrypted && dma_set_encrypted(dev, vaddr, size)) return; - __dma_direct_free_pages(dev, page, size); + + if (swiotlb_pool) + swiotlb_free_from_pool(dev, phys, size, swiotlb_pool); + else + dma_free_contiguous(dev, page, size); } =20 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 1abd3e6146f4..ac03a6856c2e 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -1809,6 +1809,12 @@ bool swiotlb_free(struct device *dev, struct page *p= age, size_t size) return true; } =20 +void swiotlb_free_from_pool(struct device *dev, phys_addr_t tlb_addr, size= _t size, + struct io_tlb_pool *pool) +{ + swiotlb_release_slots(dev, tlb_addr, pool); +} + static int rmem_swiotlb_device_init(struct reserved_mem *rmem, struct device *dev) { --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 8DECD1C5F13; Fri, 22 May 2026 04:29:25 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424166; cv=none; b=tw8tak21SCs1w7HXdfV8qqNYs205UYvKGxzEBdrnpjP1qSSKs5142MjJFoWE32WDm2CfcxIMVWq5bw11X6mnvcQSJscMgujJV/+enIcm9zhMtIJF+3DvPXMPCWQa186KriL/7phlMaTpf0KIzijsxSGwm7+cY4W30HqjgKZVnPs= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424166; c=relaxed/simple; bh=lfeMjdmsBLLh45oGEJ6h5sXG7d5ZHOMmFRwifnU/B5c=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=I4ynJ9YZ5P2j7kNq94u1TNJ4m09UWh3tIaUe6T6+CpAvowe/Q4ky1tvtsmP8t8tJCIQSS/QUO2Jk/MAlACPFnNEGrOqOmE/Hs6e73NS5Pz9Q01OgX4pR33vobS6uw6064x+WYu9XReOQK3AKbkwbUsyISsexAe2pwC6A4bm6Ejs= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=aw7yRvJC; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="aw7yRvJC" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 179781F000E9; Fri, 22 May 2026 04:29:13 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424165; bh=OPgHp7pxAl+G137GB0yt85dPbtiD0BHiuOT+gKio7vA=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=aw7yRvJCooVrFTKlzMdtEgb1kRMsEFzLc/ODc9uR9zpQKFeUd4Ld1QGBCEOI/N1iA RPG4Sc3hgUWzei8gOtyavODnhE6Uh6NhxNkpk4ENMph2sEBPBEP63Y/mHjkWK3fJ7w UlF8ojLoXfpzJu5jMzb9VhuMURrcZOtM26glZfnIo5xPPGy6IJ+izSD9bRiIE/xtoR JfsrvLFe+kSKcl+DglH3k97+xsmnIj1y3UnrSSL0Ham0uW85uyDHWPKky+sz74exGn XQFzxNIvJ9mHHoU14gDbUzI7RbiF49w+xKONCmqJhWo9NX2zKdHzQ09KRGUzAkjw0y IkaJcfK1qXq8g== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org, Jiri Pirko Subject: [PATCH v5 04/20] dma-direct: use DMA_ATTR_CC_SHARED in alloc/free paths Date: Fri, 22 May 2026 09:57:59 +0530 Message-ID: <20260522042815.370873-5-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Propagate force_dma_unencrypted() into DMA_ATTR_CC_SHARED in the dma-direct allocation path and use the attribute to drive the related decisions. This updates dma_direct_alloc(), dma_direct_free(), and dma_direct_alloc_pages() to fold the forced unencrypted case into attrs. Tested-by: Jiri Pirko Signed-off-by: Aneesh Kumar K.V (Arm) --- kernel/dma/direct.c | 53 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index fe8e83a36058..a224b1bed6f9 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -193,16 +193,31 @@ void *dma_direct_alloc(struct device *dev, size_t siz= e, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { bool remap =3D false, set_uncached =3D false; - bool mark_mem_decrypt =3D true; + bool mark_mem_decrypt =3D false; struct page *page; void *ret; =20 + /* + * DMA_ATTR_CC_SHARED is not a caller-visible dma_alloc_*() + * attribute. The direct allocator uses it internally after it has + * decided that the backing pages must be shared/decrypted, so the + * rest of the allocation path can consistently select DMA addresses, + * choose compatible pools and restore encryption on free. + */ + if (attrs & DMA_ATTR_CC_SHARED) + return NULL; + + if (force_dma_unencrypted(dev)) { + attrs |=3D DMA_ATTR_CC_SHARED; + mark_mem_decrypt =3D true; + } + size =3D PAGE_ALIGN(size); if (attrs & DMA_ATTR_NO_WARN) gfp |=3D __GFP_NOWARN; =20 - if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) && - !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) + if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_CC_SHARED)) =3D=3D + DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp); =20 if (!dev_is_dma_coherent(dev)) { @@ -236,7 +251,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, * Remapping or decrypting memory may block, allocate the memory from * the atomic pools instead if we aren't allowed block. */ - if ((remap || force_dma_unencrypted(dev)) && + if ((remap || (attrs & DMA_ATTR_CC_SHARED)) && dma_direct_use_pool(dev, gfp)) return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); =20 @@ -312,12 +327,24 @@ void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) { phys_addr_t phys; - bool mark_mem_encrypted =3D true; + bool mark_mem_encrypted =3D false; struct io_tlb_pool *swiotlb_pool; unsigned int page_order =3D get_order(size); =20 - if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) && - !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) { + /* see dma_direct_alloc() for details */ + WARN_ON(attrs & DMA_ATTR_CC_SHARED); + + /* + * if the device had requested for an unencrypted buffer, + * convert it to encrypted on free + */ + if (force_dma_unencrypted(dev)) { + attrs |=3D DMA_ATTR_CC_SHARED; + mark_mem_encrypted =3D true; + } + + if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_CC_SHARED)) =3D=3D + DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) { /* cpu_addr is a struct page cookie, not a kernel address */ dma_free_contiguous(dev, cpu_addr, size); return; @@ -366,10 +393,14 @@ void dma_direct_free(struct device *dev, size_t size, struct page *dma_direct_alloc_pages(struct device *dev, size_t size, dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) { + unsigned long attrs =3D 0; struct page *page; void *ret; =20 - if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp)) + if (force_dma_unencrypted(dev)) + attrs |=3D DMA_ATTR_CC_SHARED; + + if ((attrs & DMA_ATTR_CC_SHARED) && dma_direct_use_pool(dev, gfp)) return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); =20 if (is_swiotlb_for_alloc(dev)) { @@ -403,7 +434,11 @@ void dma_direct_free_pages(struct device *dev, size_t = size, phys_addr_t phys; void *vaddr =3D page_address(page); struct io_tlb_pool *swiotlb_pool; - bool mark_mem_encrypted =3D true; + /* + * if the device had requested for an unencrypted buffer, + * convert it to encrypted on free + */ + bool mark_mem_encrypted =3D force_dma_unencrypted(dev); =20 /* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */ if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) && --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 91ECA25B085; Fri, 22 May 2026 04:29:36 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424178; cv=none; b=NsyXA+iwQ2fztN/VauyDQr6TWl2gy2+YtDWmCH13/38siqId/fKoED+xnzXKqJEHrhkcmyMmYMKUqR+8Pl76WemJ+L4899zYX7tYLWXbZS0+ccfYcHdwH+wukTubVy4mgLzoMoli65308LN+TqeaOBcBV9R3Emce8BHLvYCKAW4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424178; c=relaxed/simple; bh=C0N7/olRWrrsijmKiKF647sYorhOZnDh3/J1loOrxB4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=RJxDGVUsh6TrrgJPy0ftdL0SEVRnM9vf+zJD38evQsdhTqnknMxWtbUcBilYnZP8jQM1eVBfDxUiVUDy+mW1QpJ7ptdLp5xcteAfcQsTInyB4uaxDdaJymeIqXrwaVmzY5oN0Zlsc+jc4pfknOOyT40FtQLoYiVIZuDU91I3T0o= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=DoGLzTbk; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="DoGLzTbk" Received: by smtp.kernel.org (Postfix) with ESMTPSA id A4B241F00A3D; Fri, 22 May 2026 04:29:25 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424176; bh=YUYF/ZWVHAsvRAz1VLuc61GkwG8ESI93ZRZ2zvCVMtI=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=DoGLzTbk/I88vJlhH7GT2e9h6X5GKFvxkpAMTuxrlLnP7SQnAmQXJ+7GvnJBdONYG YM00NN5I+qNpXNFb0AVBJmegPSAb8t0Z7ybjlH5CCRb0J1RNutvDtAKbsjbBpO3Pf+ n99DxmG3uL7wQuJAaalgW0Io3c30IBwOz05eCTpescnXEKjXYUcJXSXeViTaFzS10+ 4HygNslMM6WgjcIhdIojrhSxzlJrE7lhnHjMpqmzWsmC8Md7dSIBJrjP8sSDqih+jm sb0muav9BOfQgEZ0w9Sv/iBnMPuORN2ZWNAYBgvMok7rjy3GzHPd0mYmqdWFdpJcjz 72C0CUduodAMQ== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org, Jiri Pirko Subject: [PATCH v5 05/20] dma-pool: track decrypted atomic pools and select them via attrs Date: Fri, 22 May 2026 09:58:00 +0530 Message-ID: <20260522042815.370873-6-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Teach the atomic DMA pool code to distinguish between encrypted and unencrypted pools, and make pool allocation select the matching pool based on DMA attributes. Introduce a dma_gen_pool wrapper that records whether a pool is unencrypted, initialize that state when the atomic pools are created, and use it when expanding and resizing the pools. Update dma_alloc_from_pool() to take attrs and skip pools whose encrypted state does not match DMA_ATTR_CC_SHARED. Update dma_free_from_pool() accordingly. Also pass DMA_ATTR_CC_SHARED from the swiotlb atomic allocation path so decrypted swiotlb allocations are taken from the correct atomic pool. Tested-by: Jiri Pirko Reviewed-by: Mostafa Saleh Signed-off-by: Aneesh Kumar K.V (Arm) --- drivers/iommu/dma-iommu.c | 2 +- include/linux/dma-map-ops.h | 2 +- kernel/dma/direct.c | 11 ++- kernel/dma/pool.c | 167 +++++++++++++++++++++++------------- kernel/dma/swiotlb.c | 7 +- 5 files changed, 123 insertions(+), 66 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 54d96e847f16..c2595bee3d41 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1673,7 +1673,7 @@ void *iommu_dma_alloc(struct device *dev, size_t size= , dma_addr_t *handle, if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && !gfpflags_allow_blocking(gfp) && !coherent) page =3D dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr, - gfp, NULL); + gfp, attrs, NULL); else cpu_addr =3D iommu_dma_alloc_pages(dev, size, &page, gfp, attrs); if (!cpu_addr) diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 6a1832a73cad..696b2c3a2305 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -212,7 +212,7 @@ void *dma_common_pages_remap(struct page **pages, size_= t size, pgprot_t prot, void dma_common_free_remap(void *cpu_addr, size_t size); =20 struct page *dma_alloc_from_pool(struct device *dev, size_t size, - void **cpu_addr, gfp_t flags, + void **cpu_addr, gfp_t flags, unsigned long attrs, bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t)); bool dma_free_from_pool(struct device *dev, void *start, size_t size); =20 diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index a224b1bed6f9..dd959716df33 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -154,7 +154,7 @@ static bool dma_direct_use_pool(struct device *dev, gfp= _t gfp) } =20 static void *dma_direct_alloc_from_pool(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp) + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { struct page *page; u64 phys_limit; @@ -164,7 +164,8 @@ static void *dma_direct_alloc_from_pool(struct device *= dev, size_t size, return NULL; =20 gfp |=3D dma_direct_optimal_gfp_mask(dev, &phys_limit); - page =3D dma_alloc_from_pool(dev, size, &ret, gfp, dma_coherent_ok); + page =3D dma_alloc_from_pool(dev, size, &ret, gfp, attrs, + dma_coherent_ok); if (!page) return NULL; *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page)); @@ -253,7 +254,8 @@ void *dma_direct_alloc(struct device *dev, size_t size, */ if ((remap || (attrs & DMA_ATTR_CC_SHARED)) && dma_direct_use_pool(dev, gfp)) - return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); + return dma_direct_alloc_from_pool(dev, size, dma_handle, + gfp, attrs); =20 if (is_swiotlb_for_alloc(dev)) { page =3D dma_direct_alloc_swiotlb(dev, size); @@ -401,7 +403,8 @@ struct page *dma_direct_alloc_pages(struct device *dev,= size_t size, attrs |=3D DMA_ATTR_CC_SHARED; =20 if ((attrs & DMA_ATTR_CC_SHARED) && dma_direct_use_pool(dev, gfp)) - return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); + return dma_direct_alloc_from_pool(dev, size, dma_handle, + gfp, attrs); =20 if (is_swiotlb_for_alloc(dev)) { page =3D dma_direct_alloc_swiotlb(dev, size); diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c index 2b2fbb709242..be78474a6c49 100644 --- a/kernel/dma/pool.c +++ b/kernel/dma/pool.c @@ -12,12 +12,18 @@ #include #include #include +#include =20 -static struct gen_pool *atomic_pool_dma __ro_after_init; +struct dma_gen_pool { + bool unencrypted; + struct gen_pool *pool; +}; + +static struct dma_gen_pool atomic_pool_dma __ro_after_init; static unsigned long pool_size_dma; -static struct gen_pool *atomic_pool_dma32 __ro_after_init; +static struct dma_gen_pool atomic_pool_dma32 __ro_after_init; static unsigned long pool_size_dma32; -static struct gen_pool *atomic_pool_kernel __ro_after_init; +static struct dma_gen_pool atomic_pool_kernel __ro_after_init; static unsigned long pool_size_kernel; =20 /* Size can be defined by the coherent_pool command line */ @@ -76,11 +82,12 @@ static bool cma_in_zone(gfp_t gfp) return true; } =20 -static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size, +static int atomic_pool_expand(struct dma_gen_pool *dma_pool, size_t pool_s= ize, gfp_t gfp) { unsigned int order; struct page *page =3D NULL; + bool leak_pages =3D false; void *addr; int ret =3D -ENOMEM; =20 @@ -113,12 +120,17 @@ static int atomic_pool_expand(struct gen_pool *pool, = size_t pool_size, * Memory in the atomic DMA pools must be unencrypted, the pools do not * shrink so no re-encryption occurs in dma_direct_free(). */ - ret =3D set_memory_decrypted((unsigned long)page_to_virt(page), - 1 << order); - if (ret) - goto remove_mapping; - ret =3D gen_pool_add_virt(pool, (unsigned long)addr, page_to_phys(page), - pool_size, NUMA_NO_NODE); + if (dma_pool->unencrypted) { + ret =3D set_memory_decrypted((unsigned long)page_to_virt(page), + 1 << order); + if (ret) { + leak_pages =3D true; + goto remove_mapping; + } + } + + ret =3D gen_pool_add_virt(dma_pool->pool, (unsigned long)addr, + page_to_phys(page), pool_size, NUMA_NO_NODE); if (ret) goto encrypt_mapping; =20 @@ -126,62 +138,67 @@ static int atomic_pool_expand(struct gen_pool *pool, = size_t pool_size, return 0; =20 encrypt_mapping: - ret =3D set_memory_encrypted((unsigned long)page_to_virt(page), - 1 << order); - if (WARN_ON_ONCE(ret)) { - /* Decrypt succeeded but encrypt failed, purposely leak */ - goto out; - } + if (dma_pool->unencrypted && + set_memory_encrypted((unsigned long)page_to_virt(page), 1 << order)) + leak_pages =3D true; + remove_mapping: #ifdef CONFIG_DMA_DIRECT_REMAP dma_common_free_remap(addr, pool_size); free_page: - __free_pages(page, order); + if (!leak_pages) + __free_pages(page, order); #endif out: return ret; } =20 -static void atomic_pool_resize(struct gen_pool *pool, gfp_t gfp) +static void atomic_pool_resize(struct dma_gen_pool *dma_pool, gfp_t gfp) { - if (pool && gen_pool_avail(pool) < atomic_pool_size) - atomic_pool_expand(pool, gen_pool_size(pool), gfp); + if (dma_pool->pool && gen_pool_avail(dma_pool->pool) < atomic_pool_size) + atomic_pool_expand(dma_pool, gen_pool_size(dma_pool->pool), gfp); } =20 static void atomic_pool_work_fn(struct work_struct *work) { if (IS_ENABLED(CONFIG_ZONE_DMA)) - atomic_pool_resize(atomic_pool_dma, + atomic_pool_resize(&atomic_pool_dma, GFP_KERNEL | GFP_DMA); if (IS_ENABLED(CONFIG_ZONE_DMA32)) - atomic_pool_resize(atomic_pool_dma32, + atomic_pool_resize(&atomic_pool_dma32, GFP_KERNEL | GFP_DMA32); - atomic_pool_resize(atomic_pool_kernel, GFP_KERNEL); + atomic_pool_resize(&atomic_pool_kernel, GFP_KERNEL); } =20 -static __init struct gen_pool *__dma_atomic_pool_init(size_t pool_size, - gfp_t gfp) +static __init struct dma_gen_pool *__dma_atomic_pool_init(struct dma_gen_p= ool *dma_pool, + size_t pool_size, gfp_t gfp) { - struct gen_pool *pool; int ret; =20 - pool =3D gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE); - if (!pool) + dma_pool->pool =3D gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE); + if (!dma_pool->pool) return NULL; =20 - gen_pool_set_algo(pool, gen_pool_first_fit_order_align, NULL); + gen_pool_set_algo(dma_pool->pool, gen_pool_first_fit_order_align, NULL); + + /* if platform is using memory encryption atomic pools are by default dec= rypted. */ + if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) + dma_pool->unencrypted =3D true; + else + dma_pool->unencrypted =3D false; =20 - ret =3D atomic_pool_expand(pool, pool_size, gfp); + ret =3D atomic_pool_expand(dma_pool, pool_size, gfp); if (ret) { - gen_pool_destroy(pool); + gen_pool_destroy(dma_pool->pool); + dma_pool->pool =3D NULL; pr_err("DMA: failed to allocate %zu KiB %pGg pool for atomic allocation\= n", pool_size >> 10, &gfp); return NULL; } =20 pr_info("DMA: preallocated %zu KiB %pGg pool for atomic allocations\n", - gen_pool_size(pool) >> 10, &gfp); - return pool; + gen_pool_size(dma_pool->pool) >> 10, &gfp); + return dma_pool; } =20 #ifdef CONFIG_ZONE_DMA32 @@ -207,21 +224,22 @@ static int __init dma_atomic_pool_init(void) =20 /* All memory might be in the DMA zone(s) to begin with */ if (has_managed_zone(ZONE_NORMAL)) { - atomic_pool_kernel =3D __dma_atomic_pool_init(atomic_pool_size, - GFP_KERNEL); - if (!atomic_pool_kernel) + __dma_atomic_pool_init(&atomic_pool_kernel, atomic_pool_size, GFP_KERNEL= ); + if (!atomic_pool_kernel.pool) ret =3D -ENOMEM; } + if (has_managed_dma()) { - atomic_pool_dma =3D __dma_atomic_pool_init(atomic_pool_size, - GFP_KERNEL | GFP_DMA); - if (!atomic_pool_dma) + __dma_atomic_pool_init(&atomic_pool_dma, atomic_pool_size, + GFP_KERNEL | GFP_DMA); + if (!atomic_pool_dma.pool) ret =3D -ENOMEM; } + if (has_managed_dma32) { - atomic_pool_dma32 =3D __dma_atomic_pool_init(atomic_pool_size, - GFP_KERNEL | GFP_DMA32); - if (!atomic_pool_dma32) + __dma_atomic_pool_init(&atomic_pool_dma32, atomic_pool_size, + GFP_KERNEL | GFP_DMA32); + if (!atomic_pool_dma32.pool) ret =3D -ENOMEM; } =20 @@ -230,19 +248,44 @@ static int __init dma_atomic_pool_init(void) } postcore_initcall(dma_atomic_pool_init); =20 -static inline struct gen_pool *dma_guess_pool(struct gen_pool *prev, gfp_t= gfp) +static inline struct dma_gen_pool *__dma_guess_pool(struct dma_gen_pool *f= irst, + struct dma_gen_pool *second, struct dma_gen_pool *third) { - if (prev =3D=3D NULL) { + if (first->pool) + return first; + if (second && second->pool) + return second; + if (third && third->pool) + return third; + return NULL; +} + +static inline struct dma_gen_pool *dma_guess_pool(struct dma_gen_pool *pre= v, + gfp_t gfp) +{ + if (!prev) { if (gfp & GFP_DMA) - return atomic_pool_dma ?: atomic_pool_dma32 ?: atomic_pool_kernel; + return __dma_guess_pool(&atomic_pool_dma, + &atomic_pool_dma32, + &atomic_pool_kernel); + if (gfp & GFP_DMA32) - return atomic_pool_dma32 ?: atomic_pool_dma ?: atomic_pool_kernel; - return atomic_pool_kernel ?: atomic_pool_dma32 ?: atomic_pool_dma; + return __dma_guess_pool(&atomic_pool_dma32, + &atomic_pool_dma, + &atomic_pool_kernel); + + return __dma_guess_pool(&atomic_pool_kernel, + &atomic_pool_dma32, + &atomic_pool_dma); } - if (prev =3D=3D atomic_pool_kernel) - return atomic_pool_dma32 ? atomic_pool_dma32 : atomic_pool_dma; - if (prev =3D=3D atomic_pool_dma32) - return atomic_pool_dma; + + if (prev =3D=3D &atomic_pool_kernel) + return __dma_guess_pool(&atomic_pool_dma32, + &atomic_pool_dma, NULL); + + if (prev =3D=3D &atomic_pool_dma32) + return __dma_guess_pool(&atomic_pool_dma, NULL, NULL); + return NULL; } =20 @@ -272,16 +315,20 @@ static struct page *__dma_alloc_from_pool(struct devi= ce *dev, size_t size, } =20 struct page *dma_alloc_from_pool(struct device *dev, size_t size, - void **cpu_addr, gfp_t gfp, + void **cpu_addr, gfp_t gfp, unsigned long attrs, bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t)) { - struct gen_pool *pool =3D NULL; + struct dma_gen_pool *dma_pool =3D NULL; struct page *page; bool pool_found =3D false; =20 - while ((pool =3D dma_guess_pool(pool, gfp))) { + while ((dma_pool =3D dma_guess_pool(dma_pool, gfp))) { + + if (dma_pool->unencrypted !=3D !!(attrs & DMA_ATTR_CC_SHARED)) + continue; + pool_found =3D true; - page =3D __dma_alloc_from_pool(dev, size, pool, cpu_addr, + page =3D __dma_alloc_from_pool(dev, size, dma_pool->pool, cpu_addr, phys_addr_ok); if (page) return page; @@ -296,12 +343,14 @@ struct page *dma_alloc_from_pool(struct device *dev, = size_t size, =20 bool dma_free_from_pool(struct device *dev, void *start, size_t size) { - struct gen_pool *pool =3D NULL; + struct dma_gen_pool *dma_pool =3D NULL; + + while ((dma_pool =3D dma_guess_pool(dma_pool, 0))) { =20 - while ((pool =3D dma_guess_pool(pool, 0))) { - if (!gen_pool_has_addr(pool, (unsigned long)start, size)) + if (!gen_pool_has_addr(dma_pool->pool, (unsigned long)start, size)) continue; - gen_pool_free(pool, (unsigned long)start, size); + + gen_pool_free(dma_pool->pool, (unsigned long)start, size); return true; } =20 diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index ac03a6856c2e..be4d418d92ac 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -612,6 +612,7 @@ static struct page *swiotlb_alloc_tlb(struct device *de= v, size_t bytes, u64 phys_limit, gfp_t gfp) { struct page *page; + unsigned long attrs =3D 0; =20 /* * Allocate from the atomic pools if memory is encrypted and @@ -623,8 +624,12 @@ static struct page *swiotlb_alloc_tlb(struct device *d= ev, size_t bytes, if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL)) return NULL; =20 + /* swiotlb considered decrypted by default */ + if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) + attrs =3D DMA_ATTR_CC_SHARED; + return dma_alloc_from_pool(dev, bytes, &vaddr, gfp, - dma_coherent_ok); + attrs, dma_coherent_ok); } =20 gfp &=3D ~GFP_ZONEMASK; --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C2EF51C5F13; Fri, 22 May 2026 04:29:47 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424188; cv=none; b=F7nkP3PVPwRK968b++yScXc7gRC/aTYc2+Bx41VWHD9r0BqspbV6pPnsHoErHj6vnMThE4tiOTB+SsiaYXF8KgXA3rg/C/AzxEaOKZTMu5BfaATcbqcghN8q285MbRfTMjoUH5fj+1TZDSQrZYrY/wPKJ6JUGwaygX5iKfdc+3g= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424188; c=relaxed/simple; bh=DTZwtvzr4H/iUrmfhOfanXsmvurs6mR83G/JRtdfrsU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=g9MPh8x/c0DjwwErX1XAz7l4zsDaGdB6xsyB4zREGBkh7eG/YnVbPJHdTAJqqLgF673TxYT+nv3sU6gegRnvMSTzSWJFgAjj6Srs01zUuflb77lD24kw8pa7kjO4Zlv+rCU5ZroOSG03hfZPc3buw3a+NB3W/l06cEEKgek+nuQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=DTpcAEtz; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="DTpcAEtz" Received: by smtp.kernel.org (Postfix) with ESMTPSA id E53511F000E9; Fri, 22 May 2026 04:29:36 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424187; bh=fiD0PG9z39TyZrHJFNazn+XwQJBql3fj9afxEX8CQgQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=DTpcAEtz3uKGBFKHkFzOU4V7cNehSZ84WvKmLbDguu82gpqawSm7NQKAVGnXX8eq6 ptrDsyL2dVgsCtoWT/K17V5mHSNV9puGmMDqD2E67JUeM1Pm1pIVoCrKL8e00an2Yc 5coCreumM4fDiu7xpU8TBkCYzsqvND1lAZHsXgGmeSh5Jm+HiZUPa+M/VpM4LId2kf I8NdR1Six+SEqwDDeqnrYWcTSn/Ct18aJZCbSSror+lZuMVS2gSZ9BonkJXQM6P/NQ jZIKeAkPyiLAWKnLx1o20KEF4Jrm0zsHLtmwwkNYVHUUd/I3d8wYGQiVGd+jVlSpin fHjzgsGmyPoZg== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org Subject: [PATCH v5 06/20] dma: swiotlb: pass mapping attributes by reference Date: Fri, 22 May 2026 09:58:01 +0530 Message-ID: <20260522042815.370873-7-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Change swiotlb_tbl_map_single() to take the DMA mapping attributes by reference and update the direct callers accordingly. This is a preparatory change for a follow-up patch which updates the attributes based on the selected swiotlb pool. Keeping the signature change separate makes the follow-up patch easier to review. No functional change in this patch. Signed-off-by: Aneesh Kumar K.V (Arm) --- drivers/iommu/dma-iommu.c | 2 +- drivers/xen/swiotlb-xen.c | 2 +- include/linux/swiotlb.h | 2 +- kernel/dma/swiotlb.c | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index c2595bee3d41..725c7adb0a8d 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1180,7 +1180,7 @@ static phys_addr_t iommu_dma_map_swiotlb(struct devic= e *dev, phys_addr_t phys, trace_swiotlb_bounced(dev, phys, size); =20 phys =3D swiotlb_tbl_map_single(dev, phys, size, iova_mask(iovad), dir, - attrs); + &attrs); =20 /* * Untrusted devices should not see padding areas with random leftover diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 2cbf2b588f5b..8c4abe65cd49 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -243,7 +243,7 @@ static dma_addr_t xen_swiotlb_map_phys(struct device *d= ev, phys_addr_t phys, */ trace_swiotlb_bounced(dev, dev_addr, size); =20 - map =3D swiotlb_tbl_map_single(dev, phys, size, 0, dir, attrs); + map =3D swiotlb_tbl_map_single(dev, phys, size, 0, dir, &attrs); if (map =3D=3D (phys_addr_t)DMA_MAPPING_ERROR) return DMA_MAPPING_ERROR; =20 diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 133bb8ca9032..29187cec90d8 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -238,7 +238,7 @@ static inline phys_addr_t default_swiotlb_limit(void) =20 phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys, size_t mapping_size, unsigned int alloc_aligned_mask, - enum dma_data_direction dir, unsigned long attrs); + enum dma_data_direction dir, unsigned long *attrs); dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys, size_t size, enum dma_data_direction dir, unsigned long attrs); =20 diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index be4d418d92ac..78ce05857c00 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -1391,7 +1391,7 @@ static unsigned long mem_used(struct io_tlb_mem *mem) */ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_ad= dr, size_t mapping_size, unsigned int alloc_align_mask, - enum dma_data_direction dir, unsigned long attrs) + enum dma_data_direction dir, unsigned long *attrs) { struct io_tlb_mem *mem =3D dev->dma_io_tlb_mem; unsigned int offset; @@ -1425,7 +1425,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev= , phys_addr_t orig_addr, size =3D ALIGN(mapping_size + offset, alloc_align_mask + 1); index =3D swiotlb_find_slots(dev, orig_addr, size, alloc_align_mask, &poo= l); if (index =3D=3D -1) { - if (!(attrs & DMA_ATTR_NO_WARN)) + if (!(*attrs & DMA_ATTR_NO_WARN)) dev_warn_ratelimited(dev, "swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slo= ts)\n", size, mem->nslabs, mem_used(mem)); @@ -1604,7 +1604,7 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_= t paddr, size_t size, =20 trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size); =20 - swiotlb_addr =3D swiotlb_tbl_map_single(dev, paddr, size, 0, dir, attrs); + swiotlb_addr =3D swiotlb_tbl_map_single(dev, paddr, size, 0, dir, &attrs); if (swiotlb_addr =3D=3D (phys_addr_t)DMA_MAPPING_ERROR) return DMA_MAPPING_ERROR; =20 --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B73D5349CFC; Fri, 22 May 2026 04:29:58 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424200; cv=none; b=lhEmrYl6bNfiN39gLg3tfg7Mu9xq3Ezj/Q0XD72VzSLxyAXw7kHfh4a7T2hJzRmfI4+HRKv1afjH5XfafBJv6NPCsw5GPE0aN2+4s0eNb0J3TPi2Pu9J384CjMrj7BIzgJAYTBusOAENmfSKQamsYRxaZP4GCP3STgWicQpfqbw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424200; c=relaxed/simple; bh=5QcH+B/h5oUaiR5t19uEU3YfI1X+2v4AT3NLFSOtGfY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=mPZP4qoAKesPbQxtIpN4rfSlIbHxiA9AQwIfLgBy0HmswtCiM2twSgZCqEY/+uaWvzEYMp3mzJyOaWcU6XivVt98KaHFvgQ1ywlYODQN1GRrCyvMzrt55nqJ6uqixxbkYdyGWqaa/JAsNVh8Iu1Bbg//x2WjTUIgIbgRw4xfUm4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=OlD/bu9/; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="OlD/bu9/" Received: by smtp.kernel.org (Postfix) with ESMTPSA id F13F21F00A3E; Fri, 22 May 2026 04:29:47 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424198; bh=274NQyZowmRCaNAvfNEeUVtDPXznBjNJeJpwqwgcZAM=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=OlD/bu9/eKFBqpgnIhvNmry/tRVYRHJTuRuw4DRXTD33t5V2Sd3OenzQtvqjEK3oz cBhL68a+rqifg5ccS/NFskThqTDBYcH04c1hYbnptId8Z94Hj63K1v3OU9bcxmfAj8 C44KbydggihS5PD43UC9sbFNt8OShwN5HeRMUvfwXKu2GYSWtmWlWKty/K5aF/br7C 1R80cFJJ3NFla/tlGSFT7Kqp24/N4scym8x1hzYUFRr94NBB8orA1p5PH9Gq/FvAez CTOtszRf7tw3Vi0EqkiYFulPeA8GfvrC3DfUL0qNyoRk1aVTkC+x4V2m+497ckrCJU FuUYJKnXWMN3w== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org, Jiri Pirko Subject: [PATCH v5 07/20] dma: swiotlb: track pool encryption state and honor DMA_ATTR_CC_SHARED Date: Fri, 22 May 2026 09:58:02 +0530 Message-ID: <20260522042815.370873-8-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Teach swiotlb to distinguish between encrypted and decrypted bounce buffer pools, and make allocation and mapping paths select a pool whose state matches the requested DMA attributes. Add a unencrypted flag to io_tlb_mem, initialize it for the default and restricted pools, and propagate DMA_ATTR_CC_SHARED into swiotlb pool allocation. Reject swiotlb alloc/map requests when the selected pool does not match the required encrypted/decrypted state. Also return DMA addresses with the matching phys_to_dma_{encrypted, unencrypted} helper so the DMA address encoding stays consistent with the chosen pool. Tested-by: Jiri Pirko Signed-off-by: Aneesh Kumar K.V (Arm) --- include/linux/dma-direct.h | 10 +++ include/linux/swiotlb.h | 8 +- kernel/dma/direct.c | 13 +++- kernel/dma/swiotlb.c | 154 ++++++++++++++++++++++++++++--------- 4 files changed, 142 insertions(+), 43 deletions(-) diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h index c249912456f9..94fad4e7c11e 100644 --- a/include/linux/dma-direct.h +++ b/include/linux/dma-direct.h @@ -77,6 +77,10 @@ static inline dma_addr_t dma_range_map_max(const struct = bus_dma_region *map) #ifndef phys_to_dma_unencrypted #define phys_to_dma_unencrypted phys_to_dma #endif + +#ifndef phys_to_dma_encrypted +#define phys_to_dma_encrypted phys_to_dma +#endif #else static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t pad= dr) { @@ -90,6 +94,12 @@ static inline dma_addr_t phys_to_dma_unencrypted(struct = device *dev, { return dma_addr_unencrypted(__phys_to_dma(dev, paddr)); } + +static inline dma_addr_t phys_to_dma_encrypted(struct device *dev, + phys_addr_t paddr) +{ + return dma_addr_encrypted(__phys_to_dma(dev, paddr)); +} /* * If memory encryption is supported, phys_to_dma will set the memory encr= yption * bit in the DMA address, and dma_to_phys will clear it. diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 29187cec90d8..4dcbf3931be1 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -81,6 +81,7 @@ struct io_tlb_pool { struct list_head node; struct rcu_head rcu; bool transient; + bool unencrypted; #endif }; =20 @@ -111,6 +112,7 @@ struct io_tlb_mem { struct dentry *debugfs; bool force_bounce; bool for_alloc; + bool unencrypted; #ifdef CONFIG_SWIOTLB_DYNAMIC bool can_grow; u64 phys_limit; @@ -282,7 +284,8 @@ static inline void swiotlb_sync_single_for_cpu(struct d= evice *dev, extern void swiotlb_print_info(void); =20 #ifdef CONFIG_DMA_RESTRICTED_POOL -struct page *swiotlb_alloc(struct device *dev, size_t size); +struct page *swiotlb_alloc(struct device *dev, size_t size, + unsigned long attrs); bool swiotlb_free(struct device *dev, struct page *page, size_t size); void swiotlb_free_from_pool(struct device *dev, phys_addr_t tlb_addr, size_t size, struct io_tlb_pool *pool); @@ -292,7 +295,8 @@ static inline bool is_swiotlb_for_alloc(struct device *= dev) return dev->dma_io_tlb_mem->for_alloc; } #else -static inline struct page *swiotlb_alloc(struct device *dev, size_t size) +static inline struct page *swiotlb_alloc(struct device *dev, size_t size, + unsigned long attrs) { return NULL; } diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index dd959716df33..7cf1618a235d 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -96,9 +96,10 @@ static int dma_set_encrypted(struct device *dev, void *v= addr, size_t size) return ret; } =20 -static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t si= ze) +static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t si= ze, + unsigned long attrs) { - struct page *page =3D swiotlb_alloc(dev, size); + struct page *page =3D swiotlb_alloc(dev, size, attrs); =20 if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { swiotlb_free(dev, page, size); @@ -258,8 +259,12 @@ void *dma_direct_alloc(struct device *dev, size_t size, gfp, attrs); =20 if (is_swiotlb_for_alloc(dev)) { - page =3D dma_direct_alloc_swiotlb(dev, size); + page =3D dma_direct_alloc_swiotlb(dev, size, attrs); if (page) { + /* + * swiotlb allocations comes from pool already marked + * decrypted + */ mark_mem_decrypt =3D false; goto setup_page; } @@ -407,7 +412,7 @@ struct page *dma_direct_alloc_pages(struct device *dev,= size_t size, gfp, attrs); =20 if (is_swiotlb_for_alloc(dev)) { - page =3D dma_direct_alloc_swiotlb(dev, size); + page =3D dma_direct_alloc_swiotlb(dev, size, attrs); if (!page) return NULL; =20 diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 78ce05857c00..2bf3981db35d 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -259,10 +259,21 @@ void __init swiotlb_update_mem_attributes(void) struct io_tlb_pool *mem =3D &io_tlb_default_mem.defpool; unsigned long bytes; =20 + /* + * if platform support memory encryption, swiotlb buffers are + * decrypted by default. + */ + if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) + io_tlb_default_mem.unencrypted =3D true; + else + io_tlb_default_mem.unencrypted =3D false; + if (!mem->nslabs || mem->late_alloc) return; bytes =3D PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT); - set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT); + + if (io_tlb_default_mem.unencrypted) + set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT); } =20 static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t = start, @@ -505,8 +516,10 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask, if (!mem->slots) goto error_slots; =20 - set_memory_decrypted((unsigned long)vstart, - (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT); + if (io_tlb_default_mem.unencrypted) + set_memory_decrypted((unsigned long)vstart, + (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT); + swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true, nareas); add_mem_pool(&io_tlb_default_mem, mem); @@ -539,7 +552,9 @@ void __init swiotlb_exit(void) tbl_size =3D PAGE_ALIGN(mem->end - mem->start); slots_size =3D PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs)); =20 - set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT); + if (io_tlb_default_mem.unencrypted) + set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT); + if (mem->late_alloc) { area_order =3D get_order(array_size(sizeof(*mem->areas), mem->nareas)); @@ -563,6 +578,7 @@ void __init swiotlb_exit(void) * @gfp: GFP flags for the allocation. * @bytes: Size of the buffer. * @phys_limit: Maximum allowed physical address of the buffer. + * @unencrypted: true to allocate unencrypted memory, false for encrypted = memory * * Allocate pages from the buddy allocator. If successful, make the alloca= ted * pages decrypted that they can be used for DMA. @@ -570,7 +586,8 @@ void __init swiotlb_exit(void) * Return: Decrypted pages, %NULL on allocation failure, or ERR_PTR(-EAGAI= N) * if the allocated physical address was above @phys_limit. */ -static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limi= t) +static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, + u64 phys_limit, bool unencrypted) { unsigned int order =3D get_order(bytes); struct page *page; @@ -588,13 +605,13 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t= bytes, u64 phys_limit) } =20 vaddr =3D phys_to_virt(paddr); - if (set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes))) + if (unencrypted && set_memory_decrypted((unsigned long)vaddr, PFN_UP(byte= s))) goto error; return page; =20 error: /* Intentional leak if pages cannot be encrypted again. */ - if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes))) + if (unencrypted && !set_memory_encrypted((unsigned long)vaddr, PFN_UP(byt= es))) __free_pages(page, order); return NULL; } @@ -604,30 +621,26 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t= bytes, u64 phys_limit) * @dev: Device for which a memory pool is allocated. * @bytes: Size of the buffer. * @phys_limit: Maximum allowed physical address of the buffer. + * @attrs: DMA attributes for the allocation. * @gfp: GFP flags for the allocation. * * Return: Allocated pages, or %NULL on allocation failure. */ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes, - u64 phys_limit, gfp_t gfp) + u64 phys_limit, unsigned long attrs, gfp_t gfp) { struct page *page; - unsigned long attrs =3D 0; =20 /* * Allocate from the atomic pools if memory is encrypted and * the allocation is atomic, because decrypting may block. */ - if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) { + if (!gfpflags_allow_blocking(gfp) && (attrs & DMA_ATTR_CC_SHARED)) { void *vaddr; =20 if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL)) return NULL; =20 - /* swiotlb considered decrypted by default */ - if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) - attrs =3D DMA_ATTR_CC_SHARED; - return dma_alloc_from_pool(dev, bytes, &vaddr, gfp, attrs, dma_coherent_ok); } @@ -638,7 +651,8 @@ static struct page *swiotlb_alloc_tlb(struct device *de= v, size_t bytes, else if (phys_limit <=3D DMA_BIT_MASK(32)) gfp |=3D __GFP_DMA32; =20 - while (IS_ERR(page =3D alloc_dma_pages(gfp, bytes, phys_limit))) { + while (IS_ERR(page =3D alloc_dma_pages(gfp, bytes, phys_limit, + !!(attrs & DMA_ATTR_CC_SHARED)))) { if (IS_ENABLED(CONFIG_ZONE_DMA32) && phys_limit < DMA_BIT_MASK(64) && !(gfp & (__GFP_DMA32 | __GFP_DMA))) @@ -657,15 +671,18 @@ static struct page *swiotlb_alloc_tlb(struct device *= dev, size_t bytes, * swiotlb_free_tlb() - free a dynamically allocated IO TLB buffer * @vaddr: Virtual address of the buffer. * @bytes: Size of the buffer. + * @unencrypted: true if @vaddr was allocated decrypted and must be + * re-encrypted before being freed */ -static void swiotlb_free_tlb(void *vaddr, size_t bytes) +static void swiotlb_free_tlb(void *vaddr, size_t bytes, bool unencrypted) { if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) && dma_free_from_pool(NULL, vaddr, bytes)) return; =20 /* Intentional leak if pages cannot be encrypted again. */ - if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes))) + if (!unencrypted || + !set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes))) __free_pages(virt_to_page(vaddr), get_order(bytes)); } =20 @@ -676,6 +693,7 @@ static void swiotlb_free_tlb(void *vaddr, size_t bytes) * @nslabs: Desired (maximum) number of slabs. * @nareas: Number of areas. * @phys_limit: Maximum DMA buffer physical address. + * @attrs: DMA attributes for the allocation. * @gfp: GFP flags for the allocations. * * Allocate and initialize a new IO TLB memory pool. The actual number of @@ -686,7 +704,8 @@ static void swiotlb_free_tlb(void *vaddr, size_t bytes) */ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev, unsigned long minslabs, unsigned long nslabs, - unsigned int nareas, u64 phys_limit, gfp_t gfp) + unsigned int nareas, u64 phys_limit, + unsigned long attrs, gfp_t gfp) { struct io_tlb_pool *pool; unsigned int slot_order; @@ -704,9 +723,10 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct d= evice *dev, if (!pool) goto error; pool->areas =3D (void *)pool + sizeof(*pool); + pool->unencrypted =3D !!(attrs & DMA_ATTR_CC_SHARED); =20 tlb_size =3D nslabs << IO_TLB_SHIFT; - while (!(tlb =3D swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp))) { + while (!(tlb =3D swiotlb_alloc_tlb(dev, tlb_size, phys_limit, attrs, gfp)= )) { if (nslabs <=3D minslabs) goto error_tlb; nslabs =3D ALIGN(nslabs >> 1, IO_TLB_SEGSIZE); @@ -724,7 +744,8 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct de= vice *dev, return pool; =20 error_slots: - swiotlb_free_tlb(page_address(tlb), tlb_size); + swiotlb_free_tlb(page_address(tlb), tlb_size, + !!(attrs & DMA_ATTR_CC_SHARED)); error_tlb: kfree(pool); error: @@ -742,7 +763,9 @@ static void swiotlb_dyn_alloc(struct work_struct *work) struct io_tlb_pool *pool; =20 pool =3D swiotlb_alloc_pool(NULL, IO_TLB_MIN_SLABS, default_nslabs, - default_nareas, mem->phys_limit, GFP_KERNEL); + default_nareas, mem->phys_limit, + mem->unencrypted ? DMA_ATTR_CC_SHARED : 0, + GFP_KERNEL); if (!pool) { pr_warn_ratelimited("Failed to allocate new pool"); return; @@ -762,7 +785,7 @@ static void swiotlb_dyn_free(struct rcu_head *rcu) size_t tlb_size =3D pool->end - pool->start; =20 free_pages((unsigned long)pool->slots, get_order(slots_size)); - swiotlb_free_tlb(pool->vaddr, tlb_size); + swiotlb_free_tlb(pool->vaddr, tlb_size, pool->unencrypted); kfree(pool); } =20 @@ -1037,13 +1060,11 @@ static void dec_transient_used(struct io_tlb_mem *m= em, unsigned int nslots) * Return: Index of the first allocated slot, or -1 on error. */ static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool= *pool, - int area_index, phys_addr_t orig_addr, size_t alloc_size, - unsigned int alloc_align_mask) + int area_index, phys_addr_t orig_addr, dma_addr_t tbl_dma_addr, + size_t alloc_size, unsigned int alloc_align_mask) { struct io_tlb_area *area =3D pool->areas + area_index; unsigned long boundary_mask =3D dma_get_seg_boundary(dev); - dma_addr_t tbl_dma_addr =3D - phys_to_dma_unencrypted(dev, pool->start) & boundary_mask; unsigned long max_slots =3D get_max_slots(boundary_mask); unsigned int iotlb_align_mask =3D dma_get_min_align_mask(dev); unsigned int nslots =3D nr_slots(alloc_size), stride; @@ -1056,6 +1077,8 @@ static int swiotlb_search_pool_area(struct device *de= v, struct io_tlb_pool *pool BUG_ON(!nslots); BUG_ON(area_index >=3D pool->nareas); =20 + tbl_dma_addr &=3D boundary_mask; + /* * Historically, swiotlb allocations >=3D PAGE_SIZE were guaranteed to be * page-aligned in the absence of any other alignment requirements. @@ -1167,6 +1190,7 @@ static int swiotlb_search_area(struct device *dev, in= t start_cpu, { struct io_tlb_mem *mem =3D dev->dma_io_tlb_mem; struct io_tlb_pool *pool; + dma_addr_t tbl_dma_addr; int area_index; int index =3D -1; =20 @@ -1175,9 +1199,15 @@ static int swiotlb_search_area(struct device *dev, i= nt start_cpu, if (cpu_offset >=3D pool->nareas) continue; area_index =3D (start_cpu + cpu_offset) & (pool->nareas - 1); + + if (mem->unencrypted) + tbl_dma_addr =3D phys_to_dma_unencrypted(dev, pool->start); + else + tbl_dma_addr =3D phys_to_dma_encrypted(dev, pool->start); + index =3D swiotlb_search_pool_area(dev, pool, area_index, - orig_addr, alloc_size, - alloc_align_mask); + orig_addr, tbl_dma_addr, + alloc_size, alloc_align_mask); if (index >=3D 0) { *retpool =3D pool; break; @@ -1207,6 +1237,7 @@ static int swiotlb_find_slots(struct device *dev, phy= s_addr_t orig_addr, { struct io_tlb_mem *mem =3D dev->dma_io_tlb_mem; struct io_tlb_pool *pool; + dma_addr_t tbl_dma_addr; unsigned long nslabs; unsigned long flags; u64 phys_limit; @@ -1232,11 +1263,17 @@ static int swiotlb_find_slots(struct device *dev, p= hys_addr_t orig_addr, nslabs =3D nr_slots(alloc_size); phys_limit =3D min_not_zero(*dev->dma_mask, dev->bus_dma_limit); pool =3D swiotlb_alloc_pool(dev, nslabs, nslabs, 1, phys_limit, + mem->unencrypted ? DMA_ATTR_CC_SHARED : 0, GFP_NOWAIT); if (!pool) return -1; =20 - index =3D swiotlb_search_pool_area(dev, pool, 0, orig_addr, + if (mem->unencrypted) + tbl_dma_addr =3D phys_to_dma_unencrypted(dev, pool->start); + else + tbl_dma_addr =3D phys_to_dma_encrypted(dev, pool->start); + + index =3D swiotlb_search_pool_area(dev, pool, 0, orig_addr, tbl_dma_addr, alloc_size, alloc_align_mask); if (index < 0) { swiotlb_dyn_free(&pool->rcu); @@ -1281,15 +1318,23 @@ static int swiotlb_find_slots(struct device *dev, p= hys_addr_t orig_addr, size_t alloc_size, unsigned int alloc_align_mask, struct io_tlb_pool **retpool) { + struct io_tlb_mem *mem =3D dev->dma_io_tlb_mem; struct io_tlb_pool *pool; + dma_addr_t tbl_dma_addr; int start, i; int index; =20 - *retpool =3D pool =3D &dev->dma_io_tlb_mem->defpool; + *retpool =3D pool =3D &mem->defpool; + if (mem->unencrypted) + tbl_dma_addr =3D phys_to_dma_unencrypted(dev, pool->start); + else + tbl_dma_addr =3D phys_to_dma_encrypted(dev, pool->start); + i =3D start =3D raw_smp_processor_id() & (pool->nareas - 1); do { index =3D swiotlb_search_pool_area(dev, pool, i, orig_addr, - alloc_size, alloc_align_mask); + tbl_dma_addr, alloc_size, + alloc_align_mask); if (index >=3D 0) return index; if (++i >=3D pool->nareas) @@ -1372,9 +1417,19 @@ static unsigned long mem_used(struct io_tlb_mem *mem) * any pre- or post-padding for alignment * @alloc_align_mask: Required start and end alignment of the allocated bu= ffer * @dir: DMA direction - * @attrs: Optional DMA attributes for the map operation + * @attrs: Optional DMA attributes for the map operation, updated + * to match the selected SWIOTLB pool * * Find and allocate a suitable sequence of IO TLB slots for the request. + * The device's SWIOTLB pool must match the device's current DMA encryption + * requirements. If the device requires decrypted DMA, bouncing is done th= rough + * an unencrypted pool and the mapping is marked shared. If the device can= DMA + * to encrypted memory, bouncing is done through an encrypted pool even wh= en the + * original DMA address was unencrypted. Enabling encrypted DMA for a devi= ce is + * therefore expected to update its default io_tlb_mem to an encrypted poo= l, so + * later bounce mappings for both encrypted and decrypted original memory = use + * that encrypted pool. + * * The allocated space starts at an alignment specified by alloc_align_mas= k, * and the size of the allocated space is rounded up so that the total amo= unt * of allocated space is a multiple of (alloc_align_mask + 1). If @@ -1411,6 +1466,16 @@ phys_addr_t swiotlb_tbl_map_single(struct device *de= v, phys_addr_t orig_addr, if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) pr_warn_once("Memory encryption is active and system is using DMA bounce= buffers\n"); =20 + /* swiotlb pool is incorrect for this device */ + if (unlikely(mem->unencrypted !=3D force_dma_unencrypted(dev))) + return (phys_addr_t)DMA_MAPPING_ERROR; + + /* Force attrs to match the kind of memory in the pool */ + if (mem->unencrypted) + *attrs |=3D DMA_ATTR_CC_SHARED; + else + *attrs &=3D ~DMA_ATTR_CC_SHARED; + /* * The default swiotlb memory pool is allocated with PAGE_SIZE * alignment. If a mapping is requested with larger alignment, @@ -1608,8 +1673,11 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr= _t paddr, size_t size, if (swiotlb_addr =3D=3D (phys_addr_t)DMA_MAPPING_ERROR) return DMA_MAPPING_ERROR; =20 - /* Ensure that the address returned is DMA'ble */ - dma_addr =3D phys_to_dma_unencrypted(dev, swiotlb_addr); + if (attrs & DMA_ATTR_CC_SHARED) + dma_addr =3D phys_to_dma_unencrypted(dev, swiotlb_addr); + else + dma_addr =3D phys_to_dma_encrypted(dev, swiotlb_addr); + if (unlikely(!dma_capable(dev, dma_addr, size, true))) { __swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC, @@ -1773,7 +1841,7 @@ static inline void swiotlb_create_debugfs_files(struc= t io_tlb_mem *mem, =20 #ifdef CONFIG_DMA_RESTRICTED_POOL =20 -struct page *swiotlb_alloc(struct device *dev, size_t size) +struct page *swiotlb_alloc(struct device *dev, size_t size, unsigned long = attrs) { struct io_tlb_mem *mem =3D dev->dma_io_tlb_mem; struct io_tlb_pool *pool; @@ -1784,6 +1852,9 @@ struct page *swiotlb_alloc(struct device *dev, size_t= size) if (!mem) return NULL; =20 + if (mem->unencrypted !=3D !!(attrs & DMA_ATTR_CC_SHARED)) + return NULL; + align =3D (1 << (get_order(size) + PAGE_SHIFT)) - 1; index =3D swiotlb_find_slots(dev, 0, size, align, &pool); if (index =3D=3D -1) @@ -1859,9 +1930,18 @@ static int rmem_swiotlb_device_init(struct reserved_= mem *rmem, kfree(mem); return -ENOMEM; } + /* + * if platform supports memory encryption, + * restricted mem pool is decrypted by default + */ + if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) { + mem->unencrypted =3D true; + set_memory_decrypted((unsigned long)phys_to_virt(rmem->base), + rmem->size >> PAGE_SHIFT); + } else { + mem->unencrypted =3D false; + } =20 - set_memory_decrypted((unsigned long)phys_to_virt(rmem->base), - rmem->size >> PAGE_SHIFT); swiotlb_init_io_tlb_pool(pool, rmem->base, nslabs, false, nareas); mem->force_bounce =3D true; --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 199D7280CD2; Fri, 22 May 2026 04:30:09 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424213; cv=none; b=OcPtczcyA22BQ3F3R2mUTUHgchF+EHJwJimfBLra7LuY94fPIlYBZ0f6Ye1g4pKEPQRrfbr6+39lwdsnn0gywPXz8tl505cSlQQhOtp7sHaW0XUd2mX9MnR6G62gRaU6T8a6tRAi6f2RVmPUwgQFBwhR4E/CSDMQ9UeQRonMHDM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424213; c=relaxed/simple; bh=VO/uqouajJqMQbVSad9mMceJY+1bIm4fWezBNKlq18Q=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=HDP6HvGTuC7DOMN9ENNOEKA87fNdTOBUcvKPks5T4JJHaum9JAb9XmLSE6xQUuf4D6vk0wOnYQCyN54H64x3h2gsnhp6Mj1n6L2UaDWSqMClaQhpEkjMVd2Z+UhLkXfQXVwPg0zUMrPH7LIPyIpQ1AcT8I4t2hYvrKBd8XTMq0I= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=h2bO8JMr; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="h2bO8JMr" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 3D7A01F000E9; Fri, 22 May 2026 04:29:58 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424209; bh=KUK+7oDi4SWVkG4JnsspRP7bf98T3iHls52sNW0+aSI=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=h2bO8JMrXUTKXlYbOzGyq+pBqaAkbLLxW5FczPW1wATQAHHGgRUcMO4XuC8LxwcRL 8f7P8dd4oZECJnWUO/T4V+PILFMz3tfsFDYxyHu1FbwqDJ3IUQ36oXt9SeBWz7RNOC to8lEpTZ2flA8EoohYinnIqyhXAeBLY/KSI6WD6DYR+n//zi2aHqgVisz1o9g3/jeq xl30t9dUEDeKflAuPivOnwVnyiiI/uPww7Lvl/Xt5jhq7vZNrBK5pwmrrQTVIhzuVm VTBEU18mJTaIvOcUr19P/DByFsiMMZDPKaWRU1hf44l8Mc4LTpcrIQFAjAGY0g4ZU0 pupisVugAfirw== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org, Jiri Pirko Subject: [PATCH v5 08/20] dma-mapping: make dma_pgprot() honor DMA_ATTR_CC_SHARED Date: Fri, 22 May 2026 09:58:03 +0530 Message-ID: <20260522042815.370873-9-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Fold encrypted/decrypted pgprot selection into dma_pgprot() so callers do not need to adjust the page protection separately. Update dma_pgprot() to apply pgprot_decrypted() when DMA_ATTR_CC_SHARED is set and pgprot_encrypted() otherwise Convert the dma-direct allocation and mmap paths to pass DMA_ATTR_CC_SHARED instead of open-coding force_dma_unencrypted() handling around dma_pgprot(). Tested-by: Jiri Pirko Signed-off-by: Aneesh Kumar K.V (Arm) --- kernel/dma/direct.c | 8 +++----- kernel/dma/mapping.c | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 7cf1618a235d..429791b2599a 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -290,9 +290,6 @@ void *dma_direct_alloc(struct device *dev, size_t size, if (remap) { pgprot_t prot =3D dma_pgprot(dev, PAGE_KERNEL, attrs); =20 - if (force_dma_unencrypted(dev)) - prot =3D pgprot_decrypted(prot); - /* remove any dirty cache lines on the kernel alias */ arch_dma_prep_coherent(page, size); =20 @@ -614,9 +611,10 @@ int dma_direct_mmap(struct device *dev, struct vm_area= _struct *vma, unsigned long pfn =3D PHYS_PFN(dma_to_phys(dev, dma_addr)); int ret =3D -ENXIO; =20 - vma->vm_page_prot =3D dma_pgprot(dev, vma->vm_page_prot, attrs); if (force_dma_unencrypted(dev)) - vma->vm_page_prot =3D pgprot_decrypted(vma->vm_page_prot); + attrs |=3D DMA_ATTR_CC_SHARED; + + vma->vm_page_prot =3D dma_pgprot(dev, vma->vm_page_prot, attrs); =20 if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) return ret; diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 23ed8eb9233e..44f715f3aa2d 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -543,13 +543,21 @@ EXPORT_SYMBOL(dma_get_sgtable_attrs); */ pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs) { + pgprot_t dma_prot; + if (dev_is_dma_coherent(dev)) - return prot; + dma_prot =3D prot; #ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE - if (attrs & DMA_ATTR_WRITE_COMBINE) - return pgprot_writecombine(prot); + else if (attrs & DMA_ATTR_WRITE_COMBINE) + dma_prot =3D pgprot_writecombine(prot); #endif - return pgprot_dmacoherent(prot); + else + dma_prot =3D pgprot_dmacoherent(prot); + + if (attrs & DMA_ATTR_CC_SHARED) + return pgprot_decrypted(dma_prot); + else + return pgprot_encrypted(dma_prot); } #endif /* CONFIG_MMU */ =20 --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id F289924E4C3; Fri, 22 May 2026 04:30:20 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424222; cv=none; b=BOmFTW+QQPXGsywQklUSzPWHtvU/7GdrO5y1fkVSjJCt3Vac/NxrrJHUvTu/9SzVBfdd57G/tke8CzuqS1SHBfKhaJ0bK8dyHs3fohV7EIdEtnWuffY1kn/dJhNJwxcOkgfXm83MlkIIsAZo2kswnO3+gfsl/zYa93xTBa3XToo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424222; c=relaxed/simple; bh=yeChi67kaMST3pBN7bhm5cA3IokHAbXJE5a59Rr/JXU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=OjJ42UmPMnJ4yuao6XWuye1IcZezGRFRkR35LOcHtNBTypjByFpc+DwS4Oj0CHRsp/huy2lrecYHdMqPL/UahBZSyn1GxUrGEKl/w31DvzJ4e1gsFTLL88lyR6qfVfe68gXHi8SyOnOl02hWBp4bGers9e4BIgEnJkVDVYR8bCo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=fbaHQxI0; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="fbaHQxI0" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 4C0F61F00A3D; Fri, 22 May 2026 04:30:10 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424220; bh=xGhuUchYGN/yLwxzDZqU5Hd6NG/zeP6enhZO+IyjIRA=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=fbaHQxI0QMrsbOrLUV76IP7qV/XlrTKgrql+iJ4JYLFAQkSUTt/3JOfipOc+5vf4F jVczPyvVFDscYi2iDPIN3DBGi/fHkVV+vfq3no4HyqjRtv4K/ARJOdsCMD4xwcSs0N avIusyE3t6X8nbmHN3KnP/oXTetgcxEB8HPzHMzxoqquXPYfTQStDvryfhuqVNxp8/ b5ko+HuPMJ2zKq8yj3ordih8vfryq1a11zcmC8hSIMHd6qyW6jjb1Cyav03Woi9g8E C+l9yOW4nwX6edhqfUT5ZdvYwaxOhLPDpYNapWpvw5QrgibDB58Woe6Sd7KJ/R6EyZ D421cQf0iFKRQ== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org, Jiri Pirko Subject: [PATCH v5 09/20] dma-direct: pass attrs to dma_capable() for DMA_ATTR_CC_SHARED checks Date: Fri, 22 May 2026 09:58:04 +0530 Message-ID: <20260522042815.370873-10-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Teach dma_capable() about DMA_ATTR_CC_SHARED so the capability check can reject encrypted DMA addresses for devices that require unencrypted/shared DMA. Also propagate DMA_ATTR_CC_SHARED in swiotlb_map() when the selected SWIOTLB pool is decrypted so the capability check sees the correct DMA address attribute. Tested-by: Jiri Pirko Signed-off-by: Aneesh Kumar K.V (Arm) --- arch/x86/kernel/amd_gart_64.c | 30 ++++++++++++++++-------------- drivers/xen/swiotlb-xen.c | 6 +++--- include/linux/dma-direct.h | 10 +++++++++- kernel/dma/direct.h | 6 +++--- kernel/dma/swiotlb.c | 2 +- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c index e8000a56732e..b5f1f031d45b 100644 --- a/arch/x86/kernel/amd_gart_64.c +++ b/arch/x86/kernel/amd_gart_64.c @@ -180,22 +180,23 @@ static void iommu_full(struct device *dev, size_t siz= e, int dir) } =20 static inline int -need_iommu(struct device *dev, unsigned long addr, size_t size) +need_iommu(struct device *dev, unsigned long addr, size_t size, unsigned l= ong attrs) { - return force_iommu || !dma_capable(dev, addr, size, true); + return force_iommu || !dma_capable(dev, addr, size, true, attrs); } =20 static inline int -nonforced_iommu(struct device *dev, unsigned long addr, size_t size) +nonforced_iommu(struct device *dev, unsigned long addr, size_t size, + unsigned long attrs) { - return !dma_capable(dev, addr, size, true); + return !dma_capable(dev, addr, size, true, attrs); } =20 /* Map a single continuous physical area into the IOMMU. * Caller needs to check if the iommu is needed and flush. */ static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem, - size_t size, int dir, unsigned long align_mask) + size_t size, int dir, unsigned long align_mask, unsigned long attrs) { unsigned long npages =3D iommu_num_pages(phys_mem, size, PAGE_SIZE); unsigned long iommu_page; @@ -206,7 +207,7 @@ static dma_addr_t dma_map_area(struct device *dev, dma_= addr_t phys_mem, =20 iommu_page =3D alloc_iommu(dev, npages, align_mask); if (iommu_page =3D=3D -1) { - if (!nonforced_iommu(dev, phys_mem, size)) + if (!nonforced_iommu(dev, phys_mem, size, attrs)) return phys_mem; if (panic_on_overflow) panic("dma_map_area overflow %lu bytes\n", size); @@ -231,10 +232,10 @@ static dma_addr_t gart_map_phys(struct device *dev, p= hys_addr_t paddr, if (unlikely(attrs & DMA_ATTR_MMIO)) return DMA_MAPPING_ERROR; =20 - if (!need_iommu(dev, paddr, size)) + if (!need_iommu(dev, paddr, size, attrs)) return paddr; =20 - bus =3D dma_map_area(dev, paddr, size, dir, 0); + bus =3D dma_map_area(dev, paddr, size, dir, 0, attrs); flush_gart(); =20 return bus; @@ -289,7 +290,7 @@ static void gart_unmap_sg(struct device *dev, struct sc= atterlist *sg, int nents, =20 /* Fallback for dma_map_sg in case of overflow */ static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg, - int nents, int dir) + int nents, int dir, unsigned long attrs) { struct scatterlist *s; int i; @@ -301,8 +302,8 @@ static int dma_map_sg_nonforce(struct device *dev, stru= ct scatterlist *sg, for_each_sg(sg, s, nents, i) { unsigned long addr =3D sg_phys(s); =20 - if (nonforced_iommu(dev, addr, s->length)) { - addr =3D dma_map_area(dev, addr, s->length, dir, 0); + if (nonforced_iommu(dev, addr, s->length, attrs)) { + addr =3D dma_map_area(dev, addr, s->length, dir, 0, attrs); if (addr =3D=3D DMA_MAPPING_ERROR) { if (i > 0) gart_unmap_sg(dev, sg, i, dir, 0); @@ -401,7 +402,7 @@ static int gart_map_sg(struct device *dev, struct scatt= erlist *sg, int nents, s->dma_address =3D addr; BUG_ON(s->length =3D=3D 0); =20 - nextneed =3D need_iommu(dev, addr, s->length); + nextneed =3D need_iommu(dev, addr, s->length, attrs); =20 /* Handle the previous not yet processed entries */ if (i > start) { @@ -449,7 +450,7 @@ static int gart_map_sg(struct device *dev, struct scatt= erlist *sg, int nents, =20 /* When it was forced or merged try again in a dumb way */ if (force_iommu || iommu_merge) { - out =3D dma_map_sg_nonforce(dev, sg, nents, dir); + out =3D dma_map_sg_nonforce(dev, sg, nents, dir, attrs); if (out > 0) return out; } @@ -473,7 +474,8 @@ gart_alloc_coherent(struct device *dev, size_t size, dm= a_addr_t *dma_addr, return vaddr; =20 *dma_addr =3D dma_map_area(dev, virt_to_phys(vaddr), size, - DMA_BIDIRECTIONAL, (1UL << get_order(size)) - 1); + DMA_BIDIRECTIONAL, + (1UL << get_order(size)) - 1, attrs); flush_gart(); if (unlikely(*dma_addr =3D=3D DMA_MAPPING_ERROR)) goto out_free; diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 8c4abe65cd49..e2538824ef52 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -212,7 +212,7 @@ static dma_addr_t xen_swiotlb_map_phys(struct device *d= ev, phys_addr_t phys, BUG_ON(dir =3D=3D DMA_NONE); =20 if (attrs & DMA_ATTR_MMIO) { - if (unlikely(!dma_capable(dev, phys, size, false))) { + if (unlikely(!dma_capable(dev, phys, size, false, attrs))) { dev_err_once( dev, "DMA addr %pa+%zu overflow (mask %llx, bus limit %llx).\n", @@ -231,7 +231,7 @@ static dma_addr_t xen_swiotlb_map_phys(struct device *d= ev, phys_addr_t phys, * we can safely return the device addr and not worry about bounce * buffering it. */ - if (dma_capable(dev, dev_addr, size, true) && + if (dma_capable(dev, dev_addr, size, true, attrs) && !dma_kmalloc_needs_bounce(dev, size, dir) && !range_straddles_page_boundary(phys, size) && !xen_arch_need_swiotlb(dev, phys, dev_addr) && @@ -253,7 +253,7 @@ static dma_addr_t xen_swiotlb_map_phys(struct device *d= ev, phys_addr_t phys, /* * Ensure that the address returned is DMA'ble */ - if (unlikely(!dma_capable(dev, dev_addr, size, true))) { + if (unlikely(!dma_capable(dev, dev_addr, size, true, attrs))) { __swiotlb_tbl_unmap_single(dev, map, size, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC, swiotlb_find_pool(dev, map)); diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h index 94fad4e7c11e..daa31a1adf7b 100644 --- a/include/linux/dma-direct.h +++ b/include/linux/dma-direct.h @@ -135,12 +135,20 @@ static inline bool force_dma_unencrypted(struct devic= e *dev) #endif /* CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED */ =20 static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t= size, - bool is_ram) + bool is_ram, unsigned long attrs) { dma_addr_t end =3D addr + size - 1; =20 if (addr =3D=3D DMA_MAPPING_ERROR) return false; + /* + * The DMA address was derived from encrypted RAM, but this device + * requires unencrypted DMA addresses. Treat it as not DMA-capable + * so the caller can fall back to a suitable SWIOTLB pool. + */ + if (!(attrs & DMA_ATTR_CC_SHARED) && force_dma_unencrypted(dev)) + return false; + if (is_ram && !IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) && min(addr, end) < phys_to_dma(dev, PFN_PHYS(min_low_pfn))) return false; diff --git a/kernel/dma/direct.h b/kernel/dma/direct.h index 7140c208c123..e05dc7649366 100644 --- a/kernel/dma/direct.h +++ b/kernel/dma/direct.h @@ -101,15 +101,15 @@ static inline dma_addr_t dma_direct_map_phys(struct d= evice *dev, =20 if (attrs & DMA_ATTR_MMIO) { dma_addr =3D phys; - if (unlikely(!dma_capable(dev, dma_addr, size, false))) + if (unlikely(!dma_capable(dev, dma_addr, size, false, attrs))) goto err_overflow; } else if (attrs & DMA_ATTR_CC_SHARED) { dma_addr =3D phys_to_dma_unencrypted(dev, phys); - if (unlikely(!dma_capable(dev, dma_addr, size, false))) + if (unlikely(!dma_capable(dev, dma_addr, size, false, attrs))) goto err_overflow; } else { dma_addr =3D phys_to_dma(dev, phys); - if (unlikely(!dma_capable(dev, dma_addr, size, true)) || + if (unlikely(!dma_capable(dev, dma_addr, size, true, attrs)) || dma_kmalloc_needs_bounce(dev, size, dir)) { if (is_swiotlb_active(dev) && !(attrs & DMA_ATTR_REQUIRE_COHERENT)) diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 2bf3981db35d..f4e8b241a1c4 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -1678,7 +1678,7 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_= t paddr, size_t size, else dma_addr =3D phys_to_dma_encrypted(dev, swiotlb_addr); =20 - if (unlikely(!dma_capable(dev, dma_addr, size, true))) { + if (unlikely(!dma_capable(dev, dma_addr, size, true, attrs))) { __swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC, swiotlb_find_pool(dev, swiotlb_addr)); --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 15CA02367CF; Fri, 22 May 2026 04:30:31 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424234; cv=none; b=V2eQySuYwunD7ZmpbrT/R9Pc/+sEov1K/tdtLY9tz9RboZQTlW2hQAsFwghR+KinMmr0DjoT7SvhqmskE/prdE3u3leAi099fpas+5tn2kTeI/eGHvygMX/QhozPWR8dhUKkKwdjXBOFxc3WLCk1Z2HxKKwMzB1XREAUpljMDBg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424234; c=relaxed/simple; bh=RV9y6oCsHlefd5ymT05u5UfmofuMGtsK0RJ9r6weBKo=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=cdqSka8OXx65cWW/Kc302EwcSvrFHxAKB6Hli7d+d6dkBrveF7Le84l9IFaWTbDvhFWOiKRlBvh/kBiiz3Nj/qAE9p0VD08frxrYsbwguLcyIb8Od4Gemrn+gi5i23534udcTqOkrE9Rp8gZCF1Qnu6nam1aow7Rr+BbG/NhRZU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=KQjop4pc; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="KQjop4pc" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 5A8D31F000E9; Fri, 22 May 2026 04:30:21 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424231; bh=yz/sGqfqBvDBAnbbNxVoDgt2HJ5Km5y5+l03PeTOpg0=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=KQjop4pcYeqjmueQHL6P2mhQEsUyfDraldU4rC+JGeFegMHBg4OMorAFtxGgqLE2n Nohd/Gbt0+rx2uoWWj10KHilY7YqucmZZXGzveHLXyw1pBrFaggMKdiAY8yY1srMu7 ooV9OIFMhZg6QTxhWR8jcPB5g5pAf8mUStqNwT5js+Ayc1OzB/1yrylTUfQSgkMRM2 p4n0RF5xbk8KBT633zY2em/290EhYI35ytUCo0aSb5/6Q8gmtz7RWklKsRqauwayFA yi2WTuWw/ssg0IXD1Zp9r1cE0vlGja+2f5fRu9n4wwXO4NyCbQ4Wikc/CqNrobMtn/ h8OXSqaO7fCRw== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org, Jiri Pirko Subject: [PATCH v5 10/20] dma-direct: make dma_direct_map_phys() honor DMA_ATTR_CC_SHARED Date: Fri, 22 May 2026 09:58:05 +0530 Message-ID: <20260522042815.370873-11-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Teach dma_direct_map_phys() to select the DMA address encoding based on DMA_ATTR_CC_SHARED. Use phys_to_dma_unencrypted() for decrypted mappings and phys_to_dma_encrypted() otherwise. If a device requires unencrypted DMA but the source physical address is still encrypted, force the mapping through swiotlb so the DMA address and backing memory attributes remain consistent. Update the arm64, x86, s390 and powerpc secure-guest setup to not use swiotlb force option Tested-by: Jiri Pirko Signed-off-by: Aneesh Kumar K.V (Arm) --- Changes from v3: * Handle DMA_ATTR_MMIO --- arch/arm64/mm/init.c | 4 +-- arch/powerpc/platforms/pseries/svm.c | 2 +- arch/s390/mm/init.c | 2 +- arch/x86/kernel/pci-dma.c | 4 +-- kernel/dma/direct.c | 4 ++- kernel/dma/direct.h | 45 +++++++++++++++------------- 6 files changed, 31 insertions(+), 30 deletions(-) diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index c1b223e7cc8e..a087ac5b15f7 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -340,10 +340,8 @@ void __init arch_mm_preinit(void) unsigned int flags =3D SWIOTLB_VERBOSE; bool swiotlb =3D max_pfn > PFN_DOWN(arm64_dma_phys_limit); =20 - if (is_realm_world()) { + if (is_realm_world()) swiotlb =3D true; - flags |=3D SWIOTLB_FORCE; - } =20 if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && !swiotlb) { /* diff --git a/arch/powerpc/platforms/pseries/svm.c b/arch/powerpc/platforms/= pseries/svm.c index 384c9dc1899a..7a403dbd35ee 100644 --- a/arch/powerpc/platforms/pseries/svm.c +++ b/arch/powerpc/platforms/pseries/svm.c @@ -29,7 +29,7 @@ static int __init init_svm(void) * need to use the SWIOTLB buffer for DMA even if dma_capable() says * otherwise. */ - ppc_swiotlb_flags |=3D SWIOTLB_ANY | SWIOTLB_FORCE; + ppc_swiotlb_flags |=3D SWIOTLB_ANY; =20 /* Share the SWIOTLB buffer with the host. */ swiotlb_update_mem_attributes(); diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index ad3c6d92b801..581af1483c42 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -163,7 +163,7 @@ static void __init pv_init(void) virtio_set_mem_acc_cb(virtio_require_restricted_mem_acc); =20 /* make sure bounce buffers are shared */ - swiotlb_init(true, SWIOTLB_FORCE | SWIOTLB_VERBOSE); + swiotlb_init(true, SWIOTLB_VERBOSE); swiotlb_update_mem_attributes(); } =20 diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c index 6267363e0189..75cf8f6ae8cd 100644 --- a/arch/x86/kernel/pci-dma.c +++ b/arch/x86/kernel/pci-dma.c @@ -59,10 +59,8 @@ static void __init pci_swiotlb_detect(void) * bounce buffers as the hypervisor can't access arbitrary VM memory * that is not explicitly shared with it. */ - if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) { + if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) x86_swiotlb_enable =3D true; - x86_swiotlb_flags |=3D SWIOTLB_FORCE; - } } #else static inline void __init pci_swiotlb_detect(void) diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 429791b2599a..9e65c8432b6e 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -702,8 +702,10 @@ size_t dma_direct_max_mapping_size(struct device *dev) { /* If SWIOTLB is active, use its maximum mapping size */ if (is_swiotlb_active(dev) && - (dma_addressing_limited(dev) || is_swiotlb_force_bounce(dev))) + (dma_addressing_limited(dev) || is_swiotlb_force_bounce(dev) || + force_dma_unencrypted(dev))) return swiotlb_max_mapping_size(dev); + return SIZE_MAX; } =20 diff --git a/kernel/dma/direct.h b/kernel/dma/direct.h index e05dc7649366..f3fc28f352ba 100644 --- a/kernel/dma/direct.h +++ b/kernel/dma/direct.h @@ -88,37 +88,40 @@ static inline dma_addr_t dma_direct_map_phys(struct dev= ice *dev, { dma_addr_t dma_addr; =20 + /* + * For a device requiring unencrypted DMA, MMIO memory is treated + * as shared by default. + */ + if (force_dma_unencrypted(dev) && (attrs & DMA_ATTR_MMIO)) + attrs |=3D DMA_ATTR_CC_SHARED; + if (is_swiotlb_force_bounce(dev)) { - if (!(attrs & DMA_ATTR_CC_SHARED)) { - if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT)) - return DMA_MAPPING_ERROR; + if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT)) + return DMA_MAPPING_ERROR; =20 - return swiotlb_map(dev, phys, size, dir, attrs); - } - } else if (attrs & DMA_ATTR_CC_SHARED) { - return DMA_MAPPING_ERROR; + return swiotlb_map(dev, phys, size, dir, attrs); } =20 - if (attrs & DMA_ATTR_MMIO) { - dma_addr =3D phys; - if (unlikely(!dma_capable(dev, dma_addr, size, false, attrs))) - goto err_overflow; - } else if (attrs & DMA_ATTR_CC_SHARED) { + if (attrs & DMA_ATTR_CC_SHARED) dma_addr =3D phys_to_dma_unencrypted(dev, phys); + else + dma_addr =3D phys_to_dma_encrypted(dev, phys); + + if (attrs & DMA_ATTR_MMIO) { if (unlikely(!dma_capable(dev, dma_addr, size, false, attrs))) goto err_overflow; - } else { - dma_addr =3D phys_to_dma(dev, phys); - if (unlikely(!dma_capable(dev, dma_addr, size, true, attrs)) || - dma_kmalloc_needs_bounce(dev, size, dir)) { - if (is_swiotlb_active(dev) && - !(attrs & DMA_ATTR_REQUIRE_COHERENT)) - return swiotlb_map(dev, phys, size, dir, attrs); + goto dma_mapped; + } =20 - goto err_overflow; - } + if (unlikely(!dma_capable(dev, dma_addr, size, true, attrs)) || + dma_kmalloc_needs_bounce(dev, size, dir)) { + if (is_swiotlb_active(dev) && + !(attrs & DMA_ATTR_REQUIRE_COHERENT)) + return swiotlb_map(dev, phys, size, dir, attrs); + goto err_overflow; } =20 +dma_mapped: if (!dev_is_dma_coherent(dev) && !(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO))) { arch_sync_dma_for_device(phys, size, dir); --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 5819F1B6CE9; Fri, 22 May 2026 04:30:43 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424244; cv=none; b=A3NMcdfGHPjT54BeS//i+68VbHOzWMo2gXkIlz1PA0OvtGsc09cgQDVmSJc308pplUcJycugM1IPsgYxpmQxNWplnHIAbveix/9fgLRD96iohWEYuo5u34jCJLdW/3bIlE3ISt10Er6WGgVe7Q1obgM/waUAnpf6s/S1kgUIi5w= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424244; c=relaxed/simple; bh=2q0YAsBPsXATBZBZLW6qbEQxl/vpoDmCRFUm/ftc4pA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=bRupBJ3PVCYvFYr4j0tt4oo5vZmCiI1/Bo1PjgYWMl7scTtFMVlW+tdczIAo5Plts4Ky0+KbavNN/QJ/hslLwaS+osJkwzsVb5yxJ889cWfkH4lETTq3EbMdGM30OfynJ2OkLeJRYy3m2yJSa02kL0FaRLFgwOY4DjRNvMuQsMo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=BcStVCGY; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="BcStVCGY" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 81F821F00A3D; Fri, 22 May 2026 04:30:32 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424243; bh=UimiJKf1atAV++YvZ9iVVwY9ppNjQrEQsdOyX7IBAF0=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=BcStVCGY45gRT0Uy65Q9yDB9Q8vYJCtFILxW2DeH3ca+eqKeay+TMRJW+rcDLlchP uiUSexXj+f1vphGtExFKWaLuJ4lifT/f75SrSLaj6uFBJSRJvbpqkC+zDcEjg/Faci CzBAwdbHOrDkibhosN3DPR0XpLOnI0GQ3RPWD5a+XYn7/Frs42+iNqXHviHM0W3ACX 409hXav7LUA+ommHdAAeJZ8zSR3SWyZXjNHe2kHYzQGElrVWWRt/lz2jjHQAhviP2s gVGp512RqskgNglY48+z8v6IlWrObrVV6T6nHEMdrkb5U6eQHJoWLuaJfJDEDPHwH9 HvOmK5x6sr/7g== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org, Jiri Pirko Subject: [PATCH v5 11/20] dma-direct: set decrypted flag for remapped DMA allocations Date: Fri, 22 May 2026 09:58:06 +0530 Message-ID: <20260522042815.370873-12-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Devices that are DMA non-coherent and require a remap were skipping dma_set_decrypted(), leaving DMA buffers encrypted even when the device requires unencrypted access. Move the call after the if (remap) branch so that both the direct and remapped allocation paths correctly mark the allocation as decrypted (or fail cleanly) before use. Fix dma_direct_alloc() and dma_direct_free() to apply set_memory_*() to the linear-map alias of the backing pages instead of the remapped CPU address. Also disallow highmem pages for DMA_ATTR_CC_SHARED, because highmem buffers do not provide a usable linear-map address. Fixes: f3c962226dbe ("dma-direct: clean up the remapping checks in dma_dire= ct_alloc") Tested-by: Jiri Pirko Signed-off-by: Aneesh Kumar K.V (Arm) --- kernel/dma/direct.c | 55 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 9e65c8432b6e..d7eee77f95af 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -196,6 +196,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, { bool remap =3D false, set_uncached =3D false; bool mark_mem_decrypt =3D false; + bool allow_highmem =3D true; struct page *page; void *ret; =20 @@ -214,6 +215,15 @@ void *dma_direct_alloc(struct device *dev, size_t size, mark_mem_decrypt =3D true; } =20 + if (attrs & DMA_ATTR_CC_SHARED) + /* + * Unencrypted/shared DMA requires a linear-mapped buffer + * address to look up the PFN and set architecture-required PFN + * attributes. This is not possible with HighMem. Avoid HighMem + * allocation. + */ + allow_highmem =3D false; + size =3D PAGE_ALIGN(size); if (attrs & DMA_ATTR_NO_WARN) gfp |=3D __GFP_NOWARN; @@ -272,7 +282,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, } =20 /* we always manually zero the memory once we are done */ - page =3D __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true); + page =3D __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, allow_hig= hmem); if (!page) return NULL; =20 @@ -287,6 +297,14 @@ void *dma_direct_alloc(struct device *dev, size_t size, set_uncached =3D false; } =20 + if (mark_mem_decrypt) { + void *lm_addr; + + lm_addr =3D page_address(page); + if (set_memory_decrypted((unsigned long)lm_addr, PFN_UP(size))) + goto out_leak_pages; + } + if (remap) { pgprot_t prot =3D dma_pgprot(dev, PAGE_KERNEL, attrs); =20 @@ -297,29 +315,36 @@ void *dma_direct_alloc(struct device *dev, size_t siz= e, ret =3D dma_common_contiguous_remap(page, size, prot, __builtin_return_address(0)); if (!ret) - goto out_free_pages; + goto out_encrypt_pages; } else { ret =3D page_address(page); - if (mark_mem_decrypt && dma_set_decrypted(dev, ret, size)) - goto out_leak_pages; } =20 memset(ret, 0, size); =20 if (set_uncached) { + void *uncached_cpu_addr; + arch_dma_prep_coherent(page, size); - ret =3D arch_dma_set_uncached(ret, size); - if (IS_ERR(ret)) - goto out_encrypt_pages; + uncached_cpu_addr =3D arch_dma_set_uncached(ret, size); + if (IS_ERR(uncached_cpu_addr)) + goto out_free_remap_pages; + ret =3D uncached_cpu_addr; } =20 *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page)); return ret; =20 + +out_free_remap_pages: + if (remap) + dma_common_free_remap(ret, size); + out_encrypt_pages: - if (mark_mem_decrypt && dma_set_encrypted(dev, page_address(page), size)) - return NULL; -out_free_pages: + if (mark_mem_decrypt && + dma_set_encrypted(dev, page_address(page), size)) + goto out_leak_pages; + if (!swiotlb_free(dev, page, size)) dma_free_contiguous(dev, page, size); return NULL; @@ -384,8 +409,16 @@ void dma_direct_free(struct device *dev, size_t size, } else { if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED)) arch_dma_clear_uncached(cpu_addr, size); - if (mark_mem_encrypted && dma_set_encrypted(dev, cpu_addr, size)) + } + + if (mark_mem_encrypted) { + void *lm_addr; + + lm_addr =3D phys_to_virt(phys); + if (set_memory_encrypted((unsigned long)lm_addr, PFN_UP(size))) { + pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n"); return; + } } =20 if (swiotlb_pool) --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A877B1B6CE9; Fri, 22 May 2026 04:30:54 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424255; cv=none; b=NgGl/LFPPDIsEncR5J3D2rc1bsKj1g8H9tT50J6phxNL+QcDQZbKXAynt3YorkOriVo4/KmoBqg9TqJemWHnGnKngekJObzxEWG9qQzMSYSwH0ap5pDy/86rfgvao4LZQULWM7x1K2m+kNfVRNvmcr4Uw7XEFBUP+VUXRHQhU/I= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424255; c=relaxed/simple; bh=/8UQbJFSflm14frKSxWfIhVf5QX1/sjbU1+q7i87uEw=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=DU0SOwaR0Zt7Ic6PYgJr0Uh34hd95FzbohhXeK6iYgsiiY144GEDVQG/OL69IxDbozDWk8v3drN1pSjrY1rYW1BowMOCZ+1Yz29UcUSnwNFb3BV9+qUqpJlioa4EHWwkatpdp5EIio2ToiIYLZOuEFFj9k3nQiBC9iFNsiczeuQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=nVVwpu4U; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="nVVwpu4U" Received: by smtp.kernel.org (Postfix) with ESMTPSA id C36641F000E9; Fri, 22 May 2026 04:30:43 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424254; bh=mVHtjAJQlp0gV8B0Nq6WpIH3y5hTEG8y8BlmpuaiuX4=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=nVVwpu4U5NOqV4Ms01DgRonDxuS/cyHuEXXmiRp6ZPvczMe2Aq0PjseIgGYy1H7wl 6L6+eVeJOapMW4Ziz4RAKAPRXmc/ojvzypwHGhNXvOrZHoW//v+/uFdLphsT2FJ5ng c5nthmnYe0A68XIxH0E0z7Olac5Ld+nZHzihjKafuw0kvVBBlhgjDqyuyaBanC9WAl Me8X84fE1ushAOr06vo2BX2sEK5sO2EjJjOnlktKupeSgOYK8LUKwWvklf2fslffbt iWYSJ/68iPU83wUmNs+VE/eD0ImMJUiwAkKfY3FpXv6zLIfDPNRbi4NIgVEUe2BUpt iL4niyYWhRLVw== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org, Jiri Pirko Subject: [PATCH v5 12/20] dma-direct: select DMA address encoding from DMA_ATTR_CC_SHARED Date: Fri, 22 May 2026 09:58:07 +0530 Message-ID: <20260522042815.370873-13-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Make the dma-direct helpers derive the DMA address encoding from DMA_ATTR_CC_SHARED instead of implicitly relying on force_dma_unencrypted() inside phys_to_dma_direct() Pass an explicit unencrypted/decrypted state into phys_to_dma_direct(), make the alloc paths return DMA addresses that match the requested buffer encryption state. Also only call dma_set_decrypted() when DMA_ATTR_CC_SHARED is actually set. Tested-by: Jiri Pirko Signed-off-by: Aneesh Kumar K.V (Arm) --- kernel/dma/direct.c | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index d7eee77f95af..6f3aff8448a0 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -24,11 +24,11 @@ u64 zone_dma_limit __ro_after_init =3D DMA_BIT_MASK(24); =20 static inline dma_addr_t phys_to_dma_direct(struct device *dev, - phys_addr_t phys) + phys_addr_t phys, bool unencrypted) { - if (force_dma_unencrypted(dev)) + if (unencrypted) return phys_to_dma_unencrypted(dev, phys); - return phys_to_dma(dev, phys); + return phys_to_dma_encrypted(dev, phys); } =20 static inline struct page *dma_direct_to_page(struct device *dev, @@ -39,8 +39,9 @@ static inline struct page *dma_direct_to_page(struct devi= ce *dev, =20 u64 dma_direct_get_required_mask(struct device *dev) { + bool require_decrypted =3D force_dma_unencrypted(dev); phys_addr_t phys =3D (phys_addr_t)(max_pfn - 1) << PAGE_SHIFT; - u64 max_dma =3D phys_to_dma_direct(dev, phys); + u64 max_dma =3D phys_to_dma_direct(dev, phys, require_decrypted); =20 return (1ULL << (fls64(max_dma) - 1)) * 2 - 1; } @@ -69,7 +70,8 @@ static gfp_t dma_direct_optimal_gfp_mask(struct device *d= ev, u64 *phys_limit) =20 bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) { - dma_addr_t dma_addr =3D phys_to_dma_direct(dev, phys); + bool require_decrypted =3D force_dma_unencrypted(dev); + dma_addr_t dma_addr =3D phys_to_dma_direct(dev, phys, require_decrypted); =20 if (dma_addr =3D=3D DMA_MAPPING_ERROR) return false; @@ -79,17 +81,18 @@ bool dma_coherent_ok(struct device *dev, phys_addr_t ph= ys, size_t size) =20 static int dma_set_decrypted(struct device *dev, void *vaddr, size_t size) { - if (!force_dma_unencrypted(dev)) - return 0; - return set_memory_decrypted((unsigned long)vaddr, PFN_UP(size)); + int ret; + + ret =3D set_memory_decrypted((unsigned long)vaddr, PFN_UP(size)); + if (ret) + pr_warn_ratelimited("leaking DMA memory that can't be decrypted\n"); + return ret; } =20 static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size) { int ret; =20 - if (!force_dma_unencrypted(dev)) - return 0; ret =3D set_memory_encrypted((unsigned long)vaddr, PFN_UP(size)); if (ret) pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n"); @@ -169,7 +172,8 @@ static void *dma_direct_alloc_from_pool(struct device *= dev, size_t size, dma_coherent_ok); if (!page) return NULL; - *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page)); + *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page), + !!(attrs & DMA_ATTR_CC_SHARED)); return ret; } =20 @@ -185,9 +189,11 @@ static void *dma_direct_alloc_no_mapping(struct device= *dev, size_t size, /* remove any dirty cache lines on the kernel alias */ if (!PageHighMem(page)) arch_dma_prep_coherent(page, size); - - /* return the page pointer as the opaque cookie */ - *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page)); + /* + * return the page pointer as the opaque cookie. + * Never used for unencrypted allocation + */ + *dma_handle =3D phys_to_dma_encrypted(dev, page_to_phys(page)); return page; } =20 @@ -332,7 +338,8 @@ void *dma_direct_alloc(struct device *dev, size_t size, ret =3D uncached_cpu_addr; } =20 - *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page)); + *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page), + !!(attrs & DMA_ATTR_CC_SHARED)); return ret; =20 =20 @@ -455,11 +462,12 @@ struct page *dma_direct_alloc_pages(struct device *de= v, size_t size, return NULL; =20 ret =3D page_address(page); - if (dma_set_decrypted(dev, ret, size)) + if ((attrs & DMA_ATTR_CC_SHARED) && dma_set_decrypted(dev, ret, size)) goto out_leak_pages; setup_page: memset(ret, 0, size); - *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page)); + *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page), + !!(attrs & DMA_ATTR_CC_SHARED)); return page; out_leak_pages: return NULL; --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id BD0CA1A23A4; Fri, 22 May 2026 04:31:05 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424266; cv=none; b=RpqJCwrk5HBa4oQ7l4FpvWR+aqJwMy0tzLvT/KD8/2QoiWDrBLOLCszcYhe5REnWAB7jLKhZoAsz9eH/drM1MPAkpQXo1fEgxMa5/dBNLT3XqTFw5fh2ijCGbgaWv/Adp1GvAQYMTuHxGxftQOB3ZaUpRZPLCoUgT4paLaRCqA0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424266; c=relaxed/simple; bh=RWeuaTQuJCQ4oNrEuU7Wto5E5+vjyR3cNo5Y5QWJsgc=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=Wjz6JMYtR0FdwVkTsTl8QNDV96AnDfYB8nvSYZh2DbCGSvWluy4MdXY+o6pSPsWdaFLxFq4gR5kRWQEw/gxPUEdcs99NGeeh5nOnFNJofUPleJP5keSjgKaCCwLXf4lo2MTChqiBRbH7/0dqoRcS7lfb8XH/Wt9o96L9y8wGxAk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=QJqOdGXU; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="QJqOdGXU" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 295411F00A3D; Fri, 22 May 2026 04:30:54 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424265; bh=BRoOTjIXEIfZY33xB/HfCrHMNoIMuz43xHMqdbl/MXA=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=QJqOdGXUzoeK/2txik8hoWdiGyYw3ctMEtyPvywn3Wu8DCk3sMglax3QlyOPFYUVm 8R8/BKN2Pb5a9JjZcGIh2+8jAI+HZzKk1myOUuKKFuVs4/Xi969HK3/F0ktmoU8hwr vQazAEaw5n9TwQ9VvmkrjlczEPvakOpbQd3Z69nW22XSfcxfiRCFxQkyRiwzUbM56D Wrsqr4HHYb8mXz7NwrbgrDRNxbXy9s3hw+8PPcBENymcvkPLcF6Jwfzh3kTnal2jDl xMuN2HcG+gwdFOdUIPxeDkD4gKLcj/208Gaf84OvuNRB0uDd08BUx/z1Db+drjlkGm c2tylJmWR1M+A== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org Subject: [PATCH v5 13/20] dma-pool: fix page leak in atomic_pool_expand() cleanup Date: Fri, 22 May 2026 09:58:08 +0530 Message-ID: <20260522042815.370873-14-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" atomic_pool_expand() frees the allocated pages from the remove_mapping error path only when CONFIG_DMA_DIRECT_REMAP is enabled. When CONFIG_DMA_DIRECT_REMAP is disabled, failures after page allocation, such as gen_pool_add_virt(), jump to remove_mapping and return without freeing the pages. Move __free_pages(page, order) out of the CONFIG_DMA_DIRECT_REMAP block so that cleanup paths always release the allocation. Signed-off-by: Aneesh Kumar K.V (Arm) --- kernel/dma/pool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c index be78474a6c49..e7df8d279e75 100644 --- a/kernel/dma/pool.c +++ b/kernel/dma/pool.c @@ -146,9 +146,9 @@ static int atomic_pool_expand(struct dma_gen_pool *dma_= pool, size_t pool_size, #ifdef CONFIG_DMA_DIRECT_REMAP dma_common_free_remap(addr, pool_size); free_page: +#endif if (!leak_pages) __free_pages(page, order); -#endif out: return ret; } --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C326924E4C3; Fri, 22 May 2026 04:31:16 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424278; cv=none; b=aAwgyEVuSJDQi/CroRtAJ6Gb90bOGOSZ1uTw41RCvcCdNalS1MFgU3B6fDx/l5bWcHLtIhrEpv9g086dc8VdCoEeBkc1FVh8NNWP4xm4WsUiUIiM8foh2TAdgxqO6PcfV1lswIyNnuAKu1JQuJOR+5wcvVjm7b6pruGApHdHUmY= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424278; c=relaxed/simple; bh=hvWYfM6mSpmqOTdNzqFJw3aNpmDhkbBB/nMA1LjHLk4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=dGMZLuk6HCPwiauoQKA9yGHjQNxlY3RxZpTjPtz85EODIErgVa6O0Kua1MK1KwiweGhClXfsNxKDkIDPnpcXCNnVttsr6Ck8Iyeieg8toVLg5NyiW0ZVCgc3eiGRQucJavI44FLs+qSOghJxn8Gk3muPIr0oetoqlBGhkcT51XM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=gb+irCyV; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="gb+irCyV" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 043F61F00A3F; Fri, 22 May 2026 04:31:05 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424276; bh=9HbXDEd6lxpYbprgPYBPibGx8x7yNUBr6Iu1RKWnQ8w=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=gb+irCyV6FByu0mJApO1aqbN50d5xD/ZcdG6sC+zx1T2OAWmkFi5mhqMfz+O3/8cG tZpj382w2at6pX6+vufbwxg6y9SFI47xb2Q8ut+bnD8tZUpLZnKK4xFfWS8fa2V/bp sjVNc8uswQ1xNQh3iAjarG4AddOJRYB9lnbSuuKmFzbtzD4IiVCKjmd7hLfE/7jjSN IBKyGZNFBBOaO3SVuaMN4x3ilfUQucEI0jPEo1fgNq+zAIWGIx3lKipEzk2/1KtxEz J/n3ZZ2nJYMN051WwhyjXFINfaE9so4DZ7KECYQ3Bv+iHP/VkUcNw2pUale0bv4qjO RH+sNxwSucBJA== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org Subject: [PATCH v5 14/20] dma-direct: rename ret to cpu_addr in alloc helpers Date: Fri, 22 May 2026 09:58:09 +0530 Message-ID: <20260522042815.370873-15-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" ret in dma_direct_alloc() and dma_direct_alloc_pages() holds the returned CPU mapping, not a generic return value. Rename it to cpu_addr and update the remaining uses to match. This makes the allocation paths easier to follow and keeps the local naming consistent with what the variable actually represents. Signed-off-by: Aneesh Kumar K.V (Arm) --- kernel/dma/direct.c | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 6f3aff8448a0..6b00c7f4a78b 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -204,7 +204,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, bool mark_mem_decrypt =3D false; bool allow_highmem =3D true; struct page *page; - void *ret; + void *cpu_addr; =20 /* * DMA_ATTR_CC_SHARED is not a caller-visible dma_alloc_*() @@ -318,34 +318,33 @@ void *dma_direct_alloc(struct device *dev, size_t siz= e, arch_dma_prep_coherent(page, size); =20 /* create a coherent mapping */ - ret =3D dma_common_contiguous_remap(page, size, prot, - __builtin_return_address(0)); - if (!ret) + cpu_addr =3D dma_common_contiguous_remap(page, size, prot, + __builtin_return_address(0)); + if (!cpu_addr) goto out_encrypt_pages; } else { - ret =3D page_address(page); + cpu_addr =3D page_address(page); } =20 - memset(ret, 0, size); + memset(cpu_addr, 0, size); =20 if (set_uncached) { void *uncached_cpu_addr; =20 arch_dma_prep_coherent(page, size); - uncached_cpu_addr =3D arch_dma_set_uncached(ret, size); + uncached_cpu_addr =3D arch_dma_set_uncached(cpu_addr, size); if (IS_ERR(uncached_cpu_addr)) goto out_free_remap_pages; - ret =3D uncached_cpu_addr; + cpu_addr =3D uncached_cpu_addr; } =20 *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page), !!(attrs & DMA_ATTR_CC_SHARED)); - return ret; - + return cpu_addr; =20 out_free_remap_pages: if (remap) - dma_common_free_remap(ret, size); + dma_common_free_remap(cpu_addr, size); =20 out_encrypt_pages: if (mark_mem_decrypt && @@ -439,7 +438,7 @@ struct page *dma_direct_alloc_pages(struct device *dev,= size_t size, { unsigned long attrs =3D 0; struct page *page; - void *ret; + void *cpu_addr; =20 if (force_dma_unencrypted(dev)) attrs |=3D DMA_ATTR_CC_SHARED; @@ -453,7 +452,7 @@ struct page *dma_direct_alloc_pages(struct device *dev,= size_t size, if (!page) return NULL; =20 - ret =3D page_address(page); + cpu_addr =3D page_address(page); goto setup_page; } =20 @@ -461,11 +460,11 @@ struct page *dma_direct_alloc_pages(struct device *de= v, size_t size, if (!page) return NULL; =20 - ret =3D page_address(page); - if ((attrs & DMA_ATTR_CC_SHARED) && dma_set_decrypted(dev, ret, size)) + cpu_addr =3D page_address(page); + if ((attrs & DMA_ATTR_CC_SHARED) && dma_set_decrypted(dev, cpu_addr, size= )) goto out_leak_pages; setup_page: - memset(ret, 0, size); + memset(cpu_addr, 0, size); *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page), !!(attrs & DMA_ATTR_CC_SHARED)); return page; --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 085DB5FDA7; Fri, 22 May 2026 04:31:27 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424289; cv=none; b=GxUiUN6vA58wWBDsySMGsVdDG47hWjbkIKdhm2vifqmrg2uwpUF+2P1cUGZFzdcmco8L/ngykvuLi0VR+7CuyndGQa7iQAkHNNTAgkf2vveadyViCLqrXsY3irPzeZJx2OWjC5LPWkIZXJ5yptOh8idcG7OvWrwrjqay08oyE+Y= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424289; c=relaxed/simple; bh=dBmiagtseSJY/pXrUFuHOgZc6imP5v8LoLnFIWVWM/U=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=P3TG957kWXJH2Uulk0UEMc5d4qZDTXcVwceYqfLwV8HkHigVsygCl21zSdHzwNczVOHWSP1RAQsnIJz4aQqTjGluOJkTH7mPYREx9kccF9REAeFPir9w+l7S32NPe27iiJ9dGdp7zsmaBZkQvb2naXXbNwfs5SCdM6vrSrdnB5k= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=a6H9Hzej; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="a6H9Hzej" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 1386E1F000E9; Fri, 22 May 2026 04:31:16 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424287; bh=UPVhX+V6bjt+jEYXx2EEvMCCL/8FtnLC6xq+shkO+d4=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=a6H9HzejgTanmuvGyP7VJaa5Gx2QiHLNOK3GFUWd0ZZvC/lxq60uptNh63/1uLIDF r8wOVVv3MLgOfgtVaDvbHBKq/7aCkp+YcjtPbsHLYs4a4pvyalLKpsZmd4QVqtaVBo ztu2tfrIP3SjlJHE/nZDZaIUOFbCXMNTz7XX0a9hxs+SPMzy3P+vjSWis5FZbQpnkF 2B4rv2K5RaDKUOhKP4Q1Uiz5kc6EXjJNFu59BnZUUT3xzNpn6oTaNK9i8YJixThzJJ kGgq4gQihEdciMu0b4qIrYdc0g8hTxmYnnJfk3XnhnXtdq9j98LguDscHIfx4bYYHk qBbfoTpTTqZ1Q== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org, stable@vger.kernel.org Subject: [PATCH v5 15/20] dma-direct: return struct page from dma_direct_alloc_from_pool() Date: Fri, 22 May 2026 09:58:10 +0530 Message-ID: <20260522042815.370873-16-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Commit 5b138c534fda ("dma-direct: factor out a dma_direct_alloc_from_pool helper") changed dma_direct_alloc_from_pool() to return the CPU address from dma_alloc_from_pool(). That fits dma_direct_alloc(), but dma_direct_alloc_pages() also uses the helper and expects a struct page *. Fix this by making dma_direct_alloc_from_pool() return the struct page * again, and pass the CPU address back through an out-parameter for the dma_direct_alloc() caller. Fixes: 5b138c534fda ("dma-direct: factor out a dma_direct_alloc_from_pool h= elper") Cc: stable@vger.kernel.org Signed-off-by: Aneesh Kumar K.V (Arm) --- kernel/dma/direct.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 6b00c7f4a78b..907c6084c616 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -157,24 +157,24 @@ static bool dma_direct_use_pool(struct device *dev, g= fp_t gfp) return !gfpflags_allow_blocking(gfp) && !is_swiotlb_for_alloc(dev); } =20 -static void *dma_direct_alloc_from_pool(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) +static struct page *dma_direct_alloc_from_pool(struct device *dev, size_t = size, + dma_addr_t *dma_handle, void **cpu_addr, gfp_t gfp, + unsigned long attrs) { struct page *page; u64 phys_limit; - void *ret; =20 if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))) return NULL; =20 gfp |=3D dma_direct_optimal_gfp_mask(dev, &phys_limit); - page =3D dma_alloc_from_pool(dev, size, &ret, gfp, attrs, + page =3D dma_alloc_from_pool(dev, size, cpu_addr, gfp, attrs, dma_coherent_ok); if (!page) return NULL; *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page), !!(attrs & DMA_ATTR_CC_SHARED)); - return ret; + return page; } =20 static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size, @@ -270,9 +270,12 @@ void *dma_direct_alloc(struct device *dev, size_t size, * the atomic pools instead if we aren't allowed block. */ if ((remap || (attrs & DMA_ATTR_CC_SHARED)) && - dma_direct_use_pool(dev, gfp)) - return dma_direct_alloc_from_pool(dev, size, dma_handle, - gfp, attrs); + dma_direct_use_pool(dev, gfp)) { + page =3D dma_direct_alloc_from_pool(dev, size, + dma_handle, &cpu_addr, + gfp, attrs); + return page ? cpu_addr : NULL; + } =20 if (is_swiotlb_for_alloc(dev)) { page =3D dma_direct_alloc_swiotlb(dev, size, attrs); @@ -445,7 +448,7 @@ struct page *dma_direct_alloc_pages(struct device *dev,= size_t size, =20 if ((attrs & DMA_ATTR_CC_SHARED) && dma_direct_use_pool(dev, gfp)) return dma_direct_alloc_from_pool(dev, size, dma_handle, - gfp, attrs); + &cpu_addr, gfp, attrs); =20 if (is_swiotlb_for_alloc(dev)) { page =3D dma_direct_alloc_swiotlb(dev, size, attrs); --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B5BDC347C7; Fri, 22 May 2026 04:31:38 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424299; cv=none; b=mjnmr2rxy+S5MpjQkS5Kkcri0rSCWqWjYzP4ZeIDM0xJBXKnzNJPrqcz1LviV3xNepIJ3jz6+aDSrxLOCD/u6RvklR2qM4xCPRq+S6zV0axBNv6eKNVwgWWh+OaWV2j0PDgFU7FuwhB5oefSdVuyalK93YY/VKbY2CGX4Z5/R4s= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424299; c=relaxed/simple; bh=MOHjoD9Y04Fj2BUzGKub8PM2+Jom+FjSwfzhM/D0SaY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=NGdLTrh05CSV9Pfk/CbHjObEfhCXP9fK6q+FbB7aMxrtZBLaDbEarouAK5L90fu/56td+WaaDXLDY/JiHno8uw8bstHnD0HzQdk2OhJ4CnXAUZqoCj2URddGaujFie8dKXyYXQ+yt6Q7xbmSalyUmlGI7/2uUGW+yfiMy5s3DKo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=AnEzxNen; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="AnEzxNen" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 6CB661F00A3D; Fri, 22 May 2026 04:31:28 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424298; bh=eNxPV4X8+LJvSw+1YGCbEG9piVGDIt8Se6Gkeol8Ww0=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=AnEzxNenegVHPTY+nqLjqtzKZpEkvdGMmUqLl7CrAAfRurgXThzU7ToDaiLG38rB5 gB0hJ3EocpSYOc2M5i8i9Dp1Ui3+aFSDCE8zAYwD4S/cUQYmG1sYMCkfsxWRQy9kuZ ndgnDw20jy7R+n7mbHcpPHHqs2TxMcZGJkIod2vyMBmShf+bJjxOYp+DWQ/Et0EUM3 zoI9t8zIj/2zVJ0ozOizxl7i8+a+Qxr4J1ssSZOz+HA+DZfmqpWkefoRIKIZP/zISp rqA41qNbKiCFSBEkjyGCO7Si8F4XX/tHCby/WZearmmtasBok8wnPn+TaMvHw53sHB fCKm8mG+JiYmA== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org Subject: [PATCH v5 16/20] iommu/dma: Check atomic pool allocation result directly Date: Fri, 22 May 2026 09:58:11 +0530 Message-ID: <20260522042815.370873-17-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The non-blocking, non-coherent allocation path uses dma_alloc_from_pool(), which returns the allocated page and fills cpu_addr only on success. Do not rely on cpu_addr to detect allocation failure in this path. Check the returned page directly before using it for the IOMMU mapping. Fixes: 9420139f516d ("dma-pool: fix coherent pool allocations for IOMMU map= pings") Signed-off-by: Aneesh Kumar K.V (Arm) --- drivers/iommu/dma-iommu.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 725c7adb0a8d..52c599f4472c 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1671,13 +1671,16 @@ void *iommu_dma_alloc(struct device *dev, size_t si= ze, dma_addr_t *handle, } =20 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && - !gfpflags_allow_blocking(gfp) && !coherent) + !gfpflags_allow_blocking(gfp) && !coherent) { page =3D dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr, gfp, attrs, NULL); - else + if (!page) + return NULL; + } else { cpu_addr =3D iommu_dma_alloc_pages(dev, size, &page, gfp, attrs); - if (!cpu_addr) - return NULL; + if (!cpu_addr) + return NULL; + } =20 *handle =3D __iommu_dma_map(dev, page_to_phys(page), size, ioprot, dev->coherent_dma_mask); --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 0CCE7347C7; Fri, 22 May 2026 04:31:49 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424311; cv=none; b=KkfXxJCfykiBRi8esxuFQnoCUuMLCVrtNtKAKhvVq0yE6Rb/+LqN7xZli8wXja/CWSZVWGLtTAnrTe94+nrO9wjAloKNXcJlQoKzPie78R8dgz5JyVEyETGXSgYQTjRLvKIv6M0TiFfGGCQNrCTOTYOBjqHR+u6rbZgsZVS07B4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424311; c=relaxed/simple; bh=xr+kSch/ATQyq4vIzt/fxDyt5FaVoJkq1JcWBmHwCGM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=gHSnvwrcOXRLs/K0wRTuYxx+cHJ3nrRlFZHGLjn58c8BbV1gI27vkikHvD0SMD625QqBo8WgnpBbNIh9xBZVqEog5w/KGevUsIXu1TkxCoQN9/dwkJgM3SSdUaaxWQcuCtIfqiA/4UrSbJX/BpG1skDVUX+3IRj2p6Am6vuHBhQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=j/xCchvP; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="j/xCchvP" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 490C41F000E9; Fri, 22 May 2026 04:31:39 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424309; bh=dcmTXB2lgA7G/MmVc6Lgh462v2J6ii5qK8aSjHsJld8=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=j/xCchvPe4MfoXPQoR0SqW+SV/OSZQxiq1dm+OogpuMy4jIINMhXlPkIKXBsWNG4c chVAiBxy6kb/TGkLAS9BamKfkaYocokS55O+vAgutoYZIqTQcH4oXDc8Jwgmdtjma3 N1npDpLRGqks29WjG463px4akkaCHmNVVYHX+BHd3d9NsapQWMTAamL8ojxeT6jVWH m9y7yi/fPo6zoTg5kevEsUm+EVB6qCusnExZ6C+3mcLrAXFbYDNtuThai//rRZLtiK Ohv8iSA61InNcTuKe4hQ6rhYHHCEnX2We4GjLcNMOFfdLGd1+7zAwTnAZOW49XJhsy zu/mI96MORqvA== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org Subject: [PATCH v5 17/20] dma: swiotlb: free dynamic pools from process context Date: Fri, 22 May 2026 09:58:12 +0530 Message-ID: <20260522042815.370873-18-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" swiotlb_dyn_free() is used after removing a dynamic swiotlb pool from RCU-protected lists. It can call swiotlb_free_tlb(), which may need to restore the encryption state of an unencrypted pool with set_memory_encrypted() before freeing the pages. RCU callbacks run in atomic context, but set_memory_encrypted() is not guaranteed to be atomic-safe on all architectures. For example, page attribute updates may allocate page tables or take sleeping locks. Use queue_rcu_work() for dynamic pool freeing instead. This keeps the RCU grace period before freeing a published pool, while running the actual pool teardown from workqueue context. Use the same helper for the transient-pool error path, since that path may also be reached from atomic DMA mapping context. Signed-off-by: Aneesh Kumar K.V (Arm) --- include/linux/swiotlb.h | 4 ++-- kernel/dma/swiotlb.c | 19 +++++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 4dcbf3931be1..526f82e9da45 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -64,7 +64,7 @@ extern void __init swiotlb_update_mem_attributes(void); * @areas: Array of memory area descriptors. * @slots: Array of slot descriptors. * @node: Member of the IO TLB memory pool list. - * @rcu: RCU head for swiotlb_dyn_free(). + * @dyn_free: RCU work item used to free the pool from process context. * @transient: %true if transient memory pool. */ struct io_tlb_pool { @@ -79,7 +79,7 @@ struct io_tlb_pool { struct io_tlb_slot *slots; #ifdef CONFIG_SWIOTLB_DYNAMIC struct list_head node; - struct rcu_head rcu; + struct rcu_work dyn_free; bool transient; bool unencrypted; #endif diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index f4e8b241a1c4..4c56f64602ea 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -774,13 +774,10 @@ static void swiotlb_dyn_alloc(struct work_struct *wor= k) add_mem_pool(mem, pool); } =20 -/** - * swiotlb_dyn_free() - RCU callback to free a memory pool - * @rcu: RCU head in the corresponding struct io_tlb_pool. - */ -static void swiotlb_dyn_free(struct rcu_head *rcu) +static void swiotlb_dyn_free_work(struct work_struct *work) { - struct io_tlb_pool *pool =3D container_of(rcu, struct io_tlb_pool, rcu); + struct io_tlb_pool *pool =3D + container_of(to_rcu_work(work), struct io_tlb_pool, dyn_free); size_t slots_size =3D array_size(sizeof(*pool->slots), pool->nslabs); size_t tlb_size =3D pool->end - pool->start; =20 @@ -789,6 +786,12 @@ static void swiotlb_dyn_free(struct rcu_head *rcu) kfree(pool); } =20 +static void swiotlb_schedule_dyn_free(struct io_tlb_pool *pool) +{ + INIT_RCU_WORK(&pool->dyn_free, swiotlb_dyn_free_work); + queue_rcu_work(system_wq, &pool->dyn_free); +} + /** * __swiotlb_find_pool() - find the IO TLB pool for a physical address * @dev: Device which has mapped the DMA buffer. @@ -835,7 +838,7 @@ static void swiotlb_del_pool(struct device *dev, struct= io_tlb_pool *pool) list_del_rcu(&pool->node); spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags); =20 - call_rcu(&pool->rcu, swiotlb_dyn_free); + swiotlb_schedule_dyn_free(pool); } =20 #endif /* CONFIG_SWIOTLB_DYNAMIC */ @@ -1276,7 +1279,7 @@ static int swiotlb_find_slots(struct device *dev, phy= s_addr_t orig_addr, index =3D swiotlb_search_pool_area(dev, pool, 0, orig_addr, tbl_dma_addr, alloc_size, alloc_align_mask); if (index < 0) { - swiotlb_dyn_free(&pool->rcu); + swiotlb_schedule_dyn_free(pool); return -1; } =20 --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2AB94274FDF; Fri, 22 May 2026 04:32:00 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424322; cv=none; b=FoPqVNsTtF+sok4ufME1+KEf9WaEbQrZFPz4DWNStQ1AwkQm4/e+c+gF3+fPSz6IN4UJLrIHuGXjQDpI7WrSaUMiIdRwyhouAtRUe0VfviVCAp/taWjN2VAHFJR8xZRhLuzFFRqEHD7HstGG0es98C8ypXAGZh+uaxEJME3O6SI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424322; c=relaxed/simple; bh=1NMTpPgHDWWBijWAxwujrdNnKaXIGnJRKKqXSI9Y8oI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=EUWdDKQ7Rg/uz7lOkgmVBBwFmuFqN+1WZEptm4+FpVO5p1NkV5wgzGOmqoX4+jBA5A0KZEVk/QbtlYQ4yuBsbBM/oVlrRCsoOR5SrNPx9mu4bN3kh62eN/dy3LvZ90pRZT5RHyAKrr+oms4qBiSntZsNpmM1FmG1JqMTNw6JidQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=IyO1HZQd; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="IyO1HZQd" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 4EED21F00A3D; Fri, 22 May 2026 04:31:50 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424320; bh=84KfpthU62MYA6bQosKk2G8Mf+jCUzajJERZvfMNmbU=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=IyO1HZQdLFeS3TxuBt5U0k1v/N+HbmfY/kfiq8WqwRYFm70y9e7LyGeFJLx162/WC fYd6tjrh8yVM4DkYYoGefgymRiqQOS69uHjcJ6qO5Xe/Z80YMT0hUwwR6kB+1A0uaS c1+TRu8jmbrHLtMtOpJ6sqFWJclIHPd3N2wNpzHBbxGl8S1ohhBLsoLGcr5bwq3HhQ S8Gnx9d+083850dr8RRDCrrAXKuJDvQ5SJwTBLcyjYcoz8UxBvIn0gnaSpOgqmIywt HRUsRglJGSPxS1ReHOeTl6Zdwgqz4roG8/9myNGeynPs7DHi6gXntfKjqWemJrYQti Py20w+Z3IuQhQ== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org Subject: [PATCH v5 18/20] dma: swiotlb: handle set_memory_decrypted() failures Date: Fri, 22 May 2026 09:58:13 +0530 Message-ID: <20260522042815.370873-19-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Check the return value when converting swiotlb pools between encrypted and decrypted mappings. If the default pool cannot be decrypted after early initialization, mark the pool fully used so it cannot satisfy future bounce allocations. For late initialization, return the `set_memory_decrypted()` failure. For restricted DMA pools, fail device initialization if the reserved pool cannot be decrypted. This prevents swiotlb from using pools whose encryption attributes do not match their metadata, and avoids returning pages with uncertain encryption state back to the allocator. Signed-off-by: Aneesh Kumar K.V (Arm) --- kernel/dma/swiotlb.c | 80 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 65 insertions(+), 15 deletions(-) diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 4c56f64602ea..14d834ca298b 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -248,6 +248,23 @@ static inline unsigned long nr_slots(u64 val) return DIV_ROUND_UP(val, IO_TLB_SIZE); } =20 +static void swiotlb_mark_pool_used(struct io_tlb_pool *pool) +{ + unsigned long i; + + for (i =3D 0; i < pool->nareas; i++) { + pool->areas[i].index =3D 0; + pool->areas[i].used =3D pool->area_nslabs; + } + + for (i =3D 0; i < pool->nslabs; i++) { + pool->slots[i].list =3D 0; + pool->slots[i].orig_addr =3D INVALID_PHYS_ADDR; + pool->slots[i].alloc_size =3D 0; + pool->slots[i].pad_slots =3D 0; + } +} + /* * Early SWIOTLB allocation may be too early to allow an architecture to * perform the desired operations. This function allows the architecture = to @@ -272,8 +289,16 @@ void __init swiotlb_update_mem_attributes(void) return; bytes =3D PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT); =20 - if (io_tlb_default_mem.unencrypted) - set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT); + if (io_tlb_default_mem.unencrypted) { + int ret; + + ret =3D set_memory_decrypted((unsigned long)mem->vaddr, + bytes >> PAGE_SHIFT); + if (ret) { + pr_warn("Failed to decrypt default memory pool, disabling it\n"); + swiotlb_mark_pool_used(mem); + } + } } =20 static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t = start, @@ -442,9 +467,10 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask, { struct io_tlb_pool *mem =3D &io_tlb_default_mem.defpool; unsigned long nslabs =3D ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE); + unsigned int order, area_order, slot_order; + bool leak_pages =3D false; unsigned int nareas; unsigned char *vstart =3D NULL; - unsigned int order, area_order; bool retried =3D false; int rc =3D 0; =20 @@ -504,6 +530,7 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask, (PAGE_SIZE << order) >> 20); } =20 + rc =3D -ENOMEM; nareas =3D limit_nareas(default_nareas, nslabs); area_order =3D get_order(array_size(sizeof(*mem->areas), nareas)); mem->areas =3D (struct io_tlb_area *) @@ -511,14 +538,20 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask, if (!mem->areas) goto error_area; =20 + slot_order =3D get_order(array_size(sizeof(*mem->slots), nslabs)); mem->slots =3D (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - get_order(array_size(sizeof(*mem->slots), nslabs))); + slot_order); if (!mem->slots) goto error_slots; =20 - if (io_tlb_default_mem.unencrypted) - set_memory_decrypted((unsigned long)vstart, - (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT); + if (io_tlb_default_mem.unencrypted) { + rc =3D set_memory_decrypted((unsigned long)vstart, + (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT); + if (rc) { + leak_pages =3D true; + goto error_decrypt; + } + } =20 swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true, nareas); @@ -527,16 +560,20 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask, swiotlb_print_info(); return 0; =20 +error_decrypt: + free_pages((unsigned long)mem->slots, slot_order); error_slots: free_pages((unsigned long)mem->areas, area_order); error_area: - free_pages((unsigned long)vstart, order); - return -ENOMEM; + if (!leak_pages) + free_pages((unsigned long)vstart, order); + return rc; } =20 void __init swiotlb_exit(void) { struct io_tlb_pool *mem =3D &io_tlb_default_mem.defpool; + bool leak_pages =3D false; unsigned long tbl_vaddr; size_t tbl_size, slots_size; unsigned int area_order; @@ -552,19 +589,23 @@ void __init swiotlb_exit(void) tbl_size =3D PAGE_ALIGN(mem->end - mem->start); slots_size =3D PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs)); =20 - if (io_tlb_default_mem.unencrypted) - set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT); + if (io_tlb_default_mem.unencrypted) { + if (set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT)) + leak_pages =3D true; + } =20 if (mem->late_alloc) { area_order =3D get_order(array_size(sizeof(*mem->areas), mem->nareas)); free_pages((unsigned long)mem->areas, area_order); - free_pages(tbl_vaddr, get_order(tbl_size)); + if (!leak_pages) + free_pages(tbl_vaddr, get_order(tbl_size)); free_pages((unsigned long)mem->slots, get_order(slots_size)); } else { memblock_free(mem->areas, array_size(sizeof(*mem->areas), mem->nareas)); - memblock_phys_free(mem->start, tbl_size); + if (!leak_pages) + memblock_phys_free(mem->start, tbl_size); memblock_free(mem->slots, slots_size); } =20 @@ -1938,9 +1979,18 @@ static int rmem_swiotlb_device_init(struct reserved_= mem *rmem, * restricted mem pool is decrypted by default */ if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) { + int ret; + mem->unencrypted =3D true; - set_memory_decrypted((unsigned long)phys_to_virt(rmem->base), - rmem->size >> PAGE_SHIFT); + ret =3D set_memory_decrypted((unsigned long)phys_to_virt(rmem->base), + rmem->size >> PAGE_SHIFT); + if (ret) { + dev_err(dev, "Failed to decrypt restricted DMA pool\n"); + kfree(pool->areas); + kfree(pool->slots); + kfree(mem); + return ret; + } } else { mem->unencrypted =3D false; } --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 1F6E32D0617; Fri, 22 May 2026 04:32:12 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424333; cv=none; b=gmpZKruz4yheJYKvLrtU80SvsTT3BAciYqwlC9tgai3MHosqxDrU6E9x3i7Zvs9iPbeCJeeOZZXacp+CNoup8c6SR140Nk7PvHs3tkTnlSC+aka+Rda7byZ51DyGnQotty7N9X4dfGT0MWMQZsy5pBJmALGbWxfBBTpvsHpK8aE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424333; c=relaxed/simple; bh=Y1Mg9b9d9bIb709Ighmmiim5ypPgj3rWkPckj9AUtgs=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=gUts+xzxPH7KcfcrBIoNueGcSlUxIq1lzD3kHWCpViHZN9VbO4P3W0bPcEV5Rk4QYt2GmKjhLelqr989X5m2u/ogFgKeRcciVhk8IA0biVZ2V8fpiGLUxMLSPWrVkOxg+ftRh4rtnlSxOlcQqQvhFLCuzfHnTO7ccecVW4X0doQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=FP9J5cBM; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="FP9J5cBM" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 7FC141F000E9; Fri, 22 May 2026 04:32:01 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424332; bh=JY/iEQ1zR5tB3W7tvOEg18U1UPZnJP5nA9qclDJhKPc=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=FP9J5cBMtIrWF+YanfLJVtaal/L7De0g7NNTpiXBYsS3spBW3aKd/DrwT3ygheJCW /YjQagBXSYIQ2v3g7014Y5s6f2Ab4hJy19x6UcGGNgWnX8NCgZo/CmaMMZ5Y1ms06i kCPF5b37pZm+8cRgKLiWjp660ZTGDDyyHNSiRNcn9uzFTsXM8ukykKmrSnOyBckOvV YpcjVw6+BU47iM8oxd9+3xBNwjZsHoccsURGcqTaFAfAD0UudDFa8YuFiIV4dedX59 CfuuECchxVGDG+aPmlgg/Yu7u8rmQDsiMuPZ3WwI5iC2s9JERoajJ7bkP4n4ZHagT6 fNwVIfT1zfP+g== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org Subject: [PATCH v5 19/20] dma: free atomic pool pages by physical address Date: Fri, 22 May 2026 09:58:14 +0530 Message-ID: <20260522042815.370873-20-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" dma_direct_alloc_pages() may satisfy atomic allocations from the coherent atomic pools. The pool allocation is keyed by the virtual address stored in the gen_pool, but the pages API returns only the backing struct page. On architectures with CONFIG_DMA_DIRECT_REMAP, atomic pool chunks are added to the gen_pool using their remapped virtual address. dma_direct_free_pages() reconstructs a linear-map address with page_address(page) and passes that to dma_free_from_pool(). That address does not match the gen_pool virtual range, so the pool lookup can fail and the code can fall through to freeing a pool-owned page through the normal page allocator path. Add a page-based pool free helper that looks up the owning pool chunk by physical address, translates it back to the gen_pool virtual address, and frees that address to the pool. Use it from dma_direct_free_pages() while keeping the existing virtual-address helper for coherent allocation frees. Signed-off-by: Aneesh Kumar K.V (Arm) --- include/linux/dma-map-ops.h | 1 + kernel/dma/direct.c | 4 +-- kernel/dma/pool.c | 54 +++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 696b2c3a2305..8be059e69935 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -215,6 +215,7 @@ struct page *dma_alloc_from_pool(struct device *dev, si= ze_t size, void **cpu_addr, gfp_t flags, unsigned long attrs, bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t)); bool dma_free_from_pool(struct device *dev, void *start, size_t size); +bool dma_free_from_pool_page(struct device *dev, struct page *page, size_t= size); =20 int dma_direct_set_offset(struct device *dev, phys_addr_t cpu_start, dma_addr_t dma_start, u64 size); diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 907c6084c616..488d53ed21f3 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -488,9 +488,9 @@ void dma_direct_free_pages(struct device *dev, size_t s= ize, */ bool mark_mem_encrypted =3D force_dma_unencrypted(dev); =20 - /* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */ + /* If page is not from an atomic pool, dma_free_from_pool_page() fails */ if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) && - dma_free_from_pool(dev, vaddr, size)) + dma_free_from_pool_page(dev, page, size)) return; =20 phys =3D page_to_phys(page); diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c index e7df8d279e75..43b8101d860f 100644 --- a/kernel/dma/pool.c +++ b/kernel/dma/pool.c @@ -356,3 +356,57 @@ bool dma_free_from_pool(struct device *dev, void *star= t, size_t size) =20 return false; } + +struct dma_pool_phys_match { + phys_addr_t phys; + size_t size; + unsigned long addr; + bool found; +}; + +static void dma_pool_find_phys(struct gen_pool *pool, struct gen_pool_chun= k *chunk, + void *data) +{ + struct dma_pool_phys_match *match =3D data; + phys_addr_t end =3D match->phys + match->size - 1; + phys_addr_t chunk_end; + + if (match->found) + return; + + chunk_end =3D chunk->phys_addr + (chunk->end_addr - chunk->start_addr); + if (match->phys < chunk->phys_addr || end > chunk_end) + return; + + match->addr =3D chunk->start_addr + (match->phys - chunk->phys_addr); + match->found =3D true; +} + +static bool dma_free_from_pool_phys(struct dma_gen_pool *dma_pool, phys_ad= dr_t phys, + size_t size) +{ + struct dma_pool_phys_match match =3D { + .phys =3D phys, + .size =3D size, + }; + + gen_pool_for_each_chunk(dma_pool->pool, dma_pool_find_phys, &match); + if (!match.found) + return false; + + gen_pool_free(dma_pool->pool, match.addr, size); + return true; +} + +bool dma_free_from_pool_page(struct device *dev, struct page *page, size_t= size) +{ + struct dma_gen_pool *dma_pool =3D NULL; + phys_addr_t phys =3D page_to_phys(page); + + while ((dma_pool =3D dma_guess_pool(dma_pool, 0))) { + if (dma_free_from_pool_phys(dma_pool, phys, size)) + return true; + } + + return false; +} --=20 2.43.0 From nobody Sun May 24 20:33:05 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 379BE2367CF; Fri, 22 May 2026 04:32:23 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424344; cv=none; b=CGNdoxPVH5DfgnZ23DRUq3SSPniN4Nea7PfXhKC4WoGKvG6vNzSnX7T5nmNsVIsRAimflNV6pqStKXtW0aBInOKCnlToZVCZf/cqQyiSHXLGMzC5yFJmr0WauIK35Ueq8dbNLOF0TgbP3jkm5yYOEzh7echle+FAiK1I/LuWkjQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779424344; c=relaxed/simple; bh=7Db5F3WYzLzvHXC/08q2mn/xaHv85VelQsZvO4S1xKA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=GCYOkNp/qiv7fb3Qb/fOqSWt8dZR6qktDcPUzH5OHo3wPZIien4fCIKkgKgrWVlglwpA8X+JRos4fHE6HJzHRSaEuCCoB1O3H2UrJxgfyW5AL4gn/2pcA4/JyUGbz7ZsrXTvwe2B2+ZMWq06/ybQGAed7k3xNUd5/0EMVwG9qoE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=N1Lo4X9/; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="N1Lo4X9/" Received: by smtp.kernel.org (Postfix) with ESMTPSA id A6AC31F00A3D; Fri, 22 May 2026 04:32:12 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1779424343; bh=PnWTr5M4G1txOK8iaNSiBBsCv6diWB65cB7qFD+glq0=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=N1Lo4X9/X/2KPv5cK2+3UKHdUXj1cGz7UQ49YpR4OgSBfR0+o5/zbRv9Z7cltDb50 FwgTU/8k/2hexHUy4YbuJemEE6lSzfhi66i39msOSQ4Bz6GNyATpcOm1rMYcXf9OWR clSxELElIXK8J5eNJg8255eiDlYEt34uyOkGHxaOA8zTgaijkHvaF4GQFHLw9OI/+1 a3X51B2vggGH+Uo3Y7IFjEncwGqJZZBJcVWIhpJCeRKnOYZ/QJLuxl5i5wYmtzgx/h 1Av8uIqZ12flfvo+MlDQ6FpOex14s+fq2H0CWER71C8OgvcFFRmUhvSte73qn524+a UwGNG8l5DiBgA== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-coco@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Robin Murphy , Marek Szyprowski , Will Deacon , Marc Zyngier , Steven Price , Suzuki K Poulose , Catalin Marinas , Jiri Pirko , Jason Gunthorpe , Mostafa Saleh , Petr Tesarik , Alexey Kardashevskiy , Dan Williams , Xu Yilun , linuxppc-dev@lists.ozlabs.org, linux-s390@vger.kernel.org, Madhavan Srinivasan , Michael Ellerman , Nicholas Piggin , "Christophe Leroy (CS GROUP)" , Alexander Gordeev , Gerald Schaefer , Heiko Carstens , Vasily Gorbik , Christian Borntraeger , Sven Schnelle , x86@kernel.org Subject: [PATCH v5 20/20] swiotlb: Preserve allocation virtual address for dynamic pools Date: Fri, 22 May 2026 09:58:15 +0530 Message-ID: <20260522042815.370873-21-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org> References: <20260522042815.370873-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" swiotlb_alloc_tlb() can allocate from the DMA atomic pool when a decrypted pool is needed from atomic context. With CONFIG_DMA_DIRECT_REMAP, the atomic pool is backed by remapped virtual addresses, which are not the same as the direct-map addresses returned by phys_to_virt(). swiotlb_init_io_tlb_pool() currently reconstructs the pool virtual address from the physical start address. For atomic-pool backed allocations this stores the wrong address in pool->vaddr. Later, swiotlb_free_tlb() passes that address to dma_free_from_pool(), which will fail to recognize the chunk Pass the virtual address returned by the allocation path into swiotlb_init_io_tlb_pool(), and store that address in pool->vaddr. This keeps the pool free path using the same virtual address as the allocator. Signed-off-by: Aneesh Kumar K.V (Arm) --- kernel/dma/swiotlb.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 14d834ca298b..e4bd8c9eaeda 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -302,9 +302,9 @@ void __init swiotlb_update_mem_attributes(void) } =20 static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t = start, - unsigned long nslabs, bool late_alloc, unsigned int nareas) + void *vaddr, unsigned long nslabs, bool late_alloc, + unsigned int nareas) { - void *vaddr =3D phys_to_virt(start); unsigned long bytes =3D nslabs << IO_TLB_SHIFT, i; =20 mem->nslabs =3D nslabs; @@ -445,7 +445,7 @@ void __init swiotlb_init_remap(bool addressing_limit, u= nsigned int flags, return; } =20 - swiotlb_init_io_tlb_pool(mem, __pa(tlb), nslabs, false, nareas); + swiotlb_init_io_tlb_pool(mem, __pa(tlb), tlb, nslabs, false, nareas); add_mem_pool(&io_tlb_default_mem, mem); =20 if (flags & SWIOTLB_VERBOSE) @@ -553,7 +553,7 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask, } } =20 - swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true, + swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), vstart, nslabs, true, nareas); add_mem_pool(&io_tlb_default_mem, mem); =20 @@ -664,25 +664,26 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t= bytes, * @phys_limit: Maximum allowed physical address of the buffer. * @attrs: DMA attributes for the allocation. * @gfp: GFP flags for the allocation. + * @vaddr: Receives the virtual address for the allocated buffer. * * Return: Allocated pages, or %NULL on allocation failure. */ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes, - u64 phys_limit, unsigned long attrs, gfp_t gfp) + u64 phys_limit, unsigned long attrs, gfp_t gfp, void **vaddr) { struct page *page; =20 + *vaddr =3D NULL; + /* * Allocate from the atomic pools if memory is encrypted and * the allocation is atomic, because decrypting may block. */ if (!gfpflags_allow_blocking(gfp) && (attrs & DMA_ATTR_CC_SHARED)) { - void *vaddr; - if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL)) return NULL; =20 - return dma_alloc_from_pool(dev, bytes, &vaddr, gfp, + return dma_alloc_from_pool(dev, bytes, vaddr, gfp, attrs, dma_coherent_ok); } =20 @@ -705,6 +706,8 @@ static struct page *swiotlb_alloc_tlb(struct device *de= v, size_t bytes, return NULL; } =20 + if (page) + *vaddr =3D phys_to_virt(page_to_phys(page)); return page; } =20 @@ -750,6 +753,7 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct de= vice *dev, { struct io_tlb_pool *pool; unsigned int slot_order; + void *tlb_vaddr; struct page *tlb; size_t pool_size; size_t tlb_size; @@ -767,7 +771,8 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct de= vice *dev, pool->unencrypted =3D !!(attrs & DMA_ATTR_CC_SHARED); =20 tlb_size =3D nslabs << IO_TLB_SHIFT; - while (!(tlb =3D swiotlb_alloc_tlb(dev, tlb_size, phys_limit, attrs, gfp)= )) { + while (!(tlb =3D swiotlb_alloc_tlb(dev, tlb_size, phys_limit, attrs, gfp, + &tlb_vaddr))) { if (nslabs <=3D minslabs) goto error_tlb; nslabs =3D ALIGN(nslabs >> 1, IO_TLB_SEGSIZE); @@ -781,12 +786,12 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct = device *dev, if (!pool->slots) goto error_slots; =20 - swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), nslabs, true, nareas); + swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), tlb_vaddr, nslabs, + true, nareas); return pool; =20 error_slots: - swiotlb_free_tlb(page_address(tlb), tlb_size, - !!(attrs & DMA_ATTR_CC_SHARED)); + swiotlb_free_tlb(tlb_vaddr, tlb_size, !!(attrs & DMA_ATTR_CC_SHARED)); error_tlb: kfree(pool); error: @@ -1995,7 +2000,8 @@ static int rmem_swiotlb_device_init(struct reserved_m= em *rmem, mem->unencrypted =3D false; } =20 - swiotlb_init_io_tlb_pool(pool, rmem->base, nslabs, + swiotlb_init_io_tlb_pool(pool, rmem->base, phys_to_virt(rmem->base), + nslabs, false, nareas); mem->force_bounce =3D true; mem->for_alloc =3D true; --=20 2.43.0