From nobody Tue Jun 16 07:26:32 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id DB5D835F8A8; Fri, 17 Apr 2026 08:59:49 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776416389; cv=none; b=hk1jhx1hlBEEa0Cs5avHdnwQjhV4y3Tg2NvosLejhxFbqAHjV8iZl9xMM9D2AGt7WETWqKeNkoB2+qT/oM4xhFKRDg8C7rdig9nBGLSqYvU8qhcwGAtTi/THt7dL9QNzmhre0ckoY6Ho1C9uPwyAjqkJIRZ2IhsSS64Row5yu2U= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776416389; c=relaxed/simple; bh=SMTPZxBrGWET40HHUPAfjE8Aok3FvJ1waxDLjxRSjT4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=JkPjiTbfgkN0M7vadzDZJEZX4LNxdWYLmM8tQiM28wb9jZ9H3CcaOsfHkYZTIDC5zp5XnvXo3knY05yCdSwfsQwWbifEFDBmcQSCGDWXYxL1j1y1zSMG10Dw/ovkbiv7wZhcKCrYOK12zAlyo5vREhoAh9QN0yqD8Z8mFQRnedU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=RqGx5LZy; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="RqGx5LZy" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 73FBCC19425; Fri, 17 Apr 2026 08:59:45 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1776416389; bh=SMTPZxBrGWET40HHUPAfjE8Aok3FvJ1waxDLjxRSjT4=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=RqGx5LZy0P1djSqLPmUXT25WtLypdS74y16FcGTuMiPVXAiugQzw+25VmsigfPqVF tnG7LQ+e5e/ULL0JZJ4es5J+soRIsgLMQeo/hhbaGYSt1aArW8TvqGdSzcrcLoXLtP F6yAUOJnlphTjbnAaLS1Ra2+Zendlt7fSXp4PWTHr/CIVDad3Hq8LsvYMgcgdtTxvi /s83KE6gFVV7s7vutmgtkOeZjpyy2FEDI9AVrHD5aCDAMM2XxqBv0AGYzmtPomD4Rq lB1fDtdLzy/Z8HJVZsjl+Bq0HWfLMsg5E7u7PJoTHS7zLBrbqUjqL9arCWD+E1BTIf 6sl27qlyUQdFw== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-kernel@vger.kernel.org Cc: robin.murphy@arm.com, m.szyprowski@samsung.com, will@kernel.org, maz@kernel.org, suzuki.poulose@arm.com, catalin.marinas@arm.com, jiri@resnulli.us, jgg@ziepe.ca, aneesh.kumar@kernel.org, Mostafa Saleh , Thomas Gleixner , Steven Price Subject: [RFC PATCH 1/7] dma-direct: swiotlb: handle swiotlb alloc/free outside __dma_direct_alloc_pages Date: Fri, 17 Apr 2026 14:28:54 +0530 Message-ID: <20260417085900.3062416-2-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260417085900.3062416-1-aneesh.kumar@kernel.org> References: <20260417085900.3062416-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Move swiotlb allocation out of __dma_direct_alloc_pages() and handle it in dma_direct_alloc() / dma_direct_alloc_pages(). This is needed for follow-up changes that align shared decrypted buffers to hypervisor page size. swiotlb pool memory is decrypted as a whole and does not need per-allocation alignment handling. swiotlb backing pages are already mapped decrypted by swiotlb_update_mem_attributes() and rmem_swiotlb_device_init(), so dma-direct should not call dma_set_decrypted() on allocation nor dma_set_encrypted() on free for swiotlb-backed memory. Update alloc/free paths to detect swiotlb-backed pages and skip encrypt/decrypt transitions for those paths. Keep the existing highmem rejection in dma_direct_alloc_pages() for swiotlb allocations. Only for "restricted-dma-pool", we currently set `for_alloc =3D true`, while rmem_swiotlb_device_init() decrypts the whole pool up front. This pool is typically used together with "shared-dma-pool", where the shared region is accessed after remap/ioremap and the returned address is suitable for decrypted memory access. So existing code paths remain valid. Cc: Marc Zyngier Cc: Thomas Gleixner Cc: Catalin Marinas Cc: Will Deacon Cc: Jason Gunthorpe Cc: Marek Szyprowski Cc: Robin Murphy Cc: Steven Price Cc: Suzuki K Poulose Signed-off-by: Aneesh Kumar K.V (Arm) --- kernel/dma/direct.c | 44 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 8f43a930716d..c2a43e4ef902 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -125,9 +125,6 @@ static struct page *__dma_direct_alloc_pages(struct dev= ice *dev, size_t size, =20 WARN_ON_ONCE(!PAGE_ALIGNED(size)); =20 - if (is_swiotlb_for_alloc(dev)) - return dma_direct_alloc_swiotlb(dev, size); - gfp |=3D dma_direct_optimal_gfp_mask(dev, &phys_limit); page =3D dma_alloc_contiguous(dev, size, gfp); if (page) { @@ -204,6 +201,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { bool remap =3D false, set_uncached =3D false; + bool mark_mem_decrypt =3D true; struct page *page; void *ret; =20 @@ -250,11 +248,21 @@ void *dma_direct_alloc(struct device *dev, size_t siz= e, dma_direct_use_pool(dev, gfp)) return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); =20 + if (is_swiotlb_for_alloc(dev)) { + page =3D dma_direct_alloc_swiotlb(dev, size); + if (page) { + mark_mem_decrypt =3D false; + goto setup_page; + } + return NULL; + } + /* we always manually zero the memory once we are done */ page =3D __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true); if (!page) return NULL; =20 +setup_page: /* * dma_alloc_contiguous can return highmem pages depending on a * combination the cma=3D arguments and per-arch setup. These need to be @@ -281,7 +289,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, goto out_free_pages; } else { ret =3D page_address(page); - if (dma_set_decrypted(dev, ret, size)) + if (mark_mem_decrypt && dma_set_decrypted(dev, ret, size)) goto out_leak_pages; } =20 @@ -298,7 +306,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, return ret; =20 out_encrypt_pages: - if (dma_set_encrypted(dev, page_address(page), size)) + if (mark_mem_decrypt && dma_set_encrypted(dev, page_address(page), size)) return NULL; out_free_pages: __dma_direct_free_pages(dev, page, size); @@ -310,6 +318,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) { + bool mark_mem_encrypted =3D true; unsigned int page_order =3D get_order(size); =20 if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) && @@ -338,12 +347,15 @@ void dma_direct_free(struct device *dev, size_t size, dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size))) return; =20 + if (swiotlb_find_pool(dev, dma_to_phys(dev, dma_addr))) + mark_mem_encrypted =3D false; + if (is_vmalloc_addr(cpu_addr)) { vunmap(cpu_addr); } else { if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED)) arch_dma_clear_uncached(cpu_addr, size); - if (dma_set_encrypted(dev, cpu_addr, size)) + if (mark_mem_encrypted && dma_set_encrypted(dev, cpu_addr, size)) return; } =20 @@ -359,6 +371,19 @@ struct page *dma_direct_alloc_pages(struct device *dev= , size_t size, if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp)) return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); =20 + if (is_swiotlb_for_alloc(dev)) { + page =3D dma_direct_alloc_swiotlb(dev, size); + if (!page) + return NULL; + + if (PageHighMem(page)) { + swiotlb_free(dev, page, size); + return NULL; + } + ret =3D page_address(page); + goto setup_page; + } + page =3D __dma_direct_alloc_pages(dev, size, gfp, false); if (!page) return NULL; @@ -366,6 +391,7 @@ struct page *dma_direct_alloc_pages(struct device *dev,= size_t size, ret =3D page_address(page); if (dma_set_decrypted(dev, ret, size)) goto out_leak_pages; +setup_page: memset(ret, 0, size); *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page)); return page; @@ -378,13 +404,17 @@ void dma_direct_free_pages(struct device *dev, size_t= size, enum dma_data_direction dir) { void *vaddr =3D page_address(page); + bool mark_mem_encrypted =3D true; =20 /* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */ if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) && dma_free_from_pool(dev, vaddr, size)) return; =20 - if (dma_set_encrypted(dev, vaddr, size)) + if (swiotlb_find_pool(dev, page_to_phys(page))) + mark_mem_encrypted =3D false; + + if (mark_mem_encrypted && dma_set_encrypted(dev, vaddr, size)) return; __dma_direct_free_pages(dev, page, size); } --=20 2.43.0 From nobody Tue Jun 16 07:26:32 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B46823502AA; Fri, 17 Apr 2026 08:59:53 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776416393; cv=none; b=CUcfWRsrmqa8TPJIsRFmACri6jzfOyKb0tQaNymd6elU47DD07/+Z1ngQ9tt4EbyppKFMiBZLbI/uzJhY/4GcGmPWT8oK6vM/mT9+daKeGX2OadN1Mhcf5SJtzRsN+EDDYRGkJDPARUQ7vC8HSWZfrtxR4g8cTy0BQqWnmDe+h8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776416393; c=relaxed/simple; bh=NPbjsIwD8g92JzX7wavhGqNdM/uKMVtRfW7D98XYiN8=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=ns3XYfRA3Kuo6A4x0hEwjKtMuz61dJG3ohaPDl7gLtL6nNWfTFdAePE3zLlhMnmh7KkW3N3GCquzl3rlrw4Q2X9jrGyn/50cfNVgC+8xDtEwe+puZJPj3ovIolNBKl8k79EYm3hTXIOvtCK0hGnxUPZD9x+qaa9rA7FWoMGtMlk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Epe/MGSF; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Epe/MGSF" Received: by smtp.kernel.org (Postfix) with ESMTPSA id EA0B8C2BCB4; Fri, 17 Apr 2026 08:59:49 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1776416393; bh=NPbjsIwD8g92JzX7wavhGqNdM/uKMVtRfW7D98XYiN8=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Epe/MGSFe4LhtiLh5F3B0DyNUxwgoFjUeHAdnR/a0AoFxWN+qUWy3/H6QmDht6TZP tszrpsItkyMs2DEF8uExjRyBo4rcFTWfYtNIJP9s6ZTgqw74cyMETg+jQhwa/5KaR+ iOQ7DQZCGZ8TrFc2T8JoISKW2SCfIuWkXHwDB+pyIy82Wa9SuS5viORnWFgfsb/VWr V//1WojvoQJX5VSxF2j3zCt/9b9KVC1EIy6Xnv3vdaIvGYZjqLFeWqTzsYPwODmzWu 57gHWWOAB0JQ8tGpW/nnAvrcyqWZBHrGVGsIJtmMTeJP+TOgCfzuj5nDWvCbRIfIzK 1f1+GYdUG3S2w== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-kernel@vger.kernel.org Cc: robin.murphy@arm.com, m.szyprowski@samsung.com, will@kernel.org, maz@kernel.org, suzuki.poulose@arm.com, catalin.marinas@arm.com, jiri@resnulli.us, jgg@ziepe.ca, aneesh.kumar@kernel.org, Mostafa Saleh Subject: [RFC PATCH 2/7] dma-direct: use DMA_ATTR_CC_DECRYPTED in alloc/free paths Date: Fri, 17 Apr 2026 14:28:55 +0530 Message-ID: <20260417085900.3062416-3-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260417085900.3062416-1-aneesh.kumar@kernel.org> References: <20260417085900.3062416-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Propagate force_dma_unencrypted() into DMA_ATTR_CC_DECRYPTED in the dma-direct allocation path and use the attribute to drive the related decisions. This updates dma_direct_alloc(), dma_direct_free(), and dma_direct_alloc_pages() to fold the forced unencrypted case into attrs. Signed-off-by: Aneesh Kumar K.V (Arm) --- kernel/dma/direct.c | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index c2a43e4ef902..3932033f4d8c 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -201,16 +201,21 @@ void *dma_direct_alloc(struct device *dev, size_t siz= e, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { bool remap =3D false, set_uncached =3D false; - bool mark_mem_decrypt =3D true; + bool mark_mem_decrypt =3D !!(attrs & DMA_ATTR_CC_DECRYPTED); struct page *page; void *ret; =20 + if (force_dma_unencrypted(dev)) { + attrs |=3D DMA_ATTR_CC_DECRYPTED; + mark_mem_decrypt =3D true; + } + size =3D PAGE_ALIGN(size); if (attrs & DMA_ATTR_NO_WARN) gfp |=3D __GFP_NOWARN; =20 - if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) && - !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) + if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_CC_DECRYPTED)) =3D= =3D + DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp); =20 if (!dev_is_dma_coherent(dev)) { @@ -244,7 +249,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, * Remapping or decrypting memory may block, allocate the memory from * the atomic pools instead if we aren't allowed block. */ - if ((remap || force_dma_unencrypted(dev)) && + if ((remap || (attrs & DMA_ATTR_CC_DECRYPTED)) && dma_direct_use_pool(dev, gfp)) return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); =20 @@ -318,11 +323,20 @@ void *dma_direct_alloc(struct device *dev, size_t siz= e, void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) { - bool mark_mem_encrypted =3D true; + bool mark_mem_encrypted =3D !!(attrs & DMA_ATTR_CC_DECRYPTED); unsigned int page_order =3D get_order(size); =20 - if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) && - !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) { + /* + * if the device had requested for an unencrypted buffer, + * convert it to encrypted on free + */ + if (force_dma_unencrypted(dev)) { + attrs |=3D DMA_ATTR_CC_DECRYPTED; + mark_mem_encrypted =3D true; + } + + if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_CC_DECRYPTED)) =3D= =3D + DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) { /* cpu_addr is a struct page cookie, not a kernel address */ dma_free_contiguous(dev, cpu_addr, size); return; @@ -365,10 +379,14 @@ void dma_direct_free(struct device *dev, size_t size, struct page *dma_direct_alloc_pages(struct device *dev, size_t size, dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) { + unsigned long attrs =3D 0; struct page *page; void *ret; =20 - if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp)) + if (force_dma_unencrypted(dev)) + attrs |=3D DMA_ATTR_CC_DECRYPTED; + + if ((attrs & DMA_ATTR_CC_DECRYPTED) && dma_direct_use_pool(dev, gfp)) return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); =20 if (is_swiotlb_for_alloc(dev)) { --=20 2.43.0 From nobody Tue Jun 16 07:26:32 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D464E270EDF; Fri, 17 Apr 2026 08:59:57 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776416397; cv=none; b=qnqEB5rbmPRDA5Yms1Rmz3QehxZZgqusm3adSVSkB2G4dHKHOZochFvCAD1injQNqCpjDeMGQOl+tFouN/LgWlAf9+88cIrvNMG3kLbmqQbb+4nZYJgAjnlyh6dgqvaUmmoB8cOaxdh6uscwGoBTT5v4DwS50MTuHUez5v12tVI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776416397; c=relaxed/simple; bh=Im5Egu+28aJChtCEne+SeNAV/nf1ZLputmBfj4oRyVc=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=ny1gswm3k1QtcvDOKSocXLxDeLD40Z0afvLi7CNNWNtVW4GTdaSiKLLJ0aqnwZnmCxfxoSumqE4pDbGnlF6CGLqxQ6PejjHe++OiXnDqAuzJHeBZ/8k6ntq6gV32LiWmHKGJ6E346xPphlCKvozteZLGWNs2FRs3dHMsruk0IxU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=OSqDYucR; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="OSqDYucR" Received: by smtp.kernel.org (Postfix) with ESMTPSA id E8B1CC19425; Fri, 17 Apr 2026 08:59:53 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1776416397; bh=Im5Egu+28aJChtCEne+SeNAV/nf1ZLputmBfj4oRyVc=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=OSqDYucRSBjMSaiamJTfgg6YeVGGug8B2EHSGC5n16qIja6KcVoNPvNmvLEOxhLmA QU5cUPX9k/X9m1razPAWASI0RBNrqkuXxDRVKSnu5OB2UsMA94qvhLZMWZnWGoidzl 8QexDIwI/5Yc8gkIKJ8pApveO4AB9wAxcjtqIx/f8yVn1bpVpcH4OqsEHtAnVEOCF2 Rkf29kNznpLyqKv2rx6tDJ2JFlGZfIdzGM9O4PXD9XJRcH92VpkkgAU0bfDiLQasSW 4fwHyoFSZj//Ai/urvIVEzu5gZDiK1Q3jRWnBXc9Lu6uXBSWJmGKCRejku5WL0/oWJ B0XVW+Tk5Rg6A== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-kernel@vger.kernel.org Cc: robin.murphy@arm.com, m.szyprowski@samsung.com, will@kernel.org, maz@kernel.org, suzuki.poulose@arm.com, catalin.marinas@arm.com, jiri@resnulli.us, jgg@ziepe.ca, aneesh.kumar@kernel.org, Mostafa Saleh Subject: [RFC PATCH 3/7] dma-pool: track decrypted atomic pools and select them via attrs Date: Fri, 17 Apr 2026 14:28:56 +0530 Message-ID: <20260417085900.3062416-4-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260417085900.3062416-1-aneesh.kumar@kernel.org> References: <20260417085900.3062416-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Teach the atomic DMA pool code to distinguish between encrypted and decrypted pools, and make pool allocation select the matching pool based on DMA attributes. Introduce a dma_gen_pool wrapper that records whether a pool is decrypted, initialize that state when the atomic pools are created, and use it when expanding and resizing the pools. Update dma_alloc_from_pool() to take attrs and skip pools whose encrypted/decrypted state does not match DMA_ATTR_CC_DECRYPTED. Update dma_free_from_pool() accordingly. Also pass DMA_ATTR_CC_DECRYPTED from the swiotlb atomic allocation path so decrypted swiotlb allocations are taken from the correct atomic pool. Signed-off-by: Aneesh Kumar K.V (Arm) --- drivers/iommu/dma-iommu.c | 2 +- include/linux/dma-map-ops.h | 2 +- kernel/dma/direct.c | 11 ++- kernel/dma/pool.c | 154 +++++++++++++++++++++++------------- kernel/dma/swiotlb.c | 2 + 5 files changed, 109 insertions(+), 62 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 94d514169642..ddd5dd244a86 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1650,7 +1650,7 @@ void *iommu_dma_alloc(struct device *dev, size_t size= , dma_addr_t *handle, if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && !gfpflags_allow_blocking(gfp) && !coherent) page =3D dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr, - gfp, NULL); + gfp, attrs, NULL); else cpu_addr =3D iommu_dma_alloc_pages(dev, size, &page, gfp, attrs); if (!cpu_addr) diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 60b63756df82..72bc28c0390e 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -217,7 +217,7 @@ void *dma_common_pages_remap(struct page **pages, size_= t size, pgprot_t prot, void dma_common_free_remap(void *cpu_addr, size_t size); =20 struct page *dma_alloc_from_pool(struct device *dev, size_t size, - void **cpu_addr, gfp_t flags, + void **cpu_addr, gfp_t flags, unsigned long attrs, bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t)); bool dma_free_from_pool(struct device *dev, void *start, size_t size); =20 diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 3932033f4d8c..ba1c731e001d 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -162,7 +162,7 @@ static bool dma_direct_use_pool(struct device *dev, gfp= _t gfp) } =20 static void *dma_direct_alloc_from_pool(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp) + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { struct page *page; u64 phys_limit; @@ -172,7 +172,8 @@ static void *dma_direct_alloc_from_pool(struct device *= dev, size_t size, return NULL; =20 gfp |=3D dma_direct_optimal_gfp_mask(dev, &phys_limit); - page =3D dma_alloc_from_pool(dev, size, &ret, gfp, dma_coherent_ok); + page =3D dma_alloc_from_pool(dev, size, &ret, gfp, attrs, + dma_coherent_ok); if (!page) return NULL; *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page)); @@ -251,7 +252,8 @@ void *dma_direct_alloc(struct device *dev, size_t size, */ if ((remap || (attrs & DMA_ATTR_CC_DECRYPTED)) && dma_direct_use_pool(dev, gfp)) - return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); + return dma_direct_alloc_from_pool(dev, size, dma_handle, + gfp, attrs); =20 if (is_swiotlb_for_alloc(dev)) { page =3D dma_direct_alloc_swiotlb(dev, size); @@ -387,7 +389,8 @@ struct page *dma_direct_alloc_pages(struct device *dev,= size_t size, attrs |=3D DMA_ATTR_CC_DECRYPTED; =20 if ((attrs & DMA_ATTR_CC_DECRYPTED) && dma_direct_use_pool(dev, gfp)) - return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); + return dma_direct_alloc_from_pool(dev, size, dma_handle, + gfp, attrs); =20 if (is_swiotlb_for_alloc(dev)) { page =3D dma_direct_alloc_swiotlb(dev, size); diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c index 2b2fbb709242..e4dde3e769bf 100644 --- a/kernel/dma/pool.c +++ b/kernel/dma/pool.c @@ -12,12 +12,18 @@ #include #include #include +#include =20 -static struct gen_pool *atomic_pool_dma __ro_after_init; +struct dma_gen_pool { + bool decrypted; + struct gen_pool *pool; +}; + +static struct dma_gen_pool atomic_pool_dma __ro_after_init; static unsigned long pool_size_dma; -static struct gen_pool *atomic_pool_dma32 __ro_after_init; +static struct dma_gen_pool atomic_pool_dma32 __ro_after_init; static unsigned long pool_size_dma32; -static struct gen_pool *atomic_pool_kernel __ro_after_init; +static struct dma_gen_pool atomic_pool_kernel __ro_after_init; static unsigned long pool_size_kernel; =20 /* Size can be defined by the coherent_pool command line */ @@ -76,7 +82,7 @@ static bool cma_in_zone(gfp_t gfp) return true; } =20 -static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size, +static int atomic_pool_expand(struct dma_gen_pool *dma_pool, size_t pool_s= ize, gfp_t gfp) { unsigned int order; @@ -113,11 +119,14 @@ static int atomic_pool_expand(struct gen_pool *pool, = size_t pool_size, * Memory in the atomic DMA pools must be unencrypted, the pools do not * shrink so no re-encryption occurs in dma_direct_free(). */ - ret =3D set_memory_decrypted((unsigned long)page_to_virt(page), + if (dma_pool->decrypted) { + ret =3D set_memory_decrypted((unsigned long)page_to_virt(page), 1 << order); - if (ret) - goto remove_mapping; - ret =3D gen_pool_add_virt(pool, (unsigned long)addr, page_to_phys(page), + if (ret) + goto remove_mapping; + } + + ret =3D gen_pool_add_virt(dma_pool->pool, (unsigned long)addr, page_to_ph= ys(page), pool_size, NUMA_NO_NODE); if (ret) goto encrypt_mapping; @@ -126,11 +135,13 @@ static int atomic_pool_expand(struct gen_pool *pool, = size_t pool_size, return 0; =20 encrypt_mapping: - ret =3D set_memory_encrypted((unsigned long)page_to_virt(page), - 1 << order); - if (WARN_ON_ONCE(ret)) { - /* Decrypt succeeded but encrypt failed, purposely leak */ - goto out; + if (dma_pool->decrypted) { + ret =3D set_memory_encrypted((unsigned long)page_to_virt(page), + 1 << order); + if (WARN_ON_ONCE(ret)) { + /* Decrypt succeeded but encrypt failed, purposely leak */ + goto out; + } } remove_mapping: #ifdef CONFIG_DMA_DIRECT_REMAP @@ -142,46 +153,51 @@ static int atomic_pool_expand(struct gen_pool *pool, = size_t pool_size, return ret; } =20 -static void atomic_pool_resize(struct gen_pool *pool, gfp_t gfp) +static void atomic_pool_resize(struct dma_gen_pool *dma_pool, gfp_t gfp) { - if (pool && gen_pool_avail(pool) < atomic_pool_size) - atomic_pool_expand(pool, gen_pool_size(pool), gfp); + if (dma_pool->pool && gen_pool_avail(dma_pool->pool) < atomic_pool_size) + atomic_pool_expand(dma_pool, gen_pool_size(dma_pool->pool), gfp); } =20 static void atomic_pool_work_fn(struct work_struct *work) { if (IS_ENABLED(CONFIG_ZONE_DMA)) - atomic_pool_resize(atomic_pool_dma, + atomic_pool_resize(&atomic_pool_dma, GFP_KERNEL | GFP_DMA); if (IS_ENABLED(CONFIG_ZONE_DMA32)) - atomic_pool_resize(atomic_pool_dma32, + atomic_pool_resize(&atomic_pool_dma32, GFP_KERNEL | GFP_DMA32); - atomic_pool_resize(atomic_pool_kernel, GFP_KERNEL); + atomic_pool_resize(&atomic_pool_kernel, GFP_KERNEL); } =20 -static __init struct gen_pool *__dma_atomic_pool_init(size_t pool_size, - gfp_t gfp) +static __init struct dma_gen_pool *__dma_atomic_pool_init(struct dma_gen_p= ool *dma_pool, + size_t pool_size, gfp_t gfp) { - struct gen_pool *pool; int ret; =20 - pool =3D gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE); - if (!pool) + dma_pool->pool =3D gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE); + if (!dma_pool->pool) return NULL; =20 - gen_pool_set_algo(pool, gen_pool_first_fit_order_align, NULL); + gen_pool_set_algo(dma_pool->pool, gen_pool_first_fit_order_align, NULL); + + /* if platform is using memory encryption atomic pools are by default dec= rypted. */ + if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) + dma_pool->decrypted =3D true; + else + dma_pool->decrypted =3D false; =20 - ret =3D atomic_pool_expand(pool, pool_size, gfp); + ret =3D atomic_pool_expand(dma_pool, pool_size, gfp); if (ret) { - gen_pool_destroy(pool); + gen_pool_destroy(dma_pool->pool); pr_err("DMA: failed to allocate %zu KiB %pGg pool for atomic allocation\= n", pool_size >> 10, &gfp); return NULL; } =20 pr_info("DMA: preallocated %zu KiB %pGg pool for atomic allocations\n", - gen_pool_size(pool) >> 10, &gfp); - return pool; + gen_pool_size(dma_pool->pool) >> 10, &gfp); + return dma_pool; } =20 #ifdef CONFIG_ZONE_DMA32 @@ -207,21 +223,22 @@ static int __init dma_atomic_pool_init(void) =20 /* All memory might be in the DMA zone(s) to begin with */ if (has_managed_zone(ZONE_NORMAL)) { - atomic_pool_kernel =3D __dma_atomic_pool_init(atomic_pool_size, - GFP_KERNEL); - if (!atomic_pool_kernel) + __dma_atomic_pool_init(&atomic_pool_kernel, atomic_pool_size, GFP_KERNEL= ); + if (!atomic_pool_kernel.pool) ret =3D -ENOMEM; } + if (has_managed_dma()) { - atomic_pool_dma =3D __dma_atomic_pool_init(atomic_pool_size, - GFP_KERNEL | GFP_DMA); - if (!atomic_pool_dma) + __dma_atomic_pool_init(&atomic_pool_dma, atomic_pool_size, + GFP_KERNEL | GFP_DMA); + if (!atomic_pool_dma.pool) ret =3D -ENOMEM; } + if (has_managed_dma32) { - atomic_pool_dma32 =3D __dma_atomic_pool_init(atomic_pool_size, - GFP_KERNEL | GFP_DMA32); - if (!atomic_pool_dma32) + __dma_atomic_pool_init(&atomic_pool_dma32, atomic_pool_size, + GFP_KERNEL | GFP_DMA32); + if (!atomic_pool_dma32.pool) ret =3D -ENOMEM; } =20 @@ -230,19 +247,38 @@ static int __init dma_atomic_pool_init(void) } postcore_initcall(dma_atomic_pool_init); =20 -static inline struct gen_pool *dma_guess_pool(struct gen_pool *prev, gfp_t= gfp) +static inline struct dma_gen_pool *dma_guess_pool(struct dma_gen_pool *pre= v, gfp_t gfp) { if (prev =3D=3D NULL) { - if (gfp & GFP_DMA) - return atomic_pool_dma ?: atomic_pool_dma32 ?: atomic_pool_kernel; - if (gfp & GFP_DMA32) - return atomic_pool_dma32 ?: atomic_pool_dma ?: atomic_pool_kernel; - return atomic_pool_kernel ?: atomic_pool_dma32 ?: atomic_pool_dma; + if (gfp & GFP_DMA) { + if (atomic_pool_dma.pool) + return &atomic_pool_dma; + if (atomic_pool_dma32.pool) + return &atomic_pool_dma32; + return &atomic_pool_kernel; + } + + if (gfp & GFP_DMA32) { + if (atomic_pool_dma32.pool) + return &atomic_pool_dma32; + if (atomic_pool_dma.pool) + return &atomic_pool_dma; + return &atomic_pool_kernel; + } + if (atomic_pool_kernel.pool) + return &atomic_pool_kernel; + if (atomic_pool_dma32.pool) + return &atomic_pool_dma32; + if (atomic_pool_dma.pool) + return &atomic_pool_dma; } - if (prev =3D=3D atomic_pool_kernel) - return atomic_pool_dma32 ? atomic_pool_dma32 : atomic_pool_dma; - if (prev =3D=3D atomic_pool_dma32) - return atomic_pool_dma; + if (prev =3D=3D &atomic_pool_kernel) { + if (atomic_pool_dma32.pool) + return &atomic_pool_dma32; + return &atomic_pool_dma; + } + if (prev =3D=3D &atomic_pool_dma32) + return &atomic_pool_dma; return NULL; } =20 @@ -272,16 +308,20 @@ static struct page *__dma_alloc_from_pool(struct devi= ce *dev, size_t size, } =20 struct page *dma_alloc_from_pool(struct device *dev, size_t size, - void **cpu_addr, gfp_t gfp, + void **cpu_addr, gfp_t gfp, unsigned long attrs, bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t)) { - struct gen_pool *pool =3D NULL; + struct dma_gen_pool *dma_pool =3D NULL; struct page *page; bool pool_found =3D false; =20 - while ((pool =3D dma_guess_pool(pool, gfp))) { + while ((dma_pool =3D dma_guess_pool(dma_pool, gfp))) { + + if (dma_pool->decrypted !=3D !!(attrs & DMA_ATTR_CC_DECRYPTED)) + continue; + pool_found =3D true; - page =3D __dma_alloc_from_pool(dev, size, pool, cpu_addr, + page =3D __dma_alloc_from_pool(dev, size, dma_pool->pool, cpu_addr, phys_addr_ok); if (page) return page; @@ -296,12 +336,14 @@ struct page *dma_alloc_from_pool(struct device *dev, = size_t size, =20 bool dma_free_from_pool(struct device *dev, void *start, size_t size) { - struct gen_pool *pool =3D NULL; + struct dma_gen_pool *dma_pool =3D NULL; =20 - while ((pool =3D dma_guess_pool(pool, 0))) { - if (!gen_pool_has_addr(pool, (unsigned long)start, size)) + while ((dma_pool =3D dma_guess_pool(dma_pool, 0))) { + + if (!gen_pool_has_addr(dma_pool->pool, (unsigned long)start, size)) continue; - gen_pool_free(pool, (unsigned long)start, size); + + gen_pool_free(dma_pool->pool, (unsigned long)start, size); return true; } =20 diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 9fd73700ddcf..2373a9f7e21a 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -623,7 +623,9 @@ static struct page *swiotlb_alloc_tlb(struct device *de= v, size_t bytes, if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL)) return NULL; =20 + /* considered decrypted by default */ return dma_alloc_from_pool(dev, bytes, &vaddr, gfp, + DMA_ATTR_CC_DECRYPTED, dma_coherent_ok); } =20 --=20 2.43.0 From nobody Tue Jun 16 07:26:32 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 068D9386578; Fri, 17 Apr 2026 09:00:01 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776416402; cv=none; b=L7MtQqtDlgmQ5XyX5C8kCQPHovhWTtX1RJsujrS41pt5dVzZNSm8mzcko+1nIyyLtt72BrAnz1R7c1ROJ5jlfHMMCGj1RcGpyD1q1YBLdJjzKx0SN2m2SAttOWJAyZSBO6M+rUBkhFVEDopN/k06uCeoO3oqXymOJmZVRkYh7B0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776416402; c=relaxed/simple; bh=xbwDAYKt2y8KkY+kT/UgsbetMP8sb3BKbbJJCTr1e0g=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=sY9iYJs1VXvFDDdiLjcLi3lpYR5QJvSW4u7+NvPlU/UK1dOSbbzpsbTfk9nbOZcwONbYUruD9a7m7SPO0BRIMf2HKW2Fzvq2hxS0DqGvhbpZWqnAnZuoOsBO23AbF5wgPLpgZd7vlp3B00L5sexvSaKkKuzffOIev8Qt0UjRK4Y= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=HqHZO14C; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="HqHZO14C" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 334AFC2BCB4; Fri, 17 Apr 2026 08:59:57 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1776416401; bh=xbwDAYKt2y8KkY+kT/UgsbetMP8sb3BKbbJJCTr1e0g=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=HqHZO14CAv5Lwg5oTTArIcLJW3xkerqHetF3bJ9oBux18AdEFnwBzZO6zq9AFztfi qtlAHQEQ9+JDdZyhtW7/Z9nU5881vaitepBP7m1uL8H2dNtLiluasHgzFDLJs3ITwd aoR+5yg09F2aGWRFAqK8TTRjF6WhXLAWb5Zd6PnbodytHJSXI5xGAOj3Kcxpmn+cYg m5TnDtI7XmcAquaLzv1cIw0sEmkXP7HLnTiSe+/4GxDOHt7M7jQsChkRMV7Reqp8bd UTUzVobqLZccGEMtErHpXcOaoxomP83CDaK24aMCblb321KM8aMyC+xGtgZoE458K1 ijM6YphFxavsw== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-kernel@vger.kernel.org Cc: robin.murphy@arm.com, m.szyprowski@samsung.com, will@kernel.org, maz@kernel.org, suzuki.poulose@arm.com, catalin.marinas@arm.com, jiri@resnulli.us, jgg@ziepe.ca, aneesh.kumar@kernel.org, Mostafa Saleh Subject: [RFC PATCH 4/7] dma: swiotlb: track pool encryption state and honor DMA_ATTR_CC_DECRYPTED Date: Fri, 17 Apr 2026 14:28:57 +0530 Message-ID: <20260417085900.3062416-5-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260417085900.3062416-1-aneesh.kumar@kernel.org> References: <20260417085900.3062416-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Teach swiotlb to distinguish between encrypted and decrypted bounce buffer pools, and make allocation and mapping paths select a pool whose state matches the requested DMA attributes. Add a decrypted flag to io_tlb_mem, initialize it for the default and restricted pools, and propagate DMA_ATTR_CC_DECRYPTED into swiotlb pool allocation. Reject swiotlb alloc/map requests when the selected pool does not match the required encrypted/decrypted state. Also return DMA addresses with the matching phys_to_dma_{encrypted, unencrypted} helper so the DMA address encoding stays consistent with the chosen pool. Signed-off-by: Aneesh Kumar K.V (Arm) --- include/linux/dma-direct.h | 10 +++++ include/linux/swiotlb.h | 7 ++- kernel/dma/direct.c | 14 ++++-- kernel/dma/swiotlb.c | 89 ++++++++++++++++++++++++++++++-------- 4 files changed, 95 insertions(+), 25 deletions(-) diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h index c249912456f9..94fad4e7c11e 100644 --- a/include/linux/dma-direct.h +++ b/include/linux/dma-direct.h @@ -77,6 +77,10 @@ static inline dma_addr_t dma_range_map_max(const struct = bus_dma_region *map) #ifndef phys_to_dma_unencrypted #define phys_to_dma_unencrypted phys_to_dma #endif + +#ifndef phys_to_dma_encrypted +#define phys_to_dma_encrypted phys_to_dma +#endif #else static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t pad= dr) { @@ -90,6 +94,12 @@ static inline dma_addr_t phys_to_dma_unencrypted(struct = device *dev, { return dma_addr_unencrypted(__phys_to_dma(dev, paddr)); } + +static inline dma_addr_t phys_to_dma_encrypted(struct device *dev, + phys_addr_t paddr) +{ + return dma_addr_encrypted(__phys_to_dma(dev, paddr)); +} /* * If memory encryption is supported, phys_to_dma will set the memory encr= yption * bit in the DMA address, and dma_to_phys will clear it. diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 3dae0f592063..382753ba3f06 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -111,6 +111,7 @@ struct io_tlb_mem { struct dentry *debugfs; bool force_bounce; bool for_alloc; + bool decrypted; #ifdef CONFIG_SWIOTLB_DYNAMIC bool can_grow; u64 phys_limit; @@ -282,7 +283,8 @@ static inline void swiotlb_sync_single_for_cpu(struct d= evice *dev, extern void swiotlb_print_info(void); =20 #ifdef CONFIG_DMA_RESTRICTED_POOL -struct page *swiotlb_alloc(struct device *dev, size_t size); +struct page *swiotlb_alloc(struct device *dev, size_t size, + unsigned long attrs); bool swiotlb_free(struct device *dev, struct page *page, size_t size); =20 static inline bool is_swiotlb_for_alloc(struct device *dev) @@ -290,7 +292,8 @@ static inline bool is_swiotlb_for_alloc(struct device *= dev) return dev->dma_io_tlb_mem->for_alloc; } #else -static inline struct page *swiotlb_alloc(struct device *dev, size_t size) +static inline struct page *swiotlb_alloc(struct device *dev, size_t size, + unsigned long attrs) { return NULL; } diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index ba1c731e001d..4a4147fffc5e 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -104,9 +104,10 @@ static void __dma_direct_free_pages(struct device *dev= , struct page *page, dma_free_contiguous(dev, page, size); } =20 -static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t si= ze) +static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t si= ze, + unsigned long attrs) { - struct page *page =3D swiotlb_alloc(dev, size); + struct page *page =3D swiotlb_alloc(dev, size, attrs); =20 if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { swiotlb_free(dev, page, size); @@ -256,8 +257,12 @@ void *dma_direct_alloc(struct device *dev, size_t size, gfp, attrs); =20 if (is_swiotlb_for_alloc(dev)) { - page =3D dma_direct_alloc_swiotlb(dev, size); + page =3D dma_direct_alloc_swiotlb(dev, size, attrs); if (page) { + /* + * swiotlb allocations comes from pool already marked + * decrypted + */ mark_mem_decrypt =3D false; goto setup_page; } @@ -364,6 +369,7 @@ void dma_direct_free(struct device *dev, size_t size, return; =20 if (swiotlb_find_pool(dev, dma_to_phys(dev, dma_addr))) + /* Swiotlb doesn't need a page attribute update on free */ mark_mem_encrypted =3D false; =20 if (is_vmalloc_addr(cpu_addr)) { @@ -393,7 +399,7 @@ struct page *dma_direct_alloc_pages(struct device *dev,= size_t size, gfp, attrs); =20 if (is_swiotlb_for_alloc(dev)) { - page =3D dma_direct_alloc_swiotlb(dev, size); + page =3D dma_direct_alloc_swiotlb(dev, size, attrs); if (!page) return NULL; =20 diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 2373a9f7e21a..1b845596b68f 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -262,7 +262,18 @@ void __init swiotlb_update_mem_attributes(void) if (!mem->nslabs || mem->late_alloc) return; bytes =3D PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT); - set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT); + /* + * if platform support memory encryption, swiotlb buffers are + * decrypted by default. + */ + if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) { + io_tlb_default_mem.decrypted =3D true; + set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT); + } else { + io_tlb_default_mem.decrypted =3D false; + } + + } =20 static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t = start, @@ -505,8 +516,10 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask, if (!mem->slots) goto error_slots; =20 - set_memory_decrypted((unsigned long)vstart, - (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT); + if (io_tlb_default_mem.decrypted) + set_memory_decrypted((unsigned long)vstart, + (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT); + swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true, nareas); add_mem_pool(&io_tlb_default_mem, mem); @@ -570,7 +583,8 @@ void __init swiotlb_exit(void) * Return: Decrypted pages, %NULL on allocation failure, or ERR_PTR(-EAGAI= N) * if the allocated physical address was above @phys_limit. */ -static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limi= t) +static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, + u64 phys_limit, bool unencrypted) { unsigned int order =3D get_order(bytes); struct page *page; @@ -588,13 +602,13 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t= bytes, u64 phys_limit) } =20 vaddr =3D phys_to_virt(paddr); - if (set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes))) + if (unencrypted && set_memory_decrypted((unsigned long)vaddr, PFN_UP(byte= s))) goto error; return page; =20 error: /* Intentional leak if pages cannot be encrypted again. */ - if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes))) + if (unencrypted && !set_memory_encrypted((unsigned long)vaddr, PFN_UP(byt= es))) __free_pages(page, order); return NULL; } @@ -604,12 +618,13 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t= bytes, u64 phys_limit) * @dev: Device for which a memory pool is allocated. * @bytes: Size of the buffer. * @phys_limit: Maximum allowed physical address of the buffer. + * @attrs: DMA attributes for the allocation. * @gfp: GFP flags for the allocation. * * Return: Allocated pages, or %NULL on allocation failure. */ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes, - u64 phys_limit, gfp_t gfp) + u64 phys_limit, unsigned long attrs, gfp_t gfp) { struct page *page; =20 @@ -617,7 +632,7 @@ static struct page *swiotlb_alloc_tlb(struct device *de= v, size_t bytes, * Allocate from the atomic pools if memory is encrypted and * the allocation is atomic, because decrypting may block. */ - if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) { + if (!gfpflags_allow_blocking(gfp) && (attrs & DMA_ATTR_CC_DECRYPTED)) { void *vaddr; =20 if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL)) @@ -625,8 +640,7 @@ static struct page *swiotlb_alloc_tlb(struct device *de= v, size_t bytes, =20 /* considered decrypted by default */ return dma_alloc_from_pool(dev, bytes, &vaddr, gfp, - DMA_ATTR_CC_DECRYPTED, - dma_coherent_ok); + attrs, dma_coherent_ok); } =20 gfp &=3D ~GFP_ZONEMASK; @@ -635,7 +649,8 @@ static struct page *swiotlb_alloc_tlb(struct device *de= v, size_t bytes, else if (phys_limit <=3D DMA_BIT_MASK(32)) gfp |=3D __GFP_DMA32; =20 - while (IS_ERR(page =3D alloc_dma_pages(gfp, bytes, phys_limit))) { + while (IS_ERR(page =3D alloc_dma_pages(gfp, bytes, phys_limit, + !!(attrs & DMA_ATTR_CC_DECRYPTED)))) { if (IS_ENABLED(CONFIG_ZONE_DMA32) && phys_limit < DMA_BIT_MASK(64) && !(gfp & (__GFP_DMA32 | __GFP_DMA))) @@ -673,6 +688,7 @@ static void swiotlb_free_tlb(void *vaddr, size_t bytes) * @nslabs: Desired (maximum) number of slabs. * @nareas: Number of areas. * @phys_limit: Maximum DMA buffer physical address. + * @attrs: DMA attributes for the allocation. * @gfp: GFP flags for the allocations. * * Allocate and initialize a new IO TLB memory pool. The actual number of @@ -683,7 +699,8 @@ static void swiotlb_free_tlb(void *vaddr, size_t bytes) */ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev, unsigned long minslabs, unsigned long nslabs, - unsigned int nareas, u64 phys_limit, gfp_t gfp) + unsigned int nareas, u64 phys_limit, unsigned long attrs, + gfp_t gfp) { struct io_tlb_pool *pool; unsigned int slot_order; @@ -703,7 +720,7 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct de= vice *dev, pool->areas =3D (void *)pool + sizeof(*pool); =20 tlb_size =3D nslabs << IO_TLB_SHIFT; - while (!(tlb =3D swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp))) { + while (!(tlb =3D swiotlb_alloc_tlb(dev, tlb_size, phys_limit, attrs, gfp)= )) { if (nslabs <=3D minslabs) goto error_tlb; nslabs =3D ALIGN(nslabs >> 1, IO_TLB_SEGSIZE); @@ -739,7 +756,9 @@ static void swiotlb_dyn_alloc(struct work_struct *work) struct io_tlb_pool *pool; =20 pool =3D swiotlb_alloc_pool(NULL, IO_TLB_MIN_SLABS, default_nslabs, - default_nareas, mem->phys_limit, GFP_KERNEL); + default_nareas, mem->phys_limit, + mem->decrypted ? DMA_ATTR_CC_DECRYPTED : 0, + GFP_KERNEL); if (!pool) { pr_warn_ratelimited("Failed to allocate new pool"); return; @@ -1226,6 +1245,7 @@ static int swiotlb_find_slots(struct device *dev, phy= s_addr_t orig_addr, nslabs =3D nr_slots(alloc_size); phys_limit =3D min_not_zero(*dev->dma_mask, dev->bus_dma_limit); pool =3D swiotlb_alloc_pool(dev, nslabs, nslabs, 1, phys_limit, + mem->decrypted ? DMA_ATTR_CC_DECRYPTED : 0, GFP_NOWAIT); if (!pool) return -1; @@ -1388,6 +1408,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev= , phys_addr_t orig_addr, enum dma_data_direction dir, unsigned long attrs) { struct io_tlb_mem *mem =3D dev->dma_io_tlb_mem; + bool require_decrypted =3D false; unsigned int offset; struct io_tlb_pool *pool; unsigned int i; @@ -1405,6 +1426,17 @@ phys_addr_t swiotlb_tbl_map_single(struct device *de= v, phys_addr_t orig_addr, if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) pr_warn_once("Memory encryption is active and system is using DMA bounce= buffers\n"); =20 + /* + * if we are trying to swiotlb map a decrypted paddr or the paddr is encr= ypted + * but the device is forcing decryption, use decrypted io_tlb_mem + */ + if ((attrs & DMA_ATTR_CC_DECRYPTED) || + (!(attrs & DMA_ATTR_CC_DECRYPTED) && force_dma_unencrypted(dev))) + require_decrypted =3D true; + + if (require_decrypted !=3D mem->decrypted) + return (phys_addr_t)DMA_MAPPING_ERROR; + /* * The default swiotlb memory pool is allocated with PAGE_SIZE * alignment. If a mapping is requested with larger alignment, @@ -1602,8 +1634,14 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr= _t paddr, size_t size, if (swiotlb_addr =3D=3D (phys_addr_t)DMA_MAPPING_ERROR) return DMA_MAPPING_ERROR; =20 - /* Ensure that the address returned is DMA'ble */ - dma_addr =3D phys_to_dma_unencrypted(dev, swiotlb_addr); + /* + * Use the allocated io_tlb_mem encryption type to determine dma addr. + */ + if (dev->dma_io_tlb_mem->decrypted) + dma_addr =3D phys_to_dma_unencrypted(dev, swiotlb_addr); + else + dma_addr =3D phys_to_dma_encrypted(dev, swiotlb_addr); + if (unlikely(!dma_capable(dev, dma_addr, size, true))) { __swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC, @@ -1765,7 +1803,8 @@ static inline void swiotlb_create_debugfs_files(struc= t io_tlb_mem *mem, =20 #ifdef CONFIG_DMA_RESTRICTED_POOL =20 -struct page *swiotlb_alloc(struct device *dev, size_t size) +struct page *swiotlb_alloc(struct device *dev, size_t size, + unsigned long attrs) { struct io_tlb_mem *mem =3D dev->dma_io_tlb_mem; struct io_tlb_pool *pool; @@ -1776,6 +1815,9 @@ struct page *swiotlb_alloc(struct device *dev, size_t= size) if (!mem) return NULL; =20 + if (mem->decrypted !=3D !!(attrs & DMA_ATTR_CC_DECRYPTED)) + return NULL; + align =3D (1 << (get_order(size) + PAGE_SHIFT)) - 1; index =3D swiotlb_find_slots(dev, 0, size, align, &pool); if (index =3D=3D -1) @@ -1845,9 +1887,18 @@ static int rmem_swiotlb_device_init(struct reserved_= mem *rmem, kfree(mem); return -ENOMEM; } + /* + * if platform supports memory encryption, + * restricted mem pool is decrypted by default + */ + if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) { + mem->decrypted =3D true; + set_memory_decrypted((unsigned long)phys_to_virt(rmem->base), + rmem->size >> PAGE_SHIFT); + } else { + mem->decrypted =3D false; + } =20 - set_memory_decrypted((unsigned long)phys_to_virt(rmem->base), - rmem->size >> PAGE_SHIFT); swiotlb_init_io_tlb_pool(pool, rmem->base, nslabs, false, nareas); mem->force_bounce =3D true; --=20 2.43.0 From nobody Tue Jun 16 07:26:32 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 28F4638C400; Fri, 17 Apr 2026 09:00:05 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776416406; cv=none; b=PwpuZ79yYa1PeekWl1o9vk5mPpk3IXFI7a9KZ1z/3eUW1JLneI7i7pRVw6474Wrq9nGPMFb+pE74pIC0LOXI7XX0pVH4apbkLZtGvfCjLg4haV0+36n0IORppOyN603ZIb1Tfyenmj+ESOnnYmWpO3izLzGwg3SutOV+kfvfOYE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776416406; c=relaxed/simple; bh=hkf8yDSIaDmjpcCJ9rrjqwE41gSx/WhkacTiUgjTgZg=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=pkfotjIpvnTskLzJ11G2nuzPwc2Il7ShKBUcqOn3VVpSq4ZVWMEOwukLJF73DF6f9FoEjI99BmkL4iA8noZn8yBhPQV2Vv8szKTiCpnaUGbqtSViQRvZxcpSd5u5Kef7FzRBMZRSAINI3RA8yGpvtH0CXrF75ZTRNmRE3W/FDq8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=HAf8Fq0Y; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="HAf8Fq0Y" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 39952C19425; Fri, 17 Apr 2026 09:00:01 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1776416405; bh=hkf8yDSIaDmjpcCJ9rrjqwE41gSx/WhkacTiUgjTgZg=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=HAf8Fq0Y0czYdglv3zUFt2NDwmrtGqB94RDN8musd7ZMvTTWQOQnZcSVyimhFMRfO Ur8gD2dbPBbLe8r6E7fUM/n/FdSlPuM2JssWlx8r5Q4dEKOxZBVvpZkE3vMoyVRCrt P1hkIVdR2e4XVoFXlmqKUCkKJ7uCaRQHrx5fQy3rl+Qi8Yzd49IwrH3HNLtw6Q9Hxe JTUggQmaErONsKyHOmqFgY9memtbP9G/RWKunhCzPBTFiJ+dBpbKPmo+NDEilvym1/ 7zWh5Zu5HGuAsirr01Eo+tXE3IKAqPi4uJJFkQ/k7RY2kOIVka4V5bR/l0WE/nUnah RMZTcE37/EZ2A== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-kernel@vger.kernel.org Cc: robin.murphy@arm.com, m.szyprowski@samsung.com, will@kernel.org, maz@kernel.org, suzuki.poulose@arm.com, catalin.marinas@arm.com, jiri@resnulli.us, jgg@ziepe.ca, aneesh.kumar@kernel.org, Mostafa Saleh Subject: [RFC PATCH 5/7] dma-mapping: make dma_pgprot() honor DMA_ATTR_CC_DECRYPTED Date: Fri, 17 Apr 2026 14:28:58 +0530 Message-ID: <20260417085900.3062416-6-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260417085900.3062416-1-aneesh.kumar@kernel.org> References: <20260417085900.3062416-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Fold encrypted/decrypted pgprot selection into dma_pgprot() so callers do not need to adjust the page protection separately. Update dma_pgprot() to apply pgprot_decrypted() when DMA_ATTR_CC_DECRYPTED is set and pgprot_encrypted() otherwise Convert the dma-direct allocation and mmap paths to pass DMA_ATTR_CC_DECRYPTED instead of open-coding force_dma_unencrypted() handling around dma_pgprot(). Signed-off-by: Aneesh Kumar K.V (Arm) --- kernel/dma/direct.c | 8 +++----- kernel/dma/mapping.c | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 4a4147fffc5e..1d2c27bbf3de 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -288,9 +288,6 @@ void *dma_direct_alloc(struct device *dev, size_t size, if (remap) { pgprot_t prot =3D dma_pgprot(dev, PAGE_KERNEL, attrs); =20 - if (force_dma_unencrypted(dev)) - prot =3D pgprot_decrypted(prot); - /* remove any dirty cache lines on the kernel alias */ arch_dma_prep_coherent(page, size); =20 @@ -580,9 +577,10 @@ int dma_direct_mmap(struct device *dev, struct vm_area= _struct *vma, unsigned long pfn =3D PHYS_PFN(dma_to_phys(dev, dma_addr)); int ret =3D -ENXIO; =20 - vma->vm_page_prot =3D dma_pgprot(dev, vma->vm_page_prot, attrs); if (force_dma_unencrypted(dev)) - vma->vm_page_prot =3D pgprot_decrypted(vma->vm_page_prot); + attrs |=3D DMA_ATTR_CC_DECRYPTED; + + vma->vm_page_prot =3D dma_pgprot(dev, vma->vm_page_prot, attrs); =20 if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) return ret; diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 6d3dd0bd3a88..9f505df6ee0e 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -534,13 +534,21 @@ EXPORT_SYMBOL(dma_get_sgtable_attrs); */ pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs) { + pgprot_t dma_prot; + if (dev_is_dma_coherent(dev)) - return prot; + dma_prot =3D prot; #ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE - if (attrs & DMA_ATTR_WRITE_COMBINE) - return pgprot_writecombine(prot); + else if (attrs & DMA_ATTR_WRITE_COMBINE) + dma_prot =3D pgprot_writecombine(prot); #endif - return pgprot_dmacoherent(prot); + else + dma_prot =3D pgprot_dmacoherent(prot); + + if (attrs & DMA_ATTR_CC_DECRYPTED) + return pgprot_decrypted(dma_prot); + else + return pgprot_encrypted(dma_prot); } #endif /* CONFIG_MMU */ =20 --=20 2.43.0 From nobody Tue Jun 16 07:26:32 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id F109C3B27DC; Fri, 17 Apr 2026 09:00:09 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776416410; cv=none; b=EI+7tAoUOJBdkv4vHXWPIyu7Df/8D6KOmjn2pfGDTv4l2IC/ExsXpEP/7UPhIGYp+2uWif1CwaeoyMrCeR3uTyfICngVYgNKhUPTq+VU0e3Z7m8wKfL5ftggIgz0WvUdsoR5oFX6dfC3WNhv6/8FanicO499iB0q42uyx3KfreY= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776416410; c=relaxed/simple; bh=NYO/ZaWM8DspsP7qrAyCMciTobaVhBewQVuwza9k9yk=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=PFwSdDFzdcINgCZZR2XD5YCzuppU1LYPe6k1yY9U57+6kzVeHtu7KYExVyEfBkKmQf2UoIUaYiM0lFCrW2+BtI5IA4S6fBJwIo0z0/Jr4sx0jIBk5lO/X/DzofUPYSiYshRcAW+6CNimg/D+79z4mbZkgiHKhUZAuLdu79afjzQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=YnVbwAFN; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="YnVbwAFN" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 628DFC19425; Fri, 17 Apr 2026 09:00:06 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1776416409; bh=NYO/ZaWM8DspsP7qrAyCMciTobaVhBewQVuwza9k9yk=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=YnVbwAFNZxABj4oWCJi4Kzzfe4qynC1Y8IeQgDp9wA0ZfufMTDxTDHEr5+lyG2Fg9 T+FTKst6wohzKXlWdsVgyMAw/Un29ReEcO59EYSexa5SwC7B9owCs2K8vyMjXu/SgK pTbSS1IG+fn7fqEZXtaGponmKVWvSAFfVII1QbTHDLzhYtXEXKwxA7KJ8yteTH805i Nzh8b7eIGTR52i8SXcxAYDfx1wxb/tZ+M6Pk/NMOICkIayuaEg+pDzzsIiyKLMcnJe 5jqhLmsjit5DhXQ2oR/8yQzN0+qf88wjces8bwzD6vWqPVcWqgyDYPR6E1mEwOH8BF 8zT7oRdpWxf3g== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-kernel@vger.kernel.org Cc: robin.murphy@arm.com, m.szyprowski@samsung.com, will@kernel.org, maz@kernel.org, suzuki.poulose@arm.com, catalin.marinas@arm.com, jiri@resnulli.us, jgg@ziepe.ca, aneesh.kumar@kernel.org, Mostafa Saleh Subject: [RFC PATCH 6/7] dma-direct: make dma_direct_map_phys() honor DMA_ATTR_CC_DECRYPTED Date: Fri, 17 Apr 2026 14:28:59 +0530 Message-ID: <20260417085900.3062416-7-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260417085900.3062416-1-aneesh.kumar@kernel.org> References: <20260417085900.3062416-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Teach dma_direct_map_phys() to select the DMA address encoding based on DMA_ATTR_CC_DECRYPTED. Use phys_to_dma_unencrypted() for decrypted mappings and phys_to_dma_encrypted() otherwise. If a device requires unencrypted DMA but the source physical address is still encrypted, force the mapping through swiotlb so the DMA address and backing memory attributes remain consistent. Signed-off-by: Aneesh Kumar K.V (Arm) --- kernel/dma/direct.h | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/kernel/dma/direct.h b/kernel/dma/direct.h index 6184ff303f08..421dcfb146d8 100644 --- a/kernel/dma/direct.h +++ b/kernel/dma/direct.h @@ -81,9 +81,14 @@ static inline dma_addr_t dma_direct_map_phys(struct devi= ce *dev, phys_addr_t phys, size_t size, enum dma_data_direction dir, unsigned long attrs) { + bool force_swiotlb_map =3D false; dma_addr_t dma_addr; =20 - if (is_swiotlb_force_bounce(dev)) { + /* if phys addr attribute is encrypted but the device is forcing an encry= pted dma addr */ + if (!(attrs & DMA_ATTR_CC_DECRYPTED) && force_dma_unencrypted(dev)) + force_swiotlb_map =3D true; + + if (is_swiotlb_force_bounce(dev) || force_swiotlb_map) { if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT)) return DMA_MAPPING_ERROR; =20 @@ -94,16 +99,18 @@ static inline dma_addr_t dma_direct_map_phys(struct dev= ice *dev, dma_addr =3D phys; if (unlikely(!dma_capable(dev, dma_addr, size, false))) goto err_overflow; + } else if (attrs & DMA_ATTR_CC_DECRYPTED) { + dma_addr =3D phys_to_dma_unencrypted(dev, phys); } else { - dma_addr =3D phys_to_dma(dev, phys); - if (unlikely(!dma_capable(dev, dma_addr, size, true)) || - dma_kmalloc_needs_bounce(dev, size, dir)) { - if (is_swiotlb_active(dev) && - !(attrs & DMA_ATTR_REQUIRE_COHERENT)) - return swiotlb_map(dev, phys, size, dir, attrs); + dma_addr =3D phys_to_dma_encrypted(dev, phys); + } =20 - goto err_overflow; - } + if (unlikely(!dma_capable(dev, dma_addr, size, true)) || + dma_kmalloc_needs_bounce(dev, size, dir)) { + if (is_swiotlb_active(dev) && + !(attrs & DMA_ATTR_REQUIRE_COHERENT)) + return swiotlb_map(dev, phys, size, dir, attrs); + goto err_overflow; } =20 if (!dev_is_dma_coherent(dev) && --=20 2.43.0 From nobody Tue Jun 16 07:26:32 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 310C92ED848; Fri, 17 Apr 2026 09:00:14 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776416414; cv=none; b=irR6sHJWwp3zh1VgoZ/C9LBbBC0VMn0FHESoxFwwXmOay1P2PAyVSZWhK4AepZLnVR6EA3RdD0yXix1i7DFmYcywpQvxXAXLUntGzp6Zlo8h4Z8tIa+W4xhfS6fecUEGxTFcO9q767+hbqgk3EDyncw8jvFu0DWDkvsawmG0V+8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776416414; c=relaxed/simple; bh=biZe9ZTkJa9vfGKWLQ6grlTZUp7RS3uzG+uF8YGccLA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=TV7a0SFlo9oXhnm8dE78lDmz/0jEVTmjecBC1hV2BotV7BW4wlA8XwhrY7Aqk52WrGLhdch6rE9Foh9snG4zXdR1kku+Fec1Kifu9TlvUbwHT2A9HUOFOj19pkZaAp4PAUBflf6zj4S+tSJbPKIOiOeliptls0s6bRRu2QrZHXw= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=bH2taweh; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="bH2taweh" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 7125AC19425; Fri, 17 Apr 2026 09:00:10 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1776416413; bh=biZe9ZTkJa9vfGKWLQ6grlTZUp7RS3uzG+uF8YGccLA=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=bH2tawehIYkPEOIMDsqRO8rygdoJiHo2pISV1vIDxeS2BRob4R/6PHPBd/tcA+oej Fu4KsFix5QQGaXsn+07t5GFrr6zlB4ypmI0Ua/NOOlEu+HvPO36msTcFpmj41Hr7UO 90SQ0ZcW9cKPaDCDQwi18WT3n82K+lTJaYc8jSWLlC5V8PesQUoQUCihGauAIoZeCl ETqJAtVyb+qpRFqeynN0hF4udVbADFpP9jL8MuorWAf30ILmrJXBrmMo7UmLM1uJ1H DChWOlHKL2gilpMVR42dHFQofSueAJt87LOWifQDEDTGwIxzvvm1eH4fGLjcV0uk3X k9wWCHC8NW2zw== From: "Aneesh Kumar K.V (Arm)" To: iommu@lists.linux.dev, linux-kernel@vger.kernel.org Cc: robin.murphy@arm.com, m.szyprowski@samsung.com, will@kernel.org, maz@kernel.org, suzuki.poulose@arm.com, catalin.marinas@arm.com, jiri@resnulli.us, jgg@ziepe.ca, aneesh.kumar@kernel.org, Mostafa Saleh Subject: [RFC PATCH 7/7] dma-direct: set decrypted flag for remapped DMA allocations Date: Fri, 17 Apr 2026 14:29:00 +0530 Message-ID: <20260417085900.3062416-8-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260417085900.3062416-1-aneesh.kumar@kernel.org> References: <20260417085900.3062416-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Devices that are DMA non-coherent and require a remap were skipping dma_set_decrypted(), leaving DMA buffers encrypted even when the device requires unencrypted access. Move the call after the if (remap) branch so that both the direct and remapped allocation paths correctly mark the allocation as decrypted (or fail cleanly) before use. Architectures such as arm64 cannot mark vmap addresses as decrypted, and highmem pages necessarily require a vmap remap. As a result, such allocations cannot be safely used for unencrypted DMA. Therefore, when an unencrypted DMA buffer is requested, avoid allocating high PFNs from __dma_direct_alloc_pages(). Other architectures (e.g. x86) do not have this limitation. However, rather than making this architecture-specific, apply the restriction only when the device requires unencrypted DMA access, for simplicity. Fixes: f3c962226dbe ("dma-direct: clean up the remapping checks in dma_dire= ct_alloc") Signed-off-by: Aneesh Kumar K.V (Arm) --- kernel/dma/direct.c | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 1d2c27bbf3de..bb2a32896a9e 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -204,6 +204,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, { bool remap =3D false, set_uncached =3D false; bool mark_mem_decrypt =3D !!(attrs & DMA_ATTR_CC_DECRYPTED); + bool allow_highmem =3D true; struct page *page; void *ret; =20 @@ -212,6 +213,15 @@ void *dma_direct_alloc(struct device *dev, size_t size, mark_mem_decrypt =3D true; } =20 + if (attrs & DMA_ATTR_CC_DECRYPTED) + /* + * Unencrypted/shared DMA requires a linear-mapped buffer + * address to look up the PFN and set architecture-required PFN + * attributes. This is not possible with HighMem. Avoid HighMem + * allocation. + */ + allow_highmem =3D false; + size =3D PAGE_ALIGN(size); if (attrs & DMA_ATTR_NO_WARN) gfp |=3D __GFP_NOWARN; @@ -270,7 +280,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, } =20 /* we always manually zero the memory once we are done */ - page =3D __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true); + page =3D __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, allow_hig= hmem); if (!page) return NULL; =20 @@ -298,7 +308,13 @@ void *dma_direct_alloc(struct device *dev, size_t size, goto out_free_pages; } else { ret =3D page_address(page); - if (mark_mem_decrypt && dma_set_decrypted(dev, ret, size)) + } + + if (mark_mem_decrypt) { + void *lm_addr; + + lm_addr =3D page_address(page); + if (set_memory_decrypted((unsigned long)lm_addr, PFN_UP(size))) goto out_leak_pages; } =20 @@ -374,8 +390,16 @@ void dma_direct_free(struct device *dev, size_t size, } else { if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED)) arch_dma_clear_uncached(cpu_addr, size); - if (mark_mem_encrypted && dma_set_encrypted(dev, cpu_addr, size)) + } + + if (mark_mem_encrypted) { + void *lm_addr; + + lm_addr =3D phys_to_virt(dma_to_phys(dev, dma_addr)); + if (set_memory_encrypted((unsigned long)lm_addr, PFN_UP(size))) { + pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n"); return; + } } =20 __dma_direct_free_pages(dev, dma_direct_to_page(dev, dma_addr), size); --=20 2.43.0 From nobody Tue Jun 16 07:26:32 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id AF0D723BD02; Fri, 17 Apr 2026 09:56:20 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776419780; cv=none; b=i6te/LIlyFPsioJvahDK+c4y6sWaonI03KTlGSuUutoi9x8g8+QSM8l7t6DJLz8wOMRK9LSuXFk45d9NoAGl1oFa7sV/3tdX9AAanPS9ap3mkenLNJ9NmUH6xUA8vWspifS03T2kRY8m/Cx/g6ygqDbLL3EEqcaKAmi3NdZRU9w= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776419780; c=relaxed/simple; bh=59tfoMBqti+0Vm14vEl9sXO5vfXuwKWNjTQwMptc8Qs=; h=From:To:Cc:Subject:In-Reply-To:References:Date:Message-ID: MIME-Version:Content-Type; b=XWu/R9+t05xeVES7U0HtmviijnUOuUQGs4dREFHMvPSm5g1uwCb4ENzP3b4CUDMi/ZBo7gAzVScAAg98STKVvcD9MqTU61njPSEDR0AE9EZcU3c72xH6e3n8Y+DzUXwpkqbegDbRoM8+97Edd+aX7ae3jFK3jr8n3Kwkju9w7mc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=LcKWovGM; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="LcKWovGM" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 98548C19425; Fri, 17 Apr 2026 09:56:16 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1776419780; bh=59tfoMBqti+0Vm14vEl9sXO5vfXuwKWNjTQwMptc8Qs=; h=From:To:Cc:Subject:In-Reply-To:References:Date:From; b=LcKWovGM1ODvjmxpvalDiphgTygEAncSvruAVkOp32BU1+9xs3vzt7oGmpuH2COTe Bqw3NoRWjiK9e3LfzfYBcGP+zBS5AgBaSywvEDXSUSO1CgngE4LpVAZXsMkbQn1G/J xxtMXomrMhyTt38pVfdNpYNjC6sir/oX6CDhgBloDwtM9d9fKPxpBW+zFGbheoYEOe aZm2f8dj7g9y1XZfrfu8SG+EcTYIl3FMypfPXY1i0oO5HodJA/TeblrL5fuDeMKmdG vv/j0V+gi38p0wBWH1iHDkdFDVMheOzm9aCTIoY1w6F7bWribM4N/CerdNkNZBJazk kqTVQHraCtEDw== X-Mailer: emacs 30.2 (via feedmail 11-beta-1 I) From: Aneesh Kumar K.V To: iommu@lists.linux.dev, linux-kernel@vger.kernel.org Cc: robin.murphy@arm.com, m.szyprowski@samsung.com, will@kernel.org, maz@kernel.org, suzuki.poulose@arm.com, catalin.marinas@arm.com, jiri@resnulli.us, jgg@ziepe.ca, Mostafa Saleh Subject: [RFC PATCH] dma-direct: select DMA address encoding from DMA_ATTR_CC_DECRYPTED In-Reply-To: <20260417085900.3062416-1-aneesh.kumar@kernel.org> References: <20260417085900.3062416-1-aneesh.kumar@kernel.org> Date: Fri, 17 Apr 2026 15:26:13 +0530 Message-ID: Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: "Aneesh Kumar K.V (Arm)" Make the dma-direct helpers derive the DMA address encoding from DMA_ATTR_CC_DECRYPTED instead of implicitly relying on force_dma_unencrypted() inside phys_to_dma_direct() Pass an explicit unencrypted/decrypted state into phys_to_dma_direct(), make the alloc paths return DMA addresses that match the requested buffer encryption state. Also only call dma_set_decrypted() when DMA_ATTR_CC_DECRYPTED is actually set. Signed-off-by: Aneesh Kumar K.V (Arm) --- Missed doing this conversion in the pervious set.=20 --- kernel/dma/direct.c | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index bb2a32896a9e..70fc9ed9ad1a 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -24,11 +24,11 @@ u64 zone_dma_limit __ro_after_init =3D DMA_BIT_MASK(24); =20 static inline dma_addr_t phys_to_dma_direct(struct device *dev, - phys_addr_t phys) + phys_addr_t phys, bool unencrypted) { - if (force_dma_unencrypted(dev)) + if (unencrypted) return phys_to_dma_unencrypted(dev, phys); - return phys_to_dma(dev, phys); + return phys_to_dma_encrypted(dev, phys); } =20 static inline struct page *dma_direct_to_page(struct device *dev, @@ -39,8 +39,9 @@ static inline struct page *dma_direct_to_page(struct devi= ce *dev, =20 u64 dma_direct_get_required_mask(struct device *dev) { + bool require_decrypted =3D force_dma_unencrypted(dev); phys_addr_t phys =3D (phys_addr_t)(max_pfn - 1) << PAGE_SHIFT; - u64 max_dma =3D phys_to_dma_direct(dev, phys); + u64 max_dma =3D phys_to_dma_direct(dev, phys, require_decrypted); =20 return (1ULL << (fls64(max_dma) - 1)) * 2 - 1; } @@ -69,7 +70,8 @@ static gfp_t dma_direct_optimal_gfp_mask(struct device *d= ev, u64 *phys_limit) =20 bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) { - dma_addr_t dma_addr =3D phys_to_dma_direct(dev, phys); + bool require_decrypted =3D force_dma_unencrypted(dev); + dma_addr_t dma_addr =3D phys_to_dma_direct(dev, phys, require_decrypted); =20 if (dma_addr =3D=3D DMA_MAPPING_ERROR) return false; @@ -79,8 +81,6 @@ bool dma_coherent_ok(struct device *dev, phys_addr_t phys= , size_t size) =20 static int dma_set_decrypted(struct device *dev, void *vaddr, size_t size) { - if (!force_dma_unencrypted(dev)) - return 0; return set_memory_decrypted((unsigned long)vaddr, PFN_UP(size)); } =20 @@ -88,8 +88,6 @@ static int dma_set_encrypted(struct device *dev, void *va= ddr, size_t size) { int ret; =20 - if (!force_dma_unencrypted(dev)) - return 0; ret =3D set_memory_encrypted((unsigned long)vaddr, PFN_UP(size)); if (ret) pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n"); @@ -177,7 +175,8 @@ static void *dma_direct_alloc_from_pool(struct device *= dev, size_t size, dma_coherent_ok); if (!page) return NULL; - *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page)); + *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page), + !!(attrs & DMA_ATTR_CC_DECRYPTED)); return ret; } =20 @@ -193,9 +192,11 @@ static void *dma_direct_alloc_no_mapping(struct device= *dev, size_t size, /* remove any dirty cache lines on the kernel alias */ if (!PageHighMem(page)) arch_dma_prep_coherent(page, size); - - /* return the page pointer as the opaque cookie */ - *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page)); + /* + * return the page pointer as the opaque cookie. + * Never used for unencrypted allocation + */ + *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page), false); return page; } =20 @@ -327,7 +328,8 @@ void *dma_direct_alloc(struct device *dev, size_t size, goto out_encrypt_pages; } =20 - *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page)); + *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page), + !!(attrs & DMA_ATTR_CC_DECRYPTED)); return ret; =20 out_encrypt_pages: @@ -437,11 +439,12 @@ struct page *dma_direct_alloc_pages(struct device *de= v, size_t size, return NULL; =20 ret =3D page_address(page); - if (dma_set_decrypted(dev, ret, size)) + if ((attrs & DMA_ATTR_CC_DECRYPTED) && dma_set_decrypted(dev, ret, size)) goto out_leak_pages; setup_page: memset(ret, 0, size); - *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page)); + *dma_handle =3D phys_to_dma_direct(dev, page_to_phys(page), + !!(attrs & DMA_ATTR_CC_DECRYPTED)); return page; out_leak_pages: return NULL; @@ -451,8 +454,12 @@ void dma_direct_free_pages(struct device *dev, size_t = size, struct page *page, dma_addr_t dma_addr, enum dma_data_direction dir) { + /* + * if the device had requested for an unencrypted buffer, + * convert it to encrypted on free + */ + bool mark_mem_encrypted =3D force_dma_unencrypted(dev); void *vaddr =3D page_address(page); - bool mark_mem_encrypted =3D true; =20 /* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */ if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) && --=20 2.43.0