From nobody Tue Dec 16 17:44:47 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2745616D9A4; Wed, 24 Jul 2024 18:05:26 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1721844327; cv=none; b=lomnBNUtccLtbE6lGp5ZNJMUj8/2/gFWy2Oqx8MJuiyffV7qHhtUjgXTy9gAHWjoLNsa5UPZc6llOKOqVKwYfcR2ao66mhcHwlpYrQ/by+MLkhzbJ/Ce+iXSY0REY42jteiBcoZRxs+M4VE/m2mft/MKz29qbEytIUtA25Desy4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1721844327; c=relaxed/simple; bh=UdgZiT7A9il+QprsMyU3VdpT957eeuVrd1d0/Tm9wF0=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=Y4s61AQFMIbWs+CEyx9mAHEKeNNSwl+ljWouC8yvDjRrIKGHQKYrXZqlhU687rfRduOX0SbAZMZ21qzOfpfgDrcyA/sLc0riSKcG/FcSckmvd+3pekAJd1zxhDcI/AGSExv9so9TzN2WdSVKev0lXnRwxXHqWBMUPsT55S7JK54= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=FMrFDwwf; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="FMrFDwwf" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 0C40CC4AF13; Wed, 24 Jul 2024 18:05:25 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1721844326; bh=UdgZiT7A9il+QprsMyU3VdpT957eeuVrd1d0/Tm9wF0=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=FMrFDwwfueGXFL0AwbTzF5YEujYk43UPNZACM5I5l3RC2BIo6FTHdXq0f5BuODj5F vmaI1o6jtACnkZKX9fzMOKWoiWvaetO7BBJeQJUzMzb8PBjRtuwk3V7dHHjJLzCBYt eiwitUECZw5rkLn8x7kjDrN0hCF3Nbn++wTze+4OSgqHuU04F4Hn7rdcTyQ932GyeM 69T1K4TBuQMILd8cpLsC1fgVs5ubo32P/Hn+J59VnsWzjpmbxN9rNauvtWmvF8fhFh EsK7M9NHDpIz1Q95cgVGVmOWiNxe1L4Jg2fOaQFgjkikTt/4xGSUwZlT9j8Ug+qw0F XgZv/LJf6G1EA== From: Leon Romanovsky To: Christoph Hellwig , Robin Murphy , Joerg Roedel , Will Deacon , Marek Szyprowski Cc: Leon Romanovsky , Easwar Hariharan , linux-kernel@vger.kernel.org, iommu@lists.linux.dev, Jason Gunthorpe , Greg Kroah-Hartman Subject: [PATCH v4 1/2] dma: call unconditionally to unmap_page and unmap_sg callbacks Date: Wed, 24 Jul 2024 21:04:48 +0300 Message-ID: <595d2716d252cac013a650bb3a94555ddb0d1d43.1721818168.git.leon@kernel.org> X-Mailer: git-send-email 2.45.2 In-Reply-To: References: Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Leon Romanovsky Almost all users of ->map_page()/map_sg() callbacks implement ->unmap_page()/unmap_sg() callbacks too. One user which doesn't do it, is dummy DMA ops interface, and the use of this interface is to fail the operation and in such case, there won't be any call to ->unmap_page()/unmap_sg(). This patch removes the existence checks of ->unmap_page()/unmap_sg() and calls to it directly to create symmetrical interface to ->map_page()/map_sg(). Signed-off-by: Leon Romanovsky Signed-off-by: Leon Romanovsky Acked-by: Greg Kroah-Hartman Reviewed-by: Robin Murphy --- kernel/dma/dummy.c | 21 +++++++++++++++++++++ kernel/dma/mapping.c | 4 ++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/kernel/dma/dummy.c b/kernel/dma/dummy.c index b492d59ac77e..92de80e5b057 100644 --- a/kernel/dma/dummy.c +++ b/kernel/dma/dummy.c @@ -17,6 +17,15 @@ static dma_addr_t dma_dummy_map_page(struct device *dev,= struct page *page, { return DMA_MAPPING_ERROR; } +static void dma_dummy_unmap_page(struct device *dev, dma_addr_t dma_handle, + size_t size, enum dma_data_direction dir, unsigned long attrs) +{ + /* + * Dummy ops doesn't support map_page, so unmap_page should never be + * called. + */ + WARN_ON_ONCE(true); +} =20 static int dma_dummy_map_sg(struct device *dev, struct scatterlist *sgl, int nelems, enum dma_data_direction dir, @@ -25,6 +34,16 @@ static int dma_dummy_map_sg(struct device *dev, struct s= catterlist *sgl, return -EINVAL; } =20 +static void dma_dummy_unmap_sg(struct device *dev, struct scatterlist *sgl, + int nelems, enum dma_data_direction dir, + unsigned long attrs) +{ + /* + * Dummy ops doesn't support map_sg, so unmap_sg should never be called. + */ + WARN_ON_ONCE(true); +} + static int dma_dummy_supported(struct device *hwdev, u64 mask) { return 0; @@ -33,6 +52,8 @@ static int dma_dummy_supported(struct device *hwdev, u64 = mask) const struct dma_map_ops dma_dummy_ops =3D { .mmap =3D dma_dummy_mmap, .map_page =3D dma_dummy_map_page, + .unmap_page =3D dma_dummy_unmap_page, .map_sg =3D dma_dummy_map_sg, + .unmap_sg =3D dma_dummy_unmap_sg, .dma_supported =3D dma_dummy_supported, }; diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 81de84318ccc..6832fd6f0796 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -177,7 +177,7 @@ void dma_unmap_page_attrs(struct device *dev, dma_addr_= t addr, size_t size, if (dma_map_direct(dev, ops) || arch_dma_unmap_page_direct(dev, addr + size)) dma_direct_unmap_page(dev, addr, size, dir, attrs); - else if (ops->unmap_page) + else ops->unmap_page(dev, addr, size, dir, attrs); debug_dma_unmap_page(dev, addr, size, dir); } @@ -291,7 +291,7 @@ void dma_unmap_sg_attrs(struct device *dev, struct scat= terlist *sg, if (dma_map_direct(dev, ops) || arch_dma_unmap_sg_direct(dev, sg, nents)) dma_direct_unmap_sg(dev, sg, nents, dir, attrs); - else if (ops->unmap_sg) + else ops->unmap_sg(dev, sg, nents, dir, attrs); } EXPORT_SYMBOL(dma_unmap_sg_attrs); --=20 2.45.2 From nobody Tue Dec 16 17:44:47 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id BC24B15CD62; Wed, 24 Jul 2024 18:05:20 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1721844320; cv=none; b=eGnTm1ff9GO7PVqByq5OOSKwizkf2W0R6oB0vakRnLT/By4/I9Lnhou7f4BE51jaK86wd8g4Fom6FkvHwF3iFsqpQ6lOmRCMjY2gB6J2Vzur0g65eY5krZPx+RHah/Gnaqpu62CLabx5hkBtVEj58Z6n5uwloYBTGqdHNrD9En4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1721844320; c=relaxed/simple; bh=2hmHg8FVwTa/pXbydQXjoHCf7FovsvD2a8+bcaYC8cA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=VjgMdXXEKQ1fwtfmF6wJbQgg5ObcRBpYwEhGoraUFfJ1sEbAGjRh9z+mPbtgbG/XjM25l0ykMITCJqmRMNeerIDr0NqFIglASOMEFh0hrvO1LXOlS90+NrbMgBv2SrUcD7PFUpJErrUbhdJGbk4ZSbCi3maH7FYo/BHp3RqTBr4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=ndDG8mNX; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="ndDG8mNX" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 719A8C32781; Wed, 24 Jul 2024 18:05:19 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1721844320; bh=2hmHg8FVwTa/pXbydQXjoHCf7FovsvD2a8+bcaYC8cA=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=ndDG8mNXUYVWR7CCNEdYHq18QkIStKYE5/bYQcc0LYXaqAf32hpWdgOANCYs3cn42 PrC5STwCGg2iZxemhM5pEAoM3yy/nRE5DwSuTdZUtM7v9M0DrN6EG0C01kDlEOEXFB YLxGvh1TiHICTvEdpehGJIg9oWnGsZdJK6lUY+Pl2QVYx2xk/QMZH8kZ9H1wf3Fd3R 8AzASXtN1CfMY2EyL2mIrg+QY4WsgxxRiUX6nHKeOpOXcbJe2o1aBhRDgLS9icayUc Piv1woang25hReLNHeCANO0tpXdFZp4/Ugm1uFN/HTwwkugLuMagZLrj6Jy55it/Tu himzeF9tLkEwA== From: Leon Romanovsky To: Christoph Hellwig , Robin Murphy , Joerg Roedel , Will Deacon , Marek Szyprowski Cc: Leon Romanovsky , Easwar Hariharan , linux-kernel@vger.kernel.org, iommu@lists.linux.dev, Jason Gunthorpe , Greg Kroah-Hartman Subject: [PATCH v4 2/2] dma: add IOMMU static calls with clear default ops Date: Wed, 24 Jul 2024 21:04:49 +0300 Message-ID: X-Mailer: git-send-email 2.45.2 In-Reply-To: References: Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Leon Romanovsky Most of the arch DMA ops (which often, but not always, involve some sort of IOMMU) are using the same DMA operations, but for all modern platforms dma-iommu implementation is really matters. So let's make sure to call them directly without need to perform function pointers dereference. During system initialization, the arch can set its own DMA and in such case, the default DMA operations will be overridden. Signed-off-by: Leon Romanovsky Signed-off-by: Leon Romanovsky Acked-by: Greg Kroah-Hartman Reported-by: N=C3=ADcolas F. R. A. Prado #KernelCI Tested-by: N=C3=ADcolas F. R. A. Prado --- MAINTAINERS | 1 + drivers/iommu/Kconfig | 2 +- drivers/iommu/dma-iommu.c | 121 ++++++++++---------------- drivers/iommu/intel/Kconfig | 1 - include/linux/device.h | 5 ++ include/linux/dma-map-ops.h | 39 +++++---- include/linux/iommu-dma.h | 169 ++++++++++++++++++++++++++++++++++++ kernel/dma/Kconfig | 6 ++ kernel/dma/Makefile | 2 +- kernel/dma/mapping.c | 90 ++++++++++++++++--- 10 files changed, 327 insertions(+), 109 deletions(-) create mode 100644 include/linux/iommu-dma.h diff --git a/MAINTAINERS b/MAINTAINERS index da5352dbd4f3..1e64be463da7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11544,6 +11544,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/gi= t/iommu/linux.git F: Documentation/devicetree/bindings/iommu/ F: Documentation/userspace-api/iommu.rst F: drivers/iommu/ +F: include/linux/iommu-dma.h F: include/linux/iommu.h F: include/linux/iova.h F: include/linux/of_iommu.h diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index c04584be3089..e24cb857b66c 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -151,7 +151,7 @@ config OF_IOMMU # IOMMU-agnostic DMA-mapping layer config IOMMU_DMA def_bool ARM64 || X86 || S390 - select DMA_OPS + select DMA_OPS_HELPERS select IOMMU_API select IOMMU_IOVA select IRQ_MSI_IOMMU diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 43520e7275cc..ab2d3092ac23 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -1039,9 +1040,8 @@ static void *iommu_dma_alloc_remap(struct device *dev= , size_t size, return NULL; } =20 -static struct sg_table *iommu_dma_alloc_noncontiguous(struct device *dev, - size_t size, enum dma_data_direction dir, gfp_t gfp, - unsigned long attrs) +struct sg_table *iommu_dma_alloc_noncontiguous(struct device *dev, size_t = size, + enum dma_data_direction dir, gfp_t gfp, unsigned long attrs) { struct dma_sgt_handle *sh; =20 @@ -1058,8 +1058,8 @@ static struct sg_table *iommu_dma_alloc_noncontiguous= (struct device *dev, return &sh->sgt; } =20 -static void iommu_dma_free_noncontiguous(struct device *dev, size_t size, - struct sg_table *sgt, enum dma_data_direction dir) +void iommu_dma_free_noncontiguous(struct device *dev, size_t size, + struct sg_table *sgt, enum dma_data_direction dir) { struct dma_sgt_handle *sh =3D sgt_handle(sgt); =20 @@ -1069,8 +1069,8 @@ static void iommu_dma_free_noncontiguous(struct devic= e *dev, size_t size, kfree(sh); } =20 -static void iommu_dma_sync_single_for_cpu(struct device *dev, - dma_addr_t dma_handle, size_t size, enum dma_data_direction dir) +void iommu_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_hand= le, + size_t size, enum dma_data_direction dir) { phys_addr_t phys; =20 @@ -1085,8 +1085,8 @@ static void iommu_dma_sync_single_for_cpu(struct devi= ce *dev, swiotlb_sync_single_for_cpu(dev, phys, size, dir); } =20 -static void iommu_dma_sync_single_for_device(struct device *dev, - dma_addr_t dma_handle, size_t size, enum dma_data_direction dir) +void iommu_dma_sync_single_for_device(struct device *dev, dma_addr_t dma_h= andle, + size_t size, enum dma_data_direction dir) { phys_addr_t phys; =20 @@ -1101,9 +1101,8 @@ static void iommu_dma_sync_single_for_device(struct d= evice *dev, arch_sync_dma_for_device(phys, size, dir); } =20 -static void iommu_dma_sync_sg_for_cpu(struct device *dev, - struct scatterlist *sgl, int nelems, - enum dma_data_direction dir) +void iommu_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl, + int nelems, enum dma_data_direction dir) { struct scatterlist *sg; int i; @@ -1117,9 +1116,8 @@ static void iommu_dma_sync_sg_for_cpu(struct device *= dev, arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir); } =20 -static void iommu_dma_sync_sg_for_device(struct device *dev, - struct scatterlist *sgl, int nelems, - enum dma_data_direction dir) +void iommu_dma_sync_sg_for_device(struct device *dev, struct scatterlist *= sgl, + int nelems, enum dma_data_direction dir) { struct scatterlist *sg; int i; @@ -1134,9 +1132,9 @@ static void iommu_dma_sync_sg_for_device(struct devic= e *dev, arch_sync_dma_for_device(sg_phys(sg), sg->length, dir); } =20 -static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, enum dma_data_direction dir, - unsigned long attrs) +dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, enum dma_data_direction dir, + unsigned long attrs) { phys_addr_t phys =3D page_to_phys(page) + offset; bool coherent =3D dev_is_dma_coherent(dev); @@ -1194,8 +1192,8 @@ static dma_addr_t iommu_dma_map_page(struct device *d= ev, struct page *page, return iova; } =20 -static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, - size_t size, enum dma_data_direction dir, unsigned long attrs) +void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, + size_t size, enum dma_data_direction dir, unsigned long attrs) { struct iommu_domain *domain =3D iommu_get_dma_domain(dev); phys_addr_t phys; @@ -1348,8 +1346,8 @@ static int iommu_dma_map_sg_swiotlb(struct device *de= v, struct scatterlist *sg, * impedance-matching, to be able to hand off a suitably-aligned list, * but still preserve the original offsets and sizes for the caller. */ -static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, unsigned long attrs) +int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, + enum dma_data_direction dir, unsigned long attrs) { struct iommu_domain *domain =3D iommu_get_dma_domain(dev); struct iommu_dma_cookie *cookie =3D domain->iova_cookie; @@ -1468,8 +1466,8 @@ static int iommu_dma_map_sg(struct device *dev, struc= t scatterlist *sg, return ret; } =20 -static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, unsigned long attrs) +void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int ne= nts, + enum dma_data_direction dir, unsigned long attrs) { dma_addr_t end =3D 0, start; struct scatterlist *tmp; @@ -1518,16 +1516,17 @@ static void iommu_dma_unmap_sg(struct device *dev, = struct scatterlist *sg, __iommu_dma_unmap(dev, start, end - start); } =20 -static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t p= hys, - size_t size, enum dma_data_direction dir, unsigned long attrs) +dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, + size_t size, enum dma_data_direction dir, + unsigned long attrs) { return __iommu_dma_map(dev, phys, size, dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO, dma_get_mask(dev)); } =20 -static void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, - size_t size, enum dma_data_direction dir, unsigned long attrs) +void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, + size_t size, enum dma_data_direction dir, unsigned long attrs) { __iommu_dma_unmap(dev, handle, size); } @@ -1563,8 +1562,8 @@ static void __iommu_dma_free(struct device *dev, size= _t size, void *cpu_addr) dma_free_contiguous(dev, page, alloc_size); } =20 -static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, unsigned long attrs) +void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t handle, unsigned long attrs) { __iommu_dma_unmap(dev, handle, size); __iommu_dma_free(dev, size, cpu_addr); @@ -1607,8 +1606,8 @@ static void *iommu_dma_alloc_pages(struct device *dev= , size_t size, return NULL; } =20 -static void *iommu_dma_alloc(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, unsigned long attrs) +void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, + gfp_t gfp, unsigned long attrs) { bool coherent =3D dev_is_dma_coherent(dev); int ioprot =3D dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); @@ -1642,9 +1641,8 @@ static void *iommu_dma_alloc(struct device *dev, size= _t size, return cpu_addr; } =20 -static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs) +int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs) { unsigned long nr_pages =3D PAGE_ALIGN(size) >> PAGE_SHIFT; unsigned long pfn, off =3D vma->vm_pgoff; @@ -1673,9 +1671,8 @@ static int iommu_dma_mmap(struct device *dev, struct = vm_area_struct *vma, vma->vm_page_prot); } =20 -static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs) +int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, + void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs) { struct page *page; int ret; @@ -1700,19 +1697,19 @@ static int iommu_dma_get_sgtable(struct device *dev= , struct sg_table *sgt, return ret; } =20 -static unsigned long iommu_dma_get_merge_boundary(struct device *dev) +unsigned long iommu_dma_get_merge_boundary(struct device *dev) { struct iommu_domain *domain =3D iommu_get_dma_domain(dev); =20 return (1UL << __ffs(domain->pgsize_bitmap)) - 1; } =20 -static size_t iommu_dma_opt_mapping_size(void) +size_t iommu_dma_opt_mapping_size(void) { return iova_rcache_range(); } =20 -static size_t iommu_dma_max_mapping_size(struct device *dev) +size_t iommu_dma_max_mapping_size(struct device *dev) { if (dev_is_untrusted(dev)) return swiotlb_max_mapping_size(dev); @@ -1720,32 +1717,6 @@ static size_t iommu_dma_max_mapping_size(struct devi= ce *dev) return SIZE_MAX; } =20 -static const struct dma_map_ops iommu_dma_ops =3D { - .flags =3D DMA_F_PCI_P2PDMA_SUPPORTED | - DMA_F_CAN_SKIP_SYNC, - .alloc =3D iommu_dma_alloc, - .free =3D iommu_dma_free, - .alloc_pages_op =3D dma_common_alloc_pages, - .free_pages =3D dma_common_free_pages, - .alloc_noncontiguous =3D iommu_dma_alloc_noncontiguous, - .free_noncontiguous =3D iommu_dma_free_noncontiguous, - .mmap =3D iommu_dma_mmap, - .get_sgtable =3D iommu_dma_get_sgtable, - .map_page =3D iommu_dma_map_page, - .unmap_page =3D iommu_dma_unmap_page, - .map_sg =3D iommu_dma_map_sg, - .unmap_sg =3D iommu_dma_unmap_sg, - .sync_single_for_cpu =3D iommu_dma_sync_single_for_cpu, - .sync_single_for_device =3D iommu_dma_sync_single_for_device, - .sync_sg_for_cpu =3D iommu_dma_sync_sg_for_cpu, - .sync_sg_for_device =3D iommu_dma_sync_sg_for_device, - .map_resource =3D iommu_dma_map_resource, - .unmap_resource =3D iommu_dma_unmap_resource, - .get_merge_boundary =3D iommu_dma_get_merge_boundary, - .opt_mapping_size =3D iommu_dma_opt_mapping_size, - .max_mapping_size =3D iommu_dma_max_mapping_size, -}; - void iommu_setup_dma_ops(struct device *dev) { struct iommu_domain *domain =3D iommu_get_domain_for_dev(dev); @@ -1753,19 +1724,15 @@ void iommu_setup_dma_ops(struct device *dev) if (dev_is_pci(dev)) dev->iommu->pci_32bit_workaround =3D !iommu_dma_forcedac; =20 - if (iommu_is_dma_domain(domain)) { - if (iommu_dma_init_domain(domain, dev)) - goto out_err; - dev->dma_ops =3D &iommu_dma_ops; - } else if (dev->dma_ops =3D=3D &iommu_dma_ops) { - /* Clean up if we've switched *from* a DMA domain */ - dev->dma_ops =3D NULL; - } + dev->dma_iommu =3D iommu_is_dma_domain(domain); + if (dev->dma_iommu && iommu_dma_init_domain(domain, dev)) + goto out_err; =20 return; out_err: - pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA op= s\n", - dev_name(dev)); + pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops= \n", + dev_name(dev)); + dev->dma_iommu =3D false; } =20 static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *de= v, diff --git a/drivers/iommu/intel/Kconfig b/drivers/iommu/intel/Kconfig index f52fb39c968e..88fd32a9323c 100644 --- a/drivers/iommu/intel/Kconfig +++ b/drivers/iommu/intel/Kconfig @@ -12,7 +12,6 @@ config DMAR_DEBUG config INTEL_IOMMU bool "Support for Intel IOMMU using DMA Remapping Devices" depends on PCI_MSI && ACPI && X86 - select DMA_OPS select IOMMU_API select IOMMU_IOVA select IOMMUFD_DRIVER if IOMMUFD diff --git a/include/linux/device.h b/include/linux/device.h index ace039151cb8..e66ec47ceb09 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -707,6 +707,8 @@ struct device_physical_location { * for dma allocations. This flag is managed by the dma ops * instance from ->dma_supported. * @dma_skip_sync: DMA sync operations can be skipped for coherent buffers. + * @dma_iommu: Device is using default IOMMU implementation for DMA and + * doesn't rely on dma_ops structure. * * At the lowest level, every device in a Linux system is represented by an * instance of struct device. The device structure contains the information @@ -822,6 +824,9 @@ struct device { #ifdef CONFIG_DMA_NEED_SYNC bool dma_skip_sync:1; #endif +#ifdef CONFIG_IOMMU_DMA + bool dma_iommu : 1; +#endif }; =20 /** diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 02a1c825896b..103d9c66c445 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -13,20 +13,7 @@ struct cma; struct iommu_ops; =20 -/* - * Values for struct dma_map_ops.flags: - * - * DMA_F_PCI_P2PDMA_SUPPORTED: Indicates the dma_map_ops implementation can - * handle PCI P2PDMA pages in the map_sg/unmap_sg operation. - * DMA_F_CAN_SKIP_SYNC: DMA sync operations can be skipped if the device is - * coherent and it's not an SWIOTLB buffer. - */ -#define DMA_F_PCI_P2PDMA_SUPPORTED (1 << 0) -#define DMA_F_CAN_SKIP_SYNC (1 << 1) - struct dma_map_ops { - unsigned int flags; - void *(*alloc)(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs); @@ -114,6 +101,28 @@ static inline void set_dma_ops(struct device *dev, } #endif /* CONFIG_DMA_OPS */ =20 +#ifdef CONFIG_DMA_OPS_HELPERS +#include + +struct page *dma_common_alloc_pages(struct device *dev, size_t size, + dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp); +void dma_common_free_pages(struct device *dev, size_t size, struct page *v= addr, + dma_addr_t dma_handle, enum dma_data_direction dir); +#else /* CONFIG_DMA_OPS_HELPERS */ +static inline struct page * +dma_common_alloc_pages(struct device *dev, size_t size, dma_addr_t *dma_ha= ndle, + enum dma_data_direction dir, gfp_t gfp) +{ + return NULL; +} +static inline void dma_common_free_pages(struct device *dev, size_t size, + struct page *vaddr, + dma_addr_t dma_handle, + enum dma_data_direction dir) +{ +} +#endif /* CONFIG_DMA_OPS_HELPERS */ + #ifdef CONFIG_DMA_CMA extern struct cma *dma_contiguous_default_area; =20 @@ -239,10 +248,6 @@ int dma_common_get_sgtable(struct device *dev, struct = sg_table *sgt, int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs); -struct page *dma_common_alloc_pages(struct device *dev, size_t size, - dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp); -void dma_common_free_pages(struct device *dev, size_t size, struct page *v= addr, - dma_addr_t dma_handle, enum dma_data_direction dir); =20 struct page **dma_common_find_pages(void *cpu_addr); void *dma_common_contiguous_remap(struct page *page, size_t size, pgprot_t= prot, diff --git a/include/linux/iommu-dma.h b/include/linux/iommu-dma.h new file mode 100644 index 000000000000..622232fc9510 --- /dev/null +++ b/include/linux/iommu-dma.h @@ -0,0 +1,169 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved + * + * DMA operations that map physical memory through IOMMU. + */ +#ifndef _LINUX_IOMMU_DMA_H +#define _LINUX_IOMMU_DMA_H + +#include + +#ifdef CONFIG_IOMMU_DMA +dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, + enum dma_data_direction dir, unsigned long attrs); +void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, + size_t size, enum dma_data_direction dir, + unsigned long attrs); +int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, + enum dma_data_direction dir, unsigned long attrs); +void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int ne= nts, + enum dma_data_direction dir, unsigned long attrs); +void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, + gfp_t gfp, unsigned long attrs); +int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, + size_t size, unsigned long attrs); +int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs); +unsigned long iommu_dma_get_merge_boundary(struct device *dev); +size_t iommu_dma_opt_mapping_size(void); +size_t iommu_dma_max_mapping_size(struct device *dev); +void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t handle, unsigned long attrs); +dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, + size_t size, enum dma_data_direction dir, + unsigned long attrs); +void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, + size_t size, enum dma_data_direction dir, + unsigned long attrs); +struct sg_table *iommu_dma_alloc_noncontiguous(struct device *dev, size_t = size, + enum dma_data_direction dir, + gfp_t gfp, unsigned long attrs); +void iommu_dma_free_noncontiguous(struct device *dev, size_t size, + struct sg_table *sgt, + enum dma_data_direction dir); +void iommu_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_hand= le, + size_t size, enum dma_data_direction dir); +void iommu_dma_sync_single_for_device(struct device *dev, dma_addr_t dma_h= andle, + size_t size, enum dma_data_direction dir); +void iommu_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl, + int nelems, enum dma_data_direction dir); +void iommu_dma_sync_sg_for_device(struct device *dev, struct scatterlist *= sgl, + int nelems, enum dma_data_direction dir); +#else +static inline dma_addr_t iommu_dma_map_page(struct device *dev, + struct page *page, + unsigned long offset, size_t size, + enum dma_data_direction dir, + unsigned long attrs) +{ + return DMA_MAPPING_ERROR; +} +static inline void iommu_dma_unmap_page(struct device *dev, + dma_addr_t dma_handle, size_t size, + enum dma_data_direction dir, + unsigned long attrs) +{ +} +static inline int iommu_dma_map_sg(struct device *dev, struct scatterlist = *sg, + int nents, enum dma_data_direction dir, + unsigned long attrs) +{ + return -EINVAL; +} +static inline void iommu_dma_unmap_sg(struct device *dev, + struct scatterlist *sg, int nents, + enum dma_data_direction dir, + unsigned long attrs) +{ +} +static inline void *iommu_dma_alloc(struct device *dev, size_t size, + dma_addr_t *handle, gfp_t gfp, + unsigned long attrs) +{ + return NULL; +} +static inline int iommu_dma_mmap(struct device *dev, struct vm_area_struct= *vma, + void *cpu_addr, dma_addr_t dma_addr, + size_t size, unsigned long attrs) +{ + return -EINVAL; +} +static inline int iommu_dma_get_sgtable(struct device *dev, + struct sg_table *sgt, void *cpu_addr, + dma_addr_t dma_addr, size_t size, + unsigned long attrs) +{ + return -EINVAL; +} +static inline unsigned long iommu_dma_get_merge_boundary(struct device *de= v) +{ + return 0; +} +static inline size_t iommu_dma_opt_mapping_size(void) +{ + return 0; +} +static inline size_t iommu_dma_max_mapping_size(struct device *dev) +{ + return 0; +} +static inline void iommu_dma_free(struct device *dev, size_t size, + void *cpu_addr, dma_addr_t handle, + unsigned long attrs) +{ +} +static inline dma_addr_t iommu_dma_map_resource(struct device *dev, + phys_addr_t phys, size_t size, + enum dma_data_direction dir, + unsigned long attrs) +{ + return DMA_MAPPING_ERROR; +} +static inline void iommu_dma_unmap_resource(struct device *dev, + dma_addr_t handle, size_t size, + enum dma_data_direction dir, + unsigned long attrs) +{ +} +static inline struct sg_table * +iommu_dma_alloc_noncontiguous(struct device *dev, size_t size, + enum dma_data_direction dir, gfp_t gfp, + unsigned long attrs) +{ + return NULL; +} +static inline void iommu_dma_free_noncontiguous(struct device *dev, size_t= size, + struct sg_table *sgt, + enum dma_data_direction dir) +{ +} +static inline void iommu_dma_sync_single_for_cpu(struct device *dev, + dma_addr_t dma_handle, + size_t size, + enum dma_data_direction dir) +{ +} +static inline void iommu_dma_sync_single_for_device(struct device *dev, + dma_addr_t dma_handle, + size_t size, + enum dma_data_direction dir) +{ +} +static inline void iommu_dma_sync_sg_for_cpu(struct device *dev, + struct scatterlist *sgl, + int nelems, + enum dma_data_direction dir) +{ +} +static inline void iommu_dma_sync_sg_for_device(struct device *dev, + struct scatterlist *sgl, + int nelems, + enum dma_data_direction dir) +{ +} +#endif /* CONFIG_IOMMU_DMA */ +#endif /* _LINUX_IOMMU_DMA_H */ diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig index c06e56be0ca1..03bb925014a7 100644 --- a/kernel/dma/Kconfig +++ b/kernel/dma/Kconfig @@ -8,8 +8,14 @@ config HAS_DMA depends on !NO_DMA default y =20 +# DMA IOMMU uses common ops helpers for certain operations, so let's allow= to build +# ops_helpers.c even if DMA_OPS is not enabled +config DMA_OPS_HELPERS + bool + config DMA_OPS depends on HAS_DMA + select DMA_OPS_HELPERS bool =20 # diff --git a/kernel/dma/Makefile b/kernel/dma/Makefile index 21926e46ef4f..2e6e933cf7f3 100644 --- a/kernel/dma/Makefile +++ b/kernel/dma/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 =20 obj-$(CONFIG_HAS_DMA) +=3D mapping.o direct.o -obj-$(CONFIG_DMA_OPS) +=3D ops_helpers.o +obj-$(CONFIG_DMA_OPS_HELPERS) +=3D ops_helpers.o obj-$(CONFIG_DMA_OPS) +=3D dummy.o obj-$(CONFIG_DMA_CMA) +=3D contiguous.o obj-$(CONFIG_DMA_DECLARE_COHERENT) +=3D coherent.o diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 6832fd6f0796..02451e27e0b1 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -8,6 +8,7 @@ #include /* for max_pfn */ #include #include +#include #include #include #include @@ -113,11 +114,27 @@ void *dmam_alloc_attrs(struct device *dev, size_t siz= e, dma_addr_t *dma_handle, } EXPORT_SYMBOL(dmam_alloc_attrs); =20 +#ifdef CONFIG_IOMMU_DMA +static bool use_dma_iommu(struct device *dev) +{ + return dev->dma_iommu; +} +#else +static bool use_dma_iommu(struct device *dev) +{ + return false; +} +#endif + static bool dma_go_direct(struct device *dev, dma_addr_t mask, const struct dma_map_ops *ops) { + if (use_dma_iommu(dev)) + return false; + if (likely(!ops)) return true; + #ifdef CONFIG_DMA_OPS_BYPASS if (dev->dma_ops_bypass) return min_not_zero(mask, dev->bus_dma_limit) >=3D @@ -159,6 +176,8 @@ dma_addr_t dma_map_page_attrs(struct device *dev, struc= t page *page, if (dma_map_direct(dev, ops) || arch_dma_map_page_direct(dev, page_to_phys(page) + offset + size)) addr =3D dma_direct_map_page(dev, page, offset, size, dir, attrs); + else if (use_dma_iommu(dev)) + addr =3D iommu_dma_map_page(dev, page, offset, size, dir, attrs); else addr =3D ops->map_page(dev, page, offset, size, dir, attrs); kmsan_handle_dma(page, offset, size, dir); @@ -177,6 +196,8 @@ void dma_unmap_page_attrs(struct device *dev, dma_addr_= t addr, size_t size, if (dma_map_direct(dev, ops) || arch_dma_unmap_page_direct(dev, addr + size)) dma_direct_unmap_page(dev, addr, size, dir, attrs); + else if (use_dma_iommu(dev)) + iommu_dma_unmap_page(dev, addr, size, dir, attrs); else ops->unmap_page(dev, addr, size, dir, attrs); debug_dma_unmap_page(dev, addr, size, dir); @@ -197,6 +218,8 @@ static int __dma_map_sg_attrs(struct device *dev, struc= t scatterlist *sg, if (dma_map_direct(dev, ops) || arch_dma_map_sg_direct(dev, sg, nents)) ents =3D dma_direct_map_sg(dev, sg, nents, dir, attrs); + else if (use_dma_iommu(dev)) + ents =3D iommu_dma_map_sg(dev, sg, nents, dir, attrs); else ents =3D ops->map_sg(dev, sg, nents, dir, attrs); =20 @@ -291,7 +314,9 @@ void dma_unmap_sg_attrs(struct device *dev, struct scat= terlist *sg, if (dma_map_direct(dev, ops) || arch_dma_unmap_sg_direct(dev, sg, nents)) dma_direct_unmap_sg(dev, sg, nents, dir, attrs); - else + else if (use_dma_iommu(dev)) + iommu_dma_unmap_sg(dev, sg, nents, dir, attrs); + else if (ops->unmap_sg) ops->unmap_sg(dev, sg, nents, dir, attrs); } EXPORT_SYMBOL(dma_unmap_sg_attrs); @@ -309,6 +334,8 @@ dma_addr_t dma_map_resource(struct device *dev, phys_ad= dr_t phys_addr, =20 if (dma_map_direct(dev, ops)) addr =3D dma_direct_map_resource(dev, phys_addr, size, dir, attrs); + else if (use_dma_iommu(dev)) + addr =3D iommu_dma_map_resource(dev, phys_addr, size, dir, attrs); else if (ops->map_resource) addr =3D ops->map_resource(dev, phys_addr, size, dir, attrs); =20 @@ -323,8 +350,12 @@ void dma_unmap_resource(struct device *dev, dma_addr_t= addr, size_t size, const struct dma_map_ops *ops =3D get_dma_ops(dev); =20 BUG_ON(!valid_dma_direction(dir)); - if (!dma_map_direct(dev, ops) && ops->unmap_resource) - ops->unmap_resource(dev, addr, size, dir, attrs); + if (dma_map_direct(dev, ops)) + ; /* nothing to do: uncached and no swiotlb */ + else if (use_dma_iommu(dev)) + iommu_dma_unmap_resource(dev, addr, size, dir, attrs); + else if (ops->unmap_resource) + ops->unmap_resource(dev, addr, size, dir, attrs); debug_dma_unmap_resource(dev, addr, size, dir); } EXPORT_SYMBOL(dma_unmap_resource); @@ -338,6 +369,8 @@ void __dma_sync_single_for_cpu(struct device *dev, dma_= addr_t addr, size_t size, BUG_ON(!valid_dma_direction(dir)); if (dma_map_direct(dev, ops)) dma_direct_sync_single_for_cpu(dev, addr, size, dir); + else if (use_dma_iommu(dev)) + iommu_dma_sync_single_for_cpu(dev, addr, size, dir); else if (ops->sync_single_for_cpu) ops->sync_single_for_cpu(dev, addr, size, dir); debug_dma_sync_single_for_cpu(dev, addr, size, dir); @@ -352,6 +385,8 @@ void __dma_sync_single_for_device(struct device *dev, d= ma_addr_t addr, BUG_ON(!valid_dma_direction(dir)); if (dma_map_direct(dev, ops)) dma_direct_sync_single_for_device(dev, addr, size, dir); + else if (use_dma_iommu(dev)) + iommu_dma_sync_single_for_device(dev, addr, size, dir); else if (ops->sync_single_for_device) ops->sync_single_for_device(dev, addr, size, dir); debug_dma_sync_single_for_device(dev, addr, size, dir); @@ -366,6 +401,8 @@ void __dma_sync_sg_for_cpu(struct device *dev, struct s= catterlist *sg, BUG_ON(!valid_dma_direction(dir)); if (dma_map_direct(dev, ops)) dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir); + else if (use_dma_iommu(dev)) + iommu_dma_sync_sg_for_cpu(dev, sg, nelems, dir); else if (ops->sync_sg_for_cpu) ops->sync_sg_for_cpu(dev, sg, nelems, dir); debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir); @@ -380,6 +417,8 @@ void __dma_sync_sg_for_device(struct device *dev, struc= t scatterlist *sg, BUG_ON(!valid_dma_direction(dir)); if (dma_map_direct(dev, ops)) dma_direct_sync_sg_for_device(dev, sg, nelems, dir); + else if (use_dma_iommu(dev)) + iommu_dma_sync_sg_for_device(dev, sg, nelems, dir); else if (ops->sync_sg_for_device) ops->sync_sg_for_device(dev, sg, nelems, dir); debug_dma_sync_sg_for_device(dev, sg, nelems, dir); @@ -405,7 +444,7 @@ static void dma_setup_need_sync(struct device *dev) { const struct dma_map_ops *ops =3D get_dma_ops(dev); =20 - if (dma_map_direct(dev, ops) || (ops->flags & DMA_F_CAN_SKIP_SYNC)) + if (dma_map_direct(dev, ops) || use_dma_iommu(dev)) /* * dma_skip_sync will be reset to %false on first SWIOTLB buffer * mapping, if any. During the device initialization, it's @@ -446,6 +485,9 @@ int dma_get_sgtable_attrs(struct device *dev, struct sg= _table *sgt, if (dma_alloc_direct(dev, ops)) return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs); + if (use_dma_iommu(dev)) + return iommu_dma_get_sgtable(dev, sgt, cpu_addr, dma_addr, + size, attrs); if (!ops->get_sgtable) return -ENXIO; return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs); @@ -482,6 +524,8 @@ bool dma_can_mmap(struct device *dev) =20 if (dma_alloc_direct(dev, ops)) return dma_direct_can_mmap(dev); + if (use_dma_iommu(dev)) + return true; return ops->mmap !=3D NULL; } EXPORT_SYMBOL_GPL(dma_can_mmap); @@ -508,6 +552,9 @@ int dma_mmap_attrs(struct device *dev, struct vm_area_s= truct *vma, if (dma_alloc_direct(dev, ops)) return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); + if (use_dma_iommu(dev)) + return iommu_dma_mmap(dev, vma, cpu_addr, dma_addr, size, + attrs); if (!ops->mmap) return -ENXIO; return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs); @@ -559,6 +606,8 @@ void *dma_alloc_attrs(struct device *dev, size_t size, = dma_addr_t *dma_handle, =20 if (dma_alloc_direct(dev, ops)) cpu_addr =3D dma_direct_alloc(dev, size, dma_handle, flag, attrs); + else if (use_dma_iommu(dev)) + cpu_addr =3D iommu_dma_alloc(dev, size, dma_handle, flag, attrs); else if (ops->alloc) cpu_addr =3D ops->alloc(dev, size, dma_handle, flag, attrs); else @@ -591,6 +640,8 @@ void dma_free_attrs(struct device *dev, size_t size, vo= id *cpu_addr, debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); if (dma_alloc_direct(dev, ops)) dma_direct_free(dev, size, cpu_addr, dma_handle, attrs); + else if (use_dma_iommu(dev)) + iommu_dma_free(dev, size, cpu_addr, dma_handle, attrs); else if (ops->free) ops->free(dev, size, cpu_addr, dma_handle, attrs); } @@ -611,6 +662,8 @@ static struct page *__dma_alloc_pages(struct device *de= v, size_t size, size =3D PAGE_ALIGN(size); if (dma_alloc_direct(dev, ops)) return dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp); + if (use_dma_iommu(dev)) + return dma_common_alloc_pages(dev, size, dma_handle, dir, gfp); if (!ops->alloc_pages_op) return NULL; return ops->alloc_pages_op(dev, size, dma_handle, dir, gfp); @@ -635,6 +688,8 @@ static void __dma_free_pages(struct device *dev, size_t= size, struct page *page, size =3D PAGE_ALIGN(size); if (dma_alloc_direct(dev, ops)) dma_direct_free_pages(dev, size, page, dma_handle, dir); + else if (use_dma_iommu(dev)) + dma_common_free_pages(dev, size, page, dma_handle, dir); else if (ops->free_pages) ops->free_pages(dev, size, page, dma_handle, dir); } @@ -697,6 +752,8 @@ struct sg_table *dma_alloc_noncontiguous(struct device = *dev, size_t size, =20 if (ops && ops->alloc_noncontiguous) sgt =3D ops->alloc_noncontiguous(dev, size, dir, gfp, attrs); + else if (use_dma_iommu(dev)) + sgt =3D iommu_dma_alloc_noncontiguous(dev, size, dir, gfp, attrs); else sgt =3D alloc_single_sgt(dev, size, dir, gfp); =20 @@ -725,6 +782,8 @@ void dma_free_noncontiguous(struct device *dev, size_t = size, debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir); if (ops && ops->free_noncontiguous) ops->free_noncontiguous(dev, size, sgt, dir); + else if (use_dma_iommu(dev)) + iommu_dma_free_noncontiguous(dev, size, sgt, dir); else free_single_sgt(dev, size, sgt, dir); } @@ -772,6 +831,8 @@ static int dma_supported(struct device *dev, u64 mask) { const struct dma_map_ops *ops =3D get_dma_ops(dev); =20 + if (WARN_ON(ops && use_dma_iommu(dev))) + return false; /* * ->dma_supported sets the bypass flag, so we must always call * into the method here unless the device is truly direct mapped. @@ -787,17 +848,14 @@ bool dma_pci_p2pdma_supported(struct device *dev) { const struct dma_map_ops *ops =3D get_dma_ops(dev); =20 - /* if ops is not set, dma direct will be used which supports P2PDMA */ - if (!ops) - return true; - /* * Note: dma_ops_bypass is not checked here because P2PDMA should * not be used with dma mapping ops that do not have support even * if the specific device is bypassing them. */ =20 - return ops->flags & DMA_F_PCI_P2PDMA_SUPPORTED; + /* if ops is not set, dma direct and default IOMMU support P2PDMA */ + return !ops; } EXPORT_SYMBOL_GPL(dma_pci_p2pdma_supported); =20 @@ -865,6 +923,8 @@ size_t dma_max_mapping_size(struct device *dev) =20 if (dma_map_direct(dev, ops)) size =3D dma_direct_max_mapping_size(dev); + else if (use_dma_iommu(dev)) + size =3D iommu_dma_max_mapping_size(dev); else if (ops && ops->max_mapping_size) size =3D ops->max_mapping_size(dev); =20 @@ -877,9 +937,10 @@ size_t dma_opt_mapping_size(struct device *dev) const struct dma_map_ops *ops =3D get_dma_ops(dev); size_t size =3D SIZE_MAX; =20 - if (ops && ops->opt_mapping_size) + if (use_dma_iommu(dev)) + size =3D iommu_dma_opt_mapping_size(); + else if (ops && ops->opt_mapping_size) size =3D ops->opt_mapping_size(); - return min(dma_max_mapping_size(dev), size); } EXPORT_SYMBOL_GPL(dma_opt_mapping_size); @@ -888,7 +949,12 @@ unsigned long dma_get_merge_boundary(struct device *de= v) { const struct dma_map_ops *ops =3D get_dma_ops(dev); =20 - if (!ops || !ops->get_merge_boundary) + if (use_dma_iommu(dev)) + return iommu_dma_get_merge_boundary(dev); + + if (!ops) + return 0; /* can't merge */ + if (!ops->get_merge_boundary) return 0; /* can't merge */ =20 return ops->get_merge_boundary(dev); --=20 2.45.2