From: Jean-Philippe Brucker <jean-philippe@linaro.org>

Backport Linux commit e881e7839fba. This is the clean backport without
any changes.

Allow sharing structure definitions with the upcoming SVA support for
Arm SMMUv3, by moving them to a separate header. We could surgically
extract only what is needed but keeping all definitions in one place
looks nicer.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Link: https://lore.kernel.org/r/20200918101852.582559-8-jean-philippe@linaro.org
Signed-off-by: Will Deacon <will@kernel.org>
Origin: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git e881e7839fba
Signed-off-by: Rahul Singh <rahul.singh@arm.com>
---
Changes in v2:
 - fix commit msg
 - also move struct definition to header file to sync with Linux patch
---
 xen/drivers/passthrough/arm/smmu-v3.c | 666 +------------------------
 xen/drivers/passthrough/arm/smmu-v3.h | 674 ++++++++++++++++++++++++++
 2 files changed, 675 insertions(+), 665 deletions(-)
 create mode 100644 xen/drivers/passthrough/arm/smmu-v3.h

diff --git a/xen/drivers/passthrough/arm/smmu-v3.c b/xen/drivers/passthrough/arm/smmu-v3.c
index cee13d1fc7..85ad066266 100644
--- a/xen/drivers/passthrough/arm/smmu-v3.c
+++ b/xen/drivers/passthrough/arm/smmu-v3.c
@@ -90,6 +90,7 @@
 #include <asm/iommu_fwspec.h>
 #include <asm/platform.h>
 
+#include "smmu-v3.h"
 
 #define ARM_SMMU_VTCR_SH_IS		3
 #define ARM_SMMU_VTCR_RGN_WBWA		1
@@ -102,21 +103,10 @@
 #define ARM_SMMU_VTCR_PS_48_BIT		0x5ULL
 #define ARM_SMMU_VTCR_PS_52_BIT		0x6ULL
 
-/* Linux compatibility functions. */
-typedef paddr_t		dma_addr_t;
-typedef paddr_t		phys_addr_t;
-typedef unsigned int		gfp_t;
-
 #define platform_device		device
 
 #define GFP_KERNEL		0
 
-/* Alias to Xen lock functions */
-#define mutex spinlock
-#define mutex_init spin_lock_init
-#define mutex_lock spin_lock
-#define mutex_unlock spin_unlock
-
 /* Device logger functions */
 #define dev_name(dev)	dt_node_full_name(dev->of_node)
 #define dev_dbg(dev, fmt, ...)			\
@@ -157,12 +147,6 @@ typedef unsigned int		gfp_t;
 #define readl_relaxed_poll_timeout(addr, val, cond, delay_us, timeout_us)	\
 	readx_poll_timeout(readl_relaxed, addr, val, cond, delay_us, timeout_us)
 
-#define FIELD_PREP(_mask, _val)			\
-	(((typeof(_mask))(_val) << (ffs64(_mask) - 1)) & (_mask))
-
-#define FIELD_GET(_mask, _reg)			\
-	((typeof(_mask))(((_reg) & (_mask)) >> (ffs64(_mask) - 1)))
-
 /*
  * Helpers for DMA allocation. Just the function name is reused for
  * porting code, these allocation are not managed allocations
@@ -195,27 +179,6 @@ static void *dmam_alloc_coherent(struct device *dev, size_t size,
 	return vaddr;
 }
 
-
-/* Xen specific code. */
-struct iommu_domain {
-	/* Runtime SMMU configuration for this iommu_domain */
-	atomic_t		ref;
-	/*
-	 * Used to link iommu_domain contexts for a same domain.
-	 * There is at least one per-SMMU to used by the domain.
-	 */
-	struct list_head		list;
-};
-
-/* Describes information required for a Xen domain */
-struct arm_smmu_xen_domain {
-	spinlock_t		lock;
-
-	/* List of iommu domains associated to this domain */
-	struct list_head	contexts;
-};
-
-
 /* Keep a list of devices associated with this driver */
 static DEFINE_SPINLOCK(arm_smmu_devices_lock);
 static LIST_HEAD(arm_smmu_devices);
@@ -259,635 +222,8 @@ static int platform_get_irq_byname_optional(struct device *dev,
 }
 
 /* Start of Linux SMMUv3 code */
-/* MMIO registers */
-#define ARM_SMMU_IDR0			0x0
-#define IDR0_ST_LVL			GENMASK(28, 27)
-#define IDR0_ST_LVL_2LVL		1
-#define IDR0_STALL_MODEL		GENMASK(25, 24)
-#define IDR0_STALL_MODEL_STALL		0
-#define IDR0_STALL_MODEL_FORCE		2
-#define IDR0_TTENDIAN			GENMASK(22, 21)
-#define IDR0_TTENDIAN_MIXED		0
-#define IDR0_TTENDIAN_LE		2
-#define IDR0_TTENDIAN_BE		3
-#define IDR0_CD2L			(1 << 19)
-#define IDR0_VMID16			(1 << 18)
-#define IDR0_PRI			(1 << 16)
-#define IDR0_SEV			(1 << 14)
-#define IDR0_MSI			(1 << 13)
-#define IDR0_ASID16			(1 << 12)
-#define IDR0_ATS			(1 << 10)
-#define IDR0_HYP			(1 << 9)
-#define IDR0_COHACC			(1 << 4)
-#define IDR0_TTF			GENMASK(3, 2)
-#define IDR0_TTF_AARCH64		2
-#define IDR0_TTF_AARCH32_64		3
-#define IDR0_S1P			(1 << 1)
-#define IDR0_S2P			(1 << 0)
-
-#define ARM_SMMU_IDR1			0x4
-#define IDR1_TABLES_PRESET		(1 << 30)
-#define IDR1_QUEUES_PRESET		(1 << 29)
-#define IDR1_REL			(1 << 28)
-#define IDR1_CMDQS			GENMASK(25, 21)
-#define IDR1_EVTQS			GENMASK(20, 16)
-#define IDR1_PRIQS			GENMASK(15, 11)
-#define IDR1_SSIDSIZE			GENMASK(10, 6)
-#define IDR1_SIDSIZE			GENMASK(5, 0)
-
-#define ARM_SMMU_IDR5			0x14
-#define IDR5_STALL_MAX			GENMASK(31, 16)
-#define IDR5_GRAN64K			(1 << 6)
-#define IDR5_GRAN16K			(1 << 5)
-#define IDR5_GRAN4K			(1 << 4)
-#define IDR5_OAS			GENMASK(2, 0)
-#define IDR5_OAS_32_BIT			0
-#define IDR5_OAS_36_BIT			1
-#define IDR5_OAS_40_BIT			2
-#define IDR5_OAS_42_BIT			3
-#define IDR5_OAS_44_BIT			4
-#define IDR5_OAS_48_BIT			5
-#define IDR5_OAS_52_BIT			6
-#define IDR5_VAX			GENMASK(11, 10)
-#define IDR5_VAX_52_BIT			1
-
-#define ARM_SMMU_CR0			0x20
-#define CR0_ATSCHK			(1 << 4)
-#define CR0_CMDQEN			(1 << 3)
-#define CR0_EVTQEN			(1 << 2)
-#define CR0_PRIQEN			(1 << 1)
-#define CR0_SMMUEN			(1 << 0)
-
-#define ARM_SMMU_CR0ACK			0x24
-
-#define ARM_SMMU_CR1			0x28
-#define CR1_TABLE_SH			GENMASK(11, 10)
-#define CR1_TABLE_OC			GENMASK(9, 8)
-#define CR1_TABLE_IC			GENMASK(7, 6)
-#define CR1_QUEUE_SH			GENMASK(5, 4)
-#define CR1_QUEUE_OC			GENMASK(3, 2)
-#define CR1_QUEUE_IC			GENMASK(1, 0)
-/* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
-#define CR1_CACHE_NC			0
-#define CR1_CACHE_WB			1
-#define CR1_CACHE_WT			2
-
-#define ARM_SMMU_CR2			0x2c
-#define CR2_PTM				(1 << 2)
-#define CR2_RECINVSID			(1 << 1)
-#define CR2_E2H				(1 << 0)
-
-#define ARM_SMMU_GBPA			0x44
-#define GBPA_UPDATE			(1U << 31)
-#define GBPA_ABORT			(1 << 20)
-
-#define ARM_SMMU_IRQ_CTRL		0x50
-#define IRQ_CTRL_EVTQ_IRQEN		(1 << 2)
-#define IRQ_CTRL_PRIQ_IRQEN		(1 << 1)
-#define IRQ_CTRL_GERROR_IRQEN		(1 << 0)
-
-#define ARM_SMMU_IRQ_CTRLACK		0x54
-
-#define ARM_SMMU_GERROR			0x60
-#define GERROR_SFM_ERR			(1 << 8)
-#define GERROR_MSI_GERROR_ABT_ERR	(1 << 7)
-#define GERROR_MSI_PRIQ_ABT_ERR		(1 << 6)
-#define GERROR_MSI_EVTQ_ABT_ERR		(1 << 5)
-#define GERROR_MSI_CMDQ_ABT_ERR		(1 << 4)
-#define GERROR_PRIQ_ABT_ERR		(1 << 3)
-#define GERROR_EVTQ_ABT_ERR		(1 << 2)
-#define GERROR_CMDQ_ERR			(1 << 0)
-#define GERROR_ERR_MASK			0xfd
-
-#define ARM_SMMU_GERRORN		0x64
-
-#define ARM_SMMU_GERROR_IRQ_CFG0	0x68
-#define ARM_SMMU_GERROR_IRQ_CFG1	0x70
-#define ARM_SMMU_GERROR_IRQ_CFG2	0x74
-
-#define ARM_SMMU_STRTAB_BASE		0x80
-#define STRTAB_BASE_RA			(1UL << 62)
-#define STRTAB_BASE_ADDR_MASK		GENMASK_ULL(51, 6)
-
-#define ARM_SMMU_STRTAB_BASE_CFG	0x88
-#define STRTAB_BASE_CFG_FMT		GENMASK(17, 16)
-#define STRTAB_BASE_CFG_FMT_LINEAR	0
-#define STRTAB_BASE_CFG_FMT_2LVL	1
-#define STRTAB_BASE_CFG_SPLIT		GENMASK(10, 6)
-#define STRTAB_BASE_CFG_LOG2SIZE	GENMASK(5, 0)
-
-#define ARM_SMMU_CMDQ_BASE		0x90
-#define ARM_SMMU_CMDQ_PROD		0x98
-#define ARM_SMMU_CMDQ_CONS		0x9c
-
-#define ARM_SMMU_EVTQ_BASE		0xa0
-#define ARM_SMMU_EVTQ_PROD		0x100a8
-#define ARM_SMMU_EVTQ_CONS		0x100ac
-#define ARM_SMMU_EVTQ_IRQ_CFG0		0xb0
-#define ARM_SMMU_EVTQ_IRQ_CFG1		0xb8
-#define ARM_SMMU_EVTQ_IRQ_CFG2		0xbc
-
-#define ARM_SMMU_PRIQ_BASE		0xc0
-#define ARM_SMMU_PRIQ_PROD		0x100c8
-#define ARM_SMMU_PRIQ_CONS		0x100cc
-#define ARM_SMMU_PRIQ_IRQ_CFG0		0xd0
-#define ARM_SMMU_PRIQ_IRQ_CFG1		0xd8
-#define ARM_SMMU_PRIQ_IRQ_CFG2		0xdc
-
-#define ARM_SMMU_REG_SZ			0xe00
-
-/* Common MSI config fields */
-#define MSI_CFG0_ADDR_MASK		GENMASK_ULL(51, 2)
-#define MSI_CFG2_SH			GENMASK(5, 4)
-#define MSI_CFG2_MEMATTR		GENMASK(3, 0)
-
-/* Common memory attribute values */
-#define ARM_SMMU_SH_NSH			0
-#define ARM_SMMU_SH_OSH			2
-#define ARM_SMMU_SH_ISH			3
-#define ARM_SMMU_MEMATTR_DEVICE_nGnRE	0x1
-#define ARM_SMMU_MEMATTR_OIWB		0xf
-
-#define Q_IDX(llq, p)			((p) & ((1 << (llq)->max_n_shift) - 1))
-#define Q_WRP(llq, p)			((p) & (1 << (llq)->max_n_shift))
-#define Q_OVERFLOW_FLAG			(1U << 31)
-#define Q_OVF(p)			((p) & Q_OVERFLOW_FLAG)
-#define Q_ENT(q, p)			((q)->base +			\
-					 Q_IDX(&((q)->llq), p) *	\
-					 (q)->ent_dwords)
-
-#define Q_BASE_RWA			(1UL << 62)
-#define Q_BASE_ADDR_MASK		GENMASK_ULL(51, 5)
-#define Q_BASE_LOG2SIZE			GENMASK(4, 0)
-
-/* Ensure DMA allocations are naturally aligned */
-#ifdef CONFIG_CMA_ALIGNMENT
-#define Q_MAX_SZ_SHIFT			(PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
-#else
-#define Q_MAX_SZ_SHIFT			(PAGE_SHIFT + MAX_ORDER - 1)
-#endif
-
-/*
- * Stream table.
- *
- * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
- * 2lvl: 128k L1 entries,
- *       256 lazy entries per table (each table covers a PCI bus)
- */
-#define STRTAB_L1_SZ_SHIFT		20
-#define STRTAB_SPLIT			8
-
-#define STRTAB_L1_DESC_DWORDS		1
-#define STRTAB_L1_DESC_SPAN		GENMASK_ULL(4, 0)
-#define STRTAB_L1_DESC_L2PTR_MASK	GENMASK_ULL(51, 6)
-
-#define STRTAB_STE_DWORDS		8
-#define STRTAB_STE_0_V			(1UL << 0)
-#define STRTAB_STE_0_CFG		GENMASK_ULL(3, 1)
-#define STRTAB_STE_0_CFG_ABORT		0
-#define STRTAB_STE_0_CFG_BYPASS		4
-#define STRTAB_STE_0_CFG_S1_TRANS	5
-#define STRTAB_STE_0_CFG_S2_TRANS	6
-
-#define STRTAB_STE_0_S1FMT		GENMASK_ULL(5, 4)
-#define STRTAB_STE_0_S1FMT_LINEAR	0
-#define STRTAB_STE_0_S1FMT_64K_L2	2
-#define STRTAB_STE_0_S1CTXPTR_MASK	GENMASK_ULL(51, 6)
-#define STRTAB_STE_0_S1CDMAX		GENMASK_ULL(63, 59)
-
-#define STRTAB_STE_1_S1DSS		GENMASK_ULL(1, 0)
-#define STRTAB_STE_1_S1DSS_TERMINATE	0x0
-#define STRTAB_STE_1_S1DSS_BYPASS	0x1
-#define STRTAB_STE_1_S1DSS_SSID0	0x2
-
-#define STRTAB_STE_1_S1C_CACHE_NC	0UL
-#define STRTAB_STE_1_S1C_CACHE_WBRA	1UL
-#define STRTAB_STE_1_S1C_CACHE_WT	2UL
-#define STRTAB_STE_1_S1C_CACHE_WB	3UL
-#define STRTAB_STE_1_S1CIR		GENMASK_ULL(3, 2)
-#define STRTAB_STE_1_S1COR		GENMASK_ULL(5, 4)
-#define STRTAB_STE_1_S1CSH		GENMASK_ULL(7, 6)
-
-#define STRTAB_STE_1_S1STALLD		(1UL << 27)
-
-#define STRTAB_STE_1_EATS		GENMASK_ULL(29, 28)
-#define STRTAB_STE_1_EATS_ABT		0UL
-#define STRTAB_STE_1_EATS_TRANS		1UL
-#define STRTAB_STE_1_EATS_S1CHK		2UL
-
-#define STRTAB_STE_1_STRW		GENMASK_ULL(31, 30)
-#define STRTAB_STE_1_STRW_NSEL1		0UL
-#define STRTAB_STE_1_STRW_EL2		2UL
-
-#define STRTAB_STE_1_SHCFG		GENMASK_ULL(45, 44)
-#define STRTAB_STE_1_SHCFG_INCOMING	1UL
-
-#define STRTAB_STE_2_S2VMID		GENMASK_ULL(15, 0)
-#define STRTAB_STE_2_VTCR		GENMASK_ULL(50, 32)
-#define STRTAB_STE_2_VTCR_S2T0SZ	GENMASK_ULL(5, 0)
-#define STRTAB_STE_2_VTCR_S2SL0		GENMASK_ULL(7, 6)
-#define STRTAB_STE_2_VTCR_S2IR0		GENMASK_ULL(9, 8)
-#define STRTAB_STE_2_VTCR_S2OR0		GENMASK_ULL(11, 10)
-#define STRTAB_STE_2_VTCR_S2SH0		GENMASK_ULL(13, 12)
-#define STRTAB_STE_2_VTCR_S2TG		GENMASK_ULL(15, 14)
-#define STRTAB_STE_2_VTCR_S2PS		GENMASK_ULL(18, 16)
-#define STRTAB_STE_2_S2AA64		(1UL << 51)
-#define STRTAB_STE_2_S2ENDI		(1UL << 52)
-#define STRTAB_STE_2_S2PTW		(1UL << 54)
-#define STRTAB_STE_2_S2R		(1UL << 58)
-
-#define STRTAB_STE_3_S2TTB_MASK		GENMASK_ULL(51, 4)
-
-/*
- * Context descriptors.
- *
- * Linear: when less than 1024 SSIDs are supported
- * 2lvl: at most 1024 L1 entries,
- *       1024 lazy entries per table.
- */
-#define CTXDESC_SPLIT			10
-#define CTXDESC_L2_ENTRIES		(1 << CTXDESC_SPLIT)
-
-#define CTXDESC_L1_DESC_DWORDS		1
-#define CTXDESC_L1_DESC_V		(1UL << 0)
-#define CTXDESC_L1_DESC_L2PTR_MASK	GENMASK_ULL(51, 12)
-
-#define CTXDESC_CD_DWORDS		8
-#define CTXDESC_CD_0_TCR_T0SZ		GENMASK_ULL(5, 0)
-#define CTXDESC_CD_0_TCR_TG0		GENMASK_ULL(7, 6)
-#define CTXDESC_CD_0_TCR_IRGN0		GENMASK_ULL(9, 8)
-#define CTXDESC_CD_0_TCR_ORGN0		GENMASK_ULL(11, 10)
-#define CTXDESC_CD_0_TCR_SH0		GENMASK_ULL(13, 12)
-#define CTXDESC_CD_0_TCR_EPD0		(1ULL << 14)
-#define CTXDESC_CD_0_TCR_EPD1		(1ULL << 30)
-
-#define CTXDESC_CD_0_ENDI		(1UL << 15)
-#define CTXDESC_CD_0_V			(1UL << 31)
-
-#define CTXDESC_CD_0_TCR_IPS		GENMASK_ULL(34, 32)
-#define CTXDESC_CD_0_TCR_TBI0		(1ULL << 38)
-
-#define CTXDESC_CD_0_AA64		(1UL << 41)
-#define CTXDESC_CD_0_S			(1UL << 44)
-#define CTXDESC_CD_0_R			(1UL << 45)
-#define CTXDESC_CD_0_A			(1UL << 46)
-#define CTXDESC_CD_0_ASET		(1UL << 47)
-#define CTXDESC_CD_0_ASID		GENMASK_ULL(63, 48)
-
-#define CTXDESC_CD_1_TTB0_MASK		GENMASK_ULL(51, 4)
-
-/*
- * When the SMMU only supports linear context descriptor tables, pick a
- * reasonable size limit (64kB).
- */
-#define CTXDESC_LINEAR_CDMAX		ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3))
-
-/* Command queue */
-#define CMDQ_ENT_SZ_SHIFT		4
-#define CMDQ_ENT_DWORDS			((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
-#define CMDQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
-
-#define CMDQ_CONS_ERR			GENMASK(30, 24)
-#define CMDQ_ERR_CERROR_NONE_IDX	0
-#define CMDQ_ERR_CERROR_ILL_IDX		1
-#define CMDQ_ERR_CERROR_ABT_IDX		2
-#define CMDQ_ERR_CERROR_ATC_INV_IDX	3
-
-#define CMDQ_0_OP			GENMASK_ULL(7, 0)
-#define CMDQ_0_SSV			(1UL << 11)
-
-#define CMDQ_PREFETCH_0_SID		GENMASK_ULL(63, 32)
-#define CMDQ_PREFETCH_1_SIZE		GENMASK_ULL(4, 0)
-#define CMDQ_PREFETCH_1_ADDR_MASK	GENMASK_ULL(63, 12)
-
-#define CMDQ_CFGI_0_SSID		GENMASK_ULL(31, 12)
-#define CMDQ_CFGI_0_SID			GENMASK_ULL(63, 32)
-#define CMDQ_CFGI_1_LEAF		(1UL << 0)
-#define CMDQ_CFGI_1_RANGE		GENMASK_ULL(4, 0)
-
-#define CMDQ_TLBI_0_VMID		GENMASK_ULL(47, 32)
-#define CMDQ_TLBI_0_ASID		GENMASK_ULL(63, 48)
-#define CMDQ_TLBI_1_LEAF		(1UL << 0)
-#define CMDQ_TLBI_1_VA_MASK		GENMASK_ULL(63, 12)
-#define CMDQ_TLBI_1_IPA_MASK		GENMASK_ULL(51, 12)
-
-#define CMDQ_ATC_0_SSID			GENMASK_ULL(31, 12)
-#define CMDQ_ATC_0_SID			GENMASK_ULL(63, 32)
-#define CMDQ_ATC_0_GLOBAL		(1UL << 9)
-#define CMDQ_ATC_1_SIZE			GENMASK_ULL(5, 0)
-#define CMDQ_ATC_1_ADDR_MASK		GENMASK_ULL(63, 12)
-
-#define CMDQ_PRI_0_SSID			GENMASK_ULL(31, 12)
-#define CMDQ_PRI_0_SID			GENMASK_ULL(63, 32)
-#define CMDQ_PRI_1_GRPID		GENMASK_ULL(8, 0)
-#define CMDQ_PRI_1_RESP			GENMASK_ULL(13, 12)
-
-#define CMDQ_SYNC_0_CS			GENMASK_ULL(13, 12)
-#define CMDQ_SYNC_0_CS_NONE		0
-#define CMDQ_SYNC_0_CS_IRQ		1
-#define CMDQ_SYNC_0_CS_SEV		2
-#define CMDQ_SYNC_0_MSH			GENMASK_ULL(23, 22)
-#define CMDQ_SYNC_0_MSIATTR		GENMASK_ULL(27, 24)
-#define CMDQ_SYNC_0_MSIDATA		GENMASK_ULL(63, 32)
-#define CMDQ_SYNC_1_MSIADDR_MASK	GENMASK_ULL(51, 2)
-
-/* Event queue */
-#define EVTQ_ENT_SZ_SHIFT		5
-#define EVTQ_ENT_DWORDS			((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
-#define EVTQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
-
-#define EVTQ_0_ID			GENMASK_ULL(7, 0)
-
-/* PRI queue */
-#define PRIQ_ENT_SZ_SHIFT		4
-#define PRIQ_ENT_DWORDS			((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
-#define PRIQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
-
-#define PRIQ_0_SID			GENMASK_ULL(31, 0)
-#define PRIQ_0_SSID			GENMASK_ULL(51, 32)
-#define PRIQ_0_PERM_PRIV		(1UL << 58)
-#define PRIQ_0_PERM_EXEC		(1UL << 59)
-#define PRIQ_0_PERM_READ		(1UL << 60)
-#define PRIQ_0_PERM_WRITE		(1UL << 61)
-#define PRIQ_0_PRG_LAST			(1UL << 62)
-#define PRIQ_0_SSID_V			(1UL << 63)
-
-#define PRIQ_1_PRG_IDX			GENMASK_ULL(8, 0)
-#define PRIQ_1_ADDR_MASK		GENMASK_ULL(63, 12)
-
-/* High-level queue structures */
-#define ARM_SMMU_POLL_TIMEOUT_US	100
-#define ARM_SMMU_CMDQ_SYNC_TIMEOUT_US	1000000 /* 1s! */
-#define ARM_SMMU_CMDQ_SYNC_SPIN_COUNT	10
-
 static bool disable_bypass = 1;
 
-enum pri_resp {
-	PRI_RESP_DENY = 0,
-	PRI_RESP_FAIL = 1,
-	PRI_RESP_SUCC = 2,
-};
-
-#ifdef CONFIG_MSI
-enum arm_smmu_msi_index {
-	EVTQ_MSI_INDEX,
-	GERROR_MSI_INDEX,
-	PRIQ_MSI_INDEX,
-	ARM_SMMU_MAX_MSIS,
-};
-
-static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
-	[EVTQ_MSI_INDEX] = {
-		ARM_SMMU_EVTQ_IRQ_CFG0,
-		ARM_SMMU_EVTQ_IRQ_CFG1,
-		ARM_SMMU_EVTQ_IRQ_CFG2,
-	},
-	[GERROR_MSI_INDEX] = {
-		ARM_SMMU_GERROR_IRQ_CFG0,
-		ARM_SMMU_GERROR_IRQ_CFG1,
-		ARM_SMMU_GERROR_IRQ_CFG2,
-	},
-	[PRIQ_MSI_INDEX] = {
-		ARM_SMMU_PRIQ_IRQ_CFG0,
-		ARM_SMMU_PRIQ_IRQ_CFG1,
-		ARM_SMMU_PRIQ_IRQ_CFG2,
-	},
-};
-#endif /* CONFIG_MSI */
-
-struct arm_smmu_cmdq_ent {
-	/* Common fields */
-	u8				opcode;
-	bool				substream_valid;
-
-	/* Command-specific fields */
-	union {
-		#define CMDQ_OP_PREFETCH_CFG	0x1
-		struct {
-			u32			sid;
-			u8			size;
-			u64			addr;
-		} prefetch;
-
-		#define CMDQ_OP_CFGI_STE	0x3
-		#define CMDQ_OP_CFGI_ALL	0x4
-		struct {
-			u32			sid;
-			union {
-				bool		leaf;
-				u8		span;
-			};
-		} cfgi;
-
-		#define CMDQ_OP_TLBI_EL2_ALL	0x20
-		#define CMDQ_OP_TLBI_S12_VMALL	0x28
-		#define CMDQ_OP_TLBI_S2_IPA	0x2a
-		#define CMDQ_OP_TLBI_NSNH_ALL	0x30
-		struct {
-			u16			asid;
-			u16			vmid;
-			bool			leaf;
-			u64			addr;
-		} tlbi;
-
-		#define CMDQ_OP_ATC_INV		0x40
-		#define ATC_INV_SIZE_ALL	52
-		struct {
-			u32			sid;
-			u32			ssid;
-			u64			addr;
-			u8			size;
-			bool			global;
-		} atc;
-
-		#define CMDQ_OP_PRI_RESP	0x41
-		struct {
-			u32			sid;
-			u32			ssid;
-			u16			grpid;
-			enum pri_resp		resp;
-		} pri;
-
-		#define CMDQ_OP_CMD_SYNC	0x46
-		struct {
-			u32			msidata;
-			u64			msiaddr;
-		} sync;
-	};
-};
-
-struct arm_smmu_ll_queue {
-	u32				prod;
-	u32				cons;
-	u32				max_n_shift;
-};
-
-struct arm_smmu_queue {
-	struct arm_smmu_ll_queue	llq;
-	int				irq; /* Wired interrupt */
-
-	__le64				*base;
-	dma_addr_t			base_dma;
-	u64				q_base;
-
-	size_t				ent_dwords;
-
-	u32 __iomem			*prod_reg;
-	u32 __iomem			*cons_reg;
-};
-
-struct arm_smmu_cmdq {
-	struct arm_smmu_queue		q;
-	spinlock_t			lock;
-};
-
-struct arm_smmu_evtq {
-	struct arm_smmu_queue		q;
-	u32				max_stalls;
-};
-
-struct arm_smmu_priq {
-	struct arm_smmu_queue		q;
-};
-
-/* High-level stream table and context descriptor structures */
-struct arm_smmu_strtab_l1_desc {
-	u8				span;
-
-	__le64				*l2ptr;
-	dma_addr_t			l2ptr_dma;
-};
-
-struct arm_smmu_s2_cfg {
-	u16				vmid;
-	u64				vttbr;
-	u64				vtcr;
-};
-
-struct arm_smmu_strtab_cfg {
-	__le64				*strtab;
-	dma_addr_t			strtab_dma;
-	struct arm_smmu_strtab_l1_desc	*l1_desc;
-	unsigned int			num_l1_ents;
-
-	u64				strtab_base;
-	u32				strtab_base_cfg;
-};
-
-struct arm_lpae_s2_cfg {
-	u64			vttbr;
-	struct {
-		u32			ps:3;
-		u32			tg:2;
-		u32			sh:2;
-		u32			orgn:2;
-		u32			irgn:2;
-		u32			sl:2;
-		u32			tsz:6;
-	} vtcr;
-};
-
-/* An SMMUv3 instance */
-struct arm_smmu_device {
-	struct device			*dev;
-	void __iomem			*base;
-	void __iomem			*page1;
-
-#define ARM_SMMU_FEAT_2_LVL_STRTAB	(1 << 0)
-#define ARM_SMMU_FEAT_2_LVL_CDTAB	(1 << 1)
-#define ARM_SMMU_FEAT_TT_LE		(1 << 2)
-#define ARM_SMMU_FEAT_TT_BE		(1 << 3)
-#define ARM_SMMU_FEAT_PRI		(1 << 4)
-#define ARM_SMMU_FEAT_ATS		(1 << 5)
-#define ARM_SMMU_FEAT_SEV		(1 << 6)
-#define ARM_SMMU_FEAT_MSI		(1 << 7)
-#define ARM_SMMU_FEAT_COHERENCY		(1 << 8)
-#define ARM_SMMU_FEAT_TRANS_S1		(1 << 9)
-#define ARM_SMMU_FEAT_TRANS_S2		(1 << 10)
-#define ARM_SMMU_FEAT_STALLS		(1 << 11)
-#define ARM_SMMU_FEAT_HYP		(1 << 12)
-#define ARM_SMMU_FEAT_STALL_FORCE	(1 << 13)
-#define ARM_SMMU_FEAT_VAX		(1 << 14)
-	u32				features;
-
-#define ARM_SMMU_OPT_SKIP_PREFETCH	(1 << 0)
-#define ARM_SMMU_OPT_PAGE0_REGS_ONLY	(1 << 1)
-	u32				options;
-
-	struct arm_smmu_cmdq		cmdq;
-	struct arm_smmu_evtq		evtq;
-	struct arm_smmu_priq		priq;
-
-	int				gerr_irq;
-	int				combined_irq;
-	u32				sync_nr;
-	u8				prev_cmd_opcode;
-
-	unsigned long			ias; /* IPA */
-	unsigned long			oas; /* PA */
-	unsigned long			pgsize_bitmap;
-
-#define ARM_SMMU_MAX_VMIDS		(1 << 16)
-	unsigned int			vmid_bits;
-	DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
-
-	unsigned int			sid_bits;
-
-	struct arm_smmu_strtab_cfg	strtab_cfg;
-
-	/* Hi16xx adds an extra 32 bits of goodness to its MSI payload */
-	union {
-		u32			sync_count;
-		u64			padding;
-	};
-
-	/* Need to keep a list of SMMU devices */
-	struct list_head		devices;
-
-	/* Tasklets for handling evts/faults and pci page request IRQs*/
-	struct tasklet		evtq_irq_tasklet;
-	struct tasklet		priq_irq_tasklet;
-	struct tasklet		combined_irq_tasklet;
-};
-
-/* SMMU private data for each master */
-struct arm_smmu_master {
-	struct arm_smmu_device		*smmu;
-	struct device			*dev;
-	struct arm_smmu_domain		*domain;
-	struct list_head		domain_head;
-	u32				*sids;
-	unsigned int			num_sids;
-	bool				ats_enabled;
-};
-
-/* SMMU private data for an IOMMU domain */
-enum arm_smmu_domain_stage {
-	ARM_SMMU_DOMAIN_S1 = 0,
-	ARM_SMMU_DOMAIN_S2,
-	ARM_SMMU_DOMAIN_NESTED,
-	ARM_SMMU_DOMAIN_BYPASS,
-};
-
-struct arm_smmu_domain {
-	struct arm_smmu_device		*smmu;
-	struct mutex			init_mutex; /* Protects smmu pointer */
-
-	bool				non_strict;
-	atomic_t			nr_ats_masters;
-
-	enum arm_smmu_domain_stage	stage;
-	struct arm_smmu_s2_cfg	s2_cfg;
-
-	/* Xen domain associated with this SMMU domain */
-	struct domain		*d;
-
-	struct iommu_domain		domain;
-
-	struct list_head		devices;
-	spinlock_t			devices_lock;
-};
-
 struct arm_smmu_option_prop {
 	u32 opt;
 	const char *prop;
diff --git a/xen/drivers/passthrough/arm/smmu-v3.h b/xen/drivers/passthrough/arm/smmu-v3.h
new file mode 100644
index 0000000000..c45d2f16c4
--- /dev/null
+++ b/xen/drivers/passthrough/arm/smmu-v3.h
@@ -0,0 +1,674 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * IOMMU API for ARM architected SMMUv3 implementations.
+ *
+ * Copyright (C) 2022 ARM Limited
+ */
+
+#ifndef _ARM_SMMU_V3_H
+#define _ARM_SMMU_V3_H
+
+/* MMIO registers */
+#define ARM_SMMU_IDR0			0x0
+#define IDR0_ST_LVL			GENMASK(28, 27)
+#define IDR0_ST_LVL_2LVL		1
+#define IDR0_STALL_MODEL		GENMASK(25, 24)
+#define IDR0_STALL_MODEL_STALL		0
+#define IDR0_STALL_MODEL_FORCE		2
+#define IDR0_TTENDIAN			GENMASK(22, 21)
+#define IDR0_TTENDIAN_MIXED		0
+#define IDR0_TTENDIAN_LE		2
+#define IDR0_TTENDIAN_BE		3
+#define IDR0_CD2L			(1 << 19)
+#define IDR0_VMID16			(1 << 18)
+#define IDR0_PRI			(1 << 16)
+#define IDR0_SEV			(1 << 14)
+#define IDR0_MSI			(1 << 13)
+#define IDR0_ASID16			(1 << 12)
+#define IDR0_ATS			(1 << 10)
+#define IDR0_HYP			(1 << 9)
+#define IDR0_COHACC			(1 << 4)
+#define IDR0_TTF			GENMASK(3, 2)
+#define IDR0_TTF_AARCH64		2
+#define IDR0_TTF_AARCH32_64		3
+#define IDR0_S1P			(1 << 1)
+#define IDR0_S2P			(1 << 0)
+
+#define ARM_SMMU_IDR1			0x4
+#define IDR1_TABLES_PRESET		(1 << 30)
+#define IDR1_QUEUES_PRESET		(1 << 29)
+#define IDR1_REL			(1 << 28)
+#define IDR1_CMDQS			GENMASK(25, 21)
+#define IDR1_EVTQS			GENMASK(20, 16)
+#define IDR1_PRIQS			GENMASK(15, 11)
+#define IDR1_SSIDSIZE			GENMASK(10, 6)
+#define IDR1_SIDSIZE			GENMASK(5, 0)
+
+#define ARM_SMMU_IDR5			0x14
+#define IDR5_STALL_MAX			GENMASK(31, 16)
+#define IDR5_GRAN64K			(1 << 6)
+#define IDR5_GRAN16K			(1 << 5)
+#define IDR5_GRAN4K			(1 << 4)
+#define IDR5_OAS			GENMASK(2, 0)
+#define IDR5_OAS_32_BIT			0
+#define IDR5_OAS_36_BIT			1
+#define IDR5_OAS_40_BIT			2
+#define IDR5_OAS_42_BIT			3
+#define IDR5_OAS_44_BIT			4
+#define IDR5_OAS_48_BIT			5
+#define IDR5_OAS_52_BIT			6
+#define IDR5_VAX			GENMASK(11, 10)
+#define IDR5_VAX_52_BIT			1
+
+#define ARM_SMMU_CR0			0x20
+#define CR0_ATSCHK			(1 << 4)
+#define CR0_CMDQEN			(1 << 3)
+#define CR0_EVTQEN			(1 << 2)
+#define CR0_PRIQEN			(1 << 1)
+#define CR0_SMMUEN			(1 << 0)
+
+#define ARM_SMMU_CR0ACK			0x24
+
+#define ARM_SMMU_CR1			0x28
+#define CR1_TABLE_SH			GENMASK(11, 10)
+#define CR1_TABLE_OC			GENMASK(9, 8)
+#define CR1_TABLE_IC			GENMASK(7, 6)
+#define CR1_QUEUE_SH			GENMASK(5, 4)
+#define CR1_QUEUE_OC			GENMASK(3, 2)
+#define CR1_QUEUE_IC			GENMASK(1, 0)
+/* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
+#define CR1_CACHE_NC			0
+#define CR1_CACHE_WB			1
+#define CR1_CACHE_WT			2
+
+#define ARM_SMMU_CR2			0x2c
+#define CR2_PTM				(1 << 2)
+#define CR2_RECINVSID			(1 << 1)
+#define CR2_E2H				(1 << 0)
+
+#define ARM_SMMU_GBPA			0x44
+#define GBPA_UPDATE			(1 << 31)
+#define GBPA_ABORT			(1 << 20)
+
+#define ARM_SMMU_IRQ_CTRL		0x50
+#define IRQ_CTRL_EVTQ_IRQEN		(1 << 2)
+#define IRQ_CTRL_PRIQ_IRQEN		(1 << 1)
+#define IRQ_CTRL_GERROR_IRQEN		(1 << 0)
+
+#define ARM_SMMU_IRQ_CTRLACK		0x54
+
+#define ARM_SMMU_GERROR			0x60
+#define GERROR_SFM_ERR			(1 << 8)
+#define GERROR_MSI_GERROR_ABT_ERR	(1 << 7)
+#define GERROR_MSI_PRIQ_ABT_ERR		(1 << 6)
+#define GERROR_MSI_EVTQ_ABT_ERR		(1 << 5)
+#define GERROR_MSI_CMDQ_ABT_ERR		(1 << 4)
+#define GERROR_PRIQ_ABT_ERR		(1 << 3)
+#define GERROR_EVTQ_ABT_ERR		(1 << 2)
+#define GERROR_CMDQ_ERR			(1 << 0)
+#define GERROR_ERR_MASK			0xfd
+
+#define ARM_SMMU_GERRORN		0x64
+
+#define ARM_SMMU_GERROR_IRQ_CFG0	0x68
+#define ARM_SMMU_GERROR_IRQ_CFG1	0x70
+#define ARM_SMMU_GERROR_IRQ_CFG2	0x74
+
+#define ARM_SMMU_STRTAB_BASE		0x80
+#define STRTAB_BASE_RA			(1UL << 62)
+#define STRTAB_BASE_ADDR_MASK		GENMASK_ULL(51, 6)
+
+#define ARM_SMMU_STRTAB_BASE_CFG	0x88
+#define STRTAB_BASE_CFG_FMT		GENMASK(17, 16)
+#define STRTAB_BASE_CFG_FMT_LINEAR	0
+#define STRTAB_BASE_CFG_FMT_2LVL	1
+#define STRTAB_BASE_CFG_SPLIT		GENMASK(10, 6)
+#define STRTAB_BASE_CFG_LOG2SIZE	GENMASK(5, 0)
+
+#define ARM_SMMU_CMDQ_BASE		0x90
+#define ARM_SMMU_CMDQ_PROD		0x98
+#define ARM_SMMU_CMDQ_CONS		0x9c
+
+#define ARM_SMMU_EVTQ_BASE		0xa0
+#define ARM_SMMU_EVTQ_PROD		0x100a8
+#define ARM_SMMU_EVTQ_CONS		0x100ac
+#define ARM_SMMU_EVTQ_IRQ_CFG0		0xb0
+#define ARM_SMMU_EVTQ_IRQ_CFG1		0xb8
+#define ARM_SMMU_EVTQ_IRQ_CFG2		0xbc
+
+#define ARM_SMMU_PRIQ_BASE		0xc0
+#define ARM_SMMU_PRIQ_PROD		0x100c8
+#define ARM_SMMU_PRIQ_CONS		0x100cc
+#define ARM_SMMU_PRIQ_IRQ_CFG0		0xd0
+#define ARM_SMMU_PRIQ_IRQ_CFG1		0xd8
+#define ARM_SMMU_PRIQ_IRQ_CFG2		0xdc
+
+#define ARM_SMMU_REG_SZ			0xe00
+
+/* Common MSI config fields */
+#define MSI_CFG0_ADDR_MASK		GENMASK_ULL(51, 2)
+#define MSI_CFG2_SH			GENMASK(5, 4)
+#define MSI_CFG2_MEMATTR		GENMASK(3, 0)
+
+/* Common memory attribute values */
+#define ARM_SMMU_SH_NSH			0
+#define ARM_SMMU_SH_OSH			2
+#define ARM_SMMU_SH_ISH			3
+#define ARM_SMMU_MEMATTR_DEVICE_nGnRE	0x1
+#define ARM_SMMU_MEMATTR_OIWB		0xf
+
+#define Q_IDX(llq, p)			((p) & ((1 << (llq)->max_n_shift) - 1))
+#define Q_WRP(llq, p)			((p) & (1 << (llq)->max_n_shift))
+#define Q_OVERFLOW_FLAG			(1 << 31)
+#define Q_OVF(p)			((p) & Q_OVERFLOW_FLAG)
+#define Q_ENT(q, p)			((q)->base +			\
+					 Q_IDX(&((q)->llq), p) *	\
+					 (q)->ent_dwords)
+
+#define Q_BASE_RWA			(1UL << 62)
+#define Q_BASE_ADDR_MASK		GENMASK_ULL(51, 5)
+#define Q_BASE_LOG2SIZE			GENMASK(4, 0)
+
+/* Ensure DMA allocations are naturally aligned */
+#ifdef CONFIG_CMA_ALIGNMENT
+#define Q_MAX_SZ_SHIFT			(PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
+#else
+#define Q_MAX_SZ_SHIFT			(PAGE_SHIFT + MAX_ORDER - 1)
+#endif
+
+/*
+ * Stream table.
+ *
+ * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
+ * 2lvl: 128k L1 entries,
+ *       256 lazy entries per table (each table covers a PCI bus)
+ */
+#define STRTAB_L1_SZ_SHIFT		20
+#define STRTAB_SPLIT			8
+
+#define STRTAB_L1_DESC_DWORDS		1
+#define STRTAB_L1_DESC_SPAN		GENMASK_ULL(4, 0)
+#define STRTAB_L1_DESC_L2PTR_MASK	GENMASK_ULL(51, 6)
+
+#define STRTAB_STE_DWORDS		8
+#define STRTAB_STE_0_V			(1UL << 0)
+#define STRTAB_STE_0_CFG		GENMASK_ULL(3, 1)
+#define STRTAB_STE_0_CFG_ABORT		0
+#define STRTAB_STE_0_CFG_BYPASS		4
+#define STRTAB_STE_0_CFG_S1_TRANS	5
+#define STRTAB_STE_0_CFG_S2_TRANS	6
+
+#define STRTAB_STE_0_S1FMT		GENMASK_ULL(5, 4)
+#define STRTAB_STE_0_S1FMT_LINEAR	0
+#define STRTAB_STE_0_S1FMT_64K_L2	2
+#define STRTAB_STE_0_S1CTXPTR_MASK	GENMASK_ULL(51, 6)
+#define STRTAB_STE_0_S1CDMAX		GENMASK_ULL(63, 59)
+
+#define STRTAB_STE_1_S1DSS		GENMASK_ULL(1, 0)
+#define STRTAB_STE_1_S1DSS_TERMINATE	0x0
+#define STRTAB_STE_1_S1DSS_BYPASS	0x1
+#define STRTAB_STE_1_S1DSS_SSID0	0x2
+
+#define STRTAB_STE_1_S1C_CACHE_NC	0UL
+#define STRTAB_STE_1_S1C_CACHE_WBRA	1UL
+#define STRTAB_STE_1_S1C_CACHE_WT	2UL
+#define STRTAB_STE_1_S1C_CACHE_WB	3UL
+#define STRTAB_STE_1_S1CIR		GENMASK_ULL(3, 2)
+#define STRTAB_STE_1_S1COR		GENMASK_ULL(5, 4)
+#define STRTAB_STE_1_S1CSH		GENMASK_ULL(7, 6)
+
+#define STRTAB_STE_1_S1STALLD		(1UL << 27)
+
+#define STRTAB_STE_1_EATS		GENMASK_ULL(29, 28)
+#define STRTAB_STE_1_EATS_ABT		0UL
+#define STRTAB_STE_1_EATS_TRANS		1UL
+#define STRTAB_STE_1_EATS_S1CHK		2UL
+
+#define STRTAB_STE_1_STRW		GENMASK_ULL(31, 30)
+#define STRTAB_STE_1_STRW_NSEL1		0UL
+#define STRTAB_STE_1_STRW_EL2		2UL
+
+#define STRTAB_STE_1_SHCFG		GENMASK_ULL(45, 44)
+#define STRTAB_STE_1_SHCFG_INCOMING	1UL
+
+#define STRTAB_STE_2_S2VMID		GENMASK_ULL(15, 0)
+#define STRTAB_STE_2_VTCR		GENMASK_ULL(50, 32)
+#define STRTAB_STE_2_VTCR_S2T0SZ	GENMASK_ULL(5, 0)
+#define STRTAB_STE_2_VTCR_S2SL0		GENMASK_ULL(7, 6)
+#define STRTAB_STE_2_VTCR_S2IR0		GENMASK_ULL(9, 8)
+#define STRTAB_STE_2_VTCR_S2OR0		GENMASK_ULL(11, 10)
+#define STRTAB_STE_2_VTCR_S2SH0		GENMASK_ULL(13, 12)
+#define STRTAB_STE_2_VTCR_S2TG		GENMASK_ULL(15, 14)
+#define STRTAB_STE_2_VTCR_S2PS		GENMASK_ULL(18, 16)
+#define STRTAB_STE_2_S2AA64		(1UL << 51)
+#define STRTAB_STE_2_S2ENDI		(1UL << 52)
+#define STRTAB_STE_2_S2PTW		(1UL << 54)
+#define STRTAB_STE_2_S2R		(1UL << 58)
+
+#define STRTAB_STE_3_S2TTB_MASK		GENMASK_ULL(51, 4)
+
+/*
+ * Context descriptors.
+ *
+ * Linear: when less than 1024 SSIDs are supported
+ * 2lvl: at most 1024 L1 entries,
+ *       1024 lazy entries per table.
+ */
+#define CTXDESC_SPLIT			10
+#define CTXDESC_L2_ENTRIES		(1 << CTXDESC_SPLIT)
+
+#define CTXDESC_L1_DESC_DWORDS		1
+#define CTXDESC_L1_DESC_V		(1UL << 0)
+#define CTXDESC_L1_DESC_L2PTR_MASK	GENMASK_ULL(51, 12)
+
+#define CTXDESC_CD_DWORDS		8
+#define CTXDESC_CD_0_TCR_T0SZ		GENMASK_ULL(5, 0)
+#define CTXDESC_CD_0_TCR_TG0		GENMASK_ULL(7, 6)
+#define CTXDESC_CD_0_TCR_IRGN0		GENMASK_ULL(9, 8)
+#define CTXDESC_CD_0_TCR_ORGN0		GENMASK_ULL(11, 10)
+#define CTXDESC_CD_0_TCR_SH0		GENMASK_ULL(13, 12)
+#define CTXDESC_CD_0_TCR_EPD0		(1ULL << 14)
+#define CTXDESC_CD_0_TCR_EPD1		(1ULL << 30)
+
+#define CTXDESC_CD_0_ENDI		(1UL << 15)
+#define CTXDESC_CD_0_V			(1UL << 31)
+
+#define CTXDESC_CD_0_TCR_IPS		GENMASK_ULL(34, 32)
+#define CTXDESC_CD_0_TCR_TBI0		(1ULL << 38)
+
+#define CTXDESC_CD_0_AA64		(1UL << 41)
+#define CTXDESC_CD_0_S			(1UL << 44)
+#define CTXDESC_CD_0_R			(1UL << 45)
+#define CTXDESC_CD_0_A			(1UL << 46)
+#define CTXDESC_CD_0_ASET		(1UL << 47)
+#define CTXDESC_CD_0_ASID		GENMASK_ULL(63, 48)
+
+#define CTXDESC_CD_1_TTB0_MASK		GENMASK_ULL(51, 4)
+
+/*
+ * When the SMMU only supports linear context descriptor tables, pick a
+ * reasonable size limit (64kB).
+ */
+#define CTXDESC_LINEAR_CDMAX		ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3))
+
+/* Command queue */
+#define CMDQ_ENT_SZ_SHIFT		4
+#define CMDQ_ENT_DWORDS			((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
+#define CMDQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
+
+#define CMDQ_CONS_ERR			GENMASK(30, 24)
+#define CMDQ_ERR_CERROR_NONE_IDX	0
+#define CMDQ_ERR_CERROR_ILL_IDX		1
+#define CMDQ_ERR_CERROR_ABT_IDX		2
+#define CMDQ_ERR_CERROR_ATC_INV_IDX	3
+
+#define CMDQ_0_OP			GENMASK_ULL(7, 0)
+#define CMDQ_0_SSV			(1UL << 11)
+
+#define CMDQ_PREFETCH_0_SID		GENMASK_ULL(63, 32)
+#define CMDQ_PREFETCH_1_SIZE		GENMASK_ULL(4, 0)
+#define CMDQ_PREFETCH_1_ADDR_MASK	GENMASK_ULL(63, 12)
+
+#define CMDQ_CFGI_0_SSID		GENMASK_ULL(31, 12)
+#define CMDQ_CFGI_0_SID			GENMASK_ULL(63, 32)
+#define CMDQ_CFGI_1_LEAF		(1UL << 0)
+#define CMDQ_CFGI_1_RANGE		GENMASK_ULL(4, 0)
+
+#define CMDQ_TLBI_0_VMID		GENMASK_ULL(47, 32)
+#define CMDQ_TLBI_0_ASID		GENMASK_ULL(63, 48)
+#define CMDQ_TLBI_1_LEAF		(1UL << 0)
+#define CMDQ_TLBI_1_VA_MASK		GENMASK_ULL(63, 12)
+#define CMDQ_TLBI_1_IPA_MASK		GENMASK_ULL(51, 12)
+
+#define CMDQ_ATC_0_SSID			GENMASK_ULL(31, 12)
+#define CMDQ_ATC_0_SID			GENMASK_ULL(63, 32)
+#define CMDQ_ATC_0_GLOBAL		(1UL << 9)
+#define CMDQ_ATC_1_SIZE			GENMASK_ULL(5, 0)
+#define CMDQ_ATC_1_ADDR_MASK		GENMASK_ULL(63, 12)
+
+#define CMDQ_PRI_0_SSID			GENMASK_ULL(31, 12)
+#define CMDQ_PRI_0_SID			GENMASK_ULL(63, 32)
+#define CMDQ_PRI_1_GRPID		GENMASK_ULL(8, 0)
+#define CMDQ_PRI_1_RESP			GENMASK_ULL(13, 12)
+
+#define CMDQ_SYNC_0_CS			GENMASK_ULL(13, 12)
+#define CMDQ_SYNC_0_CS_NONE		0
+#define CMDQ_SYNC_0_CS_IRQ		1
+#define CMDQ_SYNC_0_CS_SEV		2
+#define CMDQ_SYNC_0_MSH			GENMASK_ULL(23, 22)
+#define CMDQ_SYNC_0_MSIATTR		GENMASK_ULL(27, 24)
+#define CMDQ_SYNC_0_MSIDATA		GENMASK_ULL(63, 32)
+#define CMDQ_SYNC_1_MSIADDR_MASK	GENMASK_ULL(51, 2)
+
+/* Event queue */
+#define EVTQ_ENT_SZ_SHIFT		5
+#define EVTQ_ENT_DWORDS			((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
+#define EVTQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
+
+#define EVTQ_0_ID			GENMASK_ULL(7, 0)
+
+/* PRI queue */
+#define PRIQ_ENT_SZ_SHIFT		4
+#define PRIQ_ENT_DWORDS			((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
+#define PRIQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
+
+#define PRIQ_0_SID			GENMASK_ULL(31, 0)
+#define PRIQ_0_SSID			GENMASK_ULL(51, 32)
+#define PRIQ_0_PERM_PRIV		(1UL << 58)
+#define PRIQ_0_PERM_EXEC		(1UL << 59)
+#define PRIQ_0_PERM_READ		(1UL << 60)
+#define PRIQ_0_PERM_WRITE		(1UL << 61)
+#define PRIQ_0_PRG_LAST			(1UL << 62)
+#define PRIQ_0_SSID_V			(1UL << 63)
+
+#define PRIQ_1_PRG_IDX			GENMASK_ULL(8, 0)
+#define PRIQ_1_ADDR_MASK		GENMASK_ULL(63, 12)
+
+/* High-level queue structures */
+#define ARM_SMMU_POLL_TIMEOUT_US	100
+#define ARM_SMMU_CMDQ_SYNC_TIMEOUT_US	1000000 /* 1s! */
+#define ARM_SMMU_CMDQ_SYNC_SPIN_COUNT	10
+
+#define FIELD_PREP(_mask, _val)			\
+	(((typeof(_mask))(_val) << (ffs64(_mask) - 1)) & (_mask))
+
+#define FIELD_GET(_mask, _reg)			\
+	((typeof(_mask))(((_reg) & (_mask)) >> (ffs64(_mask) - 1)))
+
+/* Linux compatibility functions. */
+typedef paddr_t		dma_addr_t;
+typedef paddr_t		phys_addr_t;
+typedef unsigned int		gfp_t;
+
+/* Alias to Xen lock functions */
+#define mutex spinlock
+#define mutex_init spin_lock_init
+#define mutex_lock spin_lock
+#define mutex_unlock spin_unlock
+
+/* SMMU private data for an IOMMU domain */
+enum arm_smmu_domain_stage {
+	ARM_SMMU_DOMAIN_S1 = 0,
+	ARM_SMMU_DOMAIN_S2,
+	ARM_SMMU_DOMAIN_NESTED,
+	ARM_SMMU_DOMAIN_BYPASS,
+};
+
+/* Xen specific code. */
+struct iommu_domain {
+	/* Runtime SMMU configuration for this iommu_domain */
+	atomic_t		ref;
+	/*
+	 * Used to link iommu_domain contexts for a same domain.
+	 * There is at least one per-SMMU to used by the domain.
+	 */
+	struct list_head		list;
+};
+
+/* Describes information required for a Xen domain */
+struct arm_smmu_xen_domain {
+	spinlock_t		lock;
+
+	/* List of iommu domains associated to this domain */
+	struct list_head	contexts;
+};
+
+enum pri_resp {
+	PRI_RESP_DENY = 0,
+	PRI_RESP_FAIL = 1,
+	PRI_RESP_SUCC = 2,
+};
+
+#ifdef CONFIG_MSI
+enum arm_smmu_msi_index {
+	EVTQ_MSI_INDEX,
+	GERROR_MSI_INDEX,
+	PRIQ_MSI_INDEX,
+	ARM_SMMU_MAX_MSIS,
+};
+
+static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
+	[EVTQ_MSI_INDEX] = {
+		ARM_SMMU_EVTQ_IRQ_CFG0,
+		ARM_SMMU_EVTQ_IRQ_CFG1,
+		ARM_SMMU_EVTQ_IRQ_CFG2,
+	},
+	[GERROR_MSI_INDEX] = {
+		ARM_SMMU_GERROR_IRQ_CFG0,
+		ARM_SMMU_GERROR_IRQ_CFG1,
+		ARM_SMMU_GERROR_IRQ_CFG2,
+	},
+	[PRIQ_MSI_INDEX] = {
+		ARM_SMMU_PRIQ_IRQ_CFG0,
+		ARM_SMMU_PRIQ_IRQ_CFG1,
+		ARM_SMMU_PRIQ_IRQ_CFG2,
+	},
+};
+#endif /* CONFIG_MSI */
+
+struct arm_smmu_cmdq_ent {
+	/* Common fields */
+	u8				opcode;
+	bool				substream_valid;
+
+	/* Command-specific fields */
+	union {
+		#define CMDQ_OP_PREFETCH_CFG	0x1
+		struct {
+			u32			sid;
+			u8			size;
+			u64			addr;
+		} prefetch;
+
+		#define CMDQ_OP_CFGI_STE	0x3
+		#define CMDQ_OP_CFGI_ALL	0x4
+		struct {
+			u32			sid;
+			union {
+				bool		leaf;
+				u8		span;
+			};
+		} cfgi;
+
+		#define CMDQ_OP_TLBI_EL2_ALL	0x20
+		#define CMDQ_OP_TLBI_S12_VMALL	0x28
+		#define CMDQ_OP_TLBI_S2_IPA	0x2a
+		#define CMDQ_OP_TLBI_NSNH_ALL	0x30
+		struct {
+			u16			asid;
+			u16			vmid;
+			bool			leaf;
+			u64			addr;
+		} tlbi;
+
+		#define CMDQ_OP_ATC_INV		0x40
+		#define ATC_INV_SIZE_ALL	52
+		struct {
+			u32			sid;
+			u32			ssid;
+			u64			addr;
+			u8			size;
+			bool			global;
+		} atc;
+
+		#define CMDQ_OP_PRI_RESP	0x41
+		struct {
+			u32			sid;
+			u32			ssid;
+			u16			grpid;
+			enum pri_resp		resp;
+		} pri;
+
+		#define CMDQ_OP_CMD_SYNC	0x46
+		struct {
+			u32			msidata;
+			u64			msiaddr;
+		} sync;
+	};
+};
+
+struct arm_smmu_ll_queue {
+	u32				prod;
+	u32				cons;
+	u32				max_n_shift;
+};
+
+struct arm_smmu_queue {
+	struct arm_smmu_ll_queue	llq;
+	int				irq; /* Wired interrupt */
+
+	__le64				*base;
+	dma_addr_t			base_dma;
+	u64				q_base;
+
+	size_t				ent_dwords;
+
+	u32 __iomem			*prod_reg;
+	u32 __iomem			*cons_reg;
+};
+
+struct arm_smmu_cmdq {
+	struct arm_smmu_queue		q;
+	spinlock_t			lock;
+};
+
+struct arm_smmu_evtq {
+	struct arm_smmu_queue		q;
+	u32				max_stalls;
+};
+
+struct arm_smmu_priq {
+	struct arm_smmu_queue		q;
+};
+
+/* High-level stream table and context descriptor structures */
+struct arm_smmu_strtab_l1_desc {
+	u8				span;
+
+	__le64				*l2ptr;
+	dma_addr_t			l2ptr_dma;
+};
+
+struct arm_smmu_s2_cfg {
+	u16				vmid;
+	u64				vttbr;
+	u64				vtcr;
+};
+
+struct arm_smmu_strtab_cfg {
+	__le64				*strtab;
+	dma_addr_t			strtab_dma;
+	struct arm_smmu_strtab_l1_desc	*l1_desc;
+	unsigned int			num_l1_ents;
+
+	u64				strtab_base;
+	u32				strtab_base_cfg;
+};
+
+struct arm_lpae_s2_cfg {
+	u64			vttbr;
+	struct {
+		u32			ps:3;
+		u32			tg:2;
+		u32			sh:2;
+		u32			orgn:2;
+		u32			irgn:2;
+		u32			sl:2;
+		u32			tsz:6;
+	} vtcr;
+};
+
+/* An SMMUv3 instance */
+struct arm_smmu_device {
+	struct device			*dev;
+	void __iomem			*base;
+	void __iomem			*page1;
+
+#define ARM_SMMU_FEAT_2_LVL_STRTAB	(1 << 0)
+#define ARM_SMMU_FEAT_2_LVL_CDTAB	(1 << 1)
+#define ARM_SMMU_FEAT_TT_LE		(1 << 2)
+#define ARM_SMMU_FEAT_TT_BE		(1 << 3)
+#define ARM_SMMU_FEAT_PRI		(1 << 4)
+#define ARM_SMMU_FEAT_ATS		(1 << 5)
+#define ARM_SMMU_FEAT_SEV		(1 << 6)
+#define ARM_SMMU_FEAT_MSI		(1 << 7)
+#define ARM_SMMU_FEAT_COHERENCY		(1 << 8)
+#define ARM_SMMU_FEAT_TRANS_S1		(1 << 9)
+#define ARM_SMMU_FEAT_TRANS_S2		(1 << 10)
+#define ARM_SMMU_FEAT_STALLS		(1 << 11)
+#define ARM_SMMU_FEAT_HYP		(1 << 12)
+#define ARM_SMMU_FEAT_STALL_FORCE	(1 << 13)
+#define ARM_SMMU_FEAT_VAX		(1 << 14)
+	u32				features;
+
+#define ARM_SMMU_OPT_SKIP_PREFETCH	(1 << 0)
+#define ARM_SMMU_OPT_PAGE0_REGS_ONLY	(1 << 1)
+	u32				options;
+
+	struct arm_smmu_cmdq		cmdq;
+	struct arm_smmu_evtq		evtq;
+	struct arm_smmu_priq		priq;
+
+	int				gerr_irq;
+	int				combined_irq;
+	u32				sync_nr;
+	u8				prev_cmd_opcode;
+
+	unsigned long			ias; /* IPA */
+	unsigned long			oas; /* PA */
+	unsigned long			pgsize_bitmap;
+
+#define ARM_SMMU_MAX_VMIDS		(1 << 16)
+	unsigned int			vmid_bits;
+	DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
+
+	unsigned int			sid_bits;
+
+	struct arm_smmu_strtab_cfg	strtab_cfg;
+
+	/* Hi16xx adds an extra 32 bits of goodness to its MSI payload */
+	union {
+		u32			sync_count;
+		u64			padding;
+	};
+
+	/* Need to keep a list of SMMU devices */
+	struct list_head		devices;
+
+	/* Tasklets for handling evts/faults and pci page request IRQs*/
+	struct tasklet		evtq_irq_tasklet;
+	struct tasklet		priq_irq_tasklet;
+	struct tasklet		combined_irq_tasklet;
+};
+
+/* SMMU private data for each master */
+struct arm_smmu_master {
+	struct arm_smmu_device		*smmu;
+	struct device			*dev;
+	struct arm_smmu_domain		*domain;
+	struct list_head		domain_head;
+	u32				*sids;
+	unsigned int			num_sids;
+	bool				ats_enabled;
+};
+
+struct arm_smmu_domain {
+	struct arm_smmu_device		*smmu;
+	struct mutex			init_mutex; /* Protects smmu pointer */
+
+	bool				non_strict;
+	atomic_t			nr_ats_masters;
+
+	enum arm_smmu_domain_stage	stage;
+	struct arm_smmu_s2_cfg	s2_cfg;
+
+	/* Xen domain associated with this SMMU domain */
+	struct domain		*d;
+
+	struct iommu_domain		domain;
+
+	struct list_head		devices;
+	spinlock_t			devices_lock;
+};
+
+#endif /* _ARM_SMMU_V3_H */
-- 
2.25.1

On Fri, 2 Sep 2022, Rahul Singh wrote:
> From: Jean-Philippe Brucker <jean-philippe@linaro.org>
> 
> Backport Linux commit e881e7839fba. This is the clean backport without
> any changes.

I don't think we can say that this is a clean backport because there are
differences between the two patches.

That said, it is just code movement, it is similar to the original
patch, and it still compiles. So I think we should change the commit
message not to say that it is a clean backport, but other than that is
fine.


> Allow sharing structure definitions with the upcoming SVA support for
> Arm SMMUv3, by moving them to a separate header. We could surgically
> extract only what is needed but keeping all definitions in one place
> looks nicer.
> 
> Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
> Reviewed-by: Eric Auger <eric.auger@redhat.com>
> Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
> Link: https://lore.kernel.org/r/20200918101852.582559-8-jean-philippe@linaro.org
> Signed-off-by: Will Deacon <will@kernel.org>
> Origin: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git e881e7839fba
> Signed-off-by: Rahul Singh <rahul.singh@arm.com>
> ---
> Changes in v2:
>  - fix commit msg
>  - also move struct definition to header file to sync with Linux patch
> ---
>  xen/drivers/passthrough/arm/smmu-v3.c | 666 +------------------------
>  xen/drivers/passthrough/arm/smmu-v3.h | 674 ++++++++++++++++++++++++++
>  2 files changed, 675 insertions(+), 665 deletions(-)
>  create mode 100644 xen/drivers/passthrough/arm/smmu-v3.h
> 
> diff --git a/xen/drivers/passthrough/arm/smmu-v3.c b/xen/drivers/passthrough/arm/smmu-v3.c
> index cee13d1fc7..85ad066266 100644
> --- a/xen/drivers/passthrough/arm/smmu-v3.c
> +++ b/xen/drivers/passthrough/arm/smmu-v3.c
> @@ -90,6 +90,7 @@
>  #include <asm/iommu_fwspec.h>
>  #include <asm/platform.h>
>  
> +#include "smmu-v3.h"
>  
>  #define ARM_SMMU_VTCR_SH_IS		3
>  #define ARM_SMMU_VTCR_RGN_WBWA		1
> @@ -102,21 +103,10 @@
>  #define ARM_SMMU_VTCR_PS_48_BIT		0x5ULL
>  #define ARM_SMMU_VTCR_PS_52_BIT		0x6ULL
>  
> -/* Linux compatibility functions. */
> -typedef paddr_t		dma_addr_t;
> -typedef paddr_t		phys_addr_t;
> -typedef unsigned int		gfp_t;
> -
>  #define platform_device		device
>  
>  #define GFP_KERNEL		0
>  
> -/* Alias to Xen lock functions */
> -#define mutex spinlock
> -#define mutex_init spin_lock_init
> -#define mutex_lock spin_lock
> -#define mutex_unlock spin_unlock
> -
>  /* Device logger functions */
>  #define dev_name(dev)	dt_node_full_name(dev->of_node)
>  #define dev_dbg(dev, fmt, ...)			\
> @@ -157,12 +147,6 @@ typedef unsigned int		gfp_t;
>  #define readl_relaxed_poll_timeout(addr, val, cond, delay_us, timeout_us)	\
>  	readx_poll_timeout(readl_relaxed, addr, val, cond, delay_us, timeout_us)
>  
> -#define FIELD_PREP(_mask, _val)			\
> -	(((typeof(_mask))(_val) << (ffs64(_mask) - 1)) & (_mask))
> -
> -#define FIELD_GET(_mask, _reg)			\
> -	((typeof(_mask))(((_reg) & (_mask)) >> (ffs64(_mask) - 1)))
> -
>  /*
>   * Helpers for DMA allocation. Just the function name is reused for
>   * porting code, these allocation are not managed allocations
> @@ -195,27 +179,6 @@ static void *dmam_alloc_coherent(struct device *dev, size_t size,
>  	return vaddr;
>  }
>  
> -
> -/* Xen specific code. */
> -struct iommu_domain {
> -	/* Runtime SMMU configuration for this iommu_domain */
> -	atomic_t		ref;
> -	/*
> -	 * Used to link iommu_domain contexts for a same domain.
> -	 * There is at least one per-SMMU to used by the domain.
> -	 */
> -	struct list_head		list;
> -};
> -
> -/* Describes information required for a Xen domain */
> -struct arm_smmu_xen_domain {
> -	spinlock_t		lock;
> -
> -	/* List of iommu domains associated to this domain */
> -	struct list_head	contexts;
> -};
> -
> -
>  /* Keep a list of devices associated with this driver */
>  static DEFINE_SPINLOCK(arm_smmu_devices_lock);
>  static LIST_HEAD(arm_smmu_devices);
> @@ -259,635 +222,8 @@ static int platform_get_irq_byname_optional(struct device *dev,
>  }
>  
>  /* Start of Linux SMMUv3 code */
> -/* MMIO registers */
> -#define ARM_SMMU_IDR0			0x0
> -#define IDR0_ST_LVL			GENMASK(28, 27)
> -#define IDR0_ST_LVL_2LVL		1
> -#define IDR0_STALL_MODEL		GENMASK(25, 24)
> -#define IDR0_STALL_MODEL_STALL		0
> -#define IDR0_STALL_MODEL_FORCE		2
> -#define IDR0_TTENDIAN			GENMASK(22, 21)
> -#define IDR0_TTENDIAN_MIXED		0
> -#define IDR0_TTENDIAN_LE		2
> -#define IDR0_TTENDIAN_BE		3
> -#define IDR0_CD2L			(1 << 19)
> -#define IDR0_VMID16			(1 << 18)
> -#define IDR0_PRI			(1 << 16)
> -#define IDR0_SEV			(1 << 14)
> -#define IDR0_MSI			(1 << 13)
> -#define IDR0_ASID16			(1 << 12)
> -#define IDR0_ATS			(1 << 10)
> -#define IDR0_HYP			(1 << 9)
> -#define IDR0_COHACC			(1 << 4)
> -#define IDR0_TTF			GENMASK(3, 2)
> -#define IDR0_TTF_AARCH64		2
> -#define IDR0_TTF_AARCH32_64		3
> -#define IDR0_S1P			(1 << 1)
> -#define IDR0_S2P			(1 << 0)
> -
> -#define ARM_SMMU_IDR1			0x4
> -#define IDR1_TABLES_PRESET		(1 << 30)
> -#define IDR1_QUEUES_PRESET		(1 << 29)
> -#define IDR1_REL			(1 << 28)
> -#define IDR1_CMDQS			GENMASK(25, 21)
> -#define IDR1_EVTQS			GENMASK(20, 16)
> -#define IDR1_PRIQS			GENMASK(15, 11)
> -#define IDR1_SSIDSIZE			GENMASK(10, 6)
> -#define IDR1_SIDSIZE			GENMASK(5, 0)
> -
> -#define ARM_SMMU_IDR5			0x14
> -#define IDR5_STALL_MAX			GENMASK(31, 16)
> -#define IDR5_GRAN64K			(1 << 6)
> -#define IDR5_GRAN16K			(1 << 5)
> -#define IDR5_GRAN4K			(1 << 4)
> -#define IDR5_OAS			GENMASK(2, 0)
> -#define IDR5_OAS_32_BIT			0
> -#define IDR5_OAS_36_BIT			1
> -#define IDR5_OAS_40_BIT			2
> -#define IDR5_OAS_42_BIT			3
> -#define IDR5_OAS_44_BIT			4
> -#define IDR5_OAS_48_BIT			5
> -#define IDR5_OAS_52_BIT			6
> -#define IDR5_VAX			GENMASK(11, 10)
> -#define IDR5_VAX_52_BIT			1
> -
> -#define ARM_SMMU_CR0			0x20
> -#define CR0_ATSCHK			(1 << 4)
> -#define CR0_CMDQEN			(1 << 3)
> -#define CR0_EVTQEN			(1 << 2)
> -#define CR0_PRIQEN			(1 << 1)
> -#define CR0_SMMUEN			(1 << 0)
> -
> -#define ARM_SMMU_CR0ACK			0x24
> -
> -#define ARM_SMMU_CR1			0x28
> -#define CR1_TABLE_SH			GENMASK(11, 10)
> -#define CR1_TABLE_OC			GENMASK(9, 8)
> -#define CR1_TABLE_IC			GENMASK(7, 6)
> -#define CR1_QUEUE_SH			GENMASK(5, 4)
> -#define CR1_QUEUE_OC			GENMASK(3, 2)
> -#define CR1_QUEUE_IC			GENMASK(1, 0)
> -/* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
> -#define CR1_CACHE_NC			0
> -#define CR1_CACHE_WB			1
> -#define CR1_CACHE_WT			2
> -
> -#define ARM_SMMU_CR2			0x2c
> -#define CR2_PTM				(1 << 2)
> -#define CR2_RECINVSID			(1 << 1)
> -#define CR2_E2H				(1 << 0)
> -
> -#define ARM_SMMU_GBPA			0x44
> -#define GBPA_UPDATE			(1U << 31)
> -#define GBPA_ABORT			(1 << 20)
> -
> -#define ARM_SMMU_IRQ_CTRL		0x50
> -#define IRQ_CTRL_EVTQ_IRQEN		(1 << 2)
> -#define IRQ_CTRL_PRIQ_IRQEN		(1 << 1)
> -#define IRQ_CTRL_GERROR_IRQEN		(1 << 0)
> -
> -#define ARM_SMMU_IRQ_CTRLACK		0x54
> -
> -#define ARM_SMMU_GERROR			0x60
> -#define GERROR_SFM_ERR			(1 << 8)
> -#define GERROR_MSI_GERROR_ABT_ERR	(1 << 7)
> -#define GERROR_MSI_PRIQ_ABT_ERR		(1 << 6)
> -#define GERROR_MSI_EVTQ_ABT_ERR		(1 << 5)
> -#define GERROR_MSI_CMDQ_ABT_ERR		(1 << 4)
> -#define GERROR_PRIQ_ABT_ERR		(1 << 3)
> -#define GERROR_EVTQ_ABT_ERR		(1 << 2)
> -#define GERROR_CMDQ_ERR			(1 << 0)
> -#define GERROR_ERR_MASK			0xfd
> -
> -#define ARM_SMMU_GERRORN		0x64
> -
> -#define ARM_SMMU_GERROR_IRQ_CFG0	0x68
> -#define ARM_SMMU_GERROR_IRQ_CFG1	0x70
> -#define ARM_SMMU_GERROR_IRQ_CFG2	0x74
> -
> -#define ARM_SMMU_STRTAB_BASE		0x80
> -#define STRTAB_BASE_RA			(1UL << 62)
> -#define STRTAB_BASE_ADDR_MASK		GENMASK_ULL(51, 6)
> -
> -#define ARM_SMMU_STRTAB_BASE_CFG	0x88
> -#define STRTAB_BASE_CFG_FMT		GENMASK(17, 16)
> -#define STRTAB_BASE_CFG_FMT_LINEAR	0
> -#define STRTAB_BASE_CFG_FMT_2LVL	1
> -#define STRTAB_BASE_CFG_SPLIT		GENMASK(10, 6)
> -#define STRTAB_BASE_CFG_LOG2SIZE	GENMASK(5, 0)
> -
> -#define ARM_SMMU_CMDQ_BASE		0x90
> -#define ARM_SMMU_CMDQ_PROD		0x98
> -#define ARM_SMMU_CMDQ_CONS		0x9c
> -
> -#define ARM_SMMU_EVTQ_BASE		0xa0
> -#define ARM_SMMU_EVTQ_PROD		0x100a8
> -#define ARM_SMMU_EVTQ_CONS		0x100ac
> -#define ARM_SMMU_EVTQ_IRQ_CFG0		0xb0
> -#define ARM_SMMU_EVTQ_IRQ_CFG1		0xb8
> -#define ARM_SMMU_EVTQ_IRQ_CFG2		0xbc
> -
> -#define ARM_SMMU_PRIQ_BASE		0xc0
> -#define ARM_SMMU_PRIQ_PROD		0x100c8
> -#define ARM_SMMU_PRIQ_CONS		0x100cc
> -#define ARM_SMMU_PRIQ_IRQ_CFG0		0xd0
> -#define ARM_SMMU_PRIQ_IRQ_CFG1		0xd8
> -#define ARM_SMMU_PRIQ_IRQ_CFG2		0xdc
> -
> -#define ARM_SMMU_REG_SZ			0xe00
> -
> -/* Common MSI config fields */
> -#define MSI_CFG0_ADDR_MASK		GENMASK_ULL(51, 2)
> -#define MSI_CFG2_SH			GENMASK(5, 4)
> -#define MSI_CFG2_MEMATTR		GENMASK(3, 0)
> -
> -/* Common memory attribute values */
> -#define ARM_SMMU_SH_NSH			0
> -#define ARM_SMMU_SH_OSH			2
> -#define ARM_SMMU_SH_ISH			3
> -#define ARM_SMMU_MEMATTR_DEVICE_nGnRE	0x1
> -#define ARM_SMMU_MEMATTR_OIWB		0xf
> -
> -#define Q_IDX(llq, p)			((p) & ((1 << (llq)->max_n_shift) - 1))
> -#define Q_WRP(llq, p)			((p) & (1 << (llq)->max_n_shift))
> -#define Q_OVERFLOW_FLAG			(1U << 31)
> -#define Q_OVF(p)			((p) & Q_OVERFLOW_FLAG)
> -#define Q_ENT(q, p)			((q)->base +			\
> -					 Q_IDX(&((q)->llq), p) *	\
> -					 (q)->ent_dwords)
> -
> -#define Q_BASE_RWA			(1UL << 62)
> -#define Q_BASE_ADDR_MASK		GENMASK_ULL(51, 5)
> -#define Q_BASE_LOG2SIZE			GENMASK(4, 0)
> -
> -/* Ensure DMA allocations are naturally aligned */
> -#ifdef CONFIG_CMA_ALIGNMENT
> -#define Q_MAX_SZ_SHIFT			(PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
> -#else
> -#define Q_MAX_SZ_SHIFT			(PAGE_SHIFT + MAX_ORDER - 1)
> -#endif
> -
> -/*
> - * Stream table.
> - *
> - * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
> - * 2lvl: 128k L1 entries,
> - *       256 lazy entries per table (each table covers a PCI bus)
> - */
> -#define STRTAB_L1_SZ_SHIFT		20
> -#define STRTAB_SPLIT			8
> -
> -#define STRTAB_L1_DESC_DWORDS		1
> -#define STRTAB_L1_DESC_SPAN		GENMASK_ULL(4, 0)
> -#define STRTAB_L1_DESC_L2PTR_MASK	GENMASK_ULL(51, 6)
> -
> -#define STRTAB_STE_DWORDS		8
> -#define STRTAB_STE_0_V			(1UL << 0)
> -#define STRTAB_STE_0_CFG		GENMASK_ULL(3, 1)
> -#define STRTAB_STE_0_CFG_ABORT		0
> -#define STRTAB_STE_0_CFG_BYPASS		4
> -#define STRTAB_STE_0_CFG_S1_TRANS	5
> -#define STRTAB_STE_0_CFG_S2_TRANS	6
> -
> -#define STRTAB_STE_0_S1FMT		GENMASK_ULL(5, 4)
> -#define STRTAB_STE_0_S1FMT_LINEAR	0
> -#define STRTAB_STE_0_S1FMT_64K_L2	2
> -#define STRTAB_STE_0_S1CTXPTR_MASK	GENMASK_ULL(51, 6)
> -#define STRTAB_STE_0_S1CDMAX		GENMASK_ULL(63, 59)
> -
> -#define STRTAB_STE_1_S1DSS		GENMASK_ULL(1, 0)
> -#define STRTAB_STE_1_S1DSS_TERMINATE	0x0
> -#define STRTAB_STE_1_S1DSS_BYPASS	0x1
> -#define STRTAB_STE_1_S1DSS_SSID0	0x2
> -
> -#define STRTAB_STE_1_S1C_CACHE_NC	0UL
> -#define STRTAB_STE_1_S1C_CACHE_WBRA	1UL
> -#define STRTAB_STE_1_S1C_CACHE_WT	2UL
> -#define STRTAB_STE_1_S1C_CACHE_WB	3UL
> -#define STRTAB_STE_1_S1CIR		GENMASK_ULL(3, 2)
> -#define STRTAB_STE_1_S1COR		GENMASK_ULL(5, 4)
> -#define STRTAB_STE_1_S1CSH		GENMASK_ULL(7, 6)
> -
> -#define STRTAB_STE_1_S1STALLD		(1UL << 27)
> -
> -#define STRTAB_STE_1_EATS		GENMASK_ULL(29, 28)
> -#define STRTAB_STE_1_EATS_ABT		0UL
> -#define STRTAB_STE_1_EATS_TRANS		1UL
> -#define STRTAB_STE_1_EATS_S1CHK		2UL
> -
> -#define STRTAB_STE_1_STRW		GENMASK_ULL(31, 30)
> -#define STRTAB_STE_1_STRW_NSEL1		0UL
> -#define STRTAB_STE_1_STRW_EL2		2UL
> -
> -#define STRTAB_STE_1_SHCFG		GENMASK_ULL(45, 44)
> -#define STRTAB_STE_1_SHCFG_INCOMING	1UL
> -
> -#define STRTAB_STE_2_S2VMID		GENMASK_ULL(15, 0)
> -#define STRTAB_STE_2_VTCR		GENMASK_ULL(50, 32)
> -#define STRTAB_STE_2_VTCR_S2T0SZ	GENMASK_ULL(5, 0)
> -#define STRTAB_STE_2_VTCR_S2SL0		GENMASK_ULL(7, 6)
> -#define STRTAB_STE_2_VTCR_S2IR0		GENMASK_ULL(9, 8)
> -#define STRTAB_STE_2_VTCR_S2OR0		GENMASK_ULL(11, 10)
> -#define STRTAB_STE_2_VTCR_S2SH0		GENMASK_ULL(13, 12)
> -#define STRTAB_STE_2_VTCR_S2TG		GENMASK_ULL(15, 14)
> -#define STRTAB_STE_2_VTCR_S2PS		GENMASK_ULL(18, 16)
> -#define STRTAB_STE_2_S2AA64		(1UL << 51)
> -#define STRTAB_STE_2_S2ENDI		(1UL << 52)
> -#define STRTAB_STE_2_S2PTW		(1UL << 54)
> -#define STRTAB_STE_2_S2R		(1UL << 58)
> -
> -#define STRTAB_STE_3_S2TTB_MASK		GENMASK_ULL(51, 4)
> -
> -/*
> - * Context descriptors.
> - *
> - * Linear: when less than 1024 SSIDs are supported
> - * 2lvl: at most 1024 L1 entries,
> - *       1024 lazy entries per table.
> - */
> -#define CTXDESC_SPLIT			10
> -#define CTXDESC_L2_ENTRIES		(1 << CTXDESC_SPLIT)
> -
> -#define CTXDESC_L1_DESC_DWORDS		1
> -#define CTXDESC_L1_DESC_V		(1UL << 0)
> -#define CTXDESC_L1_DESC_L2PTR_MASK	GENMASK_ULL(51, 12)
> -
> -#define CTXDESC_CD_DWORDS		8
> -#define CTXDESC_CD_0_TCR_T0SZ		GENMASK_ULL(5, 0)
> -#define CTXDESC_CD_0_TCR_TG0		GENMASK_ULL(7, 6)
> -#define CTXDESC_CD_0_TCR_IRGN0		GENMASK_ULL(9, 8)
> -#define CTXDESC_CD_0_TCR_ORGN0		GENMASK_ULL(11, 10)
> -#define CTXDESC_CD_0_TCR_SH0		GENMASK_ULL(13, 12)
> -#define CTXDESC_CD_0_TCR_EPD0		(1ULL << 14)
> -#define CTXDESC_CD_0_TCR_EPD1		(1ULL << 30)
> -
> -#define CTXDESC_CD_0_ENDI		(1UL << 15)
> -#define CTXDESC_CD_0_V			(1UL << 31)
> -
> -#define CTXDESC_CD_0_TCR_IPS		GENMASK_ULL(34, 32)
> -#define CTXDESC_CD_0_TCR_TBI0		(1ULL << 38)
> -
> -#define CTXDESC_CD_0_AA64		(1UL << 41)
> -#define CTXDESC_CD_0_S			(1UL << 44)
> -#define CTXDESC_CD_0_R			(1UL << 45)
> -#define CTXDESC_CD_0_A			(1UL << 46)
> -#define CTXDESC_CD_0_ASET		(1UL << 47)
> -#define CTXDESC_CD_0_ASID		GENMASK_ULL(63, 48)
> -
> -#define CTXDESC_CD_1_TTB0_MASK		GENMASK_ULL(51, 4)
> -
> -/*
> - * When the SMMU only supports linear context descriptor tables, pick a
> - * reasonable size limit (64kB).
> - */
> -#define CTXDESC_LINEAR_CDMAX		ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3))
> -
> -/* Command queue */
> -#define CMDQ_ENT_SZ_SHIFT		4
> -#define CMDQ_ENT_DWORDS			((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
> -#define CMDQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
> -
> -#define CMDQ_CONS_ERR			GENMASK(30, 24)
> -#define CMDQ_ERR_CERROR_NONE_IDX	0
> -#define CMDQ_ERR_CERROR_ILL_IDX		1
> -#define CMDQ_ERR_CERROR_ABT_IDX		2
> -#define CMDQ_ERR_CERROR_ATC_INV_IDX	3
> -
> -#define CMDQ_0_OP			GENMASK_ULL(7, 0)
> -#define CMDQ_0_SSV			(1UL << 11)
> -
> -#define CMDQ_PREFETCH_0_SID		GENMASK_ULL(63, 32)
> -#define CMDQ_PREFETCH_1_SIZE		GENMASK_ULL(4, 0)
> -#define CMDQ_PREFETCH_1_ADDR_MASK	GENMASK_ULL(63, 12)
> -
> -#define CMDQ_CFGI_0_SSID		GENMASK_ULL(31, 12)
> -#define CMDQ_CFGI_0_SID			GENMASK_ULL(63, 32)
> -#define CMDQ_CFGI_1_LEAF		(1UL << 0)
> -#define CMDQ_CFGI_1_RANGE		GENMASK_ULL(4, 0)
> -
> -#define CMDQ_TLBI_0_VMID		GENMASK_ULL(47, 32)
> -#define CMDQ_TLBI_0_ASID		GENMASK_ULL(63, 48)
> -#define CMDQ_TLBI_1_LEAF		(1UL << 0)
> -#define CMDQ_TLBI_1_VA_MASK		GENMASK_ULL(63, 12)
> -#define CMDQ_TLBI_1_IPA_MASK		GENMASK_ULL(51, 12)
> -
> -#define CMDQ_ATC_0_SSID			GENMASK_ULL(31, 12)
> -#define CMDQ_ATC_0_SID			GENMASK_ULL(63, 32)
> -#define CMDQ_ATC_0_GLOBAL		(1UL << 9)
> -#define CMDQ_ATC_1_SIZE			GENMASK_ULL(5, 0)
> -#define CMDQ_ATC_1_ADDR_MASK		GENMASK_ULL(63, 12)
> -
> -#define CMDQ_PRI_0_SSID			GENMASK_ULL(31, 12)
> -#define CMDQ_PRI_0_SID			GENMASK_ULL(63, 32)
> -#define CMDQ_PRI_1_GRPID		GENMASK_ULL(8, 0)
> -#define CMDQ_PRI_1_RESP			GENMASK_ULL(13, 12)
> -
> -#define CMDQ_SYNC_0_CS			GENMASK_ULL(13, 12)
> -#define CMDQ_SYNC_0_CS_NONE		0
> -#define CMDQ_SYNC_0_CS_IRQ		1
> -#define CMDQ_SYNC_0_CS_SEV		2
> -#define CMDQ_SYNC_0_MSH			GENMASK_ULL(23, 22)
> -#define CMDQ_SYNC_0_MSIATTR		GENMASK_ULL(27, 24)
> -#define CMDQ_SYNC_0_MSIDATA		GENMASK_ULL(63, 32)
> -#define CMDQ_SYNC_1_MSIADDR_MASK	GENMASK_ULL(51, 2)
> -
> -/* Event queue */
> -#define EVTQ_ENT_SZ_SHIFT		5
> -#define EVTQ_ENT_DWORDS			((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
> -#define EVTQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
> -
> -#define EVTQ_0_ID			GENMASK_ULL(7, 0)
> -
> -/* PRI queue */
> -#define PRIQ_ENT_SZ_SHIFT		4
> -#define PRIQ_ENT_DWORDS			((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
> -#define PRIQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
> -
> -#define PRIQ_0_SID			GENMASK_ULL(31, 0)
> -#define PRIQ_0_SSID			GENMASK_ULL(51, 32)
> -#define PRIQ_0_PERM_PRIV		(1UL << 58)
> -#define PRIQ_0_PERM_EXEC		(1UL << 59)
> -#define PRIQ_0_PERM_READ		(1UL << 60)
> -#define PRIQ_0_PERM_WRITE		(1UL << 61)
> -#define PRIQ_0_PRG_LAST			(1UL << 62)
> -#define PRIQ_0_SSID_V			(1UL << 63)
> -
> -#define PRIQ_1_PRG_IDX			GENMASK_ULL(8, 0)
> -#define PRIQ_1_ADDR_MASK		GENMASK_ULL(63, 12)
> -
> -/* High-level queue structures */
> -#define ARM_SMMU_POLL_TIMEOUT_US	100
> -#define ARM_SMMU_CMDQ_SYNC_TIMEOUT_US	1000000 /* 1s! */
> -#define ARM_SMMU_CMDQ_SYNC_SPIN_COUNT	10
> -
>  static bool disable_bypass = 1;
>  
> -enum pri_resp {
> -	PRI_RESP_DENY = 0,
> -	PRI_RESP_FAIL = 1,
> -	PRI_RESP_SUCC = 2,
> -};
> -
> -#ifdef CONFIG_MSI
> -enum arm_smmu_msi_index {
> -	EVTQ_MSI_INDEX,
> -	GERROR_MSI_INDEX,
> -	PRIQ_MSI_INDEX,
> -	ARM_SMMU_MAX_MSIS,
> -};
> -
> -static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
> -	[EVTQ_MSI_INDEX] = {
> -		ARM_SMMU_EVTQ_IRQ_CFG0,
> -		ARM_SMMU_EVTQ_IRQ_CFG1,
> -		ARM_SMMU_EVTQ_IRQ_CFG2,
> -	},
> -	[GERROR_MSI_INDEX] = {
> -		ARM_SMMU_GERROR_IRQ_CFG0,
> -		ARM_SMMU_GERROR_IRQ_CFG1,
> -		ARM_SMMU_GERROR_IRQ_CFG2,
> -	},
> -	[PRIQ_MSI_INDEX] = {
> -		ARM_SMMU_PRIQ_IRQ_CFG0,
> -		ARM_SMMU_PRIQ_IRQ_CFG1,
> -		ARM_SMMU_PRIQ_IRQ_CFG2,
> -	},
> -};
> -#endif /* CONFIG_MSI */
> -
> -struct arm_smmu_cmdq_ent {
> -	/* Common fields */
> -	u8				opcode;
> -	bool				substream_valid;
> -
> -	/* Command-specific fields */
> -	union {
> -		#define CMDQ_OP_PREFETCH_CFG	0x1
> -		struct {
> -			u32			sid;
> -			u8			size;
> -			u64			addr;
> -		} prefetch;
> -
> -		#define CMDQ_OP_CFGI_STE	0x3
> -		#define CMDQ_OP_CFGI_ALL	0x4
> -		struct {
> -			u32			sid;
> -			union {
> -				bool		leaf;
> -				u8		span;
> -			};
> -		} cfgi;
> -
> -		#define CMDQ_OP_TLBI_EL2_ALL	0x20
> -		#define CMDQ_OP_TLBI_S12_VMALL	0x28
> -		#define CMDQ_OP_TLBI_S2_IPA	0x2a
> -		#define CMDQ_OP_TLBI_NSNH_ALL	0x30
> -		struct {
> -			u16			asid;
> -			u16			vmid;
> -			bool			leaf;
> -			u64			addr;
> -		} tlbi;
> -
> -		#define CMDQ_OP_ATC_INV		0x40
> -		#define ATC_INV_SIZE_ALL	52
> -		struct {
> -			u32			sid;
> -			u32			ssid;
> -			u64			addr;
> -			u8			size;
> -			bool			global;
> -		} atc;
> -
> -		#define CMDQ_OP_PRI_RESP	0x41
> -		struct {
> -			u32			sid;
> -			u32			ssid;
> -			u16			grpid;
> -			enum pri_resp		resp;
> -		} pri;
> -
> -		#define CMDQ_OP_CMD_SYNC	0x46
> -		struct {
> -			u32			msidata;
> -			u64			msiaddr;
> -		} sync;
> -	};
> -};
> -
> -struct arm_smmu_ll_queue {
> -	u32				prod;
> -	u32				cons;
> -	u32				max_n_shift;
> -};
> -
> -struct arm_smmu_queue {
> -	struct arm_smmu_ll_queue	llq;
> -	int				irq; /* Wired interrupt */
> -
> -	__le64				*base;
> -	dma_addr_t			base_dma;
> -	u64				q_base;
> -
> -	size_t				ent_dwords;
> -
> -	u32 __iomem			*prod_reg;
> -	u32 __iomem			*cons_reg;
> -};
> -
> -struct arm_smmu_cmdq {
> -	struct arm_smmu_queue		q;
> -	spinlock_t			lock;
> -};
> -
> -struct arm_smmu_evtq {
> -	struct arm_smmu_queue		q;
> -	u32				max_stalls;
> -};
> -
> -struct arm_smmu_priq {
> -	struct arm_smmu_queue		q;
> -};
> -
> -/* High-level stream table and context descriptor structures */
> -struct arm_smmu_strtab_l1_desc {
> -	u8				span;
> -
> -	__le64				*l2ptr;
> -	dma_addr_t			l2ptr_dma;
> -};
> -
> -struct arm_smmu_s2_cfg {
> -	u16				vmid;
> -	u64				vttbr;
> -	u64				vtcr;
> -};
> -
> -struct arm_smmu_strtab_cfg {
> -	__le64				*strtab;
> -	dma_addr_t			strtab_dma;
> -	struct arm_smmu_strtab_l1_desc	*l1_desc;
> -	unsigned int			num_l1_ents;
> -
> -	u64				strtab_base;
> -	u32				strtab_base_cfg;
> -};
> -
> -struct arm_lpae_s2_cfg {
> -	u64			vttbr;
> -	struct {
> -		u32			ps:3;
> -		u32			tg:2;
> -		u32			sh:2;
> -		u32			orgn:2;
> -		u32			irgn:2;
> -		u32			sl:2;
> -		u32			tsz:6;
> -	} vtcr;
> -};
> -
> -/* An SMMUv3 instance */
> -struct arm_smmu_device {
> -	struct device			*dev;
> -	void __iomem			*base;
> -	void __iomem			*page1;
> -
> -#define ARM_SMMU_FEAT_2_LVL_STRTAB	(1 << 0)
> -#define ARM_SMMU_FEAT_2_LVL_CDTAB	(1 << 1)
> -#define ARM_SMMU_FEAT_TT_LE		(1 << 2)
> -#define ARM_SMMU_FEAT_TT_BE		(1 << 3)
> -#define ARM_SMMU_FEAT_PRI		(1 << 4)
> -#define ARM_SMMU_FEAT_ATS		(1 << 5)
> -#define ARM_SMMU_FEAT_SEV		(1 << 6)
> -#define ARM_SMMU_FEAT_MSI		(1 << 7)
> -#define ARM_SMMU_FEAT_COHERENCY		(1 << 8)
> -#define ARM_SMMU_FEAT_TRANS_S1		(1 << 9)
> -#define ARM_SMMU_FEAT_TRANS_S2		(1 << 10)
> -#define ARM_SMMU_FEAT_STALLS		(1 << 11)
> -#define ARM_SMMU_FEAT_HYP		(1 << 12)
> -#define ARM_SMMU_FEAT_STALL_FORCE	(1 << 13)
> -#define ARM_SMMU_FEAT_VAX		(1 << 14)
> -	u32				features;
> -
> -#define ARM_SMMU_OPT_SKIP_PREFETCH	(1 << 0)
> -#define ARM_SMMU_OPT_PAGE0_REGS_ONLY	(1 << 1)
> -	u32				options;
> -
> -	struct arm_smmu_cmdq		cmdq;
> -	struct arm_smmu_evtq		evtq;
> -	struct arm_smmu_priq		priq;
> -
> -	int				gerr_irq;
> -	int				combined_irq;
> -	u32				sync_nr;
> -	u8				prev_cmd_opcode;
> -
> -	unsigned long			ias; /* IPA */
> -	unsigned long			oas; /* PA */
> -	unsigned long			pgsize_bitmap;
> -
> -#define ARM_SMMU_MAX_VMIDS		(1 << 16)
> -	unsigned int			vmid_bits;
> -	DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
> -
> -	unsigned int			sid_bits;
> -
> -	struct arm_smmu_strtab_cfg	strtab_cfg;
> -
> -	/* Hi16xx adds an extra 32 bits of goodness to its MSI payload */
> -	union {
> -		u32			sync_count;
> -		u64			padding;
> -	};
> -
> -	/* Need to keep a list of SMMU devices */
> -	struct list_head		devices;
> -
> -	/* Tasklets for handling evts/faults and pci page request IRQs*/
> -	struct tasklet		evtq_irq_tasklet;
> -	struct tasklet		priq_irq_tasklet;
> -	struct tasklet		combined_irq_tasklet;
> -};
> -
> -/* SMMU private data for each master */
> -struct arm_smmu_master {
> -	struct arm_smmu_device		*smmu;
> -	struct device			*dev;
> -	struct arm_smmu_domain		*domain;
> -	struct list_head		domain_head;
> -	u32				*sids;
> -	unsigned int			num_sids;
> -	bool				ats_enabled;
> -};
> -
> -/* SMMU private data for an IOMMU domain */
> -enum arm_smmu_domain_stage {
> -	ARM_SMMU_DOMAIN_S1 = 0,
> -	ARM_SMMU_DOMAIN_S2,
> -	ARM_SMMU_DOMAIN_NESTED,
> -	ARM_SMMU_DOMAIN_BYPASS,
> -};
> -
> -struct arm_smmu_domain {
> -	struct arm_smmu_device		*smmu;
> -	struct mutex			init_mutex; /* Protects smmu pointer */
> -
> -	bool				non_strict;
> -	atomic_t			nr_ats_masters;
> -
> -	enum arm_smmu_domain_stage	stage;
> -	struct arm_smmu_s2_cfg	s2_cfg;
> -
> -	/* Xen domain associated with this SMMU domain */
> -	struct domain		*d;
> -
> -	struct iommu_domain		domain;
> -
> -	struct list_head		devices;
> -	spinlock_t			devices_lock;
> -};
> -
>  struct arm_smmu_option_prop {
>  	u32 opt;
>  	const char *prop;
> diff --git a/xen/drivers/passthrough/arm/smmu-v3.h b/xen/drivers/passthrough/arm/smmu-v3.h
> new file mode 100644
> index 0000000000..c45d2f16c4
> --- /dev/null
> +++ b/xen/drivers/passthrough/arm/smmu-v3.h
> @@ -0,0 +1,674 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/*
> + * IOMMU API for ARM architected SMMUv3 implementations.
> + *
> + * Copyright (C) 2022 ARM Limited
> + */
> +
> +#ifndef _ARM_SMMU_V3_H
> +#define _ARM_SMMU_V3_H
> +
> +/* MMIO registers */
> +#define ARM_SMMU_IDR0			0x0
> +#define IDR0_ST_LVL			GENMASK(28, 27)
> +#define IDR0_ST_LVL_2LVL		1
> +#define IDR0_STALL_MODEL		GENMASK(25, 24)
> +#define IDR0_STALL_MODEL_STALL		0
> +#define IDR0_STALL_MODEL_FORCE		2
> +#define IDR0_TTENDIAN			GENMASK(22, 21)
> +#define IDR0_TTENDIAN_MIXED		0
> +#define IDR0_TTENDIAN_LE		2
> +#define IDR0_TTENDIAN_BE		3
> +#define IDR0_CD2L			(1 << 19)
> +#define IDR0_VMID16			(1 << 18)
> +#define IDR0_PRI			(1 << 16)
> +#define IDR0_SEV			(1 << 14)
> +#define IDR0_MSI			(1 << 13)
> +#define IDR0_ASID16			(1 << 12)
> +#define IDR0_ATS			(1 << 10)
> +#define IDR0_HYP			(1 << 9)
> +#define IDR0_COHACC			(1 << 4)
> +#define IDR0_TTF			GENMASK(3, 2)
> +#define IDR0_TTF_AARCH64		2
> +#define IDR0_TTF_AARCH32_64		3
> +#define IDR0_S1P			(1 << 1)
> +#define IDR0_S2P			(1 << 0)
> +
> +#define ARM_SMMU_IDR1			0x4
> +#define IDR1_TABLES_PRESET		(1 << 30)
> +#define IDR1_QUEUES_PRESET		(1 << 29)
> +#define IDR1_REL			(1 << 28)
> +#define IDR1_CMDQS			GENMASK(25, 21)
> +#define IDR1_EVTQS			GENMASK(20, 16)
> +#define IDR1_PRIQS			GENMASK(15, 11)
> +#define IDR1_SSIDSIZE			GENMASK(10, 6)
> +#define IDR1_SIDSIZE			GENMASK(5, 0)
> +
> +#define ARM_SMMU_IDR5			0x14
> +#define IDR5_STALL_MAX			GENMASK(31, 16)
> +#define IDR5_GRAN64K			(1 << 6)
> +#define IDR5_GRAN16K			(1 << 5)
> +#define IDR5_GRAN4K			(1 << 4)
> +#define IDR5_OAS			GENMASK(2, 0)
> +#define IDR5_OAS_32_BIT			0
> +#define IDR5_OAS_36_BIT			1
> +#define IDR5_OAS_40_BIT			2
> +#define IDR5_OAS_42_BIT			3
> +#define IDR5_OAS_44_BIT			4
> +#define IDR5_OAS_48_BIT			5
> +#define IDR5_OAS_52_BIT			6
> +#define IDR5_VAX			GENMASK(11, 10)
> +#define IDR5_VAX_52_BIT			1
> +
> +#define ARM_SMMU_CR0			0x20
> +#define CR0_ATSCHK			(1 << 4)
> +#define CR0_CMDQEN			(1 << 3)
> +#define CR0_EVTQEN			(1 << 2)
> +#define CR0_PRIQEN			(1 << 1)
> +#define CR0_SMMUEN			(1 << 0)
> +
> +#define ARM_SMMU_CR0ACK			0x24
> +
> +#define ARM_SMMU_CR1			0x28
> +#define CR1_TABLE_SH			GENMASK(11, 10)
> +#define CR1_TABLE_OC			GENMASK(9, 8)
> +#define CR1_TABLE_IC			GENMASK(7, 6)
> +#define CR1_QUEUE_SH			GENMASK(5, 4)
> +#define CR1_QUEUE_OC			GENMASK(3, 2)
> +#define CR1_QUEUE_IC			GENMASK(1, 0)
> +/* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
> +#define CR1_CACHE_NC			0
> +#define CR1_CACHE_WB			1
> +#define CR1_CACHE_WT			2
> +
> +#define ARM_SMMU_CR2			0x2c
> +#define CR2_PTM				(1 << 2)
> +#define CR2_RECINVSID			(1 << 1)
> +#define CR2_E2H				(1 << 0)
> +
> +#define ARM_SMMU_GBPA			0x44
> +#define GBPA_UPDATE			(1 << 31)
> +#define GBPA_ABORT			(1 << 20)
> +
> +#define ARM_SMMU_IRQ_CTRL		0x50
> +#define IRQ_CTRL_EVTQ_IRQEN		(1 << 2)
> +#define IRQ_CTRL_PRIQ_IRQEN		(1 << 1)
> +#define IRQ_CTRL_GERROR_IRQEN		(1 << 0)
> +
> +#define ARM_SMMU_IRQ_CTRLACK		0x54
> +
> +#define ARM_SMMU_GERROR			0x60
> +#define GERROR_SFM_ERR			(1 << 8)
> +#define GERROR_MSI_GERROR_ABT_ERR	(1 << 7)
> +#define GERROR_MSI_PRIQ_ABT_ERR		(1 << 6)
> +#define GERROR_MSI_EVTQ_ABT_ERR		(1 << 5)
> +#define GERROR_MSI_CMDQ_ABT_ERR		(1 << 4)
> +#define GERROR_PRIQ_ABT_ERR		(1 << 3)
> +#define GERROR_EVTQ_ABT_ERR		(1 << 2)
> +#define GERROR_CMDQ_ERR			(1 << 0)
> +#define GERROR_ERR_MASK			0xfd
> +
> +#define ARM_SMMU_GERRORN		0x64
> +
> +#define ARM_SMMU_GERROR_IRQ_CFG0	0x68
> +#define ARM_SMMU_GERROR_IRQ_CFG1	0x70
> +#define ARM_SMMU_GERROR_IRQ_CFG2	0x74
> +
> +#define ARM_SMMU_STRTAB_BASE		0x80
> +#define STRTAB_BASE_RA			(1UL << 62)
> +#define STRTAB_BASE_ADDR_MASK		GENMASK_ULL(51, 6)
> +
> +#define ARM_SMMU_STRTAB_BASE_CFG	0x88
> +#define STRTAB_BASE_CFG_FMT		GENMASK(17, 16)
> +#define STRTAB_BASE_CFG_FMT_LINEAR	0
> +#define STRTAB_BASE_CFG_FMT_2LVL	1
> +#define STRTAB_BASE_CFG_SPLIT		GENMASK(10, 6)
> +#define STRTAB_BASE_CFG_LOG2SIZE	GENMASK(5, 0)
> +
> +#define ARM_SMMU_CMDQ_BASE		0x90
> +#define ARM_SMMU_CMDQ_PROD		0x98
> +#define ARM_SMMU_CMDQ_CONS		0x9c
> +
> +#define ARM_SMMU_EVTQ_BASE		0xa0
> +#define ARM_SMMU_EVTQ_PROD		0x100a8
> +#define ARM_SMMU_EVTQ_CONS		0x100ac
> +#define ARM_SMMU_EVTQ_IRQ_CFG0		0xb0
> +#define ARM_SMMU_EVTQ_IRQ_CFG1		0xb8
> +#define ARM_SMMU_EVTQ_IRQ_CFG2		0xbc
> +
> +#define ARM_SMMU_PRIQ_BASE		0xc0
> +#define ARM_SMMU_PRIQ_PROD		0x100c8
> +#define ARM_SMMU_PRIQ_CONS		0x100cc
> +#define ARM_SMMU_PRIQ_IRQ_CFG0		0xd0
> +#define ARM_SMMU_PRIQ_IRQ_CFG1		0xd8
> +#define ARM_SMMU_PRIQ_IRQ_CFG2		0xdc
> +
> +#define ARM_SMMU_REG_SZ			0xe00
> +
> +/* Common MSI config fields */
> +#define MSI_CFG0_ADDR_MASK		GENMASK_ULL(51, 2)
> +#define MSI_CFG2_SH			GENMASK(5, 4)
> +#define MSI_CFG2_MEMATTR		GENMASK(3, 0)
> +
> +/* Common memory attribute values */
> +#define ARM_SMMU_SH_NSH			0
> +#define ARM_SMMU_SH_OSH			2
> +#define ARM_SMMU_SH_ISH			3
> +#define ARM_SMMU_MEMATTR_DEVICE_nGnRE	0x1
> +#define ARM_SMMU_MEMATTR_OIWB		0xf
> +
> +#define Q_IDX(llq, p)			((p) & ((1 << (llq)->max_n_shift) - 1))
> +#define Q_WRP(llq, p)			((p) & (1 << (llq)->max_n_shift))
> +#define Q_OVERFLOW_FLAG			(1 << 31)
> +#define Q_OVF(p)			((p) & Q_OVERFLOW_FLAG)
> +#define Q_ENT(q, p)			((q)->base +			\
> +					 Q_IDX(&((q)->llq), p) *	\
> +					 (q)->ent_dwords)
> +
> +#define Q_BASE_RWA			(1UL << 62)
> +#define Q_BASE_ADDR_MASK		GENMASK_ULL(51, 5)
> +#define Q_BASE_LOG2SIZE			GENMASK(4, 0)
> +
> +/* Ensure DMA allocations are naturally aligned */
> +#ifdef CONFIG_CMA_ALIGNMENT
> +#define Q_MAX_SZ_SHIFT			(PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
> +#else
> +#define Q_MAX_SZ_SHIFT			(PAGE_SHIFT + MAX_ORDER - 1)
> +#endif
> +
> +/*
> + * Stream table.
> + *
> + * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
> + * 2lvl: 128k L1 entries,
> + *       256 lazy entries per table (each table covers a PCI bus)
> + */
> +#define STRTAB_L1_SZ_SHIFT		20
> +#define STRTAB_SPLIT			8
> +
> +#define STRTAB_L1_DESC_DWORDS		1
> +#define STRTAB_L1_DESC_SPAN		GENMASK_ULL(4, 0)
> +#define STRTAB_L1_DESC_L2PTR_MASK	GENMASK_ULL(51, 6)
> +
> +#define STRTAB_STE_DWORDS		8
> +#define STRTAB_STE_0_V			(1UL << 0)
> +#define STRTAB_STE_0_CFG		GENMASK_ULL(3, 1)
> +#define STRTAB_STE_0_CFG_ABORT		0
> +#define STRTAB_STE_0_CFG_BYPASS		4
> +#define STRTAB_STE_0_CFG_S1_TRANS	5
> +#define STRTAB_STE_0_CFG_S2_TRANS	6
> +
> +#define STRTAB_STE_0_S1FMT		GENMASK_ULL(5, 4)
> +#define STRTAB_STE_0_S1FMT_LINEAR	0
> +#define STRTAB_STE_0_S1FMT_64K_L2	2
> +#define STRTAB_STE_0_S1CTXPTR_MASK	GENMASK_ULL(51, 6)
> +#define STRTAB_STE_0_S1CDMAX		GENMASK_ULL(63, 59)
> +
> +#define STRTAB_STE_1_S1DSS		GENMASK_ULL(1, 0)
> +#define STRTAB_STE_1_S1DSS_TERMINATE	0x0
> +#define STRTAB_STE_1_S1DSS_BYPASS	0x1
> +#define STRTAB_STE_1_S1DSS_SSID0	0x2
> +
> +#define STRTAB_STE_1_S1C_CACHE_NC	0UL
> +#define STRTAB_STE_1_S1C_CACHE_WBRA	1UL
> +#define STRTAB_STE_1_S1C_CACHE_WT	2UL
> +#define STRTAB_STE_1_S1C_CACHE_WB	3UL
> +#define STRTAB_STE_1_S1CIR		GENMASK_ULL(3, 2)
> +#define STRTAB_STE_1_S1COR		GENMASK_ULL(5, 4)
> +#define STRTAB_STE_1_S1CSH		GENMASK_ULL(7, 6)
> +
> +#define STRTAB_STE_1_S1STALLD		(1UL << 27)
> +
> +#define STRTAB_STE_1_EATS		GENMASK_ULL(29, 28)
> +#define STRTAB_STE_1_EATS_ABT		0UL
> +#define STRTAB_STE_1_EATS_TRANS		1UL
> +#define STRTAB_STE_1_EATS_S1CHK		2UL
> +
> +#define STRTAB_STE_1_STRW		GENMASK_ULL(31, 30)
> +#define STRTAB_STE_1_STRW_NSEL1		0UL
> +#define STRTAB_STE_1_STRW_EL2		2UL
> +
> +#define STRTAB_STE_1_SHCFG		GENMASK_ULL(45, 44)
> +#define STRTAB_STE_1_SHCFG_INCOMING	1UL
> +
> +#define STRTAB_STE_2_S2VMID		GENMASK_ULL(15, 0)
> +#define STRTAB_STE_2_VTCR		GENMASK_ULL(50, 32)
> +#define STRTAB_STE_2_VTCR_S2T0SZ	GENMASK_ULL(5, 0)
> +#define STRTAB_STE_2_VTCR_S2SL0		GENMASK_ULL(7, 6)
> +#define STRTAB_STE_2_VTCR_S2IR0		GENMASK_ULL(9, 8)
> +#define STRTAB_STE_2_VTCR_S2OR0		GENMASK_ULL(11, 10)
> +#define STRTAB_STE_2_VTCR_S2SH0		GENMASK_ULL(13, 12)
> +#define STRTAB_STE_2_VTCR_S2TG		GENMASK_ULL(15, 14)
> +#define STRTAB_STE_2_VTCR_S2PS		GENMASK_ULL(18, 16)
> +#define STRTAB_STE_2_S2AA64		(1UL << 51)
> +#define STRTAB_STE_2_S2ENDI		(1UL << 52)
> +#define STRTAB_STE_2_S2PTW		(1UL << 54)
> +#define STRTAB_STE_2_S2R		(1UL << 58)
> +
> +#define STRTAB_STE_3_S2TTB_MASK		GENMASK_ULL(51, 4)
> +
> +/*
> + * Context descriptors.
> + *
> + * Linear: when less than 1024 SSIDs are supported
> + * 2lvl: at most 1024 L1 entries,
> + *       1024 lazy entries per table.
> + */
> +#define CTXDESC_SPLIT			10
> +#define CTXDESC_L2_ENTRIES		(1 << CTXDESC_SPLIT)
> +
> +#define CTXDESC_L1_DESC_DWORDS		1
> +#define CTXDESC_L1_DESC_V		(1UL << 0)
> +#define CTXDESC_L1_DESC_L2PTR_MASK	GENMASK_ULL(51, 12)
> +
> +#define CTXDESC_CD_DWORDS		8
> +#define CTXDESC_CD_0_TCR_T0SZ		GENMASK_ULL(5, 0)
> +#define CTXDESC_CD_0_TCR_TG0		GENMASK_ULL(7, 6)
> +#define CTXDESC_CD_0_TCR_IRGN0		GENMASK_ULL(9, 8)
> +#define CTXDESC_CD_0_TCR_ORGN0		GENMASK_ULL(11, 10)
> +#define CTXDESC_CD_0_TCR_SH0		GENMASK_ULL(13, 12)
> +#define CTXDESC_CD_0_TCR_EPD0		(1ULL << 14)
> +#define CTXDESC_CD_0_TCR_EPD1		(1ULL << 30)
> +
> +#define CTXDESC_CD_0_ENDI		(1UL << 15)
> +#define CTXDESC_CD_0_V			(1UL << 31)
> +
> +#define CTXDESC_CD_0_TCR_IPS		GENMASK_ULL(34, 32)
> +#define CTXDESC_CD_0_TCR_TBI0		(1ULL << 38)
> +
> +#define CTXDESC_CD_0_AA64		(1UL << 41)
> +#define CTXDESC_CD_0_S			(1UL << 44)
> +#define CTXDESC_CD_0_R			(1UL << 45)
> +#define CTXDESC_CD_0_A			(1UL << 46)
> +#define CTXDESC_CD_0_ASET		(1UL << 47)
> +#define CTXDESC_CD_0_ASID		GENMASK_ULL(63, 48)
> +
> +#define CTXDESC_CD_1_TTB0_MASK		GENMASK_ULL(51, 4)
> +
> +/*
> + * When the SMMU only supports linear context descriptor tables, pick a
> + * reasonable size limit (64kB).
> + */
> +#define CTXDESC_LINEAR_CDMAX		ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3))
> +
> +/* Command queue */
> +#define CMDQ_ENT_SZ_SHIFT		4
> +#define CMDQ_ENT_DWORDS			((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
> +#define CMDQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
> +
> +#define CMDQ_CONS_ERR			GENMASK(30, 24)
> +#define CMDQ_ERR_CERROR_NONE_IDX	0
> +#define CMDQ_ERR_CERROR_ILL_IDX		1
> +#define CMDQ_ERR_CERROR_ABT_IDX		2
> +#define CMDQ_ERR_CERROR_ATC_INV_IDX	3
> +
> +#define CMDQ_0_OP			GENMASK_ULL(7, 0)
> +#define CMDQ_0_SSV			(1UL << 11)
> +
> +#define CMDQ_PREFETCH_0_SID		GENMASK_ULL(63, 32)
> +#define CMDQ_PREFETCH_1_SIZE		GENMASK_ULL(4, 0)
> +#define CMDQ_PREFETCH_1_ADDR_MASK	GENMASK_ULL(63, 12)
> +
> +#define CMDQ_CFGI_0_SSID		GENMASK_ULL(31, 12)
> +#define CMDQ_CFGI_0_SID			GENMASK_ULL(63, 32)
> +#define CMDQ_CFGI_1_LEAF		(1UL << 0)
> +#define CMDQ_CFGI_1_RANGE		GENMASK_ULL(4, 0)
> +
> +#define CMDQ_TLBI_0_VMID		GENMASK_ULL(47, 32)
> +#define CMDQ_TLBI_0_ASID		GENMASK_ULL(63, 48)
> +#define CMDQ_TLBI_1_LEAF		(1UL << 0)
> +#define CMDQ_TLBI_1_VA_MASK		GENMASK_ULL(63, 12)
> +#define CMDQ_TLBI_1_IPA_MASK		GENMASK_ULL(51, 12)
> +
> +#define CMDQ_ATC_0_SSID			GENMASK_ULL(31, 12)
> +#define CMDQ_ATC_0_SID			GENMASK_ULL(63, 32)
> +#define CMDQ_ATC_0_GLOBAL		(1UL << 9)
> +#define CMDQ_ATC_1_SIZE			GENMASK_ULL(5, 0)
> +#define CMDQ_ATC_1_ADDR_MASK		GENMASK_ULL(63, 12)
> +
> +#define CMDQ_PRI_0_SSID			GENMASK_ULL(31, 12)
> +#define CMDQ_PRI_0_SID			GENMASK_ULL(63, 32)
> +#define CMDQ_PRI_1_GRPID		GENMASK_ULL(8, 0)
> +#define CMDQ_PRI_1_RESP			GENMASK_ULL(13, 12)
> +
> +#define CMDQ_SYNC_0_CS			GENMASK_ULL(13, 12)
> +#define CMDQ_SYNC_0_CS_NONE		0
> +#define CMDQ_SYNC_0_CS_IRQ		1
> +#define CMDQ_SYNC_0_CS_SEV		2
> +#define CMDQ_SYNC_0_MSH			GENMASK_ULL(23, 22)
> +#define CMDQ_SYNC_0_MSIATTR		GENMASK_ULL(27, 24)
> +#define CMDQ_SYNC_0_MSIDATA		GENMASK_ULL(63, 32)
> +#define CMDQ_SYNC_1_MSIADDR_MASK	GENMASK_ULL(51, 2)
> +
> +/* Event queue */
> +#define EVTQ_ENT_SZ_SHIFT		5
> +#define EVTQ_ENT_DWORDS			((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
> +#define EVTQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
> +
> +#define EVTQ_0_ID			GENMASK_ULL(7, 0)
> +
> +/* PRI queue */
> +#define PRIQ_ENT_SZ_SHIFT		4
> +#define PRIQ_ENT_DWORDS			((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
> +#define PRIQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
> +
> +#define PRIQ_0_SID			GENMASK_ULL(31, 0)
> +#define PRIQ_0_SSID			GENMASK_ULL(51, 32)
> +#define PRIQ_0_PERM_PRIV		(1UL << 58)
> +#define PRIQ_0_PERM_EXEC		(1UL << 59)
> +#define PRIQ_0_PERM_READ		(1UL << 60)
> +#define PRIQ_0_PERM_WRITE		(1UL << 61)
> +#define PRIQ_0_PRG_LAST			(1UL << 62)
> +#define PRIQ_0_SSID_V			(1UL << 63)
> +
> +#define PRIQ_1_PRG_IDX			GENMASK_ULL(8, 0)
> +#define PRIQ_1_ADDR_MASK		GENMASK_ULL(63, 12)
> +
> +/* High-level queue structures */
> +#define ARM_SMMU_POLL_TIMEOUT_US	100
> +#define ARM_SMMU_CMDQ_SYNC_TIMEOUT_US	1000000 /* 1s! */
> +#define ARM_SMMU_CMDQ_SYNC_SPIN_COUNT	10
> +
> +#define FIELD_PREP(_mask, _val)			\
> +	(((typeof(_mask))(_val) << (ffs64(_mask) - 1)) & (_mask))
> +
> +#define FIELD_GET(_mask, _reg)			\
> +	((typeof(_mask))(((_reg) & (_mask)) >> (ffs64(_mask) - 1)))
> +
> +/* Linux compatibility functions. */
> +typedef paddr_t		dma_addr_t;
> +typedef paddr_t		phys_addr_t;
> +typedef unsigned int		gfp_t;
> +
> +/* Alias to Xen lock functions */
> +#define mutex spinlock
> +#define mutex_init spin_lock_init
> +#define mutex_lock spin_lock
> +#define mutex_unlock spin_unlock
> +
> +/* SMMU private data for an IOMMU domain */
> +enum arm_smmu_domain_stage {
> +	ARM_SMMU_DOMAIN_S1 = 0,
> +	ARM_SMMU_DOMAIN_S2,
> +	ARM_SMMU_DOMAIN_NESTED,
> +	ARM_SMMU_DOMAIN_BYPASS,
> +};
> +
> +/* Xen specific code. */
> +struct iommu_domain {
> +	/* Runtime SMMU configuration for this iommu_domain */
> +	atomic_t		ref;
> +	/*
> +	 * Used to link iommu_domain contexts for a same domain.
> +	 * There is at least one per-SMMU to used by the domain.
> +	 */
> +	struct list_head		list;
> +};
> +
> +/* Describes information required for a Xen domain */
> +struct arm_smmu_xen_domain {
> +	spinlock_t		lock;
> +
> +	/* List of iommu domains associated to this domain */
> +	struct list_head	contexts;
> +};
> +
> +enum pri_resp {
> +	PRI_RESP_DENY = 0,
> +	PRI_RESP_FAIL = 1,
> +	PRI_RESP_SUCC = 2,
> +};
> +
> +#ifdef CONFIG_MSI
> +enum arm_smmu_msi_index {
> +	EVTQ_MSI_INDEX,
> +	GERROR_MSI_INDEX,
> +	PRIQ_MSI_INDEX,
> +	ARM_SMMU_MAX_MSIS,
> +};
> +
> +static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
> +	[EVTQ_MSI_INDEX] = {
> +		ARM_SMMU_EVTQ_IRQ_CFG0,
> +		ARM_SMMU_EVTQ_IRQ_CFG1,
> +		ARM_SMMU_EVTQ_IRQ_CFG2,
> +	},
> +	[GERROR_MSI_INDEX] = {
> +		ARM_SMMU_GERROR_IRQ_CFG0,
> +		ARM_SMMU_GERROR_IRQ_CFG1,
> +		ARM_SMMU_GERROR_IRQ_CFG2,
> +	},
> +	[PRIQ_MSI_INDEX] = {
> +		ARM_SMMU_PRIQ_IRQ_CFG0,
> +		ARM_SMMU_PRIQ_IRQ_CFG1,
> +		ARM_SMMU_PRIQ_IRQ_CFG2,
> +	},
> +};
> +#endif /* CONFIG_MSI */
> +
> +struct arm_smmu_cmdq_ent {
> +	/* Common fields */
> +	u8				opcode;
> +	bool				substream_valid;
> +
> +	/* Command-specific fields */
> +	union {
> +		#define CMDQ_OP_PREFETCH_CFG	0x1
> +		struct {
> +			u32			sid;
> +			u8			size;
> +			u64			addr;
> +		} prefetch;
> +
> +		#define CMDQ_OP_CFGI_STE	0x3
> +		#define CMDQ_OP_CFGI_ALL	0x4
> +		struct {
> +			u32			sid;
> +			union {
> +				bool		leaf;
> +				u8		span;
> +			};
> +		} cfgi;
> +
> +		#define CMDQ_OP_TLBI_EL2_ALL	0x20
> +		#define CMDQ_OP_TLBI_S12_VMALL	0x28
> +		#define CMDQ_OP_TLBI_S2_IPA	0x2a
> +		#define CMDQ_OP_TLBI_NSNH_ALL	0x30
> +		struct {
> +			u16			asid;
> +			u16			vmid;
> +			bool			leaf;
> +			u64			addr;
> +		} tlbi;
> +
> +		#define CMDQ_OP_ATC_INV		0x40
> +		#define ATC_INV_SIZE_ALL	52
> +		struct {
> +			u32			sid;
> +			u32			ssid;
> +			u64			addr;
> +			u8			size;
> +			bool			global;
> +		} atc;
> +
> +		#define CMDQ_OP_PRI_RESP	0x41
> +		struct {
> +			u32			sid;
> +			u32			ssid;
> +			u16			grpid;
> +			enum pri_resp		resp;
> +		} pri;
> +
> +		#define CMDQ_OP_CMD_SYNC	0x46
> +		struct {
> +			u32			msidata;
> +			u64			msiaddr;
> +		} sync;
> +	};
> +};
> +
> +struct arm_smmu_ll_queue {
> +	u32				prod;
> +	u32				cons;
> +	u32				max_n_shift;
> +};
> +
> +struct arm_smmu_queue {
> +	struct arm_smmu_ll_queue	llq;
> +	int				irq; /* Wired interrupt */
> +
> +	__le64				*base;
> +	dma_addr_t			base_dma;
> +	u64				q_base;
> +
> +	size_t				ent_dwords;
> +
> +	u32 __iomem			*prod_reg;
> +	u32 __iomem			*cons_reg;
> +};
> +
> +struct arm_smmu_cmdq {
> +	struct arm_smmu_queue		q;
> +	spinlock_t			lock;
> +};
> +
> +struct arm_smmu_evtq {
> +	struct arm_smmu_queue		q;
> +	u32				max_stalls;
> +};
> +
> +struct arm_smmu_priq {
> +	struct arm_smmu_queue		q;
> +};
> +
> +/* High-level stream table and context descriptor structures */
> +struct arm_smmu_strtab_l1_desc {
> +	u8				span;
> +
> +	__le64				*l2ptr;
> +	dma_addr_t			l2ptr_dma;
> +};
> +
> +struct arm_smmu_s2_cfg {
> +	u16				vmid;
> +	u64				vttbr;
> +	u64				vtcr;
> +};
> +
> +struct arm_smmu_strtab_cfg {
> +	__le64				*strtab;
> +	dma_addr_t			strtab_dma;
> +	struct arm_smmu_strtab_l1_desc	*l1_desc;
> +	unsigned int			num_l1_ents;
> +
> +	u64				strtab_base;
> +	u32				strtab_base_cfg;
> +};
> +
> +struct arm_lpae_s2_cfg {
> +	u64			vttbr;
> +	struct {
> +		u32			ps:3;
> +		u32			tg:2;
> +		u32			sh:2;
> +		u32			orgn:2;
> +		u32			irgn:2;
> +		u32			sl:2;
> +		u32			tsz:6;
> +	} vtcr;
> +};
> +
> +/* An SMMUv3 instance */
> +struct arm_smmu_device {
> +	struct device			*dev;
> +	void __iomem			*base;
> +	void __iomem			*page1;
> +
> +#define ARM_SMMU_FEAT_2_LVL_STRTAB	(1 << 0)
> +#define ARM_SMMU_FEAT_2_LVL_CDTAB	(1 << 1)
> +#define ARM_SMMU_FEAT_TT_LE		(1 << 2)
> +#define ARM_SMMU_FEAT_TT_BE		(1 << 3)
> +#define ARM_SMMU_FEAT_PRI		(1 << 4)
> +#define ARM_SMMU_FEAT_ATS		(1 << 5)
> +#define ARM_SMMU_FEAT_SEV		(1 << 6)
> +#define ARM_SMMU_FEAT_MSI		(1 << 7)
> +#define ARM_SMMU_FEAT_COHERENCY		(1 << 8)
> +#define ARM_SMMU_FEAT_TRANS_S1		(1 << 9)
> +#define ARM_SMMU_FEAT_TRANS_S2		(1 << 10)
> +#define ARM_SMMU_FEAT_STALLS		(1 << 11)
> +#define ARM_SMMU_FEAT_HYP		(1 << 12)
> +#define ARM_SMMU_FEAT_STALL_FORCE	(1 << 13)
> +#define ARM_SMMU_FEAT_VAX		(1 << 14)
> +	u32				features;
> +
> +#define ARM_SMMU_OPT_SKIP_PREFETCH	(1 << 0)
> +#define ARM_SMMU_OPT_PAGE0_REGS_ONLY	(1 << 1)
> +	u32				options;
> +
> +	struct arm_smmu_cmdq		cmdq;
> +	struct arm_smmu_evtq		evtq;
> +	struct arm_smmu_priq		priq;
> +
> +	int				gerr_irq;
> +	int				combined_irq;
> +	u32				sync_nr;
> +	u8				prev_cmd_opcode;
> +
> +	unsigned long			ias; /* IPA */
> +	unsigned long			oas; /* PA */
> +	unsigned long			pgsize_bitmap;
> +
> +#define ARM_SMMU_MAX_VMIDS		(1 << 16)
> +	unsigned int			vmid_bits;
> +	DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
> +
> +	unsigned int			sid_bits;
> +
> +	struct arm_smmu_strtab_cfg	strtab_cfg;
> +
> +	/* Hi16xx adds an extra 32 bits of goodness to its MSI payload */
> +	union {
> +		u32			sync_count;
> +		u64			padding;
> +	};
> +
> +	/* Need to keep a list of SMMU devices */
> +	struct list_head		devices;
> +
> +	/* Tasklets for handling evts/faults and pci page request IRQs*/
> +	struct tasklet		evtq_irq_tasklet;
> +	struct tasklet		priq_irq_tasklet;
> +	struct tasklet		combined_irq_tasklet;
> +};
> +
> +/* SMMU private data for each master */
> +struct arm_smmu_master {
> +	struct arm_smmu_device		*smmu;
> +	struct device			*dev;
> +	struct arm_smmu_domain		*domain;
> +	struct list_head		domain_head;
> +	u32				*sids;
> +	unsigned int			num_sids;
> +	bool				ats_enabled;
> +};
> +
> +struct arm_smmu_domain {
> +	struct arm_smmu_device		*smmu;
> +	struct mutex			init_mutex; /* Protects smmu pointer */
> +
> +	bool				non_strict;
> +	atomic_t			nr_ats_masters;
> +
> +	enum arm_smmu_domain_stage	stage;
> +	struct arm_smmu_s2_cfg	s2_cfg;
> +
> +	/* Xen domain associated with this SMMU domain */
> +	struct domain		*d;
> +
> +	struct iommu_domain		domain;
> +
> +	struct list_head		devices;
> +	spinlock_t			devices_lock;
> +};
> +
> +#endif /* _ARM_SMMU_V3_H */
> -- 
> 2.25.1
>

Hi Stefano,

> On 3 Sep 2022, at 12:29 am, Stefano Stabellini <sstabellini@kernel.org> wrote:
> 
> On Fri, 2 Sep 2022, Rahul Singh wrote:
>> From: Jean-Philippe Brucker <jean-philippe@linaro.org>
>> 
>> Backport Linux commit e881e7839fba. This is the clean backport without
>> any changes.
> 
> I don't think we can say that this is a clean backport because there are
> differences between the two patches.
> 
> That said, it is just code movement, it is similar to the original
> patch, and it still compiles. So I think we should change the commit
> message not to say that it is a clean backport, but other than that is
> fine.
> 

Ok. I will remove the clean backport msg from commit msg in next version.

Regards,
Rahul