From: Tomasz Jeznach <tjeznach@rivosinc.com>
Add PCIe Address Translation Services (ATS) capabilities to the IOMMU.
This will add support for ATS translation requests in Fault/Event
queues, Page-request queue and IOATC invalidations.
Signed-off-by: Tomasz Jeznach <tjeznach@rivosinc.com>
Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
---
hw/riscv/riscv-iommu-bits.h | 43 +++++++++++-
hw/riscv/riscv-iommu.c | 129 +++++++++++++++++++++++++++++++++++-
hw/riscv/riscv-iommu.h | 1 +
hw/riscv/trace-events | 3 +
4 files changed, 173 insertions(+), 3 deletions(-)
diff --git a/hw/riscv/riscv-iommu-bits.h b/hw/riscv/riscv-iommu-bits.h
index a4def7b8ec..e253b29b16 100644
--- a/hw/riscv/riscv-iommu-bits.h
+++ b/hw/riscv/riscv-iommu-bits.h
@@ -81,6 +81,7 @@ struct riscv_iommu_pq_record {
#define RISCV_IOMMU_CAP_SV57X4 BIT_ULL(19)
#define RISCV_IOMMU_CAP_MSI_FLAT BIT_ULL(22)
#define RISCV_IOMMU_CAP_MSI_MRIF BIT_ULL(23)
+#define RISCV_IOMMU_CAP_ATS BIT_ULL(25)
#define RISCV_IOMMU_CAP_T2GPA BIT_ULL(26)
#define RISCV_IOMMU_CAP_IGS GENMASK_ULL(29, 28)
#define RISCV_IOMMU_CAP_PAS GENMASK_ULL(37, 32)
@@ -209,6 +210,7 @@ struct riscv_iommu_dc {
/* Translation control fields */
#define RISCV_IOMMU_DC_TC_V BIT_ULL(0)
+#define RISCV_IOMMU_DC_TC_EN_ATS BIT_ULL(1)
#define RISCV_IOMMU_DC_TC_EN_PRI BIT_ULL(2)
#define RISCV_IOMMU_DC_TC_T2GPA BIT_ULL(3)
#define RISCV_IOMMU_DC_TC_DTF BIT_ULL(4)
@@ -270,6 +272,20 @@ struct riscv_iommu_command {
#define RISCV_IOMMU_CMD_IODIR_DV BIT_ULL(33)
#define RISCV_IOMMU_CMD_IODIR_DID GENMASK_ULL(63, 40)
+/* 3.1.4 I/O MMU PCIe ATS */
+#define RISCV_IOMMU_CMD_ATS_OPCODE 4
+#define RISCV_IOMMU_CMD_ATS_FUNC_INVAL 0
+#define RISCV_IOMMU_CMD_ATS_FUNC_PRGR 1
+#define RISCV_IOMMU_CMD_ATS_PID GENMASK_ULL(31, 12)
+#define RISCV_IOMMU_CMD_ATS_PV BIT_ULL(32)
+#define RISCV_IOMMU_CMD_ATS_DSV BIT_ULL(33)
+#define RISCV_IOMMU_CMD_ATS_RID GENMASK_ULL(55, 40)
+#define RISCV_IOMMU_CMD_ATS_DSEG GENMASK_ULL(63, 56)
+/* dword1 is the ATS payload, two different payload types for INVAL and PRGR */
+
+/* ATS.PRGR payload */
+#define RISCV_IOMMU_CMD_ATS_PRGR_RESP_CODE GENMASK_ULL(47, 44)
+
enum riscv_iommu_dc_fsc_atp_modes {
RISCV_IOMMU_DC_FSC_MODE_BARE = 0,
RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV32 = 8,
@@ -334,7 +350,32 @@ enum riscv_iommu_fq_ttypes {
RISCV_IOMMU_FQ_TTYPE_TADDR_INST_FETCH = 5,
RISCV_IOMMU_FQ_TTYPE_TADDR_RD = 6,
RISCV_IOMMU_FQ_TTYPE_TADDR_WR = 7,
- RISCV_IOMMU_FW_TTYPE_PCIE_MSG_REQ = 8,
+ RISCV_IOMMU_FQ_TTYPE_PCIE_ATS_REQ = 8,
+ RISCV_IOMMU_FW_TTYPE_PCIE_MSG_REQ = 9,
+};
+
+/* Header fields */
+#define RISCV_IOMMU_PREQ_HDR_PID GENMASK_ULL(31, 12)
+#define RISCV_IOMMU_PREQ_HDR_PV BIT_ULL(32)
+#define RISCV_IOMMU_PREQ_HDR_PRIV BIT_ULL(33)
+#define RISCV_IOMMU_PREQ_HDR_EXEC BIT_ULL(34)
+#define RISCV_IOMMU_PREQ_HDR_DID GENMASK_ULL(63, 40)
+
+/* Payload fields */
+#define RISCV_IOMMU_PREQ_PAYLOAD_R BIT_ULL(0)
+#define RISCV_IOMMU_PREQ_PAYLOAD_W BIT_ULL(1)
+#define RISCV_IOMMU_PREQ_PAYLOAD_L BIT_ULL(2)
+#define RISCV_IOMMU_PREQ_PAYLOAD_M GENMASK_ULL(2, 0)
+#define RISCV_IOMMU_PREQ_PRG_INDEX GENMASK_ULL(11, 3)
+#define RISCV_IOMMU_PREQ_UADDR GENMASK_ULL(63, 12)
+
+
+/*
+ * struct riscv_iommu_msi_pte - MSI Page Table Entry
+ */
+struct riscv_iommu_msi_pte {
+ uint64_t pte;
+ uint64_t mrif_info;
};
/* Fields on pte */
diff --git a/hw/riscv/riscv-iommu.c b/hw/riscv/riscv-iommu.c
index 11c418b548..3516b82081 100644
--- a/hw/riscv/riscv-iommu.c
+++ b/hw/riscv/riscv-iommu.c
@@ -641,6 +641,20 @@ static bool riscv_iommu_validate_device_ctx(RISCVIOMMUState *s,
RISCVIOMMUContext *ctx)
{
uint32_t fsc_mode, msi_mode;
+ uint64_t gatp;
+
+ if (!(s->cap & RISCV_IOMMU_CAP_ATS) &&
+ (ctx->tc & RISCV_IOMMU_DC_TC_EN_ATS ||
+ ctx->tc & RISCV_IOMMU_DC_TC_EN_PRI ||
+ ctx->tc & RISCV_IOMMU_DC_TC_PRPR)) {
+ return false;
+ }
+
+ if (!(ctx->tc & RISCV_IOMMU_DC_TC_EN_ATS) &&
+ (ctx->tc & RISCV_IOMMU_DC_TC_T2GPA ||
+ ctx->tc & RISCV_IOMMU_DC_TC_EN_PRI)) {
+ return false;
+ }
if (!(ctx->tc & RISCV_IOMMU_DC_TC_EN_PRI) &&
ctx->tc & RISCV_IOMMU_DC_TC_PRPR) {
@@ -661,6 +675,12 @@ static bool riscv_iommu_validate_device_ctx(RISCVIOMMUState *s,
}
}
+ gatp = get_field(ctx->gatp, RISCV_IOMMU_ATP_MODE_FIELD);
+ if (ctx->tc & RISCV_IOMMU_DC_TC_T2GPA &&
+ gatp == RISCV_IOMMU_DC_IOHGATP_MODE_BARE) {
+ return false;
+ }
+
fsc_mode = get_field(ctx->satp, RISCV_IOMMU_DC_FSC_MODE);
if (ctx->tc & RISCV_IOMMU_DC_TC_PDTV) {
@@ -754,7 +774,12 @@ static int riscv_iommu_ctx_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx)
RISCV_IOMMU_DC_IOHGATP_MODE_BARE);
ctx->satp = set_field(0, RISCV_IOMMU_ATP_MODE_FIELD,
RISCV_IOMMU_DC_FSC_MODE_BARE);
+
ctx->tc = RISCV_IOMMU_DC_TC_V;
+ if (s->enable_ats) {
+ ctx->tc |= RISCV_IOMMU_DC_TC_EN_ATS;
+ }
+
ctx->ta = 0;
ctx->msiptp = 0;
return 0;
@@ -1191,6 +1216,16 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,
enable_pri = (iotlb->perm == IOMMU_NONE) && (ctx->tc & BIT_ULL(32));
enable_pasid = (ctx->tc & RISCV_IOMMU_DC_TC_PDTV);
+ /* Check for ATS request. */
+ if (iotlb->perm == IOMMU_NONE) {
+ /* Check if ATS is disabled. */
+ if (!(ctx->tc & RISCV_IOMMU_DC_TC_EN_ATS)) {
+ enable_pri = false;
+ fault = RISCV_IOMMU_FQ_CAUSE_TTYPE_BLOCKED;
+ goto done;
+ }
+ }
+
iot = riscv_iommu_iot_lookup(ctx, iot_cache, iotlb->iova);
perm = iot ? iot->perm : IOMMU_NONE;
if (perm != IOMMU_NONE) {
@@ -1236,11 +1271,11 @@ done:
}
if (fault) {
- unsigned ttype;
+ unsigned ttype = RISCV_IOMMU_FQ_TTYPE_PCIE_ATS_REQ;
if (iotlb->perm & IOMMU_RW) {
ttype = RISCV_IOMMU_FQ_TTYPE_UADDR_WR;
- } else {
+ } else if (iotlb->perm & IOMMU_RO) {
ttype = RISCV_IOMMU_FQ_TTYPE_UADDR_RD;
}
@@ -1268,6 +1303,73 @@ static MemTxResult riscv_iommu_iofence(RISCVIOMMUState *s, bool notify,
MEMTXATTRS_UNSPECIFIED);
}
+static void riscv_iommu_ats(RISCVIOMMUState *s,
+ struct riscv_iommu_command *cmd, IOMMUNotifierFlag flag,
+ IOMMUAccessFlags perm,
+ void (*trace_fn)(const char *id))
+{
+ RISCVIOMMUSpace *as = NULL;
+ IOMMUNotifier *n;
+ IOMMUTLBEvent event;
+ uint32_t pasid;
+ uint32_t devid;
+ const bool pv = cmd->dword0 & RISCV_IOMMU_CMD_ATS_PV;
+
+ if (cmd->dword0 & RISCV_IOMMU_CMD_ATS_DSV) {
+ /* Use device segment and requester id */
+ devid = get_field(cmd->dword0,
+ RISCV_IOMMU_CMD_ATS_DSEG | RISCV_IOMMU_CMD_ATS_RID);
+ } else {
+ devid = get_field(cmd->dword0, RISCV_IOMMU_CMD_ATS_RID);
+ }
+
+ pasid = get_field(cmd->dword0, RISCV_IOMMU_CMD_ATS_PID);
+
+ qemu_mutex_lock(&s->core_lock);
+ QLIST_FOREACH(as, &s->spaces, list) {
+ if (as->devid == devid) {
+ break;
+ }
+ }
+ qemu_mutex_unlock(&s->core_lock);
+
+ if (!as || !as->notifier) {
+ return;
+ }
+
+ event.type = flag;
+ event.entry.perm = perm;
+ event.entry.target_as = s->target_as;
+
+ IOMMU_NOTIFIER_FOREACH(n, &as->iova_mr) {
+ if (!pv || n->iommu_idx == pasid) {
+ event.entry.iova = n->start;
+ event.entry.addr_mask = n->end - n->start;
+ trace_fn(as->iova_mr.parent_obj.name);
+ memory_region_notify_iommu_one(n, &event);
+ }
+ }
+}
+
+static void riscv_iommu_ats_inval(RISCVIOMMUState *s,
+ struct riscv_iommu_command *cmd)
+{
+ return riscv_iommu_ats(s, cmd, IOMMU_NOTIFIER_DEVIOTLB_UNMAP, IOMMU_NONE,
+ trace_riscv_iommu_ats_inval);
+}
+
+static void riscv_iommu_ats_prgr(RISCVIOMMUState *s,
+ struct riscv_iommu_command *cmd)
+{
+ unsigned resp_code = get_field(cmd->dword1,
+ RISCV_IOMMU_CMD_ATS_PRGR_RESP_CODE);
+
+ /* Using the access flag to carry response code information */
+ IOMMUAccessFlags perm = resp_code ? IOMMU_NONE : IOMMU_RW;
+ return riscv_iommu_ats(s, cmd, IOMMU_NOTIFIER_MAP, perm,
+ trace_riscv_iommu_ats_prgr);
+}
+
static void riscv_iommu_process_ddtp(RISCVIOMMUState *s)
{
uint64_t old_ddtp = s->ddtp;
@@ -1423,6 +1525,25 @@ static void riscv_iommu_process_cq_tail(RISCVIOMMUState *s)
get_field(cmd.dword0, RISCV_IOMMU_CMD_IODIR_PID));
break;
+ /* ATS commands */
+ case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_ATS_FUNC_INVAL,
+ RISCV_IOMMU_CMD_ATS_OPCODE):
+ if (!s->enable_ats) {
+ goto cmd_ill;
+ }
+
+ riscv_iommu_ats_inval(s, &cmd);
+ break;
+
+ case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_ATS_FUNC_PRGR,
+ RISCV_IOMMU_CMD_ATS_OPCODE):
+ if (!s->enable_ats) {
+ goto cmd_ill;
+ }
+
+ riscv_iommu_ats_prgr(s, &cmd);
+ break;
+
default:
cmd_ill:
/* Invalid instruction, do not advance instruction index. */
@@ -1818,6 +1939,9 @@ static void riscv_iommu_realize(DeviceState *dev, Error **errp)
if (s->enable_msi) {
s->cap |= RISCV_IOMMU_CAP_MSI_FLAT | RISCV_IOMMU_CAP_MSI_MRIF;
}
+ if (s->enable_ats) {
+ s->cap |= RISCV_IOMMU_CAP_ATS;
+ }
if (s->enable_s_stage) {
s->cap |= RISCV_IOMMU_CAP_SV32 | RISCV_IOMMU_CAP_SV39 |
RISCV_IOMMU_CAP_SV48 | RISCV_IOMMU_CAP_SV57;
@@ -1925,6 +2049,7 @@ static Property riscv_iommu_properties[] = {
DEFINE_PROP_UINT32("ioatc-limit", RISCVIOMMUState, iot_limit,
LIMIT_CACHE_IOT),
DEFINE_PROP_BOOL("intremap", RISCVIOMMUState, enable_msi, TRUE),
+ DEFINE_PROP_BOOL("ats", RISCVIOMMUState, enable_ats, TRUE),
DEFINE_PROP_BOOL("off", RISCVIOMMUState, enable_off, TRUE),
DEFINE_PROP_BOOL("s-stage", RISCVIOMMUState, enable_s_stage, TRUE),
DEFINE_PROP_BOOL("g-stage", RISCVIOMMUState, enable_g_stage, TRUE),
diff --git a/hw/riscv/riscv-iommu.h b/hw/riscv/riscv-iommu.h
index c24e3e4c16..26236c3cee 100644
--- a/hw/riscv/riscv-iommu.h
+++ b/hw/riscv/riscv-iommu.h
@@ -38,6 +38,7 @@ struct RISCVIOMMUState {
bool enable_off; /* Enable out-of-reset OFF mode (DMA disabled) */
bool enable_msi; /* Enable MSI remapping */
+ bool enable_ats; /* Enable ATS support */
bool enable_s_stage; /* Enable S/VS-Stage translation */
bool enable_g_stage; /* Enable G-Stage translation */
diff --git a/hw/riscv/trace-events b/hw/riscv/trace-events
index 42a97caffa..4b486b6420 100644
--- a/hw/riscv/trace-events
+++ b/hw/riscv/trace-events
@@ -9,3 +9,6 @@ riscv_iommu_msi(const char *id, unsigned b, unsigned d, unsigned f, uint64_t iov
riscv_iommu_cmd(const char *id, uint64_t l, uint64_t u) "%s: command 0x%"PRIx64" 0x%"PRIx64
riscv_iommu_notifier_add(const char *id) "%s: dev-iotlb notifier added"
riscv_iommu_notifier_del(const char *id) "%s: dev-iotlb notifier removed"
+riscv_iommu_ats(const char *id, unsigned b, unsigned d, unsigned f, uint64_t iova) "%s: translate request %04x:%02x.%u iova: 0x%"PRIx64
+riscv_iommu_ats_inval(const char *id) "%s: dev-iotlb invalidate"
+riscv_iommu_ats_prgr(const char *id) "%s: dev-iotlb page request group response"
--
2.44.0Reviewed-by: Frank Chang <frank.chang@sifive.com>
Daniel Henrique Barboza <dbarboza@ventanamicro.com> 於 2024年5月24日 週五 上午1:41寫道：
>
> From: Tomasz Jeznach <tjeznach@rivosinc.com>
>
> Add PCIe Address Translation Services (ATS) capabilities to the IOMMU.
> This will add support for ATS translation requests in Fault/Event
> queues, Page-request queue and IOATC invalidations.
>
> Signed-off-by: Tomasz Jeznach <tjeznach@rivosinc.com>
> Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
> ---
> hw/riscv/riscv-iommu-bits.h | 43 +++++++++++-
> hw/riscv/riscv-iommu.c | 129 +++++++++++++++++++++++++++++++++++-
> hw/riscv/riscv-iommu.h | 1 +
> hw/riscv/trace-events | 3 +
> 4 files changed, 173 insertions(+), 3 deletions(-)
>
> diff --git a/hw/riscv/riscv-iommu-bits.h b/hw/riscv/riscv-iommu-bits.h
> index a4def7b8ec..e253b29b16 100644
> --- a/hw/riscv/riscv-iommu-bits.h
> +++ b/hw/riscv/riscv-iommu-bits.h
> @@ -81,6 +81,7 @@ struct riscv_iommu_pq_record {
> #define RISCV_IOMMU_CAP_SV57X4 BIT_ULL(19)
> #define RISCV_IOMMU_CAP_MSI_FLAT BIT_ULL(22)
> #define RISCV_IOMMU_CAP_MSI_MRIF BIT_ULL(23)
> +#define RISCV_IOMMU_CAP_ATS BIT_ULL(25)
> #define RISCV_IOMMU_CAP_T2GPA BIT_ULL(26)
> #define RISCV_IOMMU_CAP_IGS GENMASK_ULL(29, 28)
> #define RISCV_IOMMU_CAP_PAS GENMASK_ULL(37, 32)
> @@ -209,6 +210,7 @@ struct riscv_iommu_dc {
>
> /* Translation control fields */
> #define RISCV_IOMMU_DC_TC_V BIT_ULL(0)
> +#define RISCV_IOMMU_DC_TC_EN_ATS BIT_ULL(1)
> #define RISCV_IOMMU_DC_TC_EN_PRI BIT_ULL(2)
> #define RISCV_IOMMU_DC_TC_T2GPA BIT_ULL(3)
> #define RISCV_IOMMU_DC_TC_DTF BIT_ULL(4)
> @@ -270,6 +272,20 @@ struct riscv_iommu_command {
> #define RISCV_IOMMU_CMD_IODIR_DV BIT_ULL(33)
> #define RISCV_IOMMU_CMD_IODIR_DID GENMASK_ULL(63, 40)
>
> +/* 3.1.4 I/O MMU PCIe ATS */
> +#define RISCV_IOMMU_CMD_ATS_OPCODE 4
> +#define RISCV_IOMMU_CMD_ATS_FUNC_INVAL 0
> +#define RISCV_IOMMU_CMD_ATS_FUNC_PRGR 1
> +#define RISCV_IOMMU_CMD_ATS_PID GENMASK_ULL(31, 12)
> +#define RISCV_IOMMU_CMD_ATS_PV BIT_ULL(32)
> +#define RISCV_IOMMU_CMD_ATS_DSV BIT_ULL(33)
> +#define RISCV_IOMMU_CMD_ATS_RID GENMASK_ULL(55, 40)
> +#define RISCV_IOMMU_CMD_ATS_DSEG GENMASK_ULL(63, 56)
> +/* dword1 is the ATS payload, two different payload types for INVAL and PRGR */
> +
> +/* ATS.PRGR payload */
> +#define RISCV_IOMMU_CMD_ATS_PRGR_RESP_CODE GENMASK_ULL(47, 44)
> +
> enum riscv_iommu_dc_fsc_atp_modes {
> RISCV_IOMMU_DC_FSC_MODE_BARE = 0,
> RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV32 = 8,
> @@ -334,7 +350,32 @@ enum riscv_iommu_fq_ttypes {
> RISCV_IOMMU_FQ_TTYPE_TADDR_INST_FETCH = 5,
> RISCV_IOMMU_FQ_TTYPE_TADDR_RD = 6,
> RISCV_IOMMU_FQ_TTYPE_TADDR_WR = 7,
> - RISCV_IOMMU_FW_TTYPE_PCIE_MSG_REQ = 8,
> + RISCV_IOMMU_FQ_TTYPE_PCIE_ATS_REQ = 8,
> + RISCV_IOMMU_FW_TTYPE_PCIE_MSG_REQ = 9,
> +};
> +
> +/* Header fields */
> +#define RISCV_IOMMU_PREQ_HDR_PID GENMASK_ULL(31, 12)
> +#define RISCV_IOMMU_PREQ_HDR_PV BIT_ULL(32)
> +#define RISCV_IOMMU_PREQ_HDR_PRIV BIT_ULL(33)
> +#define RISCV_IOMMU_PREQ_HDR_EXEC BIT_ULL(34)
> +#define RISCV_IOMMU_PREQ_HDR_DID GENMASK_ULL(63, 40)
> +
> +/* Payload fields */
> +#define RISCV_IOMMU_PREQ_PAYLOAD_R BIT_ULL(0)
> +#define RISCV_IOMMU_PREQ_PAYLOAD_W BIT_ULL(1)
> +#define RISCV_IOMMU_PREQ_PAYLOAD_L BIT_ULL(2)
> +#define RISCV_IOMMU_PREQ_PAYLOAD_M GENMASK_ULL(2, 0)
> +#define RISCV_IOMMU_PREQ_PRG_INDEX GENMASK_ULL(11, 3)
> +#define RISCV_IOMMU_PREQ_UADDR GENMASK_ULL(63, 12)
> +
> +
> +/*
> + * struct riscv_iommu_msi_pte - MSI Page Table Entry
> + */
> +struct riscv_iommu_msi_pte {
> + uint64_t pte;
> + uint64_t mrif_info;
> };
>
> /* Fields on pte */
> diff --git a/hw/riscv/riscv-iommu.c b/hw/riscv/riscv-iommu.c
> index 11c418b548..3516b82081 100644
> --- a/hw/riscv/riscv-iommu.c
> +++ b/hw/riscv/riscv-iommu.c
> @@ -641,6 +641,20 @@ static bool riscv_iommu_validate_device_ctx(RISCVIOMMUState *s,
> RISCVIOMMUContext *ctx)
> {
> uint32_t fsc_mode, msi_mode;
> + uint64_t gatp;
> +
> + if (!(s->cap & RISCV_IOMMU_CAP_ATS) &&
> + (ctx->tc & RISCV_IOMMU_DC_TC_EN_ATS ||
> + ctx->tc & RISCV_IOMMU_DC_TC_EN_PRI ||
> + ctx->tc & RISCV_IOMMU_DC_TC_PRPR)) {
> + return false;
> + }
> +
> + if (!(ctx->tc & RISCV_IOMMU_DC_TC_EN_ATS) &&
> + (ctx->tc & RISCV_IOMMU_DC_TC_T2GPA ||
> + ctx->tc & RISCV_IOMMU_DC_TC_EN_PRI)) {
> + return false;
> + }
>
> if (!(ctx->tc & RISCV_IOMMU_DC_TC_EN_PRI) &&
> ctx->tc & RISCV_IOMMU_DC_TC_PRPR) {
> @@ -661,6 +675,12 @@ static bool riscv_iommu_validate_device_ctx(RISCVIOMMUState *s,
> }
> }
>
> + gatp = get_field(ctx->gatp, RISCV_IOMMU_ATP_MODE_FIELD);
> + if (ctx->tc & RISCV_IOMMU_DC_TC_T2GPA &&
> + gatp == RISCV_IOMMU_DC_IOHGATP_MODE_BARE) {
> + return false;
> + }
> +
> fsc_mode = get_field(ctx->satp, RISCV_IOMMU_DC_FSC_MODE);
>
> if (ctx->tc & RISCV_IOMMU_DC_TC_PDTV) {
> @@ -754,7 +774,12 @@ static int riscv_iommu_ctx_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx)
> RISCV_IOMMU_DC_IOHGATP_MODE_BARE);
> ctx->satp = set_field(0, RISCV_IOMMU_ATP_MODE_FIELD,
> RISCV_IOMMU_DC_FSC_MODE_BARE);
> +
> ctx->tc = RISCV_IOMMU_DC_TC_V;
> + if (s->enable_ats) {
> + ctx->tc |= RISCV_IOMMU_DC_TC_EN_ATS;
> + }
> +
> ctx->ta = 0;
> ctx->msiptp = 0;
> return 0;
> @@ -1191,6 +1216,16 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,
> enable_pri = (iotlb->perm == IOMMU_NONE) && (ctx->tc & BIT_ULL(32));
> enable_pasid = (ctx->tc & RISCV_IOMMU_DC_TC_PDTV);
>
> + /* Check for ATS request. */
> + if (iotlb->perm == IOMMU_NONE) {
> + /* Check if ATS is disabled. */
> + if (!(ctx->tc & RISCV_IOMMU_DC_TC_EN_ATS)) {
> + enable_pri = false;
> + fault = RISCV_IOMMU_FQ_CAUSE_TTYPE_BLOCKED;
> + goto done;
> + }
> + }
> +
> iot = riscv_iommu_iot_lookup(ctx, iot_cache, iotlb->iova);
> perm = iot ? iot->perm : IOMMU_NONE;
> if (perm != IOMMU_NONE) {
> @@ -1236,11 +1271,11 @@ done:
> }
>
> if (fault) {
> - unsigned ttype;
> + unsigned ttype = RISCV_IOMMU_FQ_TTYPE_PCIE_ATS_REQ;
>
> if (iotlb->perm & IOMMU_RW) {
> ttype = RISCV_IOMMU_FQ_TTYPE_UADDR_WR;
> - } else {
> + } else if (iotlb->perm & IOMMU_RO) {
> ttype = RISCV_IOMMU_FQ_TTYPE_UADDR_RD;
> }
>
> @@ -1268,6 +1303,73 @@ static MemTxResult riscv_iommu_iofence(RISCVIOMMUState *s, bool notify,
> MEMTXATTRS_UNSPECIFIED);
> }
>
> +static void riscv_iommu_ats(RISCVIOMMUState *s,
> + struct riscv_iommu_command *cmd, IOMMUNotifierFlag flag,
> + IOMMUAccessFlags perm,
> + void (*trace_fn)(const char *id))
> +{
> + RISCVIOMMUSpace *as = NULL;
> + IOMMUNotifier *n;
> + IOMMUTLBEvent event;
> + uint32_t pasid;
> + uint32_t devid;
> + const bool pv = cmd->dword0 & RISCV_IOMMU_CMD_ATS_PV;
> +
> + if (cmd->dword0 & RISCV_IOMMU_CMD_ATS_DSV) {
> + /* Use device segment and requester id */
> + devid = get_field(cmd->dword0,
> + RISCV_IOMMU_CMD_ATS_DSEG | RISCV_IOMMU_CMD_ATS_RID);
> + } else {
> + devid = get_field(cmd->dword0, RISCV_IOMMU_CMD_ATS_RID);
> + }
> +
> + pasid = get_field(cmd->dword0, RISCV_IOMMU_CMD_ATS_PID);
> +
> + qemu_mutex_lock(&s->core_lock);
> + QLIST_FOREACH(as, &s->spaces, list) {
> + if (as->devid == devid) {
> + break;
> + }
> + }
> + qemu_mutex_unlock(&s->core_lock);
> +
> + if (!as || !as->notifier) {
> + return;
> + }
> +
> + event.type = flag;
> + event.entry.perm = perm;
> + event.entry.target_as = s->target_as;
> +
> + IOMMU_NOTIFIER_FOREACH(n, &as->iova_mr) {
> + if (!pv || n->iommu_idx == pasid) {
> + event.entry.iova = n->start;
> + event.entry.addr_mask = n->end - n->start;
> + trace_fn(as->iova_mr.parent_obj.name);
> + memory_region_notify_iommu_one(n, &event);
> + }
> + }
> +}
> +
> +static void riscv_iommu_ats_inval(RISCVIOMMUState *s,
> + struct riscv_iommu_command *cmd)
> +{
> + return riscv_iommu_ats(s, cmd, IOMMU_NOTIFIER_DEVIOTLB_UNMAP, IOMMU_NONE,
> + trace_riscv_iommu_ats_inval);
> +}
> +
> +static void riscv_iommu_ats_prgr(RISCVIOMMUState *s,
> + struct riscv_iommu_command *cmd)
> +{
> + unsigned resp_code = get_field(cmd->dword1,
> + RISCV_IOMMU_CMD_ATS_PRGR_RESP_CODE);
> +
> + /* Using the access flag to carry response code information */
> + IOMMUAccessFlags perm = resp_code ? IOMMU_NONE : IOMMU_RW;
> + return riscv_iommu_ats(s, cmd, IOMMU_NOTIFIER_MAP, perm,
> + trace_riscv_iommu_ats_prgr);
> +}
> +
> static void riscv_iommu_process_ddtp(RISCVIOMMUState *s)
> {
> uint64_t old_ddtp = s->ddtp;
> @@ -1423,6 +1525,25 @@ static void riscv_iommu_process_cq_tail(RISCVIOMMUState *s)
> get_field(cmd.dword0, RISCV_IOMMU_CMD_IODIR_PID));
> break;
>
> + /* ATS commands */
> + case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_ATS_FUNC_INVAL,
> + RISCV_IOMMU_CMD_ATS_OPCODE):
> + if (!s->enable_ats) {
> + goto cmd_ill;
> + }
> +
> + riscv_iommu_ats_inval(s, &cmd);
> + break;
> +
> + case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_ATS_FUNC_PRGR,
> + RISCV_IOMMU_CMD_ATS_OPCODE):
> + if (!s->enable_ats) {
> + goto cmd_ill;
> + }
> +
> + riscv_iommu_ats_prgr(s, &cmd);
> + break;
> +
> default:
> cmd_ill:
> /* Invalid instruction, do not advance instruction index. */
> @@ -1818,6 +1939,9 @@ static void riscv_iommu_realize(DeviceState *dev, Error **errp)
> if (s->enable_msi) {
> s->cap |= RISCV_IOMMU_CAP_MSI_FLAT | RISCV_IOMMU_CAP_MSI_MRIF;
> }
> + if (s->enable_ats) {
> + s->cap |= RISCV_IOMMU_CAP_ATS;
> + }
> if (s->enable_s_stage) {
> s->cap |= RISCV_IOMMU_CAP_SV32 | RISCV_IOMMU_CAP_SV39 |
> RISCV_IOMMU_CAP_SV48 | RISCV_IOMMU_CAP_SV57;
> @@ -1925,6 +2049,7 @@ static Property riscv_iommu_properties[] = {
> DEFINE_PROP_UINT32("ioatc-limit", RISCVIOMMUState, iot_limit,
> LIMIT_CACHE_IOT),
> DEFINE_PROP_BOOL("intremap", RISCVIOMMUState, enable_msi, TRUE),
> + DEFINE_PROP_BOOL("ats", RISCVIOMMUState, enable_ats, TRUE),
> DEFINE_PROP_BOOL("off", RISCVIOMMUState, enable_off, TRUE),
> DEFINE_PROP_BOOL("s-stage", RISCVIOMMUState, enable_s_stage, TRUE),
> DEFINE_PROP_BOOL("g-stage", RISCVIOMMUState, enable_g_stage, TRUE),
> diff --git a/hw/riscv/riscv-iommu.h b/hw/riscv/riscv-iommu.h
> index c24e3e4c16..26236c3cee 100644
> --- a/hw/riscv/riscv-iommu.h
> +++ b/hw/riscv/riscv-iommu.h
> @@ -38,6 +38,7 @@ struct RISCVIOMMUState {
>
> bool enable_off; /* Enable out-of-reset OFF mode (DMA disabled) */
> bool enable_msi; /* Enable MSI remapping */
> + bool enable_ats; /* Enable ATS support */
> bool enable_s_stage; /* Enable S/VS-Stage translation */
> bool enable_g_stage; /* Enable G-Stage translation */
>
> diff --git a/hw/riscv/trace-events b/hw/riscv/trace-events
> index 42a97caffa..4b486b6420 100644
> --- a/hw/riscv/trace-events
> +++ b/hw/riscv/trace-events
> @@ -9,3 +9,6 @@ riscv_iommu_msi(const char *id, unsigned b, unsigned d, unsigned f, uint64_t iov
> riscv_iommu_cmd(const char *id, uint64_t l, uint64_t u) "%s: command 0x%"PRIx64" 0x%"PRIx64
> riscv_iommu_notifier_add(const char *id) "%s: dev-iotlb notifier added"
> riscv_iommu_notifier_del(const char *id) "%s: dev-iotlb notifier removed"
> +riscv_iommu_ats(const char *id, unsigned b, unsigned d, unsigned f, uint64_t iova) "%s: translate request %04x:%02x.%u iova: 0x%"PRIx64
> +riscv_iommu_ats_inval(const char *id) "%s: dev-iotlb invalidate"
> +riscv_iommu_ats_prgr(const char *id) "%s: dev-iotlb page request group response"
> --
> 2.44.0
>
>
© 2016 - 2024 Red Hat, Inc.