From: Frank Chang <frank.chang@sifive.com> Support the true Zicclsm extension so that we can trap misaligned accesses when Zicclsm is disabled. To enable/disable Zicclsm, simply set zicclsm=[true|false], e.g.: -cpu rv64,zicclsm=[true|false] QEMU will raise a misaligned load/store exception when executing misaligned load/store instructions if Zicclsm is disabled. Changelog: v5: * Add Zicclsm to RVA22U64 profile explicitly. * Rebase to the latest riscv-to-apply.next. v4: * Align ROM reset vector data at 8-byte aligned offsets. v3: * Enable Zicclsm for the compatible CPUs. * Rebase to the latest riscv-to-apply.next. v2: * Use (size_memop(size) | mo_endian_env(env)) to calculate MemOp. * Use (log2_esz << MO_ASHIFT) to calculate aligment MemOp for vector load/store whole register instructions. Frank Chang (7): target/riscv: Add Zicclsm CPU option target/riscv: Support raising misaligned exceptions for scalar loads/stores target/riscv: Support raising misaligned exceptions for vector loads/stores target/riscv: Enable Zicclsm for the compatible CPUs hw/riscv: sifive_u: Align ROM reset vector data target/riscv: Update Zicclsm ISA string and expose it as a CPU property target/riscv: Enable Zicclsm for RVA22U64 profile hw/riscv/sifive_u.c | 11 ++-- target/riscv/cpu.c | 10 +++- target/riscv/cpu_cfg_fields.h.inc | 2 + target/riscv/tcg/insn_trans/trans_rvi.c.inc | 6 ++ target/riscv/tcg/insn_trans/trans_rvv.c.inc | 18 +++++- target/riscv/tcg/vector_helper.c | 65 ++++++++++++++++----- 6 files changed, 88 insertions(+), 24 deletions(-) -- 2.43.0 From: Frank Chang <frank.chang@sifive.com> Add Zicclsm CPU option so that user can turn on/off misaligned loads and stores support. Signed-off-by: Frank Chang <frank.chang@sifive.com> Reviewed-by: Max Chou <max.chou@sifive.com> Reviewed-by: Alistair Francis <alistair.francis@wdc.com> --- target/riscv/cpu_cfg_fields.h.inc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/target/riscv/cpu_cfg_fields.h.inc b/target/riscv/cpu_cfg_fields.h.inc index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/cpu_cfg_fields.h.inc +++ b/target/riscv/cpu_cfg_fields.h.inc @@ -XXX,XX +XXX,XX @@ BOOL_FIELD(has_priv_1_11) /* Always enabled for TCG if has_priv_1_11 */ BOOL_FIELD(ext_ziccrse) +BOOL_FIELD(ext_zicclsm) + /* Vendor-specific custom extensions */ BOOL_FIELD(ext_xtheadba) BOOL_FIELD(ext_xtheadbb) -- 2.43.0 From: Frank Chang <frank.chang@sifive.com> When the Zicclsm extension is not enabled, raise misaligned load/store exceptions for misaligned accesses from scalar load/store instructions. Signed-off-by: Frank Chang <frank.chang@sifive.com> Reviewed-by: Max Chou <max.chou@sifive.com> Reviewed-by: Alistair Francis <alistair.francis@wdc.com> --- target/riscv/tcg/insn_trans/trans_rvi.c.inc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/target/riscv/tcg/insn_trans/trans_rvi.c.inc b/target/riscv/tcg/insn_trans/trans_rvi.c.inc index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/tcg/insn_trans/trans_rvi.c.inc +++ b/target/riscv/tcg/insn_trans/trans_rvi.c.inc @@ -XXX,XX +XXX,XX @@ static bool gen_load(DisasContext *ctx, arg_lb *a, MemOp memop) if (ctx->cfg_ptr->ext_zama16b) { memop |= MO_ATOM_WITHIN16; } + if (!ctx->cfg_ptr->ext_zicclsm) { + memop |= MO_ALIGN; + } decode_save_opc(ctx, 0); if (get_xl(ctx) == MXL_RV128) { out = gen_load_i128(ctx, a, memop); @@ -XXX,XX +XXX,XX @@ static bool gen_store(DisasContext *ctx, arg_sb *a, MemOp memop) if (ctx->cfg_ptr->ext_zama16b) { memop |= MO_ATOM_WITHIN16; } + if (!ctx->cfg_ptr->ext_zicclsm) { + memop |= MO_ALIGN; + } decode_save_opc(ctx, 0); if (get_xl(ctx) == MXL_RV128) { return gen_store_i128(ctx, a, memop); -- 2.43.0 From: Frank Chang <frank.chang@sifive.com> When the Zicclsm extension is not enabled, raise misaligned load/store exceptions for misaligned accesses from vector load/store instructions. We will skip the host fast-path and fall back to the slow TLB-path to raise misaligned load/store exceptions for the misaligned accesses when Zicclsm extension is disabled. Signed-off-by: Frank Chang <frank.chang@sifive.com> Reviewed-by: Max Chou <max.chou@sifive.com> Acked-by: Alistair Francis <alistair.francis@wdc.com> --- target/riscv/tcg/insn_trans/trans_rvv.c.inc | 18 +++++- target/riscv/tcg/vector_helper.c | 65 ++++++++++++++++----- 2 files changed, 65 insertions(+), 18 deletions(-) diff --git a/target/riscv/tcg/insn_trans/trans_rvv.c.inc b/target/riscv/tcg/insn_trans/trans_rvv.c.inc index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/tcg/insn_trans/trans_rvv.c.inc +++ b/target/riscv/tcg/insn_trans/trans_rvv.c.inc @@ -XXX,XX +XXX,XX @@ static bool ldst_whole_trans(uint32_t vd, uint32_t rs1, uint32_t nf, * Use the helper function if either: * - vstart is not 0. */ - bool use_helper_fn = !s->vstart_eq_zero; if (!use_helper_fn) { uint32_t size = s->cfg_ptr->vlenb * nf; TCGv_i64 t8 = tcg_temp_new_i64(); MemOp atomicity = MO_ATOM_NONE; + MemOp alignment = MO_UNALN; + + /* + * If Zicclsm is disabled, require alignment based on element size. + * Use MO_ALIGN_* based on log2_esz (0 = MO_UNALN, 1 = MO_ALIGN_2, etc). + */ + if (!s->cfg_ptr->ext_zicclsm) { + alignment = log2_esz << MO_ASHIFT; + } + if (log2_esz == 0) { atomicity = MO_ATOM_NONE; } else { atomicity = MO_ATOM_IFALIGN_PAIR; } + for (int i = 0; i < size; i += 8) { TCGv addr = get_address(s, rs1, i); if (is_load) { - tcg_gen_qemu_ld_i64(t8, addr, s->mem_idx, MO_LEUQ | atomicity); + tcg_gen_qemu_ld_i64(t8, addr, s->mem_idx, + MO_LEUQ | atomicity | alignment); tcg_gen_st_i64(t8, tcg_env, vreg_ofs(s, vd) + i); } else { tcg_gen_ld_i64(t8, tcg_env, vreg_ofs(s, vd) + i); - tcg_gen_qemu_st_i64(t8, addr, s->mem_idx, MO_LEUQ | atomicity); + tcg_gen_qemu_st_i64(t8, addr, s->mem_idx, + MO_LEUQ | atomicity | alignment); } if (i == size - 8) { tcg_gen_movi_i32(cpu_vstart, 0); diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/tcg/vector_helper.c +++ b/target/riscv/tcg/vector_helper.c @@ -XXX,XX +XXX,XX @@ static inline void vext_set_elem_mask(void *v0, int index, ((uint64_t *)v0)[idx] = deposit64(old, pos, 1, value); } +static inline MemOpIdx vext_make_memop_idx(CPURISCVState *env, size_t size) +{ + int mmu_idx = riscv_env_mmu_index(env, false); + MemOp memop = size_memop(size) | mo_endian_env(env); + + if (!riscv_cpu_cfg(env)->ext_zicclsm) { + memop |= MO_ALIGN; + } + + return make_memop_idx(memop, mmu_idx); +} + /* elements operations for load and store */ typedef void vext_ldst_elem_fn_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx, void *vd, uintptr_t retaddr); typedef void vext_ldst_elem_fn_host(void *vd, uint32_t idx, void *host); -#define GEN_VEXT_LD_ELEM(NAME, ETYPE, H, LDSUF) \ +#define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF) \ static inline QEMU_ALWAYS_INLINE \ void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \ uint32_t idx, void *vd, uintptr_t retaddr) \ { \ ETYPE *cur = ((ETYPE *)vd + H(idx)); \ - *cur = cpu_##LDSUF##_data_ra(env, addr, retaddr); \ + MemOpIdx oi = vext_make_memop_idx(env, sizeof(ETYPE)); \ + *cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr); \ } \ - \ + +#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF) \ static inline QEMU_ALWAYS_INLINE \ void NAME##_host(void *vd, uint32_t idx, void *host) \ { \ @@ -XXX,XX +XXX,XX @@ void NAME##_host(void *vd, uint32_t idx, void *host) \ *cur = (ETYPE)LDSUF##_p(host); \ } -GEN_VEXT_LD_ELEM(lde_b, uint8_t, H1, ldub) -GEN_VEXT_LD_ELEM(lde_h, uint16_t, H2, lduw_le) -GEN_VEXT_LD_ELEM(lde_w, uint32_t, H4, ldl_le) -GEN_VEXT_LD_ELEM(lde_d, uint64_t, H8, ldq_le) +GEN_VEXT_TLB_LD_ELEM(lde_b, uint8_t, H1, ldb) +GEN_VEXT_TLB_LD_ELEM(lde_h, uint16_t, H2, ldw) +GEN_VEXT_TLB_LD_ELEM(lde_w, uint32_t, H4, ldl) +GEN_VEXT_TLB_LD_ELEM(lde_d, uint64_t, H8, ldq) -#define GEN_VEXT_ST_ELEM(NAME, ETYPE, H, STSUF) \ +GEN_VEXT_HOST_LD_ELEM(lde_b, uint8_t, H1, ldub) +GEN_VEXT_HOST_LD_ELEM(lde_h, uint16_t, H2, lduw_le) +GEN_VEXT_HOST_LD_ELEM(lde_w, uint32_t, H4, ldl_le) +GEN_VEXT_HOST_LD_ELEM(lde_d, uint64_t, H8, ldq_le) + +#define GEN_VEXT_TLB_ST_ELEM(NAME, ETYPE, H, STSUF) \ static inline QEMU_ALWAYS_INLINE \ void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \ uint32_t idx, void *vd, uintptr_t retaddr) \ { \ ETYPE data = *((ETYPE *)vd + H(idx)); \ - cpu_##STSUF##_data_ra(env, addr, data, retaddr); \ + MemOpIdx oi = vext_make_memop_idx(env, sizeof(ETYPE)); \ + cpu_##STSUF##_mmu(env, addr, data, oi, retaddr); \ } \ - \ + +#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF) \ static inline QEMU_ALWAYS_INLINE \ void NAME##_host(void *vd, uint32_t idx, void *host) \ { \ @@ -XXX,XX +XXX,XX @@ void NAME##_host(void *vd, uint32_t idx, void *host) \ STSUF##_p(host, data); \ } -GEN_VEXT_ST_ELEM(ste_b, uint8_t, H1, stb) -GEN_VEXT_ST_ELEM(ste_h, uint16_t, H2, stw_le) -GEN_VEXT_ST_ELEM(ste_w, uint32_t, H4, stl_le) -GEN_VEXT_ST_ELEM(ste_d, uint64_t, H8, stq_le) +GEN_VEXT_TLB_ST_ELEM(ste_b, uint8_t, H1, stb) +GEN_VEXT_TLB_ST_ELEM(ste_h, uint16_t, H2, stw) +GEN_VEXT_TLB_ST_ELEM(ste_w, uint32_t, H4, stl) +GEN_VEXT_TLB_ST_ELEM(ste_d, uint64_t, H8, stq) + +GEN_VEXT_HOST_ST_ELEM(ste_b, uint8_t, H1, stb) +GEN_VEXT_HOST_ST_ELEM(ste_h, uint16_t, H2, stw_le) +GEN_VEXT_HOST_ST_ELEM(ste_w, uint32_t, H4, stl_le) +GEN_VEXT_HOST_ST_ELEM(ste_d, uint64_t, H8, stq_le) static inline QEMU_ALWAYS_INLINE void vext_continuous_ldst_tlb(CPURISCVState *env, vext_ldst_elem_fn_tlb *ldst_tlb, @@ -XXX,XX +XXX,XX @@ vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr, probe_pages(env, addr, size, ra, access_type, mmu_index, &host, &flags, true); - if (flags == 0) { + bool misaligned = addr & (esz - 1); + + /* + * Allow the host fast-pash when: + * 1. Page permission/pmp/watchpoint are checked and we have a contigous + * host mapping. + * 2. Zicclsm is enabled or load/store is not a misaligned access. + * Otherwise, we will fall back to the slow TLB-path. + */ + if (flags == 0 && (riscv_cpu_cfg(env)->ext_zicclsm || !misaligned)) { if (nf == 1) { vext_continuous_ldst_host(env, ldst_host, vd, evl, env->vstart, host, esz, is_load); -- 2.43.0 From: Frank Chang <frank.chang@sifive.com> Now that Zicclsm is supported, let's enable it for the following compatible CPUs: - Base 32 CPU (to be backward compatible) - Base 64 CPU (to be backward compatible) - XuanTie (T-Head) C908 - Tenstorrent Ascalon - Ventana Veyron V1 - XiangShan Kunminghu - MIPS P8700 (ISA doesn't include Zicclsm, but their datasheet claims that it has unaligned load/store support in hardware) Signed-off-by: Frank Chang <frank.chang@sifive.com> Acked-by: Alistair Francis <alistair.francis@wdc.com> --- target/riscv/cpu.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/cpu.c +++ b/target/riscv/cpu.c @@ -XXX,XX +XXX,XX @@ static const TypeInfo riscv_cpu_type_infos[] = { .cfg.ext_zicbom = true, .cfg.ext_zicbop = true, .cfg.ext_zicboz = true, + .cfg.ext_zicclsm = true, .cfg.ext_zicntr = true, .cfg.ext_zicsr = true, .cfg.ext_zifencei = true, @@ -XXX,XX +XXX,XX @@ static const TypeInfo riscv_cpu_type_infos[] = { .cfg.ext_zicbom = true, .cfg.ext_zicbop = true, .cfg.ext_zicboz = true, + .cfg.ext_zicclsm = true, .cfg.ext_zicntr = true, .cfg.ext_zicsr = true, .cfg.ext_zifencei = true, @@ -XXX,XX +XXX,XX @@ static const TypeInfo riscv_cpu_type_infos[] = { .cfg.ext_zbs = true, .cfg.ext_zkt = true, .cfg.ext_zbkc = true, + .cfg.ext_zicclsm = true, .cfg.ext_zicsr = true, .cfg.ext_zifencei = true, .cfg.ext_zihintpause = true, @@ -XXX,XX +XXX,XX @@ static const TypeInfo riscv_cpu_type_infos[] = { .cfg.ext_zicbom = true, .cfg.ext_zicbop = true, .cfg.ext_zicboz = true, + .cfg.ext_zicclsm = true, .cfg.ext_zicntr = true, .cfg.ext_zicond = true, .cfg.ext_zicsr = true, @@ -XXX,XX +XXX,XX @@ static const TypeInfo riscv_cpu_type_infos[] = { /* ISA extensions */ .cfg.mmu = true, + .cfg.ext_zicclsm = true, .cfg.ext_zifencei = true, .cfg.ext_zicsr = true, .cfg.pmp = true, @@ -XXX,XX +XXX,XX @@ static const TypeInfo riscv_cpu_type_infos[] = { * The RISC-V Instruction Set Manual: Volume I * Unprivileged Architecture */ + .cfg.ext_zicclsm = true, .cfg.ext_zicntr = true, .cfg.ext_zihpm = true, .cfg.ext_zihintntl = true, @@ -XXX,XX +XXX,XX @@ static const TypeInfo riscv_cpu_type_infos[] = { .misa_ext = RVI | RVM | RVA | RVF | RVD | RVC | RVS | RVU, .priv_spec = PRIV_VERSION_1_12_0, .cfg.max_satp_mode = VM_1_10_SV48, + .cfg.ext_zicclsm = true, .cfg.ext_zifencei = true, .cfg.ext_zicsr = true, .cfg.mmu = true, -- 2.43.0 From: Frank Chang <frank.chang@sifive.com> The SiFive U ROM reset vector data needs proper 8-byte alignment for RV64 ld instructions. Without Zicclsm support, misaligned loads will cause exceptions. Add padding to ensure start_addr and fdt_load_addr are placed at 8-byte aligned offsets and adjust the load instruction offsets to match the new data layout. Signed-off-by: Frank Chang <frank.chang@sifive.com> --- hw/riscv/sifive_u.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/hw/riscv/sifive_u.c b/hw/riscv/sifive_u.c index XXXXXXX..XXXXXXX 100644 --- a/hw/riscv/sifive_u.c +++ b/hw/riscv/sifive_u.c @@ -XXX,XX +XXX,XX @@ static void sifive_u_machine_init(MachineState *machine) 0, 0, 0x00028067, /* jr t0 */ + 0x00000000, /* padding for alignment */ start_addr, /* start: .dword */ start_addr_hi32, fdt_load_addr, /* fdt_laddr: .dword */ fdt_load_addr_hi32, - 0x00000000, /* fw_dyn: */ }; + if (riscv_is_32bit(&s->soc.u_cpus)) { - reset_vec[4] = 0x0202a583; /* lw a1, 32(t0) */ - reset_vec[5] = 0x0182a283; /* lw t0, 24(t0) */ + reset_vec[4] = 0x0242a583; /* lw a1, 36(t0) */ + reset_vec[5] = 0x01c2a283; /* lw t0, 28(t0) */ } else { - reset_vec[4] = 0x0202b583; /* ld a1, 32(t0) */ - reset_vec[5] = 0x0182b283; /* ld t0, 24(t0) */ + reset_vec[4] = 0x0242b583; /* ld a1, 36(t0) */ + reset_vec[5] = 0x01c2b283; /* ld t0, 28(t0) */ } -- 2.43.0 From: Frank Chang <frank.chang@sifive.com> Update Zicclsm ISA string and expose it as a CPU property to allow user to turn on/off Zicclsm extension. Signed-off-by: Frank Chang <frank.chang@sifive.com> Reviewed-by: Max Chou <max.chou@sifive.com> Reviewed-by: Alistair Francis <alistair.francis@wdc.com> --- target/riscv/cpu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/cpu.c +++ b/target/riscv/cpu.c @@ -XXX,XX +XXX,XX @@ const RISCVIsaExtData isa_edata_arr[] = { ISA_EXT_DATA_ENTRY(zicboz, PRIV_VERSION_1_12_0, ext_zicboz), ISA_INTERNAL_EXT_DATA_ENTRY(ziccamoa, PRIV_VERSION_1_11_0, has_priv_1_11), ISA_INTERNAL_EXT_DATA_ENTRY(ziccif, PRIV_VERSION_1_11_0, has_priv_1_11), - ISA_INTERNAL_EXT_DATA_ENTRY(zicclsm, PRIV_VERSION_1_11_0, has_priv_1_11), + ISA_EXT_DATA_ENTRY(zicclsm, PRIV_VERSION_1_11_0, ext_zicclsm), ISA_EXT_DATA_ENTRY(ziccrse, PRIV_VERSION_1_11_0, ext_ziccrse), ISA_EXT_DATA_ENTRY(zicfilp, PRIV_VERSION_1_12_0, ext_zicfilp), ISA_EXT_DATA_ENTRY(zicfiss, PRIV_VERSION_1_13_0, ext_zicfiss), -- 2.43.0 From: Frank Chang <frank.chang@sifive.com> Zicclsm extension is mandatory for the RVA22U64 profile. Previously, Zicclsm was enabled automatically when has_priv_1_11 was true. Now that Zicclsm has been converted to an explicit CPU option, it must be explicitly added to the RVA22U64 profile's extension list to ensure the profile remains compliant with the specification. Signed-off-by: Frank Chang <frank.chang@sifive.com> --- target/riscv/cpu.c | 1 + 1 file changed, 1 insertion(+) diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/cpu.c +++ b/target/riscv/cpu.c @@ -XXX,XX +XXX,XX @@ static RISCVCPUProfile RVA22U64 = { CPU_CFG_OFFSET(ext_zkt), CPU_CFG_OFFSET(ext_zicntr), CPU_CFG_OFFSET(ext_zihpm), CPU_CFG_OFFSET(ext_zicbom), CPU_CFG_OFFSET(ext_zicbop), CPU_CFG_OFFSET(ext_zicboz), + CPU_CFG_OFFSET(ext_zicclsm), /* mandatory named features for this profile */ CPU_CFG_OFFSET(ext_zic64b), -- 2.43.0 From: Frank Chang <frank.chang@sifive.com> Support the true Zicclsm extension so that we can trap misaligned accesses when Zicclsm is disabled. To enable/disable Zicclsm, simply set zicclsm=[true|false], e.g.: -cpu rv64,zicclsm=[true|false] QEMU will raise a misaligned load/store exception when executing misaligned load/store instructions if Zicclsm is disabled. Changelog: v6: * Reorder and squash Zicclsm CPU property and RVA22U64 profile patches to make every single commit to pass `make check`. v5: * Add Zicclsm to RVA22U64 profile explicitly. * Rebase to the latest riscv-to-apply.next. v4: * Align ROM reset vector data at 8-byte aligned offsets. v3: * Enable Zicclsm for the compatible CPUs. * Rebase to the latest riscv-to-apply.next. v2: * Use (size_memop(size) | mo_endian_env(env)) to calculate MemOp. * Use (log2_esz << MO_ASHIFT) to calculate aligment MemOp for vector load/store whole register instructions. Frank Chang (6): target/riscv: Add Zicclsm CPU option target/riscv: Enable Zicclsm for the compatible CPUs hw/riscv: sifive_u: Align ROM reset vector data target/riscv: Support raising misaligned exceptions for scalar loads/stores target/riscv: Support raising misaligned exceptions for vector loads/stores target/riscv: Expose Zicclsm as a CPU property and update RVA22U64 profile hw/riscv/sifive_u.c | 11 ++-- target/riscv/cpu.c | 10 +++- target/riscv/cpu_cfg_fields.h.inc | 2 + target/riscv/tcg/insn_trans/trans_rvi.c.inc | 6 ++ target/riscv/tcg/insn_trans/trans_rvv.c.inc | 18 +++++- target/riscv/tcg/vector_helper.c | 65 ++++++++++++++++----- 6 files changed, 88 insertions(+), 24 deletions(-) -- 2.43.0 From: Frank Chang <frank.chang@sifive.com> Add Zicclsm CPU option so that user can turn on/off misaligned loads and stores support. Signed-off-by: Frank Chang <frank.chang@sifive.com> Reviewed-by: Max Chou <max.chou@sifive.com> Reviewed-by: Alistair Francis <alistair.francis@wdc.com> --- target/riscv/cpu_cfg_fields.h.inc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/target/riscv/cpu_cfg_fields.h.inc b/target/riscv/cpu_cfg_fields.h.inc index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/cpu_cfg_fields.h.inc +++ b/target/riscv/cpu_cfg_fields.h.inc @@ -XXX,XX +XXX,XX @@ BOOL_FIELD(has_priv_1_11) /* Always enabled for TCG if has_priv_1_11 */ BOOL_FIELD(ext_ziccrse) +BOOL_FIELD(ext_zicclsm) + /* Vendor-specific custom extensions */ BOOL_FIELD(ext_xtheadba) BOOL_FIELD(ext_xtheadbb) -- 2.43.0 From: Frank Chang <frank.chang@sifive.com> To support Zicclsm, let's enable Zicclsm for the following compatible CPUs: - Base 32 CPU (to be backward compatible) - Base 64 CPU (to be backward compatible) - XuanTie (T-Head) C908 - Tenstorrent Ascalon - Ventana Veyron V1 - XiangShan Kunminghu - MIPS P8700 (ISA doesn't include Zicclsm, but their datasheet claims that it has unaligned load/store support in hardware) Signed-off-by: Frank Chang <frank.chang@sifive.com> Acked-by: Alistair Francis <alistair.francis@wdc.com> --- target/riscv/cpu.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/cpu.c +++ b/target/riscv/cpu.c @@ -XXX,XX +XXX,XX @@ static const TypeInfo riscv_cpu_type_infos[] = { .cfg.ext_zicbom = true, .cfg.ext_zicbop = true, .cfg.ext_zicboz = true, + .cfg.ext_zicclsm = true, .cfg.ext_zicntr = true, .cfg.ext_zicsr = true, .cfg.ext_zifencei = true, @@ -XXX,XX +XXX,XX @@ static const TypeInfo riscv_cpu_type_infos[] = { .cfg.ext_zicbom = true, .cfg.ext_zicbop = true, .cfg.ext_zicboz = true, + .cfg.ext_zicclsm = true, .cfg.ext_zicntr = true, .cfg.ext_zicsr = true, .cfg.ext_zifencei = true, @@ -XXX,XX +XXX,XX @@ static const TypeInfo riscv_cpu_type_infos[] = { .cfg.ext_zbs = true, .cfg.ext_zkt = true, .cfg.ext_zbkc = true, + .cfg.ext_zicclsm = true, .cfg.ext_zicsr = true, .cfg.ext_zifencei = true, .cfg.ext_zihintpause = true, @@ -XXX,XX +XXX,XX @@ static const TypeInfo riscv_cpu_type_infos[] = { .cfg.ext_zicbom = true, .cfg.ext_zicbop = true, .cfg.ext_zicboz = true, + .cfg.ext_zicclsm = true, .cfg.ext_zicntr = true, .cfg.ext_zicond = true, .cfg.ext_zicsr = true, @@ -XXX,XX +XXX,XX @@ static const TypeInfo riscv_cpu_type_infos[] = { /* ISA extensions */ .cfg.mmu = true, + .cfg.ext_zicclsm = true, .cfg.ext_zifencei = true, .cfg.ext_zicsr = true, .cfg.pmp = true, @@ -XXX,XX +XXX,XX @@ static const TypeInfo riscv_cpu_type_infos[] = { * The RISC-V Instruction Set Manual: Volume I * Unprivileged Architecture */ + .cfg.ext_zicclsm = true, .cfg.ext_zicntr = true, .cfg.ext_zihpm = true, .cfg.ext_zihintntl = true, @@ -XXX,XX +XXX,XX @@ static const TypeInfo riscv_cpu_type_infos[] = { .misa_ext = RVI | RVM | RVA | RVF | RVD | RVC | RVS | RVU, .priv_spec = PRIV_VERSION_1_12_0, .cfg.max_satp_mode = VM_1_10_SV48, + .cfg.ext_zicclsm = true, .cfg.ext_zifencei = true, .cfg.ext_zicsr = true, .cfg.mmu = true, -- 2.43.0 From: Frank Chang <frank.chang@sifive.com> The SiFive U ROM reset vector data needs proper 8-byte alignment for RV64 ld instructions. The misaligned load will cause exception when Zicclsm is supported as SiFive U CPU doesn't support hardware misaligned loads and stores. Add padding to ensure start_addr and fdt_load_addr are placed at 8-byte aligned offsets and adjust the load instruction offsets to match the new data layout. Signed-off-by: Frank Chang <frank.chang@sifive.com> --- hw/riscv/sifive_u.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/hw/riscv/sifive_u.c b/hw/riscv/sifive_u.c index XXXXXXX..XXXXXXX 100644 --- a/hw/riscv/sifive_u.c +++ b/hw/riscv/sifive_u.c @@ -XXX,XX +XXX,XX @@ static void sifive_u_machine_init(MachineState *machine) 0, 0, 0x00028067, /* jr t0 */ + 0x00000000, /* padding for alignment */ start_addr, /* start: .dword */ start_addr_hi32, fdt_load_addr, /* fdt_laddr: .dword */ fdt_load_addr_hi32, - 0x00000000, /* fw_dyn: */ }; + if (riscv_is_32bit(&s->soc.u_cpus)) { - reset_vec[4] = 0x0202a583; /* lw a1, 32(t0) */ - reset_vec[5] = 0x0182a283; /* lw t0, 24(t0) */ + reset_vec[4] = 0x0242a583; /* lw a1, 36(t0) */ + reset_vec[5] = 0x01c2a283; /* lw t0, 28(t0) */ } else { - reset_vec[4] = 0x0202b583; /* ld a1, 32(t0) */ - reset_vec[5] = 0x0182b283; /* ld t0, 24(t0) */ + reset_vec[4] = 0x0242b583; /* ld a1, 36(t0) */ + reset_vec[5] = 0x01c2b283; /* ld t0, 28(t0) */ } -- 2.43.0 From: Frank Chang <frank.chang@sifive.com> When the Zicclsm extension is not enabled, raise misaligned load/store exceptions for misaligned accesses from scalar load/store instructions. Signed-off-by: Frank Chang <frank.chang@sifive.com> Reviewed-by: Max Chou <max.chou@sifive.com> Reviewed-by: Alistair Francis <alistair.francis@wdc.com> --- target/riscv/tcg/insn_trans/trans_rvi.c.inc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/target/riscv/tcg/insn_trans/trans_rvi.c.inc b/target/riscv/tcg/insn_trans/trans_rvi.c.inc index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/tcg/insn_trans/trans_rvi.c.inc +++ b/target/riscv/tcg/insn_trans/trans_rvi.c.inc @@ -XXX,XX +XXX,XX @@ static bool gen_load(DisasContext *ctx, arg_lb *a, MemOp memop) if (ctx->cfg_ptr->ext_zama16b) { memop |= MO_ATOM_WITHIN16; } + if (!ctx->cfg_ptr->ext_zicclsm) { + memop |= MO_ALIGN; + } decode_save_opc(ctx, 0); if (get_xl(ctx) == MXL_RV128) { out = gen_load_i128(ctx, a, memop); @@ -XXX,XX +XXX,XX @@ static bool gen_store(DisasContext *ctx, arg_sb *a, MemOp memop) if (ctx->cfg_ptr->ext_zama16b) { memop |= MO_ATOM_WITHIN16; } + if (!ctx->cfg_ptr->ext_zicclsm) { + memop |= MO_ALIGN; + } decode_save_opc(ctx, 0); if (get_xl(ctx) == MXL_RV128) { return gen_store_i128(ctx, a, memop); -- 2.43.0 From: Frank Chang <frank.chang@sifive.com> When the Zicclsm extension is not enabled, raise misaligned load/store exceptions for misaligned accesses from vector load/store instructions. We will skip the host fast-path and fall back to the slow TLB-path to raise misaligned load/store exceptions for the misaligned accesses when Zicclsm extension is disabled. Signed-off-by: Frank Chang <frank.chang@sifive.com> Reviewed-by: Max Chou <max.chou@sifive.com> Acked-by: Alistair Francis <alistair.francis@wdc.com> --- target/riscv/tcg/insn_trans/trans_rvv.c.inc | 18 +++++- target/riscv/tcg/vector_helper.c | 65 ++++++++++++++++----- 2 files changed, 65 insertions(+), 18 deletions(-) diff --git a/target/riscv/tcg/insn_trans/trans_rvv.c.inc b/target/riscv/tcg/insn_trans/trans_rvv.c.inc index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/tcg/insn_trans/trans_rvv.c.inc +++ b/target/riscv/tcg/insn_trans/trans_rvv.c.inc @@ -XXX,XX +XXX,XX @@ static bool ldst_whole_trans(uint32_t vd, uint32_t rs1, uint32_t nf, * Use the helper function if either: * - vstart is not 0. */ - bool use_helper_fn = !s->vstart_eq_zero; if (!use_helper_fn) { uint32_t size = s->cfg_ptr->vlenb * nf; TCGv_i64 t8 = tcg_temp_new_i64(); MemOp atomicity = MO_ATOM_NONE; + MemOp alignment = MO_UNALN; + + /* + * If Zicclsm is disabled, require alignment based on element size. + * Use MO_ALIGN_* based on log2_esz (0 = MO_UNALN, 1 = MO_ALIGN_2, etc). + */ + if (!s->cfg_ptr->ext_zicclsm) { + alignment = log2_esz << MO_ASHIFT; + } + if (log2_esz == 0) { atomicity = MO_ATOM_NONE; } else { atomicity = MO_ATOM_IFALIGN_PAIR; } + for (int i = 0; i < size; i += 8) { TCGv addr = get_address(s, rs1, i); if (is_load) { - tcg_gen_qemu_ld_i64(t8, addr, s->mem_idx, MO_LEUQ | atomicity); + tcg_gen_qemu_ld_i64(t8, addr, s->mem_idx, + MO_LEUQ | atomicity | alignment); tcg_gen_st_i64(t8, tcg_env, vreg_ofs(s, vd) + i); } else { tcg_gen_ld_i64(t8, tcg_env, vreg_ofs(s, vd) + i); - tcg_gen_qemu_st_i64(t8, addr, s->mem_idx, MO_LEUQ | atomicity); + tcg_gen_qemu_st_i64(t8, addr, s->mem_idx, + MO_LEUQ | atomicity | alignment); } if (i == size - 8) { tcg_gen_movi_i32(cpu_vstart, 0); diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/tcg/vector_helper.c +++ b/target/riscv/tcg/vector_helper.c @@ -XXX,XX +XXX,XX @@ static inline void vext_set_elem_mask(void *v0, int index, ((uint64_t *)v0)[idx] = deposit64(old, pos, 1, value); } +static inline MemOpIdx vext_make_memop_idx(CPURISCVState *env, size_t size) +{ + int mmu_idx = riscv_env_mmu_index(env, false); + MemOp memop = size_memop(size) | mo_endian_env(env); + + if (!riscv_cpu_cfg(env)->ext_zicclsm) { + memop |= MO_ALIGN; + } + + return make_memop_idx(memop, mmu_idx); +} + /* elements operations for load and store */ typedef void vext_ldst_elem_fn_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx, void *vd, uintptr_t retaddr); typedef void vext_ldst_elem_fn_host(void *vd, uint32_t idx, void *host); -#define GEN_VEXT_LD_ELEM(NAME, ETYPE, H, LDSUF) \ +#define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF) \ static inline QEMU_ALWAYS_INLINE \ void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \ uint32_t idx, void *vd, uintptr_t retaddr) \ { \ ETYPE *cur = ((ETYPE *)vd + H(idx)); \ - *cur = cpu_##LDSUF##_data_ra(env, addr, retaddr); \ + MemOpIdx oi = vext_make_memop_idx(env, sizeof(ETYPE)); \ + *cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr); \ } \ - \ + +#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF) \ static inline QEMU_ALWAYS_INLINE \ void NAME##_host(void *vd, uint32_t idx, void *host) \ { \ @@ -XXX,XX +XXX,XX @@ void NAME##_host(void *vd, uint32_t idx, void *host) \ *cur = (ETYPE)LDSUF##_p(host); \ } -GEN_VEXT_LD_ELEM(lde_b, uint8_t, H1, ldub) -GEN_VEXT_LD_ELEM(lde_h, uint16_t, H2, lduw_le) -GEN_VEXT_LD_ELEM(lde_w, uint32_t, H4, ldl_le) -GEN_VEXT_LD_ELEM(lde_d, uint64_t, H8, ldq_le) +GEN_VEXT_TLB_LD_ELEM(lde_b, uint8_t, H1, ldb) +GEN_VEXT_TLB_LD_ELEM(lde_h, uint16_t, H2, ldw) +GEN_VEXT_TLB_LD_ELEM(lde_w, uint32_t, H4, ldl) +GEN_VEXT_TLB_LD_ELEM(lde_d, uint64_t, H8, ldq) -#define GEN_VEXT_ST_ELEM(NAME, ETYPE, H, STSUF) \ +GEN_VEXT_HOST_LD_ELEM(lde_b, uint8_t, H1, ldub) +GEN_VEXT_HOST_LD_ELEM(lde_h, uint16_t, H2, lduw_le) +GEN_VEXT_HOST_LD_ELEM(lde_w, uint32_t, H4, ldl_le) +GEN_VEXT_HOST_LD_ELEM(lde_d, uint64_t, H8, ldq_le) + +#define GEN_VEXT_TLB_ST_ELEM(NAME, ETYPE, H, STSUF) \ static inline QEMU_ALWAYS_INLINE \ void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \ uint32_t idx, void *vd, uintptr_t retaddr) \ { \ ETYPE data = *((ETYPE *)vd + H(idx)); \ - cpu_##STSUF##_data_ra(env, addr, data, retaddr); \ + MemOpIdx oi = vext_make_memop_idx(env, sizeof(ETYPE)); \ + cpu_##STSUF##_mmu(env, addr, data, oi, retaddr); \ } \ - \ + +#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF) \ static inline QEMU_ALWAYS_INLINE \ void NAME##_host(void *vd, uint32_t idx, void *host) \ { \ @@ -XXX,XX +XXX,XX @@ void NAME##_host(void *vd, uint32_t idx, void *host) \ STSUF##_p(host, data); \ } -GEN_VEXT_ST_ELEM(ste_b, uint8_t, H1, stb) -GEN_VEXT_ST_ELEM(ste_h, uint16_t, H2, stw_le) -GEN_VEXT_ST_ELEM(ste_w, uint32_t, H4, stl_le) -GEN_VEXT_ST_ELEM(ste_d, uint64_t, H8, stq_le) +GEN_VEXT_TLB_ST_ELEM(ste_b, uint8_t, H1, stb) +GEN_VEXT_TLB_ST_ELEM(ste_h, uint16_t, H2, stw) +GEN_VEXT_TLB_ST_ELEM(ste_w, uint32_t, H4, stl) +GEN_VEXT_TLB_ST_ELEM(ste_d, uint64_t, H8, stq) + +GEN_VEXT_HOST_ST_ELEM(ste_b, uint8_t, H1, stb) +GEN_VEXT_HOST_ST_ELEM(ste_h, uint16_t, H2, stw_le) +GEN_VEXT_HOST_ST_ELEM(ste_w, uint32_t, H4, stl_le) +GEN_VEXT_HOST_ST_ELEM(ste_d, uint64_t, H8, stq_le) static inline QEMU_ALWAYS_INLINE void vext_continuous_ldst_tlb(CPURISCVState *env, vext_ldst_elem_fn_tlb *ldst_tlb, @@ -XXX,XX +XXX,XX @@ vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr, probe_pages(env, addr, size, ra, access_type, mmu_index, &host, &flags, true); - if (flags == 0) { + bool misaligned = addr & (esz - 1); + + /* + * Allow the host fast-pash when: + * 1. Page permission/pmp/watchpoint are checked and we have a contigous + * host mapping. + * 2. Zicclsm is enabled or load/store is not a misaligned access. + * Otherwise, we will fall back to the slow TLB-path. + */ + if (flags == 0 && (riscv_cpu_cfg(env)->ext_zicclsm || !misaligned)) { if (nf == 1) { vext_continuous_ldst_host(env, ldst_host, vd, evl, env->vstart, host, esz, is_load); -- 2.43.0 From: Frank Chang <frank.chang@sifive.com> Update Zicclsm ISA string and expose it as a CPU property to allow user to turn on/off Zicclsm extension. In addition, Zicclsm extension is mandatory for the RVA22U64 profile. Previously, Zicclsm was enabled automatically when has_priv_1_11 was true. Now that Zicclsm has been converted to an explicit CPU option, it must be explicitly added to the RVA22U64 profile's extension list to ensure the profile remains compliant with the specification. Signed-off-by: Frank Chang <frank.chang@sifive.com> --- target/riscv/cpu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/cpu.c +++ b/target/riscv/cpu.c @@ -XXX,XX +XXX,XX @@ const RISCVIsaExtData isa_edata_arr[] = { ISA_EXT_DATA_ENTRY(zicboz, PRIV_VERSION_1_12_0, ext_zicboz), ISA_INTERNAL_EXT_DATA_ENTRY(ziccamoa, PRIV_VERSION_1_11_0, has_priv_1_11), ISA_INTERNAL_EXT_DATA_ENTRY(ziccif, PRIV_VERSION_1_11_0, has_priv_1_11), - ISA_INTERNAL_EXT_DATA_ENTRY(zicclsm, PRIV_VERSION_1_11_0, has_priv_1_11), + ISA_EXT_DATA_ENTRY(zicclsm, PRIV_VERSION_1_11_0, ext_zicclsm), ISA_EXT_DATA_ENTRY(ziccrse, PRIV_VERSION_1_11_0, ext_ziccrse), ISA_EXT_DATA_ENTRY(zicfilp, PRIV_VERSION_1_12_0, ext_zicfilp), ISA_EXT_DATA_ENTRY(zicfiss, PRIV_VERSION_1_13_0, ext_zicfiss), @@ -XXX,XX +XXX,XX @@ static RISCVCPUProfile RVA22U64 = { CPU_CFG_OFFSET(ext_zkt), CPU_CFG_OFFSET(ext_zicntr), CPU_CFG_OFFSET(ext_zihpm), CPU_CFG_OFFSET(ext_zicbom), CPU_CFG_OFFSET(ext_zicbop), CPU_CFG_OFFSET(ext_zicboz), + CPU_CFG_OFFSET(ext_zicclsm), /* mandatory named features for this profile */ CPU_CFG_OFFSET(ext_zic64b), -- 2.43.0