The following changes since commit 45240eed4f064576d589ea60ebadf3c11d7ab891: Merge remote-tracking branch 'remotes/armbru/tags/pull-yank-2021-01-13' into staging (2021-01-13 14:19:24 +0000) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20210113 for you to fetch changes up to 4cacecaaa2bbf8af0967bd3eee43297fada475a9: decodetree: Open files with encoding='utf-8' (2021-01-13 08:39:08 -1000) ---------------------------------------------------------------- Improvements to tcg constant handling. Force utf8 for decodetree. ---------------------------------------------------------------- Philippe Mathieu-Daudé (1): decodetree: Open files with encoding='utf-8' Richard Henderson (23): tcg: Use tcg_out_dupi_vec from temp_load tcg: Increase tcg_out_dupi_vec immediate to int64_t tcg: Consolidate 3 bits into enum TCGTempKind tcg: Add temp_readonly tcg: Expand TCGTemp.val to 64-bits tcg: Rename struct tcg_temp_info to TempOptInfo tcg: Expand TempOptInfo to 64-bits tcg: Introduce TYPE_CONST temporaries tcg/optimize: Improve find_better_copy tcg/optimize: Adjust TempOptInfo allocation tcg/optimize: Use tcg_constant_internal with constant folding tcg: Convert tcg_gen_dupi_vec to TCG_CONST tcg: Use tcg_constant_i32 with icount expander tcg: Use tcg_constant_{i32,i64} with tcg int expanders tcg: Use tcg_constant_{i32,i64} with tcg plugins tcg: Use tcg_constant_{i32,i64,vec} with gvec expanders tcg/tci: Add special tci_movi_{i32,i64} opcodes tcg: Remove movi and dupi opcodes tcg: Add tcg_reg_alloc_dup2 tcg/i386: Use tcg_constant_vec with tcg vec expanders tcg: Remove tcg_gen_dup{8,16,32,64}i_vec tcg/ppc: Use tcg_constant_vec with tcg vec expanders tcg/aarch64: Use tcg_constant_vec with tcg vec expanders include/exec/gen-icount.h | 25 +-- include/tcg/tcg-op.h | 17 +- include/tcg/tcg-opc.h | 11 +- include/tcg/tcg.h | 50 ++++- accel/tcg/plugin-gen.c | 49 ++--- tcg/optimize.c | 249 +++++++++++----------- tcg/tcg-op-gvec.c | 129 +++++------- tcg/tcg-op-vec.c | 52 +---- tcg/tcg-op.c | 227 ++++++++++---------- tcg/tcg.c | 488 +++++++++++++++++++++++++++++++++---------- tcg/tci.c | 4 +- tcg/aarch64/tcg-target.c.inc | 32 +-- tcg/arm/tcg-target.c.inc | 1 - tcg/i386/tcg-target.c.inc | 112 ++++++---- tcg/mips/tcg-target.c.inc | 2 - tcg/ppc/tcg-target.c.inc | 90 ++++---- tcg/riscv/tcg-target.c.inc | 2 - tcg/s390/tcg-target.c.inc | 2 - tcg/sparc/tcg-target.c.inc | 2 - tcg/tci/tcg-target.c.inc | 6 +- scripts/decodetree.py | 9 +- 21 files changed, 890 insertions(+), 669 deletions(-) Having dupi pass though movi is confusing and arguably wrong. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 6 +++- tcg/aarch64/tcg-target.c.inc | 7 ---- tcg/i386/tcg-target.c.inc | 63 ++++++++++++++++++++++++------------ tcg/ppc/tcg-target.c.inc | 6 ---- 4 files changed, 47 insertions(+), 35 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, case TEMP_VAL_CONST: reg = tcg_reg_alloc(s, desired_regs, allocated_regs, preferred_regs, ts->indirect_base); - tcg_out_movi(s, ts->type, reg, ts->val); + if (ts->type <= TCG_TYPE_I64) { + tcg_out_movi(s, ts->type, reg, ts->val); + } else { + tcg_out_dupi_vec(s, ts->type, reg, ts->val); + } ts->mem_coherent = 0; break; case TEMP_VAL_MEM: diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, case TCG_TYPE_I64: tcg_debug_assert(rd < 32); break; - - case TCG_TYPE_V64: - case TCG_TYPE_V128: - tcg_debug_assert(rd >= 32); - tcg_out_dupi_vec(s, type, rd, value); - return; - default: g_assert_not_reached(); } diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_dupi_vec(TCGContext *s, TCGType type, } } -static void tcg_out_movi(TCGContext *s, TCGType type, - TCGReg ret, tcg_target_long arg) +static void tcg_out_movi_vec(TCGContext *s, TCGType type, + TCGReg ret, tcg_target_long arg) +{ + if (arg == 0) { + tcg_out_vex_modrm(s, OPC_PXOR, ret, ret, ret); + return; + } + if (arg == -1) { + tcg_out_vex_modrm(s, OPC_PCMPEQB, ret, ret, ret); + return; + } + + int rexw = (type == TCG_TYPE_I32 ? 0 : P_REXW); + tcg_out_vex_modrm_pool(s, OPC_MOVD_VyEy + rexw, ret); + if (TCG_TARGET_REG_BITS == 64) { + new_pool_label(s, arg, R_386_PC32, s->code_ptr - 4, -4); + } else { + new_pool_label(s, arg, R_386_32, s->code_ptr - 4, 0); + } +} + +static void tcg_out_movi_int(TCGContext *s, TCGType type, + TCGReg ret, tcg_target_long arg) { tcg_target_long diff; - switch (type) { - case TCG_TYPE_I32: -#if TCG_TARGET_REG_BITS == 64 - case TCG_TYPE_I64: -#endif - if (ret < 16) { - break; - } - /* fallthru */ - case TCG_TYPE_V64: - case TCG_TYPE_V128: - case TCG_TYPE_V256: - tcg_debug_assert(ret >= 16); - tcg_out_dupi_vec(s, type, ret, arg); - return; - default: - g_assert_not_reached(); - } - if (arg == 0) { tgen_arithr(s, ARITH_XOR, ret, ret); return; @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, tcg_out64(s, arg); } +static void tcg_out_movi(TCGContext *s, TCGType type, + TCGReg ret, tcg_target_long arg) +{ + switch (type) { + case TCG_TYPE_I32: +#if TCG_TARGET_REG_BITS == 64 + case TCG_TYPE_I64: +#endif + if (ret < 16) { + tcg_out_movi_int(s, type, ret, arg); + } else { + tcg_out_movi_vec(s, type, ret, arg); + } + break; + default: + g_assert_not_reached(); + } +} + static inline void tcg_out_pushi(TCGContext *s, tcg_target_long val) { if (val == (int8_t)val) { diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg ret, tcg_out_movi_int(s, type, ret, arg, false); break; - case TCG_TYPE_V64: - case TCG_TYPE_V128: - tcg_debug_assert(ret >= TCG_REG_V0); - tcg_out_dupi_vec(s, type, ret, arg); - break; - default: g_assert_not_reached(); } -- 2.25.1 While we don't store more than tcg_target_long in TCGTemp, we shouldn't be limited to that for code generation. We will be able to use this for INDEX_op_dup2_vec with 2 constants. Also pass along the minimal vece that may be said to apply to the constant. This allows some simplification in the various backends. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 31 +++++++++++++++++++++++++----- tcg/aarch64/tcg-target.c.inc | 12 ++++++------ tcg/i386/tcg-target.c.inc | 22 ++++++++++++--------- tcg/ppc/tcg-target.c.inc | 37 +++++++++++++++++++++++------------- 4 files changed, 69 insertions(+), 33 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, TCGReg dst, TCGReg src); static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, TCGReg dst, TCGReg base, intptr_t offset); -static void tcg_out_dupi_vec(TCGContext *s, TCGType type, - TCGReg dst, tcg_target_long arg); +static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, + TCGReg dst, int64_t arg); static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl, unsigned vece, const TCGArg *args, const int *const_args); @@ -XXX,XX +XXX,XX @@ static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, { g_assert_not_reached(); } -static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, - TCGReg dst, tcg_target_long arg) +static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, + TCGReg dst, int64_t arg) { g_assert_not_reached(); } @@ -XXX,XX +XXX,XX @@ static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, if (ts->type <= TCG_TYPE_I64) { tcg_out_movi(s, ts->type, reg, ts->val); } else { - tcg_out_dupi_vec(s, ts->type, reg, ts->val); + uint64_t val = ts->val; + MemOp vece = MO_64; + + /* + * Find the minimal vector element that matches the constant. + * The targets will, in general, have to do this search anyway, + * do this generically. + */ + if (TCG_TARGET_REG_BITS == 32) { + val = dup_const(MO_32, val); + vece = MO_32; + } + if (val == dup_const(MO_8, val)) { + vece = MO_8; + } else if (val == dup_const(MO_16, val)) { + vece = MO_16; + } else if (TCG_TARGET_REG_BITS == 64 && + val == dup_const(MO_32, val)) { + vece = MO_32; + } + + tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val); } ts->mem_coherent = 0; break; diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_logicali(TCGContext *s, AArch64Insn insn, TCGType ext, tcg_out_insn_3404(s, insn, ext, rd, rn, ext, r, c); } -static void tcg_out_dupi_vec(TCGContext *s, TCGType type, - TCGReg rd, tcg_target_long v64) +static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, + TCGReg rd, int64_t v64) { bool q = type == TCG_TYPE_V128; int cmode, imm8, i; /* Test all bytes equal first. */ - if (v64 == dup_const(MO_8, v64)) { + if (vece == MO_8) { imm8 = (uint8_t)v64; tcg_out_insn(s, 3606, MOVI, q, rd, 0, 0xe, imm8); return; @@ -XXX,XX +XXX,XX @@ static void tcg_out_dupi_vec(TCGContext *s, TCGType type, * cannot find an expansion there's no point checking a larger * width because we already know by replication it cannot match. */ - if (v64 == dup_const(MO_16, v64)) { + if (vece == MO_16) { uint16_t v16 = v64; if (is_shimm16(v16, &cmode, &imm8)) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_dupi_vec(TCGContext *s, TCGType type, tcg_out_insn(s, 3606, MOVI, q, rd, 0, 0x8, v16 & 0xff); tcg_out_insn(s, 3606, ORR, q, rd, 0, 0xa, v16 >> 8); return; - } else if (v64 == dup_const(MO_32, v64)) { + } else if (vece == MO_32) { uint32_t v32 = v64; uint32_t n32 = ~v32; @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, tcg_out_insn_3617(s, insn, is_q, vece, a0, a1); break; } - tcg_out_dupi_vec(s, type, TCG_VEC_TMP, 0); + tcg_out_dupi_vec(s, type, MO_8, TCG_VEC_TMP, 0); a2 = TCG_VEC_TMP; } insn = cmp_insn[cond]; diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, return true; } -static void tcg_out_dupi_vec(TCGContext *s, TCGType type, - TCGReg ret, tcg_target_long arg) +static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, + TCGReg ret, int64_t arg) { int vex_l = (type == TCG_TYPE_V256 ? P_VEXL : 0); @@ -XXX,XX +XXX,XX @@ static void tcg_out_dupi_vec(TCGContext *s, TCGType type, return; } - if (TCG_TARGET_REG_BITS == 64) { + if (TCG_TARGET_REG_BITS == 32 && vece < MO_64) { + if (have_avx2) { + tcg_out_vex_modrm_pool(s, OPC_VPBROADCASTD + vex_l, ret); + } else { + tcg_out_vex_modrm_pool(s, OPC_VBROADCASTSS, ret); + } + new_pool_label(s, arg, R_386_32, s->code_ptr - 4, 0); + } else { if (type == TCG_TYPE_V64) { tcg_out_vex_modrm_pool(s, OPC_MOVQ_VqWq, ret); } else if (have_avx2) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_dupi_vec(TCGContext *s, TCGType type, } else { tcg_out_vex_modrm_pool(s, OPC_MOVDDUP, ret); } - new_pool_label(s, arg, R_386_PC32, s->code_ptr - 4, -4); - } else { - if (have_avx2) { - tcg_out_vex_modrm_pool(s, OPC_VPBROADCASTD + vex_l, ret); + if (TCG_TARGET_REG_BITS == 64) { + new_pool_label(s, arg, R_386_PC32, s->code_ptr - 4, -4); } else { - tcg_out_vex_modrm_pool(s, OPC_VBROADCASTSS, ret); + new_pool_l2(s, R_386_32, s->code_ptr - 4, 0, arg, arg >> 32); } - new_pool_label(s, arg, R_386_32, s->code_ptr - 4, 0); } } diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, } } -static void tcg_out_dupi_vec(TCGContext *s, TCGType type, TCGReg ret, - tcg_target_long val) +static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, + TCGReg ret, int64_t val) { uint32_t load_insn; int rel, low; intptr_t add; - low = (int8_t)val; - if (low >= -16 && low < 16) { - if (val == (tcg_target_long)dup_const(MO_8, low)) { + switch (vece) { + case MO_8: + low = (int8_t)val; + if (low >= -16 && low < 16) { tcg_out32(s, VSPLTISB | VRT(ret) | ((val & 31) << 16)); return; } - if (val == (tcg_target_long)dup_const(MO_16, low)) { + if (have_isa_3_00) { + tcg_out32(s, XXSPLTIB | VRT(ret) | ((val & 0xff) << 11)); + return; + } + break; + + case MO_16: + low = (int16_t)val; + if (low >= -16 && low < 16) { tcg_out32(s, VSPLTISH | VRT(ret) | ((val & 31) << 16)); return; } - if (val == (tcg_target_long)dup_const(MO_32, low)) { + break; + + case MO_32: + low = (int32_t)val; + if (low >= -16 && low < 16) { tcg_out32(s, VSPLTISW | VRT(ret) | ((val & 31) << 16)); return; } - } - if (have_isa_3_00 && val == (tcg_target_long)dup_const(MO_8, val)) { - tcg_out32(s, XXSPLTIB | VRT(ret) | ((val & 0xff) << 11)); - return; + break; } /* @@ -XXX,XX +XXX,XX @@ static void tcg_out_dupi_vec(TCGContext *s, TCGType type, TCGReg ret, if (TCG_TARGET_REG_BITS == 64) { new_pool_label(s, val, rel, s->code_ptr, add); } else { - new_pool_l2(s, rel, s->code_ptr, add, val, val); + new_pool_l2(s, rel, s->code_ptr, add, val >> 32, val); } } else { load_insn = LVX | VRT(ret) | RB(TCG_REG_TMP1); if (TCG_TARGET_REG_BITS == 64) { new_pool_l2(s, rel, s->code_ptr, add, val, val); } else { - new_pool_l4(s, rel, s->code_ptr, add, val, val, val, val); + new_pool_l4(s, rel, s->code_ptr, add, + val >> 32, val, val >> 32, val); } } -- 2.25.1 The temp_fixed, temp_global, temp_local bits are all related. Combine them into a single enumeration. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg.h | 20 +++++--- tcg/optimize.c | 8 +-- tcg/tcg.c | 126 ++++++++++++++++++++++++++++------------------ 3 files changed, 92 insertions(+), 62 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ typedef enum TCGTempVal { TEMP_VAL_CONST, } TCGTempVal; +typedef enum TCGTempKind { + /* Temp is dead at the end of all basic blocks. */ + TEMP_NORMAL, + /* Temp is saved across basic blocks but dead at the end of TBs. */ + TEMP_LOCAL, + /* Temp is saved across both basic blocks and translation blocks. */ + TEMP_GLOBAL, + /* Temp is in a fixed register. */ + TEMP_FIXED, +} TCGTempKind; + typedef struct TCGTemp { TCGReg reg:8; TCGTempVal val_type:8; TCGType base_type:8; TCGType type:8; - unsigned int fixed_reg:1; + TCGTempKind kind:3; unsigned int indirect_reg:1; unsigned int indirect_base:1; unsigned int mem_coherent:1; unsigned int mem_allocated:1; - /* If true, the temp is saved across both basic blocks and - translation blocks. */ - unsigned int temp_global:1; - /* If true, the temp is saved across basic blocks but dead - at the end of translation blocks. If false, the temp is - dead at the end of basic blocks. */ - unsigned int temp_local:1; unsigned int temp_allocated:1; tcg_target_long val; diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static TCGTemp *find_better_copy(TCGContext *s, TCGTemp *ts) TCGTemp *i; /* If this is already a global, we can't do better. */ - if (ts->temp_global) { + if (ts->kind >= TEMP_GLOBAL) { return ts; } /* Search for a global first. */ for (i = ts_info(ts)->next_copy; i != ts; i = ts_info(i)->next_copy) { - if (i->temp_global) { + if (i->kind >= TEMP_GLOBAL) { return i; } } /* If it is a temp, search for a temp local. */ - if (!ts->temp_local) { + if (ts->kind == TEMP_NORMAL) { for (i = ts_info(ts)->next_copy; i != ts; i = ts_info(i)->next_copy) { - if (ts->temp_local) { + if (i->kind >= TEMP_LOCAL) { return i; } } diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static inline TCGTemp *tcg_global_alloc(TCGContext *s) tcg_debug_assert(s->nb_globals == s->nb_temps); s->nb_globals++; ts = tcg_temp_alloc(s); - ts->temp_global = 1; + ts->kind = TEMP_GLOBAL; return ts; } @@ -XXX,XX +XXX,XX @@ static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, ts = tcg_global_alloc(s); ts->base_type = type; ts->type = type; - ts->fixed_reg = 1; + ts->kind = TEMP_FIXED; ts->reg = reg; ts->name = name; tcg_regset_set_reg(s->reserved_regs, reg); @@ -XXX,XX +XXX,XX @@ TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, bigendian = 1; #endif - if (!base_ts->fixed_reg) { + if (base_ts->kind != TEMP_FIXED) { /* We do not support double-indirect registers. */ tcg_debug_assert(!base_ts->indirect_reg); base_ts->indirect_base = 1; @@ -XXX,XX +XXX,XX @@ TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local) { TCGContext *s = tcg_ctx; + TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL; TCGTemp *ts; int idx, k; @@ -XXX,XX +XXX,XX @@ TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local) ts = &s->temps[idx]; ts->temp_allocated = 1; tcg_debug_assert(ts->base_type == type); - tcg_debug_assert(ts->temp_local == temp_local); + tcg_debug_assert(ts->kind == kind); } else { ts = tcg_temp_alloc(s); if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { @@ -XXX,XX +XXX,XX @@ TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local) ts->base_type = type; ts->type = TCG_TYPE_I32; ts->temp_allocated = 1; - ts->temp_local = temp_local; + ts->kind = kind; tcg_debug_assert(ts2 == ts + 1); ts2->base_type = TCG_TYPE_I64; ts2->type = TCG_TYPE_I32; ts2->temp_allocated = 1; - ts2->temp_local = temp_local; + ts2->kind = kind; } else { ts->base_type = type; ts->type = type; ts->temp_allocated = 1; - ts->temp_local = temp_local; + ts->kind = kind; } } @@ -XXX,XX +XXX,XX @@ void tcg_temp_free_internal(TCGTemp *ts) } #endif - tcg_debug_assert(ts->temp_global == 0); + tcg_debug_assert(ts->kind < TEMP_GLOBAL); tcg_debug_assert(ts->temp_allocated != 0); ts->temp_allocated = 0; idx = temp_idx(ts); - k = ts->base_type + (ts->temp_local ? TCG_TYPE_COUNT : 0); + k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT); set_bit(idx, s->free_temps[k].l); } @@ -XXX,XX +XXX,XX @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) static void tcg_reg_alloc_start(TCGContext *s) { int i, n; - TCGTemp *ts; - for (i = 0, n = s->nb_globals; i < n; i++) { - ts = &s->temps[i]; - ts->val_type = (ts->fixed_reg ? TEMP_VAL_REG : TEMP_VAL_MEM); - } - for (n = s->nb_temps; i < n; i++) { - ts = &s->temps[i]; - ts->val_type = (ts->temp_local ? TEMP_VAL_MEM : TEMP_VAL_DEAD); - ts->mem_allocated = 0; - ts->fixed_reg = 0; + for (i = 0, n = s->nb_temps; i < n; i++) { + TCGTemp *ts = &s->temps[i]; + TCGTempVal val = TEMP_VAL_MEM; + + switch (ts->kind) { + case TEMP_FIXED: + val = TEMP_VAL_REG; + break; + case TEMP_GLOBAL: + break; + case TEMP_NORMAL: + val = TEMP_VAL_DEAD; + /* fall through */ + case TEMP_LOCAL: + ts->mem_allocated = 0; + break; + default: + g_assert_not_reached(); + } + ts->val_type = val; } memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); @@ -XXX,XX +XXX,XX @@ static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, { int idx = temp_idx(ts); - if (ts->temp_global) { + switch (ts->kind) { + case TEMP_FIXED: + case TEMP_GLOBAL: pstrcpy(buf, buf_size, ts->name); - } else if (ts->temp_local) { + break; + case TEMP_LOCAL: snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); - } else { + break; + case TEMP_NORMAL: snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); + break; } return buf; } @@ -XXX,XX +XXX,XX @@ static void la_bb_end(TCGContext *s, int ng, int nt) { int i; - for (i = 0; i < ng; ++i) { - s->temps[i].state = TS_DEAD | TS_MEM; - la_reset_pref(&s->temps[i]); - } - for (i = ng; i < nt; ++i) { - s->temps[i].state = (s->temps[i].temp_local - ? TS_DEAD | TS_MEM - : TS_DEAD); - la_reset_pref(&s->temps[i]); + for (i = 0; i < nt; ++i) { + TCGTemp *ts = &s->temps[i]; + int state; + + switch (ts->kind) { + case TEMP_FIXED: + case TEMP_GLOBAL: + case TEMP_LOCAL: + state = TS_DEAD | TS_MEM; + break; + case TEMP_NORMAL: + state = TS_DEAD; + break; + default: + g_assert_not_reached(); + } + ts->state = state; + la_reset_pref(ts); } } @@ -XXX,XX +XXX,XX @@ static void la_bb_sync(TCGContext *s, int ng, int nt) la_global_sync(s, ng); for (int i = ng; i < nt; ++i) { - if (s->temps[i].temp_local) { + if (s->temps[i].kind == TEMP_LOCAL) { int state = s->temps[i].state; s->temps[i].state = state | TS_MEM; if (state != TS_DEAD) { @@ -XXX,XX +XXX,XX @@ static void check_regs(TCGContext *s) } for (k = 0; k < s->nb_temps; k++) { ts = &s->temps[k]; - if (ts->val_type == TEMP_VAL_REG && !ts->fixed_reg + if (ts->val_type == TEMP_VAL_REG + && ts->kind != TEMP_FIXED && s->reg_to_temp[ts->reg] != ts) { printf("Inconsistency for temp %s:\n", tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts)); @@ -XXX,XX +XXX,XX @@ static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); mark it free; otherwise mark it dead. */ static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) { - if (ts->fixed_reg) { + if (ts->kind == TEMP_FIXED) { return; } if (ts->val_type == TEMP_VAL_REG) { s->reg_to_temp[ts->reg] = NULL; } ts->val_type = (free_or_dead < 0 - || ts->temp_local - || ts->temp_global + || ts->kind != TEMP_NORMAL ? TEMP_VAL_MEM : TEMP_VAL_DEAD); } @@ -XXX,XX +XXX,XX @@ static inline void temp_dead(TCGContext *s, TCGTemp *ts) static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs, TCGRegSet preferred_regs, int free_or_dead) { - if (ts->fixed_reg) { + if (ts->kind == TEMP_FIXED) { return; } if (!ts->mem_coherent) { @@ -XXX,XX +XXX,XX @@ static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) { /* The liveness analysis already ensures that globals are back in memory. Keep an tcg_debug_assert for safety. */ - tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || ts->fixed_reg); + tcg_debug_assert(ts->val_type == TEMP_VAL_MEM + || ts->kind == TEMP_FIXED); } /* save globals to their canonical location and assume they can be @@ -XXX,XX +XXX,XX @@ static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) for (i = 0, n = s->nb_globals; i < n; i++) { TCGTemp *ts = &s->temps[i]; tcg_debug_assert(ts->val_type != TEMP_VAL_REG - || ts->fixed_reg + || ts->kind == TEMP_FIXED || ts->mem_coherent); } } @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) for (i = s->nb_globals; i < s->nb_temps; i++) { TCGTemp *ts = &s->temps[i]; - if (ts->temp_local) { + if (ts->kind == TEMP_LOCAL) { temp_save(s, ts, allocated_regs); } else { /* The liveness analysis already ensures that temps are dead. @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs) * The liveness analysis already ensures that temps are dead. * Keep tcg_debug_asserts for safety. */ - if (ts->temp_local) { + if (ts->kind == TEMP_LOCAL) { tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent); } else { tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, TCGRegSet preferred_regs) { /* ENV should not be modified. */ - tcg_debug_assert(!ots->fixed_reg); + tcg_debug_assert(ots->kind != TEMP_FIXED); /* The movi is not explicitly generated here. */ if (ots->val_type == TEMP_VAL_REG) { @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) ts = arg_temp(op->args[1]); /* ENV should not be modified. */ - tcg_debug_assert(!ots->fixed_reg); + tcg_debug_assert(ots->kind != TEMP_FIXED); /* Note that otype != itype for no-op truncation. */ otype = ots->type; @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) } temp_dead(s, ots); } else { - if (IS_DEAD_ARG(1) && !ts->fixed_reg) { + if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) { /* the mov can be suppressed */ if (ots->val_type == TEMP_VAL_REG) { s->reg_to_temp[ots->reg] = NULL; @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) * Store the source register into the destination slot * and leave the destination temp as TEMP_VAL_MEM. */ - assert(!ots->fixed_reg); + assert(ots->kind != TEMP_FIXED); if (!ts->mem_allocated) { temp_allocate_frame(s, ots); } @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) its = arg_temp(op->args[1]); /* ENV should not be modified. */ - tcg_debug_assert(!ots->fixed_reg); + tcg_debug_assert(ots->kind != TEMP_FIXED); itype = its->type; vece = TCGOP_VECE(op); @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) i_preferred_regs = o_preferred_regs = 0; if (arg_ct->ialias) { o_preferred_regs = op->output_pref[arg_ct->alias_index]; - if (ts->fixed_reg) { + if (ts->kind == TEMP_FIXED) { /* if fixed register, we must allocate a new register if the alias is not the same register */ if (arg != op->args[arg_ct->alias_index]) { @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) ts = arg_temp(arg); /* ENV should not be modified. */ - tcg_debug_assert(!ts->fixed_reg); + tcg_debug_assert(ts->kind != TEMP_FIXED); if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { reg = new_args[arg_ct->alias_index]; @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) ts = arg_temp(op->args[i]); /* ENV should not be modified. */ - tcg_debug_assert(!ts->fixed_reg); + tcg_debug_assert(ts->kind != TEMP_FIXED); if (NEED_SYNC_ARG(i)) { temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i)); @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) ts = arg_temp(arg); /* ENV should not be modified. */ - tcg_debug_assert(!ts->fixed_reg); + tcg_debug_assert(ts->kind != TEMP_FIXED); reg = tcg_target_call_oarg_regs[i]; tcg_debug_assert(s->reg_to_temp[reg] == NULL); -- 2.25.1 In most, but not all, places that we check for TEMP_FIXED, we are really testing that we do not modify the temporary. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg.h | 5 +++++ tcg/tcg.c | 21 ++++++++++----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ struct TCGContext { target_ulong gen_insn_data[TCG_MAX_INSNS][TARGET_INSN_START_WORDS]; }; +static inline bool temp_readonly(TCGTemp *ts) +{ + return ts->kind == TEMP_FIXED; +} + extern TCGContext tcg_init_ctx; extern __thread TCGContext *tcg_ctx; extern const void *tcg_code_gen_epilogue; diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); mark it free; otherwise mark it dead. */ static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) { - if (ts->kind == TEMP_FIXED) { + if (temp_readonly(ts)) { return; } if (ts->val_type == TEMP_VAL_REG) { @@ -XXX,XX +XXX,XX @@ static inline void temp_dead(TCGContext *s, TCGTemp *ts) static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs, TCGRegSet preferred_regs, int free_or_dead) { - if (ts->kind == TEMP_FIXED) { + if (temp_readonly(ts)) { return; } if (!ts->mem_coherent) { @@ -XXX,XX +XXX,XX @@ static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) { /* The liveness analysis already ensures that globals are back in memory. Keep an tcg_debug_assert for safety. */ - tcg_debug_assert(ts->val_type == TEMP_VAL_MEM - || ts->kind == TEMP_FIXED); + tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts)); } /* save globals to their canonical location and assume they can be @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, TCGRegSet preferred_regs) { /* ENV should not be modified. */ - tcg_debug_assert(ots->kind != TEMP_FIXED); + tcg_debug_assert(!temp_readonly(ots)); /* The movi is not explicitly generated here. */ if (ots->val_type == TEMP_VAL_REG) { @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) ts = arg_temp(op->args[1]); /* ENV should not be modified. */ - tcg_debug_assert(ots->kind != TEMP_FIXED); + tcg_debug_assert(!temp_readonly(ots)); /* Note that otype != itype for no-op truncation. */ otype = ots->type; @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) * Store the source register into the destination slot * and leave the destination temp as TEMP_VAL_MEM. */ - assert(ots->kind != TEMP_FIXED); + assert(!temp_readonly(ots)); if (!ts->mem_allocated) { temp_allocate_frame(s, ots); } @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) its = arg_temp(op->args[1]); /* ENV should not be modified. */ - tcg_debug_assert(ots->kind != TEMP_FIXED); + tcg_debug_assert(!temp_readonly(ots)); itype = its->type; vece = TCGOP_VECE(op); @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) ts = arg_temp(arg); /* ENV should not be modified. */ - tcg_debug_assert(ts->kind != TEMP_FIXED); + tcg_debug_assert(!temp_readonly(ts)); if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { reg = new_args[arg_ct->alias_index]; @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) ts = arg_temp(op->args[i]); /* ENV should not be modified. */ - tcg_debug_assert(ts->kind != TEMP_FIXED); + tcg_debug_assert(!temp_readonly(ts)); if (NEED_SYNC_ARG(i)) { temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i)); @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) ts = arg_temp(arg); /* ENV should not be modified. */ - tcg_debug_assert(ts->kind != TEMP_FIXED); + tcg_debug_assert(!temp_readonly(ts)); reg = tcg_target_call_oarg_regs[i]; tcg_debug_assert(s->reg_to_temp[reg] == NULL); -- 2.25.1 This will reduce the differences between 32-bit and 64-bit hosts, allowing full 64-bit constants to be created with the same interface. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg.h | 2 +- tcg/tcg.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ typedef struct TCGTemp { unsigned int mem_allocated:1; unsigned int temp_allocated:1; - tcg_target_long val; + int64_t val; struct TCGTemp *mem_base; intptr_t mem_offset; const char *name; diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void dump_regs(TCGContext *s) tcg_target_reg_names[ts->mem_base->reg]); break; case TEMP_VAL_CONST: - printf("$0x%" TCG_PRIlx, ts->val); + printf("$0x%" PRIx64, ts->val); break; case TEMP_VAL_DEAD: printf("D"); -- 2.25.1 Fix this name vs our coding style. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ glue(glue(case INDEX_op_, x), _i64): \ glue(glue(case INDEX_op_, x), _vec) -struct tcg_temp_info { +typedef struct TempOptInfo { bool is_const; TCGTemp *prev_copy; TCGTemp *next_copy; tcg_target_ulong val; tcg_target_ulong mask; -}; +} TempOptInfo; -static inline struct tcg_temp_info *ts_info(TCGTemp *ts) +static inline TempOptInfo *ts_info(TCGTemp *ts) { return ts->state_ptr; } -static inline struct tcg_temp_info *arg_info(TCGArg arg) +static inline TempOptInfo *arg_info(TCGArg arg) { return ts_info(arg_temp(arg)); } @@ -XXX,XX +XXX,XX @@ static inline bool ts_is_copy(TCGTemp *ts) /* Reset TEMP's state, possibly removing the temp for the list of copies. */ static void reset_ts(TCGTemp *ts) { - struct tcg_temp_info *ti = ts_info(ts); - struct tcg_temp_info *pi = ts_info(ti->prev_copy); - struct tcg_temp_info *ni = ts_info(ti->next_copy); + TempOptInfo *ti = ts_info(ts); + TempOptInfo *pi = ts_info(ti->prev_copy); + TempOptInfo *ni = ts_info(ti->next_copy); ni->prev_copy = ti->prev_copy; pi->next_copy = ti->next_copy; @@ -XXX,XX +XXX,XX @@ static void reset_temp(TCGArg arg) } /* Initialize and activate a temporary. */ -static void init_ts_info(struct tcg_temp_info *infos, +static void init_ts_info(TempOptInfo *infos, TCGTempSet *temps_used, TCGTemp *ts) { size_t idx = temp_idx(ts); if (!test_bit(idx, temps_used->l)) { - struct tcg_temp_info *ti = &infos[idx]; + TempOptInfo *ti = &infos[idx]; ts->state_ptr = ti; ti->next_copy = ts; @@ -XXX,XX +XXX,XX @@ static void init_ts_info(struct tcg_temp_info *infos, } } -static void init_arg_info(struct tcg_temp_info *infos, +static void init_arg_info(TempOptInfo *infos, TCGTempSet *temps_used, TCGArg arg) { init_ts_info(infos, temps_used, arg_temp(arg)); @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_movi(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg val) const TCGOpDef *def; TCGOpcode new_op; tcg_target_ulong mask; - struct tcg_temp_info *di = arg_info(dst); + TempOptInfo *di = arg_info(dst); def = &tcg_op_defs[op->opc]; if (def->flags & TCG_OPF_VECTOR) { @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) TCGTemp *dst_ts = arg_temp(dst); TCGTemp *src_ts = arg_temp(src); const TCGOpDef *def; - struct tcg_temp_info *di; - struct tcg_temp_info *si; + TempOptInfo *di; + TempOptInfo *si; tcg_target_ulong mask; TCGOpcode new_op; @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) di->mask = mask; if (src_ts->type == dst_ts->type) { - struct tcg_temp_info *ni = ts_info(si->next_copy); + TempOptInfo *ni = ts_info(si->next_copy); di->next_copy = si->next_copy; di->prev_copy = src_ts; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) { int nb_temps, nb_globals; TCGOp *op, *op_next, *prev_mb = NULL; - struct tcg_temp_info *infos; + TempOptInfo *infos; TCGTempSet temps_used; /* Array VALS has an element for each temp. @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) nb_temps = s->nb_temps; nb_globals = s->nb_globals; bitmap_zero(temps_used.l, nb_temps); - infos = tcg_malloc(sizeof(struct tcg_temp_info) * nb_temps); + infos = tcg_malloc(sizeof(TempOptInfo) * nb_temps); QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { tcg_target_ulong mask, partmask, affected; -- 2.25.1 This propagates the extended value of TCGTemp.val that we did before. In addition, it will be required for vector constants. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { bool is_const; TCGTemp *prev_copy; TCGTemp *next_copy; - tcg_target_ulong val; - tcg_target_ulong mask; + uint64_t val; + uint64_t mask; } TempOptInfo; static inline TempOptInfo *ts_info(TCGTemp *ts) @@ -XXX,XX +XXX,XX @@ static bool args_are_copies(TCGArg arg1, TCGArg arg2) return ts_are_copies(arg_temp(arg1), arg_temp(arg2)); } -static void tcg_opt_gen_movi(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg val) +static void tcg_opt_gen_movi(TCGContext *s, TCGOp *op, TCGArg dst, uint64_t val) { const TCGOpDef *def; TCGOpcode new_op; - tcg_target_ulong mask; + uint64_t mask; TempOptInfo *di = arg_info(dst); def = &tcg_op_defs[op->opc]; @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) const TCGOpDef *def; TempOptInfo *di; TempOptInfo *si; - tcg_target_ulong mask; + uint64_t mask; TCGOpcode new_op; if (ts_are_copies(dst_ts, src_ts)) { @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) } } -static TCGArg do_constant_folding_2(TCGOpcode op, TCGArg x, TCGArg y) +static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y) { uint64_t l64, h64; @@ -XXX,XX +XXX,XX @@ static TCGArg do_constant_folding_2(TCGOpcode op, TCGArg x, TCGArg y) } } -static TCGArg do_constant_folding(TCGOpcode op, TCGArg x, TCGArg y) +static uint64_t do_constant_folding(TCGOpcode op, uint64_t x, uint64_t y) { const TCGOpDef *def = &tcg_op_defs[op]; - TCGArg res = do_constant_folding_2(op, x, y); + uint64_t res = do_constant_folding_2(op, x, y); if (!(def->flags & TCG_OPF_64BIT)) { res = (int32_t)res; } @@ -XXX,XX +XXX,XX @@ static bool do_constant_folding_cond_eq(TCGCond c) static TCGArg do_constant_folding_cond(TCGOpcode op, TCGArg x, TCGArg y, TCGCond c) { - tcg_target_ulong xv = arg_info(x)->val; - tcg_target_ulong yv = arg_info(y)->val; + uint64_t xv = arg_info(x)->val; + uint64_t yv = arg_info(y)->val; + if (arg_is_const(x) && arg_is_const(y)) { const TCGOpDef *def = &tcg_op_defs[op]; tcg_debug_assert(!(def->flags & TCG_OPF_VECTOR)); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) infos = tcg_malloc(sizeof(TempOptInfo) * nb_temps); QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { - tcg_target_ulong mask, partmask, affected; + uint64_t mask, partmask, affected, tmp; int nb_oargs, nb_iargs, i; - TCGArg tmp; TCGOpcode opc = op->opc; const TCGOpDef *def = &tcg_op_defs[opc]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(extract2): if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - TCGArg v1 = arg_info(op->args[1])->val; - TCGArg v2 = arg_info(op->args[2])->val; + uint64_t v1 = arg_info(op->args[1])->val; + uint64_t v2 = arg_info(op->args[2])->val; + int shr = op->args[3]; if (opc == INDEX_op_extract2_i64) { - tmp = (v1 >> op->args[3]) | (v2 << (64 - op->args[3])); + tmp = (v1 >> shr) | (v2 << (64 - shr)); } else { - tmp = (int32_t)(((uint32_t)v1 >> op->args[3]) | - ((uint32_t)v2 << (32 - op->args[3]))); + tmp = (int32_t)(((uint32_t)v1 >> shr) | + ((uint32_t)v2 << (32 - shr))); } tcg_opt_gen_movi(s, op, op->args[0], tmp); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { - tcg_target_ulong tv = arg_info(op->args[3])->val; - tcg_target_ulong fv = arg_info(op->args[4])->val; + uint64_t tv = arg_info(op->args[3])->val; + uint64_t fv = arg_info(op->args[4])->val; TCGCond cond = op->args[5]; + if (fv == 1 && tv == 0) { cond = tcg_invert_cond(cond); } else if (!(tv == 1 && fv == 0)) { -- 2.25.1 These will hold a single constant for the duration of the TB. They are hashed, so that each value has one temp across the TB. Not used yet, this is all infrastructure. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg.h | 24 ++++- tcg/optimize.c | 13 ++- tcg/tcg.c | 224 ++++++++++++++++++++++++++++++++++++---------- 3 files changed, 211 insertions(+), 50 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ typedef enum TCGTempKind { TEMP_GLOBAL, /* Temp is in a fixed register. */ TEMP_FIXED, + /* Temp is a fixed constant. */ + TEMP_CONST, } TCGTempKind; typedef struct TCGTemp { @@ -XXX,XX +XXX,XX @@ struct TCGContext { QSIMPLEQ_HEAD(, TCGOp) plugin_ops; #endif + GHashTable *const_table[TCG_TYPE_COUNT]; TCGTempSet free_temps[TCG_TYPE_COUNT * 2]; TCGTemp temps[TCG_MAX_TEMPS]; /* globals first, temps after */ @@ -XXX,XX +XXX,XX @@ struct TCGContext { static inline bool temp_readonly(TCGTemp *ts) { - return ts->kind == TEMP_FIXED; + return ts->kind >= TEMP_FIXED; } extern TCGContext tcg_init_ctx; @@ -XXX,XX +XXX,XX @@ TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *op, TCGOpcode opc); void tcg_optimize(TCGContext *s); +/* Allocate a new temporary and initialize it with a constant. */ TCGv_i32 tcg_const_i32(int32_t val); TCGv_i64 tcg_const_i64(int64_t val); TCGv_i32 tcg_const_local_i32(int32_t val); @@ -XXX,XX +XXX,XX @@ TCGv_vec tcg_const_ones_vec(TCGType); TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec); TCGv_vec tcg_const_ones_vec_matching(TCGv_vec); +/* + * Locate or create a read-only temporary that is a constant. + * This kind of temporary need not and should not be freed. + */ +TCGTemp *tcg_constant_internal(TCGType type, int64_t val); + +static inline TCGv_i32 tcg_constant_i32(int32_t val) +{ + return temp_tcgv_i32(tcg_constant_internal(TCG_TYPE_I32, val)); +} + +static inline TCGv_i64 tcg_constant_i64(int64_t val) +{ + return temp_tcgv_i64(tcg_constant_internal(TCG_TYPE_I64, val)); +} + +TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val); + #if UINTPTR_MAX == UINT32_MAX # define tcg_const_ptr(x) ((TCGv_ptr)tcg_const_i32((intptr_t)(x))) # define tcg_const_local_ptr(x) ((TCGv_ptr)tcg_const_local_i32((intptr_t)(x))) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void init_ts_info(TempOptInfo *infos, ts->state_ptr = ti; ti->next_copy = ts; ti->prev_copy = ts; - ti->is_const = false; - ti->mask = -1; + if (ts->kind == TEMP_CONST) { + ti->is_const = true; + ti->val = ti->mask = ts->val; + if (TCG_TARGET_REG_BITS > 32 && ts->type == TCG_TYPE_I32) { + /* High bits of a 32-bit quantity are garbage. */ + ti->mask |= ~0xffffffffull; + } + } else { + ti->is_const = false; + ti->mask = -1; + } set_bit(idx, temps_used->l); } } diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ void tcg_func_start(TCGContext *s) /* No temps have been previously allocated for size or locality. */ memset(s->free_temps, 0, sizeof(s->free_temps)); + /* No constant temps have been previously allocated. */ + for (int i = 0; i < TCG_TYPE_COUNT; ++i) { + if (s->const_table[i]) { + g_hash_table_remove_all(s->const_table[i]); + } + } + s->nb_ops = 0; s->nb_labels = 0; s->current_frame_offset = s->frame_start; @@ -XXX,XX +XXX,XX @@ TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, bigendian = 1; #endif - if (base_ts->kind != TEMP_FIXED) { + switch (base_ts->kind) { + case TEMP_FIXED: + break; + case TEMP_GLOBAL: /* We do not support double-indirect registers. */ tcg_debug_assert(!base_ts->indirect_reg); base_ts->indirect_base = 1; s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64 ? 2 : 1); indirect_reg = 1; + break; + default: + g_assert_not_reached(); } if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { @@ -XXX,XX +XXX,XX @@ void tcg_temp_free_internal(TCGTemp *ts) TCGContext *s = tcg_ctx; int k, idx; + /* In order to simplify users of tcg_constant_*, silently ignore free. */ + if (ts->kind == TEMP_CONST) { + return; + } + #if defined(CONFIG_DEBUG_TCG) s->temps_in_use--; if (s->temps_in_use < 0) { @@ -XXX,XX +XXX,XX @@ void tcg_temp_free_internal(TCGTemp *ts) set_bit(idx, s->free_temps[k].l); } +TCGTemp *tcg_constant_internal(TCGType type, int64_t val) +{ + TCGContext *s = tcg_ctx; + GHashTable *h = s->const_table[type]; + TCGTemp *ts; + + if (h == NULL) { + h = g_hash_table_new(g_int64_hash, g_int64_equal); + s->const_table[type] = h; + } + + ts = g_hash_table_lookup(h, &val); + if (ts == NULL) { + ts = tcg_temp_alloc(s); + + if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { + TCGTemp *ts2 = tcg_temp_alloc(s); + + ts->base_type = TCG_TYPE_I64; + ts->type = TCG_TYPE_I32; + ts->kind = TEMP_CONST; + ts->temp_allocated = 1; + /* + * Retain the full value of the 64-bit constant in the low + * part, so that the hash table works. Actual uses will + * truncate the value to the low part. + */ + ts->val = val; + + tcg_debug_assert(ts2 == ts + 1); + ts2->base_type = TCG_TYPE_I64; + ts2->type = TCG_TYPE_I32; + ts2->kind = TEMP_CONST; + ts2->temp_allocated = 1; + ts2->val = val >> 32; + } else { + ts->base_type = type; + ts->type = type; + ts->kind = TEMP_CONST; + ts->temp_allocated = 1; + ts->val = val; + } + g_hash_table_insert(h, &ts->val, ts); + } + + return ts; +} + +TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val) +{ + val = dup_const(vece, val); + return temp_tcgv_vec(tcg_constant_internal(type, val)); +} + TCGv_i32 tcg_const_i32(int32_t val) { TCGv_i32 t0; @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_start(TCGContext *s) TCGTempVal val = TEMP_VAL_MEM; switch (ts->kind) { + case TEMP_CONST: + val = TEMP_VAL_CONST; + break; case TEMP_FIXED: val = TEMP_VAL_REG; break; @@ -XXX,XX +XXX,XX @@ static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, case TEMP_NORMAL: snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); break; + case TEMP_CONST: + switch (ts->type) { + case TCG_TYPE_I32: + snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val); + break; +#if TCG_TARGET_REG_BITS > 32 + case TCG_TYPE_I64: + snprintf(buf, buf_size, "$0x%" PRIx64, ts->val); + break; +#endif + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + snprintf(buf, buf_size, "v%d$0x%" PRIx64, + 64 << (ts->type - TCG_TYPE_V64), ts->val); + break; + default: + g_assert_not_reached(); + } + break; } return buf; } @@ -XXX,XX +XXX,XX @@ static void la_bb_end(TCGContext *s, int ng, int nt) state = TS_DEAD | TS_MEM; break; case TEMP_NORMAL: + case TEMP_CONST: state = TS_DEAD; break; default: @@ -XXX,XX +XXX,XX @@ static void la_bb_sync(TCGContext *s, int ng, int nt) la_global_sync(s, ng); for (int i = ng; i < nt; ++i) { - if (s->temps[i].kind == TEMP_LOCAL) { - int state = s->temps[i].state; - s->temps[i].state = state | TS_MEM; + TCGTemp *ts = &s->temps[i]; + int state; + + switch (ts->kind) { + case TEMP_LOCAL: + state = ts->state; + ts->state = state | TS_MEM; if (state != TS_DEAD) { continue; } - } else { + break; + case TEMP_NORMAL: s->temps[i].state = TS_DEAD; + break; + case TEMP_CONST: + continue; + default: + g_assert_not_reached(); } la_reset_pref(&s->temps[i]); } @@ -XXX,XX +XXX,XX @@ static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); mark it free; otherwise mark it dead. */ static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) { - if (temp_readonly(ts)) { + TCGTempVal new_type; + + switch (ts->kind) { + case TEMP_FIXED: return; + case TEMP_GLOBAL: + case TEMP_LOCAL: + new_type = TEMP_VAL_MEM; + break; + case TEMP_NORMAL: + new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD; + break; + case TEMP_CONST: + new_type = TEMP_VAL_CONST; + break; + default: + g_assert_not_reached(); } if (ts->val_type == TEMP_VAL_REG) { s->reg_to_temp[ts->reg] = NULL; } - ts->val_type = (free_or_dead < 0 - || ts->kind != TEMP_NORMAL - ? TEMP_VAL_MEM : TEMP_VAL_DEAD); + ts->val_type = new_type; } /* Mark a temporary as dead. */ @@ -XXX,XX +XXX,XX @@ static inline void temp_dead(TCGContext *s, TCGTemp *ts) static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs, TCGRegSet preferred_regs, int free_or_dead) { - if (temp_readonly(ts)) { - return; - } - if (!ts->mem_coherent) { + if (!temp_readonly(ts) && !ts->mem_coherent) { if (!ts->mem_allocated) { temp_allocate_frame(s, ts); } @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) for (i = s->nb_globals; i < s->nb_temps; i++) { TCGTemp *ts = &s->temps[i]; - if (ts->kind == TEMP_LOCAL) { + + switch (ts->kind) { + case TEMP_LOCAL: temp_save(s, ts, allocated_regs); - } else { + break; + case TEMP_NORMAL: /* The liveness analysis already ensures that temps are dead. Keep an tcg_debug_assert for safety. */ tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); + break; + case TEMP_CONST: + /* Similarly, we should have freed any allocated register. */ + tcg_debug_assert(ts->val_type == TEMP_VAL_CONST); + break; + default: + g_assert_not_reached(); } } @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs) * The liveness analysis already ensures that temps are dead. * Keep tcg_debug_asserts for safety. */ - if (ts->kind == TEMP_LOCAL) { + switch (ts->kind) { + case TEMP_LOCAL: tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent); - } else { + break; + case TEMP_NORMAL: tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); + break; + case TEMP_CONST: + break; + default: + g_assert_not_reached(); } } } @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) i_preferred_regs = o_preferred_regs = 0; if (arg_ct->ialias) { o_preferred_regs = op->output_pref[arg_ct->alias_index]; - if (ts->kind == TEMP_FIXED) { - /* if fixed register, we must allocate a new register - if the alias is not the same register */ - if (arg != op->args[arg_ct->alias_index]) { - goto allocate_in_reg; - } - } else { - /* if the input is aliased to an output and if it is - not dead after the instruction, we must allocate - a new register and move it */ - if (!IS_DEAD_ARG(i)) { - goto allocate_in_reg; - } - /* check if the current register has already been allocated - for another input aliased to an output */ - if (ts->val_type == TEMP_VAL_REG) { - int k2, i2; - reg = ts->reg; - for (k2 = 0 ; k2 < k ; k2++) { - i2 = def->args_ct[nb_oargs + k2].sort_index; - if (def->args_ct[i2].ialias && reg == new_args[i2]) { - goto allocate_in_reg; - } + /* + * If the input is readonly, then it cannot also be an + * output and aliased to itself. If the input is not + * dead after the instruction, we must allocate a new + * register and move it. + */ + if (temp_readonly(ts) || !IS_DEAD_ARG(i)) { + goto allocate_in_reg; + } + + /* + * Check if the current register has already been allocated + * for another input aliased to an output. + */ + if (ts->val_type == TEMP_VAL_REG) { + reg = ts->reg; + for (int k2 = 0; k2 < k; k2++) { + int i2 = def->args_ct[nb_oargs + k2].sort_index; + if (def->args_ct[i2].ialias && reg == new_args[i2]) { + goto allocate_in_reg; } } - i_preferred_regs = o_preferred_regs; } + i_preferred_regs = o_preferred_regs; } temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs); reg = ts->reg; - if (tcg_regset_test_reg(arg_ct->regs, reg)) { - /* nothing to do : the constraint is satisfied */ - } else { - allocate_in_reg: - /* allocate a new register matching the constraint - and move the temporary register into it */ + if (!tcg_regset_test_reg(arg_ct->regs, reg)) { + allocate_in_reg: + /* + * Allocate a new register matching the constraint + * and move the temporary register into it. + */ temp_load(s, ts, tcg_target_available_regs[ts->type], i_allocated_regs, 0); reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs, -- 2.25.1 Prefer TEMP_CONST over anything else. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void init_arg_info(TempOptInfo *infos, static TCGTemp *find_better_copy(TCGContext *s, TCGTemp *ts) { - TCGTemp *i; + TCGTemp *i, *g, *l; - /* If this is already a global, we can't do better. */ - if (ts->kind >= TEMP_GLOBAL) { + /* If this is already readonly, we can't do better. */ + if (temp_readonly(ts)) { return ts; } - /* Search for a global first. */ + g = l = NULL; for (i = ts_info(ts)->next_copy; i != ts; i = ts_info(i)->next_copy) { - if (i->kind >= TEMP_GLOBAL) { + if (temp_readonly(i)) { return i; - } - } - - /* If it is a temp, search for a temp local. */ - if (ts->kind == TEMP_NORMAL) { - for (i = ts_info(ts)->next_copy; i != ts; i = ts_info(i)->next_copy) { - if (i->kind >= TEMP_LOCAL) { - return i; + } else if (i->kind > ts->kind) { + if (i->kind == TEMP_GLOBAL) { + g = i; + } else if (i->kind == TEMP_LOCAL) { + l = i; } } } - /* Failure to find a better representation, return the same temp. */ - return ts; + /* If we didn't find a better representation, return the same temp. */ + return g ? g : l ? l : ts; } static bool ts_are_copies(TCGTemp *ts1, TCGTemp *ts2) -- 2.25.1 Do not allocate a large block for indexing. Instead, allocate for each temporary as they are seen. In general, this will use less memory, if we consider that most TBs do not touch every target register. This also allows us to allocate TempOptInfo for new temps created during optimization. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 60 ++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void reset_temp(TCGArg arg) } /* Initialize and activate a temporary. */ -static void init_ts_info(TempOptInfo *infos, - TCGTempSet *temps_used, TCGTemp *ts) +static void init_ts_info(TCGTempSet *temps_used, TCGTemp *ts) { size_t idx = temp_idx(ts); - if (!test_bit(idx, temps_used->l)) { - TempOptInfo *ti = &infos[idx]; + TempOptInfo *ti; + if (test_bit(idx, temps_used->l)) { + return; + } + set_bit(idx, temps_used->l); + + ti = ts->state_ptr; + if (ti == NULL) { + ti = tcg_malloc(sizeof(TempOptInfo)); ts->state_ptr = ti; - ti->next_copy = ts; - ti->prev_copy = ts; - if (ts->kind == TEMP_CONST) { - ti->is_const = true; - ti->val = ti->mask = ts->val; - if (TCG_TARGET_REG_BITS > 32 && ts->type == TCG_TYPE_I32) { - /* High bits of a 32-bit quantity are garbage. */ - ti->mask |= ~0xffffffffull; - } - } else { - ti->is_const = false; - ti->mask = -1; + } + + ti->next_copy = ts; + ti->prev_copy = ts; + if (ts->kind == TEMP_CONST) { + ti->is_const = true; + ti->val = ts->val; + ti->mask = ts->val; + if (TCG_TARGET_REG_BITS > 32 && ts->type == TCG_TYPE_I32) { + /* High bits of a 32-bit quantity are garbage. */ + ti->mask |= ~0xffffffffull; } - set_bit(idx, temps_used->l); + } else { + ti->is_const = false; + ti->mask = -1; } } -static void init_arg_info(TempOptInfo *infos, - TCGTempSet *temps_used, TCGArg arg) +static void init_arg_info(TCGTempSet *temps_used, TCGArg arg) { - init_ts_info(infos, temps_used, arg_temp(arg)); + init_ts_info(temps_used, arg_temp(arg)); } static TCGTemp *find_better_copy(TCGContext *s, TCGTemp *ts) @@ -XXX,XX +XXX,XX @@ static bool swap_commutative2(TCGArg *p1, TCGArg *p2) /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { - int nb_temps, nb_globals; + int nb_temps, nb_globals, i; TCGOp *op, *op_next, *prev_mb = NULL; - TempOptInfo *infos; TCGTempSet temps_used; /* Array VALS has an element for each temp. @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) nb_temps = s->nb_temps; nb_globals = s->nb_globals; + bitmap_zero(temps_used.l, nb_temps); - infos = tcg_malloc(sizeof(TempOptInfo) * nb_temps); + for (i = 0; i < nb_temps; ++i) { + s->temps[i].state_ptr = NULL; + } QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { uint64_t mask, partmask, affected, tmp; - int nb_oargs, nb_iargs, i; + int nb_oargs, nb_iargs; TCGOpcode opc = op->opc; const TCGOpDef *def = &tcg_op_defs[opc]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) for (i = 0; i < nb_oargs + nb_iargs; i++) { TCGTemp *ts = arg_temp(op->args[i]); if (ts) { - init_ts_info(infos, &temps_used, ts); + init_ts_info(&temps_used, ts); } } } else { nb_oargs = def->nb_oargs; nb_iargs = def->nb_iargs; for (i = 0; i < nb_oargs + nb_iargs; i++) { - init_arg_info(infos, &temps_used, op->args[i]); + init_arg_info(&temps_used, op->args[i]); } } -- 2.25.1 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 108 ++++++++++++++++++++++--------------------------- 1 file changed, 49 insertions(+), 59 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool args_are_copies(TCGArg arg1, TCGArg arg2) return ts_are_copies(arg_temp(arg1), arg_temp(arg2)); } -static void tcg_opt_gen_movi(TCGContext *s, TCGOp *op, TCGArg dst, uint64_t val) -{ - const TCGOpDef *def; - TCGOpcode new_op; - uint64_t mask; - TempOptInfo *di = arg_info(dst); - - def = &tcg_op_defs[op->opc]; - if (def->flags & TCG_OPF_VECTOR) { - new_op = INDEX_op_dupi_vec; - } else if (def->flags & TCG_OPF_64BIT) { - new_op = INDEX_op_movi_i64; - } else { - new_op = INDEX_op_movi_i32; - } - op->opc = new_op; - /* TCGOP_VECL and TCGOP_VECE remain unchanged. */ - op->args[0] = dst; - op->args[1] = val; - - reset_temp(dst); - di->is_const = true; - di->val = val; - mask = val; - if (TCG_TARGET_REG_BITS > 32 && new_op == INDEX_op_movi_i32) { - /* High bits of the destination are now garbage. */ - mask |= ~0xffffffffull; - } - di->mask = mask; -} - static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) { TCGTemp *dst_ts = arg_temp(dst); @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) } } +static void tcg_opt_gen_movi(TCGContext *s, TCGTempSet *temps_used, + TCGOp *op, TCGArg dst, uint64_t val) +{ + const TCGOpDef *def = &tcg_op_defs[op->opc]; + TCGType type; + TCGTemp *tv; + + if (def->flags & TCG_OPF_VECTOR) { + type = TCGOP_VECL(op) + TCG_TYPE_V64; + } else if (def->flags & TCG_OPF_64BIT) { + type = TCG_TYPE_I64; + } else { + type = TCG_TYPE_I32; + } + + /* Convert movi to mov with constant temp. */ + tv = tcg_constant_internal(type, val); + init_ts_info(temps_used, tv); + tcg_opt_gen_mov(s, op, dst, temp_arg(tv)); +} + static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y) { uint64_t l64, h64; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) nb_temps = s->nb_temps; nb_globals = s->nb_globals; - bitmap_zero(temps_used.l, nb_temps); + memset(&temps_used, 0, sizeof(temps_used)); for (i = 0; i < nb_temps; ++i) { s->temps[i].state_ptr = NULL; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(rotr): if (arg_is_const(op->args[1]) && arg_info(op->args[1])->val == 0) { - tcg_opt_gen_movi(s, op, op->args[0], 0); + tcg_opt_gen_movi(s, &temps_used, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (partmask == 0) { tcg_debug_assert(nb_oargs == 1); - tcg_opt_gen_movi(s, op, op->args[0], 0); + tcg_opt_gen_movi(s, &temps_used, op, op->args[0], 0); continue; } if (affected == 0) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(mulsh): if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == 0) { - tcg_opt_gen_movi(s, op, op->args[0], 0); + tcg_opt_gen_movi(s, &temps_used, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(sub): CASE_OP_32_64_VEC(xor): if (args_are_copies(op->args[1], op->args[2])) { - tcg_opt_gen_movi(s, op, op->args[0], 0); + tcg_opt_gen_movi(s, &temps_used, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; CASE_OP_32_64(movi): case INDEX_op_dupi_vec: - tcg_opt_gen_movi(s, op, op->args[0], op->args[1]); + tcg_opt_gen_movi(s, &temps_used, op, op->args[0], op->args[1]); break; case INDEX_op_dup_vec: if (arg_is_const(op->args[1])) { tmp = arg_info(op->args[1])->val; tmp = dup_const(TCGOP_VECE(op), tmp); - tcg_opt_gen_movi(s, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { tmp = arg_info(op->args[1])->val; if (tmp == arg_info(op->args[2])->val) { - tcg_opt_gen_movi(s, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); break; } } else if (args_are_copies(op->args[1], op->args[2])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_extrh_i64_i32: if (arg_is_const(op->args[1])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, 0); - tcg_opt_gen_movi(s, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, arg_info(op->args[2])->val); - tcg_opt_gen_movi(s, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) TCGArg v = arg_info(op->args[1])->val; if (v != 0) { tmp = do_constant_folding(opc, v, 0); - tcg_opt_gen_movi(s, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); } else { tcg_opt_gen_mov(s, op, op->args[0], op->args[2]); } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = deposit64(arg_info(op->args[1])->val, op->args[3], op->args[4], arg_info(op->args[2])->val); - tcg_opt_gen_movi(s, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = extract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); - tcg_opt_gen_movi(s, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = sextract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); - tcg_opt_gen_movi(s, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = (int32_t)(((uint32_t)v1 >> shr) | ((uint32_t)v2 << (32 - shr))); } - tcg_opt_gen_movi(s, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[3]); if (tmp != 2) { - tcg_opt_gen_movi(s, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[1], op->args[2]); if (tmp != 2) { if (tmp) { - bitmap_zero(temps_used.l, nb_temps); + memset(&temps_used, 0, sizeof(temps_used)); op->opc = INDEX_op_br; op->args[0] = op->args[3]; } else { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) uint64_t a = ((uint64_t)ah << 32) | al; uint64_t b = ((uint64_t)bh << 32) | bl; TCGArg rl, rh; - TCGOp *op2 = tcg_op_insert_before(s, op, INDEX_op_movi_i32); + TCGOp *op2 = tcg_op_insert_before(s, op, INDEX_op_mov_i32); if (opc == INDEX_op_add2_i32) { a += b; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(s, op, rl, (int32_t)a); - tcg_opt_gen_movi(s, op2, rh, (int32_t)(a >> 32)); + tcg_opt_gen_movi(s, &temps_used, op, rl, (int32_t)a); + tcg_opt_gen_movi(s, &temps_used, op2, rh, (int32_t)(a >> 32)); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) uint32_t b = arg_info(op->args[3])->val; uint64_t r = (uint64_t)a * b; TCGArg rl, rh; - TCGOp *op2 = tcg_op_insert_before(s, op, INDEX_op_movi_i32); + TCGOp *op2 = tcg_op_insert_before(s, op, INDEX_op_mov_i32); rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(s, op, rl, (int32_t)r); - tcg_opt_gen_movi(s, op2, rh, (int32_t)(r >> 32)); + tcg_opt_gen_movi(s, &temps_used, op, rl, (int32_t)r); + tcg_opt_gen_movi(s, &temps_used, op2, rh, (int32_t)(r >> 32)); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (tmp != 2) { if (tmp) { do_brcond_true: - bitmap_zero(temps_used.l, nb_temps); + memset(&temps_used, 0, sizeof(temps_used)); op->opc = INDEX_op_br; op->args[0] = op->args[5]; } else { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* Simplify LT/GE comparisons vs zero to a single compare vs the high word of the input. */ do_brcond_high: - bitmap_zero(temps_used.l, nb_temps); + memset(&temps_used, 0, sizeof(temps_used)); op->opc = INDEX_op_brcond_i32; op->args[0] = op->args[1]; op->args[1] = op->args[3]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) goto do_default; } do_brcond_low: - bitmap_zero(temps_used.l, nb_temps); + memset(&temps_used, 0, sizeof(temps_used)); op->opc = INDEX_op_brcond_i32; op->args[1] = op->args[2]; op->args[2] = op->args[4]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[5]); if (tmp != 2) { do_setcond_const: - tcg_opt_gen_movi(s, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); } else if ((op->args[5] == TCG_COND_LT || op->args[5] == TCG_COND_GE) && arg_is_const(op->args[3]) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) block, otherwise we only trash the output args. "mask" is the non-zero bits mask for the first output arg. */ if (def->flags & TCG_OPF_BB_END) { - bitmap_zero(temps_used.l, nb_temps); + memset(&temps_used, 0, sizeof(temps_used)); } else { do_reset_output: for (i = 0; i < nb_oargs; i++) { -- 2.25.1 Because we now store uint64_t in TCGTemp, we can now always store the full 64-bit duplicate immediate. So remove the difference between 32- and 64-bit hosts. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 9 ++++----- tcg/tcg-op-vec.c | 39 ++++++++++----------------------------- tcg/tcg.c | 7 +------ 3 files changed, 15 insertions(+), 40 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_dup2_vec: assert(TCG_TARGET_REG_BITS == 32); if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - tmp = arg_info(op->args[1])->val; - if (tmp == arg_info(op->args[2])->val) { - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); - break; - } + tcg_opt_gen_movi(s, &temps_used, op, op->args[0], + deposit64(arg_info(op->args[1])->val, 32, 32, + arg_info(op->args[2])->val)); + break; } else if (args_are_copies(op->args[1], op->args[2])) { op->opc = INDEX_op_dup_vec; TCGOP_VECE(op) = MO_32; diff --git a/tcg/tcg-op-vec.c b/tcg/tcg-op-vec.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-op-vec.c +++ b/tcg/tcg-op-vec.c @@ -XXX,XX +XXX,XX @@ void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a) } } -#define MO_REG (TCG_TARGET_REG_BITS == 64 ? MO_64 : MO_32) - -static void do_dupi_vec(TCGv_vec r, unsigned vece, TCGArg a) -{ - TCGTemp *rt = tcgv_vec_temp(r); - vec_gen_2(INDEX_op_dupi_vec, rt->base_type, vece, temp_arg(rt), a); -} - TCGv_vec tcg_const_zeros_vec(TCGType type) { TCGv_vec ret = tcg_temp_new_vec(type); - do_dupi_vec(ret, MO_REG, 0); + tcg_gen_dupi_vec(MO_64, ret, 0); return ret; } TCGv_vec tcg_const_ones_vec(TCGType type) { TCGv_vec ret = tcg_temp_new_vec(type); - do_dupi_vec(ret, MO_REG, -1); + tcg_gen_dupi_vec(MO_64, ret, -1); return ret; } @@ -XXX,XX +XXX,XX @@ TCGv_vec tcg_const_ones_vec_matching(TCGv_vec m) void tcg_gen_dup64i_vec(TCGv_vec r, uint64_t a) { - if (TCG_TARGET_REG_BITS == 64) { - do_dupi_vec(r, MO_64, a); - } else if (a == dup_const(MO_32, a)) { - do_dupi_vec(r, MO_32, a); - } else { - TCGv_i64 c = tcg_const_i64(a); - tcg_gen_dup_i64_vec(MO_64, r, c); - tcg_temp_free_i64(c); - } + tcg_gen_dupi_vec(MO_64, r, a); } void tcg_gen_dup32i_vec(TCGv_vec r, uint32_t a) { - do_dupi_vec(r, MO_REG, dup_const(MO_32, a)); + tcg_gen_dupi_vec(MO_32, r, a); } void tcg_gen_dup16i_vec(TCGv_vec r, uint32_t a) { - do_dupi_vec(r, MO_REG, dup_const(MO_16, a)); + tcg_gen_dupi_vec(MO_16, r, a); } void tcg_gen_dup8i_vec(TCGv_vec r, uint32_t a) { - do_dupi_vec(r, MO_REG, dup_const(MO_8, a)); + tcg_gen_dupi_vec(MO_8, r, a); } void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a) { - if (vece == MO_64) { - tcg_gen_dup64i_vec(r, a); - } else { - do_dupi_vec(r, MO_REG, dup_const(vece, a)); - } + TCGTemp *rt = tcgv_vec_temp(r); + tcg_gen_mov_vec(r, tcg_constant_vec(rt->base_type, vece, a)); } void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a) @@ -XXX,XX +XXX,XX @@ void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a) if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) { tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1); } else { - do_dupi_vec(t, MO_REG, 0); - tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a, t); + tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a, + tcg_constant_vec(type, vece, 0)); } tcg_gen_xor_vec(vece, r, a, t); tcg_gen_sub_vec(vece, r, r, t); diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, * The targets will, in general, have to do this search anyway, * do this generically. */ - if (TCG_TARGET_REG_BITS == 32) { - val = dup_const(MO_32, val); - vece = MO_32; - } if (val == dup_const(MO_8, val)) { vece = MO_8; } else if (val == dup_const(MO_16, val)) { vece = MO_16; - } else if (TCG_TARGET_REG_BITS == 64 && - val == dup_const(MO_32, val)) { + } else if (val == dup_const(MO_32, val)) { vece = MO_32; } -- 2.25.1 We must do this before we adjust tcg_out_movi_i32, lest the under-the-hood poking that we do for icount be broken. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/gen-icount.h | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/include/exec/gen-icount.h b/include/exec/gen-icount.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/gen-icount.h +++ b/include/exec/gen-icount.h @@ -XXX,XX +XXX,XX @@ static inline void gen_io_end(void) static inline void gen_tb_start(const TranslationBlock *tb) { - TCGv_i32 count, imm; + TCGv_i32 count; tcg_ctx->exitreq_label = gen_new_label(); if (tb_cflags(tb) & CF_USE_ICOUNT) { @@ -XXX,XX +XXX,XX @@ static inline void gen_tb_start(const TranslationBlock *tb) offsetof(ArchCPU, env)); if (tb_cflags(tb) & CF_USE_ICOUNT) { - imm = tcg_temp_new_i32(); - /* We emit a movi with a dummy immediate argument. Keep the insn index - * of the movi so that we later (when we know the actual insn count) - * can update the immediate argument with the actual insn count. */ - tcg_gen_movi_i32(imm, 0xdeadbeef); + /* + * We emit a sub with a dummy immediate argument. Keep the insn index + * of the sub so that we later (when we know the actual insn count) + * can update the argument with the actual insn count. + */ + tcg_gen_sub_i32(count, count, tcg_constant_i32(0)); icount_start_insn = tcg_last_op(); - - tcg_gen_sub_i32(count, count, imm); - tcg_temp_free_i32(imm); } tcg_gen_brcondi_i32(TCG_COND_LT, count, 0, tcg_ctx->exitreq_label); @@ -XXX,XX +XXX,XX @@ static inline void gen_tb_start(const TranslationBlock *tb) static inline void gen_tb_end(const TranslationBlock *tb, int num_insns) { if (tb_cflags(tb) & CF_USE_ICOUNT) { - /* Update the num_insn immediate parameter now that we know - * the actual insn count. */ - tcg_set_insn_param(icount_start_insn, 1, num_insns); + /* + * Update the num_insn immediate parameter now that we know + * the actual insn count. + */ + tcg_set_insn_param(icount_start_insn, 2, + tcgv_i32_arg(tcg_constant_i32(num_insns))); } gen_set_label(tcg_ctx->exitreq_label); -- 2.25.1 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg-op.h | 13 +-- tcg/tcg-op.c | 227 ++++++++++++++++++++----------------------- 2 files changed, 109 insertions(+), 131 deletions(-) diff --git a/include/tcg/tcg-op.h b/include/tcg/tcg-op.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg-op.h +++ b/include/tcg/tcg-op.h @@ -XXX,XX +XXX,XX @@ void tcg_gen_mb(TCGBar); /* 32 bit ops */ +void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg); void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2); void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); @@ -XXX,XX +XXX,XX @@ static inline void tcg_gen_mov_i32(TCGv_i32 ret, TCGv_i32 arg) } } -static inline void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg) -{ - tcg_gen_op2i_i32(INDEX_op_movi_i32, ret, arg); -} - static inline void tcg_gen_ld8u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) { @@ -XXX,XX +XXX,XX @@ static inline void tcg_gen_not_i32(TCGv_i32 ret, TCGv_i32 arg) /* 64 bit ops */ +void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg); void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2); void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); @@ -XXX,XX +XXX,XX @@ static inline void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) } } -static inline void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) -{ - tcg_gen_op2i_i64(INDEX_op_movi_i64, ret, arg); -} - static inline void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) { @@ -XXX,XX +XXX,XX @@ static inline void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) void tcg_gen_discard_i64(TCGv_i64 arg); void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg); -void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg); void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -XXX,XX +XXX,XX @@ void tcg_gen_mb(TCGBar mb_type) /* 32 bit ops */ +void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg) +{ + tcg_gen_mov_i32(ret, tcg_constant_i32(arg)); +} + void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) { /* some cases can be optimized here */ if (arg2 == 0) { tcg_gen_mov_i32(ret, arg1); } else { - TCGv_i32 t0 = tcg_const_i32(arg2); - tcg_gen_add_i32(ret, arg1, t0); - tcg_temp_free_i32(t0); + tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2)); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) /* Don't recurse with tcg_gen_neg_i32. */ tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2); } else { - TCGv_i32 t0 = tcg_const_i32(arg1); - tcg_gen_sub_i32(ret, t0, arg2); - tcg_temp_free_i32(t0); + tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) if (arg2 == 0) { tcg_gen_mov_i32(ret, arg1); } else { - TCGv_i32 t0 = tcg_const_i32(arg2); - tcg_gen_sub_i32(ret, arg1, t0); - tcg_temp_free_i32(t0); + tcg_gen_sub_i32(ret, arg1, tcg_constant_i32(arg2)); } } void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) { - TCGv_i32 t0; /* Some cases can be optimized here. */ switch (arg2) { case 0: @@ -XXX,XX +XXX,XX @@ void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) } break; } - t0 = tcg_const_i32(arg2); - tcg_gen_and_i32(ret, arg1, t0); - tcg_temp_free_i32(t0); + + tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2)); } void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) @@ -XXX,XX +XXX,XX @@ void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) } else if (arg2 == 0) { tcg_gen_mov_i32(ret, arg1); } else { - TCGv_i32 t0 = tcg_const_i32(arg2); - tcg_gen_or_i32(ret, arg1, t0); - tcg_temp_free_i32(t0); + tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2)); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) /* Don't recurse with tcg_gen_not_i32. */ tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1); } else { - TCGv_i32 t0 = tcg_const_i32(arg2); - tcg_gen_xor_i32(ret, arg1, t0); - tcg_temp_free_i32(t0); + tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2)); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) if (arg2 == 0) { tcg_gen_mov_i32(ret, arg1); } else { - TCGv_i32 t0 = tcg_const_i32(arg2); - tcg_gen_shl_i32(ret, arg1, t0); - tcg_temp_free_i32(t0); + tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2)); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) if (arg2 == 0) { tcg_gen_mov_i32(ret, arg1); } else { - TCGv_i32 t0 = tcg_const_i32(arg2); - tcg_gen_shr_i32(ret, arg1, t0); - tcg_temp_free_i32(t0); + tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2)); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) if (arg2 == 0) { tcg_gen_mov_i32(ret, arg1); } else { - TCGv_i32 t0 = tcg_const_i32(arg2); - tcg_gen_sar_i32(ret, arg1, t0); - tcg_temp_free_i32(t0); + tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2)); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l) if (cond == TCG_COND_ALWAYS) { tcg_gen_br(l); } else if (cond != TCG_COND_NEVER) { - TCGv_i32 t0 = tcg_const_i32(arg2); - tcg_gen_brcond_i32(cond, arg1, t0, l); - tcg_temp_free_i32(t0); + tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) { - TCGv_i32 t0 = tcg_const_i32(arg2); - tcg_gen_setcond_i32(cond, ret, arg1, t0); - tcg_temp_free_i32(t0); + tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2)); } void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) @@ -XXX,XX +XXX,XX @@ void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) } else if (is_power_of_2(arg2)) { tcg_gen_shli_i32(ret, arg1, ctz32(arg2)); } else { - TCGv_i32 t0 = tcg_const_i32(arg2); - tcg_gen_mul_i32(ret, arg1, t0); - tcg_temp_free_i32(t0); + tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2)); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) { - TCGv_i32 t = tcg_const_i32(arg2); - tcg_gen_clz_i32(ret, arg1, t); - tcg_temp_free_i32(t); + tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2)); } void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) @@ -XXX,XX +XXX,XX @@ void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) tcg_gen_clzi_i32(t, t, 32); tcg_gen_xori_i32(t, t, 31); } - z = tcg_const_i32(0); + z = tcg_constant_i32(0); tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); tcg_temp_free_i32(t); - tcg_temp_free_i32(z); } else { gen_helper_ctz_i32(ret, arg1, arg2); } @@ -XXX,XX +XXX,XX @@ void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) tcg_gen_ctpop_i32(ret, t); tcg_temp_free_i32(t); } else { - TCGv_i32 t = tcg_const_i32(arg2); - tcg_gen_ctz_i32(ret, arg1, t); - tcg_temp_free_i32(t); + tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2)); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) if (arg2 == 0) { tcg_gen_mov_i32(ret, arg1); } else if (TCG_TARGET_HAS_rot_i32) { - TCGv_i32 t0 = tcg_const_i32(arg2); - tcg_gen_rotl_i32(ret, arg1, t0); - tcg_temp_free_i32(t0); + tcg_gen_rotl_i32(ret, arg1, tcg_constant_i32(arg2)); } else { TCGv_i32 t0, t1; t0 = tcg_temp_new_i32(); @@ -XXX,XX +XXX,XX @@ void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, tcg_gen_andi_i32(ret, arg, (1u << len) - 1); } else if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) { - TCGv_i32 zero = tcg_const_i32(0); + TCGv_i32 zero = tcg_constant_i32(0); tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len); - tcg_temp_free_i32(zero); } else { /* To help two-operand hosts we prefer to zero-extend first, which allows ARG to stay live. */ @@ -XXX,XX +XXX,XX @@ void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) } else { TCGv_i32 t0 = tcg_temp_new_i32(); TCGv_i32 t1 = tcg_temp_new_i32(); - TCGv_i32 t2 = tcg_const_i32(0x00ff00ff); + TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff); /* arg = abcd */ tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */ @@ -XXX,XX +XXX,XX @@ void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) tcg_temp_free_i32(t0); tcg_temp_free_i32(t1); - tcg_temp_free_i32(t2); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_discard_i64(TCGv_i64 arg) void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) { - tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); - tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); + TCGTemp *ts = tcgv_i64_temp(arg); + + /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */ + if (ts->kind == TEMP_CONST) { + tcg_gen_movi_i64(ret, ts->val); + } else { + tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); + tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); + } } void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) @@ -XXX,XX +XXX,XX @@ void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) tcg_temp_free_i64(t0); tcg_temp_free_i32(t1); } + +#else + +void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) +{ + tcg_gen_mov_i64(ret, tcg_constant_i64(arg)); +} + #endif /* TCG_TARGET_REG_SIZE == 32 */ void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) @@ -XXX,XX +XXX,XX @@ void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) /* some cases can be optimized here */ if (arg2 == 0) { tcg_gen_mov_i64(ret, arg1); + } else if (TCG_TARGET_REG_BITS == 64) { + tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2)); } else { - TCGv_i64 t0 = tcg_const_i64(arg2); - tcg_gen_add_i64(ret, arg1, t0); - tcg_temp_free_i64(t0); + tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), + TCGV_LOW(arg1), TCGV_HIGH(arg1), + tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32)); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2) if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) { /* Don't recurse with tcg_gen_neg_i64. */ tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2); + } else if (TCG_TARGET_REG_BITS == 64) { + tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2); } else { - TCGv_i64 t0 = tcg_const_i64(arg1); - tcg_gen_sub_i64(ret, t0, arg2); - tcg_temp_free_i64(t0); + tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), + tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32), + TCGV_LOW(arg2), TCGV_HIGH(arg2)); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) /* some cases can be optimized here */ if (arg2 == 0) { tcg_gen_mov_i64(ret, arg1); + } else if (TCG_TARGET_REG_BITS == 64) { + tcg_gen_sub_i64(ret, arg1, tcg_constant_i64(arg2)); } else { - TCGv_i64 t0 = tcg_const_i64(arg2); - tcg_gen_sub_i64(ret, arg1, t0); - tcg_temp_free_i64(t0); + tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), + TCGV_LOW(arg1), TCGV_HIGH(arg1), + tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32)); } } void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) { - TCGv_i64 t0; - if (TCG_TARGET_REG_BITS == 32) { tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); @@ -XXX,XX +XXX,XX @@ void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) } break; } - t0 = tcg_const_i64(arg2); - tcg_gen_and_i64(ret, arg1, t0); - tcg_temp_free_i64(t0); + + tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2)); } void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) @@ -XXX,XX +XXX,XX @@ void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) } else if (arg2 == 0) { tcg_gen_mov_i64(ret, arg1); } else { - TCGv_i64 t0 = tcg_const_i64(arg2); - tcg_gen_or_i64(ret, arg1, t0); - tcg_temp_free_i64(t0); + tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2)); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) /* Don't recurse with tcg_gen_not_i64. */ tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1); } else { - TCGv_i64 t0 = tcg_const_i64(arg2); - tcg_gen_xor_i64(ret, arg1, t0); - tcg_temp_free_i64(t0); + tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2)); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) } else if (arg2 == 0) { tcg_gen_mov_i64(ret, arg1); } else { - TCGv_i64 t0 = tcg_const_i64(arg2); - tcg_gen_shl_i64(ret, arg1, t0); - tcg_temp_free_i64(t0); + tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2)); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) } else if (arg2 == 0) { tcg_gen_mov_i64(ret, arg1); } else { - TCGv_i64 t0 = tcg_const_i64(arg2); - tcg_gen_shr_i64(ret, arg1, t0); - tcg_temp_free_i64(t0); + tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2)); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) } else if (arg2 == 0) { tcg_gen_mov_i64(ret, arg1); } else { - TCGv_i64 t0 = tcg_const_i64(arg2); - tcg_gen_sar_i64(ret, arg1, t0); - tcg_temp_free_i64(t0); + tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2)); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) { - if (cond == TCG_COND_ALWAYS) { + if (TCG_TARGET_REG_BITS == 64) { + tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l); + } else if (cond == TCG_COND_ALWAYS) { tcg_gen_br(l); } else if (cond != TCG_COND_NEVER) { - TCGv_i64 t0 = tcg_const_i64(arg2); - tcg_gen_brcond_i64(cond, arg1, t0, l); - tcg_temp_free_i64(t0); + l->refs++; + tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, + TCGV_LOW(arg1), TCGV_HIGH(arg1), + tcg_constant_i32(arg2), + tcg_constant_i32(arg2 >> 32), + cond, label_arg(l)); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) { - TCGv_i64 t0 = tcg_const_i64(arg2); - tcg_gen_setcond_i64(cond, ret, arg1, t0); - tcg_temp_free_i64(t0); + if (TCG_TARGET_REG_BITS == 64) { + tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2)); + } else if (cond == TCG_COND_ALWAYS) { + tcg_gen_movi_i64(ret, 1); + } else if (cond == TCG_COND_NEVER) { + tcg_gen_movi_i64(ret, 0); + } else { + tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), + TCGV_LOW(arg1), TCGV_HIGH(arg1), + tcg_constant_i32(arg2), + tcg_constant_i32(arg2 >> 32), cond); + tcg_gen_movi_i32(TCGV_HIGH(ret), 0); + } } void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) @@ -XXX,XX +XXX,XX @@ void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg) } else { TCGv_i64 t0 = tcg_temp_new_i64(); TCGv_i64 t1 = tcg_temp_new_i64(); - TCGv_i64 t2 = tcg_const_i64(0x00ff00ff); + TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff); /* arg = ....abcd */ tcg_gen_shri_i64(t0, arg, 8); /* t0 = .....abc */ @@ -XXX,XX +XXX,XX @@ void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg) tcg_temp_free_i64(t0); tcg_temp_free_i64(t1); - tcg_temp_free_i64(t2); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_clz_i32 && arg2 <= 0xffffffffu) { - TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32); - tcg_gen_clz_i32(t, TCGV_LOW(arg1), t); + TCGv_i32 t = tcg_temp_new_i32(); + tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32); tcg_gen_addi_i32(t, t, 32); tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t); tcg_gen_movi_i32(TCGV_HIGH(ret), 0); tcg_temp_free_i32(t); } else { - TCGv_i64 t = tcg_const_i64(arg2); - tcg_gen_clz_i64(ret, arg1, t); - tcg_temp_free_i64(t); + TCGv_i64 t0 = tcg_const_i64(arg2); + tcg_gen_clz_i64(ret, arg1, t0); + tcg_temp_free_i64(t0); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) tcg_gen_clzi_i64(t, t, 64); tcg_gen_xori_i64(t, t, 63); } - z = tcg_const_i64(0); + z = tcg_constant_i64(0); tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); tcg_temp_free_i64(t); tcg_temp_free_i64(z); @@ -XXX,XX +XXX,XX @@ void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctz_i32 && arg2 <= 0xffffffffu) { - TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32); - tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32); + TCGv_i32 t32 = tcg_temp_new_i32(); + tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32); tcg_gen_addi_i32(t32, t32, 32); tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32); tcg_gen_movi_i32(TCGV_HIGH(ret), 0); @@ -XXX,XX +XXX,XX @@ void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) tcg_gen_ctpop_i64(ret, t); tcg_temp_free_i64(t); } else { - TCGv_i64 t64 = tcg_const_i64(arg2); - tcg_gen_ctz_i64(ret, arg1, t64); - tcg_temp_free_i64(t64); + TCGv_i64 t0 = tcg_const_i64(arg2); + tcg_gen_ctz_i64(ret, arg1, t0); + tcg_temp_free_i64(t0); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) if (arg2 == 0) { tcg_gen_mov_i64(ret, arg1); } else if (TCG_TARGET_HAS_rot_i64) { - TCGv_i64 t0 = tcg_const_i64(arg2); - tcg_gen_rotl_i64(ret, arg1, t0); - tcg_temp_free_i64(t0); + tcg_gen_rotl_i64(ret, arg1, tcg_constant_i64(arg2)); } else { TCGv_i64 t0, t1; t0 = tcg_temp_new_i64(); @@ -XXX,XX +XXX,XX @@ void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); } else if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) { - TCGv_i64 zero = tcg_const_i64(0); + TCGv_i64 zero = tcg_constant_i64(0); tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len); - tcg_temp_free_i64(zero); } else { if (TCG_TARGET_REG_BITS == 32) { if (ofs >= 32) { @@ -XXX,XX +XXX,XX @@ void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, #ifdef CONFIG_SOFTMMU { - TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); - gen(retv, cpu_env, addr, cmpv, newv, oi); - tcg_temp_free_i32(oi); + TCGMemOpIdx oi = make_memop_idx(memop & ~MO_SIGN, idx); + gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi)); } #else gen(retv, cpu_env, addr, cmpv, newv); @@ -XXX,XX +XXX,XX @@ void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, #ifdef CONFIG_SOFTMMU { - TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx)); - gen(retv, cpu_env, addr, cmpv, newv, oi); - tcg_temp_free_i32(oi); + TCGMemOpIdx oi = make_memop_idx(memop, idx); + gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi)); } #else gen(retv, cpu_env, addr, cmpv, newv); @@ -XXX,XX +XXX,XX @@ static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, #ifdef CONFIG_SOFTMMU { - TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); - gen(ret, cpu_env, addr, val, oi); - tcg_temp_free_i32(oi); + TCGMemOpIdx oi = make_memop_idx(memop & ~MO_SIGN, idx); + gen(ret, cpu_env, addr, val, tcg_constant_i32(oi)); } #else gen(ret, cpu_env, addr, val); @@ -XXX,XX +XXX,XX @@ static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, #ifdef CONFIG_SOFTMMU { - TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); - gen(ret, cpu_env, addr, val, oi); - tcg_temp_free_i32(oi); + TCGMemOpIdx oi = make_memop_idx(memop & ~MO_SIGN, idx); + gen(ret, cpu_env, addr, val, tcg_constant_i32(oi)); } #else gen(ret, cpu_env, addr, val); -- 2.25.1 Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/plugin-gen.c | 49 +++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -XXX,XX +XXX,XX @@ static TCGOp *copy_extu_i32_i64(TCGOp **begin_op, TCGOp *op) if (TCG_TARGET_REG_BITS == 32) { /* mov_i32 */ op = copy_op(begin_op, op, INDEX_op_mov_i32); - /* movi_i32 */ - op = copy_op(begin_op, op, INDEX_op_movi_i32); + /* mov_i32 w/ $0 */ + op = copy_op(begin_op, op, INDEX_op_mov_i32); } else { /* extu_i32_i64 */ op = copy_op(begin_op, op, INDEX_op_extu_i32_i64); @@ -XXX,XX +XXX,XX @@ static TCGOp *copy_mov_i64(TCGOp **begin_op, TCGOp *op) return op; } -static TCGOp *copy_movi_i64(TCGOp **begin_op, TCGOp *op, uint64_t v) -{ - if (TCG_TARGET_REG_BITS == 32) { - /* 2x movi_i32 */ - op = copy_op(begin_op, op, INDEX_op_movi_i32); - op->args[1] = v; - - op = copy_op(begin_op, op, INDEX_op_movi_i32); - op->args[1] = v >> 32; - } else { - /* movi_i64 */ - op = copy_op(begin_op, op, INDEX_op_movi_i64); - op->args[1] = v; - } - return op; -} - static TCGOp *copy_const_ptr(TCGOp **begin_op, TCGOp *op, void *ptr) { if (UINTPTR_MAX == UINT32_MAX) { - /* movi_i32 */ - op = copy_op(begin_op, op, INDEX_op_movi_i32); - op->args[1] = (uintptr_t)ptr; + /* mov_i32 */ + op = copy_op(begin_op, op, INDEX_op_mov_i32); + op->args[1] = tcgv_i32_arg(tcg_constant_i32((uintptr_t)ptr)); } else { - /* movi_i64 */ - op = copy_movi_i64(begin_op, op, (uint64_t)(uintptr_t)ptr); + /* mov_i64 */ + op = copy_op(begin_op, op, INDEX_op_mov_i64); + op->args[1] = tcgv_i64_arg(tcg_constant_i64((uintptr_t)ptr)); } return op; } static TCGOp *copy_const_i64(TCGOp **begin_op, TCGOp *op, uint64_t v) { - return copy_movi_i64(begin_op, op, v); + if (TCG_TARGET_REG_BITS == 32) { + /* 2x mov_i32 */ + op = copy_op(begin_op, op, INDEX_op_mov_i32); + op->args[1] = tcgv_i32_arg(tcg_constant_i32(v)); + op = copy_op(begin_op, op, INDEX_op_mov_i32); + op->args[1] = tcgv_i32_arg(tcg_constant_i32(v >> 32)); + } else { + /* mov_i64 */ + op = copy_op(begin_op, op, INDEX_op_mov_i64); + op->args[1] = tcgv_i64_arg(tcg_constant_i64(v)); + } + return op; } static TCGOp *copy_extu_tl_i64(TCGOp **begin_op, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static TCGOp *append_mem_cb(const struct qemu_plugin_dyn_cb *cb, tcg_debug_assert(type == PLUGIN_GEN_CB_MEM); - /* const_i32 == movi_i32 ("info", so it remains as is) */ - op = copy_op(&begin_op, op, INDEX_op_movi_i32); + /* const_i32 == mov_i32 ("info", so it remains as is) */ + op = copy_op(&begin_op, op, INDEX_op_mov_i32); /* const_ptr */ op = copy_const_ptr(&begin_op, op, cb->userp); -- 2.25.1 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg.h | 1 + tcg/tcg-op-gvec.c | 129 ++++++++++++++++++---------------------------- tcg/tcg.c | 8 +++ 3 files changed, 60 insertions(+), 78 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ static inline TCGv_i64 tcg_constant_i64(int64_t val) } TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val); +TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val); #if UINTPTR_MAX == UINT32_MAX # define tcg_const_ptr(x) ((TCGv_ptr)tcg_const_i32((intptr_t)(x))) diff --git a/tcg/tcg-op-gvec.c b/tcg/tcg-op-gvec.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-op-gvec.c +++ b/tcg/tcg-op-gvec.c @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_2_ool(uint32_t dofs, uint32_t aofs, gen_helper_gvec_2 *fn) { TCGv_ptr a0, a1; - TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data)); + TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data)); a0 = tcg_temp_new_ptr(); a1 = tcg_temp_new_ptr(); @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_2_ool(uint32_t dofs, uint32_t aofs, tcg_temp_free_ptr(a0); tcg_temp_free_ptr(a1); - tcg_temp_free_i32(desc); } /* Generate a call to a gvec-style helper with two vector operands @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_2i_ool(uint32_t dofs, uint32_t aofs, TCGv_i64 c, gen_helper_gvec_2i *fn) { TCGv_ptr a0, a1; - TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data)); + TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data)); a0 = tcg_temp_new_ptr(); a1 = tcg_temp_new_ptr(); @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_2i_ool(uint32_t dofs, uint32_t aofs, TCGv_i64 c, tcg_temp_free_ptr(a0); tcg_temp_free_ptr(a1); - tcg_temp_free_i32(desc); } /* Generate a call to a gvec-style helper with three vector operands. */ @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_3_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs, gen_helper_gvec_3 *fn) { TCGv_ptr a0, a1, a2; - TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data)); + TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data)); a0 = tcg_temp_new_ptr(); a1 = tcg_temp_new_ptr(); @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_3_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs, tcg_temp_free_ptr(a0); tcg_temp_free_ptr(a1); tcg_temp_free_ptr(a2); - tcg_temp_free_i32(desc); } /* Generate a call to a gvec-style helper with four vector operands. */ @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_4_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs, int32_t data, gen_helper_gvec_4 *fn) { TCGv_ptr a0, a1, a2, a3; - TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data)); + TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data)); a0 = tcg_temp_new_ptr(); a1 = tcg_temp_new_ptr(); @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_4_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs, tcg_temp_free_ptr(a1); tcg_temp_free_ptr(a2); tcg_temp_free_ptr(a3); - tcg_temp_free_i32(desc); } /* Generate a call to a gvec-style helper with five vector operands. */ @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_5_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs, uint32_t maxsz, int32_t data, gen_helper_gvec_5 *fn) { TCGv_ptr a0, a1, a2, a3, a4; - TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data)); + TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data)); a0 = tcg_temp_new_ptr(); a1 = tcg_temp_new_ptr(); @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_5_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs, tcg_temp_free_ptr(a2); tcg_temp_free_ptr(a3); tcg_temp_free_ptr(a4); - tcg_temp_free_i32(desc); } /* Generate a call to a gvec-style helper with three vector operands @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_2_ptr(uint32_t dofs, uint32_t aofs, int32_t data, gen_helper_gvec_2_ptr *fn) { TCGv_ptr a0, a1; - TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data)); + TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data)); a0 = tcg_temp_new_ptr(); a1 = tcg_temp_new_ptr(); @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_2_ptr(uint32_t dofs, uint32_t aofs, tcg_temp_free_ptr(a0); tcg_temp_free_ptr(a1); - tcg_temp_free_i32(desc); } /* Generate a call to a gvec-style helper with three vector operands @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_3_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs, int32_t data, gen_helper_gvec_3_ptr *fn) { TCGv_ptr a0, a1, a2; - TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data)); + TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data)); a0 = tcg_temp_new_ptr(); a1 = tcg_temp_new_ptr(); @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_3_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs, tcg_temp_free_ptr(a0); tcg_temp_free_ptr(a1); tcg_temp_free_ptr(a2); - tcg_temp_free_i32(desc); } /* Generate a call to a gvec-style helper with four vector operands @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_4_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs, gen_helper_gvec_4_ptr *fn) { TCGv_ptr a0, a1, a2, a3; - TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data)); + TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data)); a0 = tcg_temp_new_ptr(); a1 = tcg_temp_new_ptr(); @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_4_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs, tcg_temp_free_ptr(a1); tcg_temp_free_ptr(a2); tcg_temp_free_ptr(a3); - tcg_temp_free_i32(desc); } /* Generate a call to a gvec-style helper with five vector operands @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_5_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs, gen_helper_gvec_5_ptr *fn) { TCGv_ptr a0, a1, a2, a3, a4; - TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data)); + TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data)); a0 = tcg_temp_new_ptr(); a1 = tcg_temp_new_ptr(); @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_5_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs, tcg_temp_free_ptr(a2); tcg_temp_free_ptr(a3); tcg_temp_free_ptr(a4); - tcg_temp_free_i32(desc); } /* Return true if we want to implement something of OPRSZ bytes @@ -XXX,XX +XXX,XX @@ static void do_dup(unsigned vece, uint32_t dofs, uint32_t oprsz, || (TCG_TARGET_REG_BITS == 64 && (in_c == 0 || in_c == -1 || !check_size_impl(oprsz, 4)))) { - t_64 = tcg_const_i64(in_c); + t_64 = tcg_constant_i64(in_c); } else { - t_32 = tcg_const_i32(in_c); + t_32 = tcg_constant_i32(in_c); } } @@ -XXX,XX +XXX,XX @@ static void do_dup(unsigned vece, uint32_t dofs, uint32_t oprsz, t_val = tcg_temp_new_i32(); tcg_gen_extrl_i64_i32(t_val, in_64); } else { - t_val = tcg_const_i32(in_c); + t_val = tcg_constant_i32(in_c); } gen_helper_memset(t_ptr, t_ptr, t_val, t_size); - if (!in_32) { + if (in_64) { tcg_temp_free_i32(t_val); } tcg_temp_free_ptr(t_size); @@ -XXX,XX +XXX,XX @@ static void do_dup(unsigned vece, uint32_t dofs, uint32_t oprsz, return; } - t_desc = tcg_const_i32(simd_desc(oprsz, maxsz, 0)); + t_desc = tcg_constant_i32(simd_desc(oprsz, maxsz, 0)); if (vece == MO_64) { if (in_64) { gen_helper_gvec_dup64(t_ptr, t_desc, in_64); } else { - t_64 = tcg_const_i64(in_c); + t_64 = tcg_constant_i64(in_c); gen_helper_gvec_dup64(t_ptr, t_desc, t_64); - tcg_temp_free_i64(t_64); } } else { typedef void dup_fn(TCGv_ptr, TCGv_i32, TCGv_i32); @@ -XXX,XX +XXX,XX @@ static void do_dup(unsigned vece, uint32_t dofs, uint32_t oprsz, if (in_32) { fns[vece](t_ptr, t_desc, in_32); - } else { + } else if (in_64) { t_32 = tcg_temp_new_i32(); - if (in_64) { - tcg_gen_extrl_i64_i32(t_32, in_64); - } else if (vece == MO_8) { - tcg_gen_movi_i32(t_32, in_c & 0xff); - } else if (vece == MO_16) { - tcg_gen_movi_i32(t_32, in_c & 0xffff); - } else { - tcg_gen_movi_i32(t_32, in_c); - } + tcg_gen_extrl_i64_i32(t_32, in_64); fns[vece](t_ptr, t_desc, t_32); tcg_temp_free_i32(t_32); + } else { + if (vece == MO_8) { + in_c &= 0xff; + } else if (vece == MO_16) { + in_c &= 0xffff; + } + t_32 = tcg_constant_i32(in_c); + fns[vece](t_ptr, t_desc, t_32); } } tcg_temp_free_ptr(t_ptr); - tcg_temp_free_i32(t_desc); return; done: @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_2i(uint32_t dofs, uint32_t aofs, uint32_t oprsz, if (g->fno) { tcg_gen_gvec_2_ool(dofs, aofs, oprsz, maxsz, c, g->fno); } else { - TCGv_i64 tcg_c = tcg_const_i64(c); + TCGv_i64 tcg_c = tcg_constant_i64(c); tcg_gen_gvec_2i_ool(dofs, aofs, tcg_c, oprsz, maxsz, c, g->fnoi); - tcg_temp_free_i64(tcg_c); } oprsz = maxsz; } @@ -XXX,XX +XXX,XX @@ static void gen_addv_mask(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 m) void tcg_gen_vec_add8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b) { - TCGv_i64 m = tcg_const_i64(dup_const(MO_8, 0x80)); + TCGv_i64 m = tcg_constant_i64(dup_const(MO_8, 0x80)); gen_addv_mask(d, a, b, m); - tcg_temp_free_i64(m); } void tcg_gen_vec_add16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b) { - TCGv_i64 m = tcg_const_i64(dup_const(MO_16, 0x8000)); + TCGv_i64 m = tcg_constant_i64(dup_const(MO_16, 0x8000)); gen_addv_mask(d, a, b, m); - tcg_temp_free_i64(m); } void tcg_gen_vec_add32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b) @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_adds(unsigned vece, uint32_t dofs, uint32_t aofs, void tcg_gen_gvec_addi(unsigned vece, uint32_t dofs, uint32_t aofs, int64_t c, uint32_t oprsz, uint32_t maxsz) { - TCGv_i64 tmp = tcg_const_i64(c); + TCGv_i64 tmp = tcg_constant_i64(c); tcg_gen_gvec_adds(vece, dofs, aofs, tmp, oprsz, maxsz); - tcg_temp_free_i64(tmp); } static const TCGOpcode vecop_list_sub[] = { INDEX_op_sub_vec, 0 }; @@ -XXX,XX +XXX,XX @@ static void gen_subv_mask(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 m) void tcg_gen_vec_sub8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b) { - TCGv_i64 m = tcg_const_i64(dup_const(MO_8, 0x80)); + TCGv_i64 m = tcg_constant_i64(dup_const(MO_8, 0x80)); gen_subv_mask(d, a, b, m); - tcg_temp_free_i64(m); } void tcg_gen_vec_sub16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b) { - TCGv_i64 m = tcg_const_i64(dup_const(MO_16, 0x8000)); + TCGv_i64 m = tcg_constant_i64(dup_const(MO_16, 0x8000)); gen_subv_mask(d, a, b, m); - tcg_temp_free_i64(m); } void tcg_gen_vec_sub32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b) @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_muls(unsigned vece, uint32_t dofs, uint32_t aofs, void tcg_gen_gvec_muli(unsigned vece, uint32_t dofs, uint32_t aofs, int64_t c, uint32_t oprsz, uint32_t maxsz) { - TCGv_i64 tmp = tcg_const_i64(c); + TCGv_i64 tmp = tcg_constant_i64(c); tcg_gen_gvec_muls(vece, dofs, aofs, tmp, oprsz, maxsz); - tcg_temp_free_i64(tmp); } void tcg_gen_gvec_ssadd(unsigned vece, uint32_t dofs, uint32_t aofs, @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_sssub(unsigned vece, uint32_t dofs, uint32_t aofs, static void tcg_gen_usadd_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b) { - TCGv_i32 max = tcg_const_i32(-1); + TCGv_i32 max = tcg_constant_i32(-1); tcg_gen_add_i32(d, a, b); tcg_gen_movcond_i32(TCG_COND_LTU, d, d, a, max, d); - tcg_temp_free_i32(max); } static void tcg_gen_usadd_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b) { - TCGv_i64 max = tcg_const_i64(-1); + TCGv_i64 max = tcg_constant_i64(-1); tcg_gen_add_i64(d, a, b); tcg_gen_movcond_i64(TCG_COND_LTU, d, d, a, max, d); - tcg_temp_free_i64(max); } void tcg_gen_gvec_usadd(unsigned vece, uint32_t dofs, uint32_t aofs, @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_usadd(unsigned vece, uint32_t dofs, uint32_t aofs, static void tcg_gen_ussub_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b) { - TCGv_i32 min = tcg_const_i32(0); + TCGv_i32 min = tcg_constant_i32(0); tcg_gen_sub_i32(d, a, b); tcg_gen_movcond_i32(TCG_COND_LTU, d, a, b, min, d); - tcg_temp_free_i32(min); } static void tcg_gen_ussub_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b) { - TCGv_i64 min = tcg_const_i64(0); + TCGv_i64 min = tcg_constant_i64(0); tcg_gen_sub_i64(d, a, b); tcg_gen_movcond_i64(TCG_COND_LTU, d, a, b, min, d); - tcg_temp_free_i64(min); } void tcg_gen_gvec_ussub(unsigned vece, uint32_t dofs, uint32_t aofs, @@ -XXX,XX +XXX,XX @@ static void gen_negv_mask(TCGv_i64 d, TCGv_i64 b, TCGv_i64 m) void tcg_gen_vec_neg8_i64(TCGv_i64 d, TCGv_i64 b) { - TCGv_i64 m = tcg_const_i64(dup_const(MO_8, 0x80)); + TCGv_i64 m = tcg_constant_i64(dup_const(MO_8, 0x80)); gen_negv_mask(d, b, m); - tcg_temp_free_i64(m); } void tcg_gen_vec_neg16_i64(TCGv_i64 d, TCGv_i64 b) { - TCGv_i64 m = tcg_const_i64(dup_const(MO_16, 0x8000)); + TCGv_i64 m = tcg_constant_i64(dup_const(MO_16, 0x8000)); gen_negv_mask(d, b, m); - tcg_temp_free_i64(m); } void tcg_gen_vec_neg32_i64(TCGv_i64 d, TCGv_i64 b) @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_ands(unsigned vece, uint32_t dofs, uint32_t aofs, void tcg_gen_gvec_andi(unsigned vece, uint32_t dofs, uint32_t aofs, int64_t c, uint32_t oprsz, uint32_t maxsz) { - TCGv_i64 tmp = tcg_const_i64(dup_const(vece, c)); + TCGv_i64 tmp = tcg_constant_i64(dup_const(vece, c)); tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ands); - tcg_temp_free_i64(tmp); } static const GVecGen2s gop_xors = { @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_xors(unsigned vece, uint32_t dofs, uint32_t aofs, void tcg_gen_gvec_xori(unsigned vece, uint32_t dofs, uint32_t aofs, int64_t c, uint32_t oprsz, uint32_t maxsz) { - TCGv_i64 tmp = tcg_const_i64(dup_const(vece, c)); + TCGv_i64 tmp = tcg_constant_i64(dup_const(vece, c)); tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_xors); - tcg_temp_free_i64(tmp); } static const GVecGen2s gop_ors = { @@ -XXX,XX +XXX,XX @@ void tcg_gen_gvec_ors(unsigned vece, uint32_t dofs, uint32_t aofs, void tcg_gen_gvec_ori(unsigned vece, uint32_t dofs, uint32_t aofs, int64_t c, uint32_t oprsz, uint32_t maxsz) { - TCGv_i64 tmp = tcg_const_i64(dup_const(vece, c)); + TCGv_i64 tmp = tcg_constant_i64(dup_const(vece, c)); tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ors); - tcg_temp_free_i64(tmp); } void tcg_gen_vec_shl8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c) @@ -XXX,XX +XXX,XX @@ static void tcg_gen_shlv_mod_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b) { TCGv_vec t = tcg_temp_new_vec_matching(d); + TCGv_vec m = tcg_constant_vec_matching(d, vece, (8 << vece) - 1); - tcg_gen_dupi_vec(vece, t, (8 << vece) - 1); - tcg_gen_and_vec(vece, t, t, b); + tcg_gen_and_vec(vece, t, b, m); tcg_gen_shlv_vec(vece, d, a, t); tcg_temp_free_vec(t); } @@ -XXX,XX +XXX,XX @@ static void tcg_gen_shrv_mod_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b) { TCGv_vec t = tcg_temp_new_vec_matching(d); + TCGv_vec m = tcg_constant_vec_matching(d, vece, (8 << vece) - 1); - tcg_gen_dupi_vec(vece, t, (8 << vece) - 1); - tcg_gen_and_vec(vece, t, t, b); + tcg_gen_and_vec(vece, t, b, m); tcg_gen_shrv_vec(vece, d, a, t); tcg_temp_free_vec(t); } @@ -XXX,XX +XXX,XX @@ static void tcg_gen_sarv_mod_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b) { TCGv_vec t = tcg_temp_new_vec_matching(d); + TCGv_vec m = tcg_constant_vec_matching(d, vece, (8 << vece) - 1); - tcg_gen_dupi_vec(vece, t, (8 << vece) - 1); - tcg_gen_and_vec(vece, t, t, b); + tcg_gen_and_vec(vece, t, b, m); tcg_gen_sarv_vec(vece, d, a, t); tcg_temp_free_vec(t); } @@ -XXX,XX +XXX,XX @@ static void tcg_gen_rotlv_mod_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b) { TCGv_vec t = tcg_temp_new_vec_matching(d); + TCGv_vec m = tcg_constant_vec_matching(d, vece, (8 << vece) - 1); - tcg_gen_dupi_vec(vece, t, (8 << vece) - 1); - tcg_gen_and_vec(vece, t, t, b); + tcg_gen_and_vec(vece, t, b, m); tcg_gen_rotlv_vec(vece, d, a, t); tcg_temp_free_vec(t); } @@ -XXX,XX +XXX,XX @@ static void tcg_gen_rotrv_mod_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b) { TCGv_vec t = tcg_temp_new_vec_matching(d); + TCGv_vec m = tcg_constant_vec_matching(d, vece, (8 << vece) - 1); - tcg_gen_dupi_vec(vece, t, (8 << vece) - 1); - tcg_gen_and_vec(vece, t, t, b); + tcg_gen_and_vec(vece, t, b, m); tcg_gen_rotrv_vec(vece, d, a, t); tcg_temp_free_vec(t); } diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val) return temp_tcgv_vec(tcg_constant_internal(type, val)); } +TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val) +{ + TCGTemp *t = tcgv_vec_temp(match); + + tcg_debug_assert(t->temp_allocated != 0); + return tcg_constant_vec(t->base_type, vece, val); +} + TCGv_i32 tcg_const_i32(int32_t val) { TCGv_i32 t0; -- 2.25.1 The normal movi opcodes are going away. We need something for TCI to use internally. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg-opc.h | 8 ++++++++ tcg/tci.c | 4 ++-- tcg/tci/tcg-target.c.inc | 4 ++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/include/tcg/tcg-opc.h b/include/tcg/tcg-opc.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg-opc.h +++ b/include/tcg/tcg-opc.h @@ -XXX,XX +XXX,XX @@ DEF(last_generic, 0, 0, 0, TCG_OPF_NOT_PRESENT) #include "tcg-target.opc.h" #endif +#ifdef TCG_TARGET_INTERPRETER +/* These opcodes are only for use between the tci generator and interpreter. */ +DEF(tci_movi_i32, 1, 0, 1, TCG_OPF_NOT_PRESENT) +#if TCG_TARGET_REG_BITS == 64 +DEF(tci_movi_i64, 1, 0, 1, TCG_OPF_64BIT | TCG_OPF_NOT_PRESENT) +#endif +#endif + #undef TLADDR_ARGS #undef DATA64_ARGS #undef IMPL diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, t1 = tci_read_r32(regs, &tb_ptr); tci_write_reg32(regs, t0, t1); break; - case INDEX_op_movi_i32: + case INDEX_op_tci_movi_i32: t0 = *tb_ptr++; t1 = tci_read_i32(&tb_ptr); tci_write_reg32(regs, t0, t1); @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, t1 = tci_read_r64(regs, &tb_ptr); tci_write_reg64(regs, t0, t1); break; - case INDEX_op_movi_i64: + case INDEX_op_tci_movi_i64: t0 = *tb_ptr++; t1 = tci_read_i64(&tb_ptr); tci_write_reg64(regs, t0, t1); diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, uint8_t *old_code_ptr = s->code_ptr; uint32_t arg32 = arg; if (type == TCG_TYPE_I32 || arg == arg32) { - tcg_out_op_t(s, INDEX_op_movi_i32); + tcg_out_op_t(s, INDEX_op_tci_movi_i32); tcg_out_r(s, t0); tcg_out32(s, arg32); } else { tcg_debug_assert(type == TCG_TYPE_I64); #if TCG_TARGET_REG_BITS == 64 - tcg_out_op_t(s, INDEX_op_movi_i64); + tcg_out_op_t(s, INDEX_op_tci_movi_i64); tcg_out_r(s, t0); tcg_out64(s, arg); #else -- 2.25.1 These are now completely covered by mov from a TYPE_CONST temporary. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Aleksandar Markovic <aleksandar.qemu.devel@gmail.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg-opc.h | 3 --- tcg/optimize.c | 4 ---- tcg/tcg-op-vec.c | 1 - tcg/tcg.c | 18 +----------------- tcg/aarch64/tcg-target.c.inc | 3 --- tcg/arm/tcg-target.c.inc | 1 - tcg/i386/tcg-target.c.inc | 3 --- tcg/mips/tcg-target.c.inc | 2 -- tcg/ppc/tcg-target.c.inc | 3 --- tcg/riscv/tcg-target.c.inc | 2 -- tcg/s390/tcg-target.c.inc | 2 -- tcg/sparc/tcg-target.c.inc | 2 -- tcg/tci/tcg-target.c.inc | 2 -- 13 files changed, 1 insertion(+), 45 deletions(-) diff --git a/include/tcg/tcg-opc.h b/include/tcg/tcg-opc.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg-opc.h +++ b/include/tcg/tcg-opc.h @@ -XXX,XX +XXX,XX @@ DEF(br, 0, 0, 1, TCG_OPF_BB_END) DEF(mb, 0, 0, 1, 0) DEF(mov_i32, 1, 1, 0, TCG_OPF_NOT_PRESENT) -DEF(movi_i32, 1, 0, 1, TCG_OPF_NOT_PRESENT) DEF(setcond_i32, 1, 2, 1, 0) DEF(movcond_i32, 1, 4, 1, IMPL(TCG_TARGET_HAS_movcond_i32)) /* load/store */ @@ -XXX,XX +XXX,XX @@ DEF(ctz_i32, 1, 2, 0, IMPL(TCG_TARGET_HAS_ctz_i32)) DEF(ctpop_i32, 1, 1, 0, IMPL(TCG_TARGET_HAS_ctpop_i32)) DEF(mov_i64, 1, 1, 0, TCG_OPF_64BIT | TCG_OPF_NOT_PRESENT) -DEF(movi_i64, 1, 0, 1, TCG_OPF_64BIT | TCG_OPF_NOT_PRESENT) DEF(setcond_i64, 1, 2, 1, IMPL64) DEF(movcond_i64, 1, 4, 1, IMPL64 | IMPL(TCG_TARGET_HAS_movcond_i64)) /* load/store */ @@ -XXX,XX +XXX,XX @@ DEF(qemu_st8_i32, 0, TLADDR_ARGS + 1, 1, #define IMPLVEC TCG_OPF_VECTOR | IMPL(TCG_TARGET_MAYBE_vec) DEF(mov_vec, 1, 1, 0, TCG_OPF_VECTOR | TCG_OPF_NOT_PRESENT) -DEF(dupi_vec, 1, 0, 1, TCG_OPF_VECTOR | TCG_OPF_NOT_PRESENT) DEF(dup_vec, 1, 1, 0, IMPLVEC) DEF(dup2_vec, 1, 2, 0, IMPLVEC | IMPL(TCG_TARGET_REG_BITS == 32)) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(mov): tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); break; - CASE_OP_32_64(movi): - case INDEX_op_dupi_vec: - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], op->args[1]); - break; case INDEX_op_dup_vec: if (arg_is_const(op->args[1])) { diff --git a/tcg/tcg-op-vec.c b/tcg/tcg-op-vec.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-op-vec.c +++ b/tcg/tcg-op-vec.c @@ -XXX,XX +XXX,XX @@ bool tcg_can_emit_vecop_list(const TCGOpcode *list, case INDEX_op_xor_vec: case INDEX_op_mov_vec: case INDEX_op_dup_vec: - case INDEX_op_dupi_vec: case INDEX_op_dup2_vec: case INDEX_op_ld_vec: case INDEX_op_st_vec: diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ bool tcg_op_supported(TCGOpcode op) return TCG_TARGET_HAS_goto_ptr; case INDEX_op_mov_i32: - case INDEX_op_movi_i32: case INDEX_op_setcond_i32: case INDEX_op_brcond_i32: case INDEX_op_ld8u_i32: @@ -XXX,XX +XXX,XX @@ bool tcg_op_supported(TCGOpcode op) return TCG_TARGET_REG_BITS == 32; case INDEX_op_mov_i64: - case INDEX_op_movi_i64: case INDEX_op_setcond_i64: case INDEX_op_brcond_i64: case INDEX_op_ld8u_i64: @@ -XXX,XX +XXX,XX @@ bool tcg_op_supported(TCGOpcode op) case INDEX_op_mov_vec: case INDEX_op_dup_vec: - case INDEX_op_dupi_vec: case INDEX_op_dupm_vec: case INDEX_op_ld_vec: case INDEX_op_st_vec: @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs) } /* - * Specialized code generation for INDEX_op_movi_*. + * Specialized code generation for INDEX_op_mov_* with a constant. */ static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, tcg_target_ulong val, TCGLifeData arg_life, @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, } } -static void tcg_reg_alloc_movi(TCGContext *s, const TCGOp *op) -{ - TCGTemp *ots = arg_temp(op->args[0]); - tcg_target_ulong val = op->args[1]; - - tcg_reg_alloc_do_movi(s, ots, val, op->life, op->output_pref[0]); -} - /* * Specialized code generation for INDEX_op_mov_*. */ @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb) case INDEX_op_mov_vec: tcg_reg_alloc_mov(s, op); break; - case INDEX_op_movi_i32: - case INDEX_op_movi_i64: - case INDEX_op_dupi_vec: - tcg_reg_alloc_movi(s, op); - break; case INDEX_op_dup_vec: tcg_reg_alloc_dup(s, op); break; diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: - case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi. */ - case INDEX_op_movi_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ default: g_assert_not_reached(); @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */ - case INDEX_op_dupi_vec: /* Always emitted via tcg_out_movi. */ case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */ default: g_assert_not_reached(); diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ - case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi. */ case INDEX_op_call: /* Always emitted via tcg_out_call. */ default: tcg_abort(); diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: - case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi. */ - case INDEX_op_movi_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ default: tcg_abort(); @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */ - case INDEX_op_dupi_vec: /* Always emitted via tcg_out_movi. */ case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */ default: g_assert_not_reached(); diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: - case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi. */ - case INDEX_op_movi_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ default: tcg_abort(); diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: - case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi. */ - case INDEX_op_movi_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ default: tcg_abort(); @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, return; case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */ - case INDEX_op_dupi_vec: /* Always emitted via tcg_out_movi. */ case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */ default: g_assert_not_reached(); diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: - case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi. */ - case INDEX_op_movi_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ default: g_assert_not_reached(); diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.c.inc +++ b/tcg/s390/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: - case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi. */ - case INDEX_op_movi_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ default: tcg_abort(); diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: - case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi. */ - case INDEX_op_movi_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ default: tcg_abort(); diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, break; case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: - case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi. */ - case INDEX_op_movi_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ default: tcg_abort(); -- 2.25.1 There are several ways we can expand a vector dup of a 64-bit element on a 32-bit host. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) } } +static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) +{ + const TCGLifeData arg_life = op->life; + TCGTemp *ots, *itsl, *itsh; + TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64; + + /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */ + tcg_debug_assert(TCG_TARGET_REG_BITS == 32); + tcg_debug_assert(TCGOP_VECE(op) == MO_64); + + ots = arg_temp(op->args[0]); + itsl = arg_temp(op->args[1]); + itsh = arg_temp(op->args[2]); + + /* ENV should not be modified. */ + tcg_debug_assert(!temp_readonly(ots)); + + /* Allocate the output register now. */ + if (ots->val_type != TEMP_VAL_REG) { + TCGRegSet allocated_regs = s->reserved_regs; + TCGRegSet dup_out_regs = + tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; + + /* Make sure to not spill the input registers. */ + if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) { + tcg_regset_set_reg(allocated_regs, itsl->reg); + } + if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) { + tcg_regset_set_reg(allocated_regs, itsh->reg); + } + + ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, + op->output_pref[0], ots->indirect_base); + ots->val_type = TEMP_VAL_REG; + ots->mem_coherent = 0; + s->reg_to_temp[ots->reg] = ots; + } + + /* Promote dup2 of immediates to dupi_vec. */ + if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) { + uint64_t val = deposit64(itsl->val, 32, 32, itsh->val); + MemOp vece = MO_64; + + if (val == dup_const(MO_8, val)) { + vece = MO_8; + } else if (val == dup_const(MO_16, val)) { + vece = MO_16; + } else if (val == dup_const(MO_32, val)) { + vece = MO_32; + } + + tcg_out_dupi_vec(s, vtype, vece, ots->reg, val); + goto done; + } + + /* If the two inputs form one 64-bit value, try dupm_vec. */ + if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) { + if (!itsl->mem_coherent) { + temp_sync(s, itsl, s->reserved_regs, 0, 0); + } + if (!itsh->mem_coherent) { + temp_sync(s, itsh, s->reserved_regs, 0, 0); + } +#ifdef HOST_WORDS_BIGENDIAN + TCGTemp *its = itsh; +#else + TCGTemp *its = itsl; +#endif + if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg, + its->mem_base->reg, its->mem_offset)) { + goto done; + } + } + + /* Fall back to generic expansion. */ + return false; + + done: + if (IS_DEAD_ARG(1)) { + temp_dead(s, itsl); + } + if (IS_DEAD_ARG(2)) { + temp_dead(s, itsh); + } + if (NEED_SYNC_ARG(0)) { + temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0)); + } else if (IS_DEAD_ARG(0)) { + temp_dead(s, ots); + } + return true; +} + #ifdef TCG_TARGET_STACK_GROWSUP #define STACK_DIR(x) (-(x)) #else @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb) case INDEX_op_call: tcg_reg_alloc_call(s, op); break; + case INDEX_op_dup2_vec: + if (tcg_reg_alloc_dup2(s, op)) { + break; + } + /* fall through */ default: /* Sanity check that we've not introduced any unhandled opcodes. */ tcg_debug_assert(tcg_op_supported(opc)); -- 2.25.1 Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void expand_vec_rotv(TCGType type, unsigned vece, TCGv_vec v0, static void expand_vec_mul(TCGType type, unsigned vece, TCGv_vec v0, TCGv_vec v1, TCGv_vec v2) { - TCGv_vec t1, t2, t3, t4; + TCGv_vec t1, t2, t3, t4, zero; tcg_debug_assert(vece == MO_8); @@ -XXX,XX +XXX,XX @@ static void expand_vec_mul(TCGType type, unsigned vece, case TCG_TYPE_V64: t1 = tcg_temp_new_vec(TCG_TYPE_V128); t2 = tcg_temp_new_vec(TCG_TYPE_V128); - tcg_gen_dup16i_vec(t2, 0); + zero = tcg_constant_vec(TCG_TYPE_V128, MO_8, 0); vec_gen_3(INDEX_op_x86_punpckl_vec, TCG_TYPE_V128, MO_8, - tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(t2)); + tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(zero)); vec_gen_3(INDEX_op_x86_punpckl_vec, TCG_TYPE_V128, MO_8, - tcgv_vec_arg(t2), tcgv_vec_arg(t2), tcgv_vec_arg(v2)); + tcgv_vec_arg(t2), tcgv_vec_arg(zero), tcgv_vec_arg(v2)); tcg_gen_mul_vec(MO_16, t1, t1, t2); tcg_gen_shri_vec(MO_16, t1, t1, 8); vec_gen_3(INDEX_op_x86_packus_vec, TCG_TYPE_V128, MO_8, @@ -XXX,XX +XXX,XX @@ static void expand_vec_mul(TCGType type, unsigned vece, t2 = tcg_temp_new_vec(type); t3 = tcg_temp_new_vec(type); t4 = tcg_temp_new_vec(type); - tcg_gen_dup16i_vec(t4, 0); + zero = tcg_constant_vec(TCG_TYPE_V128, MO_8, 0); vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8, - tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(t4)); + tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(zero)); vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8, - tcgv_vec_arg(t2), tcgv_vec_arg(t4), tcgv_vec_arg(v2)); + tcgv_vec_arg(t2), tcgv_vec_arg(zero), tcgv_vec_arg(v2)); vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8, - tcgv_vec_arg(t3), tcgv_vec_arg(v1), tcgv_vec_arg(t4)); + tcgv_vec_arg(t3), tcgv_vec_arg(v1), tcgv_vec_arg(zero)); vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8, - tcgv_vec_arg(t4), tcgv_vec_arg(t4), tcgv_vec_arg(v2)); + tcgv_vec_arg(t4), tcgv_vec_arg(zero), tcgv_vec_arg(v2)); tcg_gen_mul_vec(MO_16, t1, t1, t2); tcg_gen_mul_vec(MO_16, t3, t3, t4); tcg_gen_shri_vec(MO_16, t1, t1, 8); @@ -XXX,XX +XXX,XX @@ static bool expand_vec_cmp_noinv(TCGType type, unsigned vece, TCGv_vec v0, NEED_UMIN = 8, NEED_UMAX = 16, }; - TCGv_vec t1, t2; + TCGv_vec t1, t2, t3; uint8_t fixup; switch (cond) { @@ -XXX,XX +XXX,XX @@ static bool expand_vec_cmp_noinv(TCGType type, unsigned vece, TCGv_vec v0, } else if (fixup & NEED_BIAS) { t1 = tcg_temp_new_vec(type); t2 = tcg_temp_new_vec(type); - tcg_gen_dupi_vec(vece, t2, 1ull << ((8 << vece) - 1)); - tcg_gen_sub_vec(vece, t1, v1, t2); - tcg_gen_sub_vec(vece, t2, v2, t2); + t3 = tcg_constant_vec(type, vece, 1ull << ((8 << vece) - 1)); + tcg_gen_sub_vec(vece, t1, v1, t3); + tcg_gen_sub_vec(vece, t2, v2, t3); v1 = t1; v2 = t2; cond = tcg_signed_cond(cond); -- 2.25.1 These interfaces have been replaced by tcg_gen_dupi_vec and tcg_constant_vec. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg-op.h | 4 ---- tcg/tcg-op-vec.c | 20 -------------------- 2 files changed, 24 deletions(-) diff --git a/include/tcg/tcg-op.h b/include/tcg/tcg-op.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg-op.h +++ b/include/tcg/tcg-op.h @@ -XXX,XX +XXX,XX @@ void tcg_gen_mov_vec(TCGv_vec, TCGv_vec); void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec, TCGv_i32); void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec, TCGv_i64); void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec, TCGv_ptr, tcg_target_long); -void tcg_gen_dup8i_vec(TCGv_vec, uint32_t); -void tcg_gen_dup16i_vec(TCGv_vec, uint32_t); -void tcg_gen_dup32i_vec(TCGv_vec, uint32_t); -void tcg_gen_dup64i_vec(TCGv_vec, uint64_t); void tcg_gen_dupi_vec(unsigned vece, TCGv_vec, uint64_t); void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); diff --git a/tcg/tcg-op-vec.c b/tcg/tcg-op-vec.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-op-vec.c +++ b/tcg/tcg-op-vec.c @@ -XXX,XX +XXX,XX @@ TCGv_vec tcg_const_ones_vec_matching(TCGv_vec m) return tcg_const_ones_vec(t->base_type); } -void tcg_gen_dup64i_vec(TCGv_vec r, uint64_t a) -{ - tcg_gen_dupi_vec(MO_64, r, a); -} - -void tcg_gen_dup32i_vec(TCGv_vec r, uint32_t a) -{ - tcg_gen_dupi_vec(MO_32, r, a); -} - -void tcg_gen_dup16i_vec(TCGv_vec r, uint32_t a) -{ - tcg_gen_dupi_vec(MO_16, r, a); -} - -void tcg_gen_dup8i_vec(TCGv_vec r, uint32_t a) -{ - tcg_gen_dupi_vec(MO_8, r, a); -} - void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a) { TCGTemp *rt = tcgv_vec_temp(r); -- 2.25.1 Improve expand_vec_shi to use sign-extraction for MO_32. This allows a single VSPLTISB instruction to load all of the valid shift constants. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/ppc/tcg-target.c.inc | 44 ++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, static void expand_vec_shi(TCGType type, unsigned vece, TCGv_vec v0, TCGv_vec v1, TCGArg imm, TCGOpcode opci) { - TCGv_vec t1 = tcg_temp_new_vec(type); + TCGv_vec t1; - /* Splat w/bytes for xxspltib. */ - tcg_gen_dupi_vec(MO_8, t1, imm & ((8 << vece) - 1)); + if (vece == MO_32) { + /* + * Only 5 bits are significant, and VSPLTISB can represent -16..15. + * So using negative numbers gets us the 4th bit easily. + */ + imm = sextract32(imm, 0, 5); + } else { + imm &= (8 << vece) - 1; + } + + /* Splat w/bytes for xxspltib when 2.07 allows MO_64. */ + t1 = tcg_constant_vec(type, MO_8, imm); vec_gen_3(opci, type, vece, tcgv_vec_arg(v0), tcgv_vec_arg(v1), tcgv_vec_arg(t1)); - tcg_temp_free_vec(t1); } static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0, @@ -XXX,XX +XXX,XX @@ static void expand_vec_mul(TCGType type, unsigned vece, TCGv_vec v0, { TCGv_vec t1 = tcg_temp_new_vec(type); TCGv_vec t2 = tcg_temp_new_vec(type); - TCGv_vec t3, t4; + TCGv_vec c0, c16; switch (vece) { case MO_8: @@ -XXX,XX +XXX,XX @@ static void expand_vec_mul(TCGType type, unsigned vece, TCGv_vec v0, case MO_32: tcg_debug_assert(!have_isa_2_07); - t3 = tcg_temp_new_vec(type); - t4 = tcg_temp_new_vec(type); - tcg_gen_dupi_vec(MO_8, t4, -16); + /* + * Only 5 bits are significant, and VSPLTISB can represent -16..15. + * So using -16 is a quick way to represent 16. + */ + c16 = tcg_constant_vec(type, MO_8, -16); + c0 = tcg_constant_vec(type, MO_8, 0); + vec_gen_3(INDEX_op_rotlv_vec, type, MO_32, tcgv_vec_arg(t1), - tcgv_vec_arg(v2), tcgv_vec_arg(t4)); + tcgv_vec_arg(v2), tcgv_vec_arg(c16)); vec_gen_3(INDEX_op_ppc_mulou_vec, type, MO_16, tcgv_vec_arg(t2), tcgv_vec_arg(v1), tcgv_vec_arg(v2)); - tcg_gen_dupi_vec(MO_8, t3, 0); - vec_gen_4(INDEX_op_ppc_msum_vec, type, MO_16, tcgv_vec_arg(t3), - tcgv_vec_arg(v1), tcgv_vec_arg(t1), tcgv_vec_arg(t3)); - vec_gen_3(INDEX_op_shlv_vec, type, MO_32, tcgv_vec_arg(t3), - tcgv_vec_arg(t3), tcgv_vec_arg(t4)); - tcg_gen_add_vec(MO_32, v0, t2, t3); - tcg_temp_free_vec(t3); - tcg_temp_free_vec(t4); + vec_gen_4(INDEX_op_ppc_msum_vec, type, MO_16, tcgv_vec_arg(t1), + tcgv_vec_arg(v1), tcgv_vec_arg(t1), tcgv_vec_arg(c0)); + vec_gen_3(INDEX_op_shlv_vec, type, MO_32, tcgv_vec_arg(t1), + tcgv_vec_arg(t1), tcgv_vec_arg(c16)); + tcg_gen_add_vec(MO_32, v0, t1, t2); break; default: -- 2.25.1 Improve rotrv_vec to reduce "t1 = -v2, t2 = t1 + c" to "t1 = -v2, t2 = c - v2". This avoids a serial dependency between t1 and t2. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target.c.inc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, TCGArg a0, ...) { va_list va; - TCGv_vec v0, v1, v2, t1, t2; + TCGv_vec v0, v1, v2, t1, t2, c1; TCGArg a2; va_start(va, a0); @@ -XXX,XX +XXX,XX @@ void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, case INDEX_op_rotlv_vec: t1 = tcg_temp_new_vec(type); - tcg_gen_dupi_vec(vece, t1, 8 << vece); - tcg_gen_sub_vec(vece, t1, v2, t1); + c1 = tcg_constant_vec(type, vece, 8 << vece); + tcg_gen_sub_vec(vece, t1, v2, c1); /* Right shifts are negative left shifts for AArch64. */ vec_gen_3(INDEX_op_shlv_vec, type, vece, tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(t1)); @@ -XXX,XX +XXX,XX @@ void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, case INDEX_op_rotrv_vec: t1 = tcg_temp_new_vec(type); t2 = tcg_temp_new_vec(type); + c1 = tcg_constant_vec(type, vece, 8 << vece); tcg_gen_neg_vec(vece, t1, v2); - tcg_gen_dupi_vec(vece, t2, 8 << vece); - tcg_gen_add_vec(vece, t2, t1, t2); + tcg_gen_sub_vec(vece, t2, c1, v2); /* Right shifts are negative left shifts for AArch64. */ vec_gen_3(INDEX_op_shlv_vec, type, vece, tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(t1)); -- 2.25.1 From: Philippe Mathieu-Daudé <f4bug@amsat.org> When decodetree.py was added in commit 568ae7efae7, QEMU was using Python 2 which happily reads UTF-8 files in text mode. Python 3 requires either UTF-8 locale or an explicit encoding passed to open(). Now that Python 3 is required, explicit UTF-8 encoding for decodetree source files. To avoid further problems with the user locale, also explicit UTF-8 encoding for the generated C files. Explicit both input/output are plain text by using the 't' mode. This fixes: $ /usr/bin/python3 scripts/decodetree.py test.decode Traceback (most recent call last): File "scripts/decodetree.py", line 1397, in <module> main() File "scripts/decodetree.py", line 1308, in main parse_file(f, toppat) File "scripts/decodetree.py", line 994, in parse_file for line in f: File "/usr/lib/python3.6/encodings/ascii.py", line 26, in decode return codecs.ascii_decode(input, self.errors)[0] UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 80: ordinal not in range(128) Reported-by: Peter Maydell <peter.maydell@linaro.org> Suggested-by: Yonggang Luo <luoyonggang@gmail.com> Reviewed-by: Eduardo Habkost <ehabkost@redhat.com> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Message-Id: <20210110000240.761122-1-f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- scripts/decodetree.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/decodetree.py b/scripts/decodetree.py index XXXXXXX..XXXXXXX 100644 --- a/scripts/decodetree.py +++ b/scripts/decodetree.py @@ -XXX,XX +XXX,XX @@ # See the syntax and semantics in docs/devel/decodetree.rst. # +import io import os import re import sys @@ -XXX,XX +XXX,XX @@ def main(): for filename in args: input_file = filename - f = open(filename, 'r') + f = open(filename, 'rt', encoding='utf-8') parse_file(f, toppat) f.close() @@ -XXX,XX +XXX,XX @@ def main(): prop_size(stree) if output_file: - output_fd = open(output_file, 'w') + output_fd = open(output_file, 'wt', encoding='utf-8') else: - output_fd = sys.stdout + output_fd = io.TextIOWrapper(sys.stdout.buffer, + encoding=sys.stdout.encoding, + errors="ignore") output_autogen() for n in sorted(arguments.keys()): -- 2.25.1 The following changes since commit e93ded1bf6c94ab95015b33e188bc8b0b0c32670: Merge tag 'testing-pull-request-2022-08-30' of https://gitlab.com/thuth/qemu into staging (2022-08-31 18:19:03 -0400) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20220901 for you to fetch changes up to 20011be2e30b8aa8ef1fc258485f00c688703deb: target/riscv: Make translator stop before the end of a page (2022-09-01 07:43:08 +0100) ---------------------------------------------------------------- Respect PROT_EXEC in user-only mode. Fix s390x, i386 and riscv for translations crossing a page. ---------------------------------------------------------------- Ilya Leoshkevich (4): linux-user: Clear translations on mprotect() accel/tcg: Introduce is_same_page() target/s390x: Make translator stop before the end of a page target/i386: Make translator stop before the end of a page Richard Henderson (16): linux-user/arm: Mark the commpage executable linux-user/hppa: Allocate page zero as a commpage linux-user/x86_64: Allocate vsyscall page as a commpage linux-user: Honor PT_GNU_STACK tests/tcg/i386: Move smc_code2 to an executable section accel/tcg: Properly implement get_page_addr_code for user-only accel/tcg: Unlock mmap_lock after longjmp accel/tcg: Make tb_htable_lookup static accel/tcg: Move qemu_ram_addr_from_host_nofail to physmem.c accel/tcg: Use probe_access_internal for softmmu get_page_addr_code_hostp accel/tcg: Document the faulting lookup in tb_lookup_cmp accel/tcg: Remove translator_ldsw accel/tcg: Add pc and host_pc params to gen_intermediate_code accel/tcg: Add fast path for translator_ld* target/riscv: Add MAX_INSN_LEN and insn_len target/riscv: Make translator stop before the end of a page include/elf.h | 1 + include/exec/cpu-common.h | 1 + include/exec/exec-all.h | 89 ++++++++---------------- include/exec/translator.h | 96 ++++++++++++++++--------- linux-user/arm/target_cpu.h | 4 +- linux-user/qemu.h | 1 + accel/tcg/cpu-exec.c | 143 ++++++++++++++++++++------------------ accel/tcg/cputlb.c | 93 +++++++------------------ accel/tcg/translate-all.c | 29 ++++---- accel/tcg/translator.c | 135 ++++++++++++++++++++++++++--------- accel/tcg/user-exec.c | 17 ++++- linux-user/elfload.c | 82 ++++++++++++++++++++-- linux-user/mmap.c | 6 +- softmmu/physmem.c | 12 ++++ target/alpha/translate.c | 5 +- target/arm/translate.c | 5 +- target/avr/translate.c | 5 +- target/cris/translate.c | 5 +- target/hexagon/translate.c | 6 +- target/hppa/translate.c | 5 +- target/i386/tcg/translate.c | 71 +++++++++++-------- target/loongarch/translate.c | 6 +- target/m68k/translate.c | 5 +- target/microblaze/translate.c | 5 +- target/mips/tcg/translate.c | 5 +- target/nios2/translate.c | 5 +- target/openrisc/translate.c | 6 +- target/ppc/translate.c | 5 +- target/riscv/translate.c | 32 +++++++-- target/rx/translate.c | 5 +- target/s390x/tcg/translate.c | 20 ++++-- target/sh4/translate.c | 5 +- target/sparc/translate.c | 5 +- target/tricore/translate.c | 6 +- target/xtensa/translate.c | 6 +- tests/tcg/i386/test-i386.c | 2 +- tests/tcg/riscv64/noexec.c | 79 +++++++++++++++++++++ tests/tcg/s390x/noexec.c | 106 ++++++++++++++++++++++++++++ tests/tcg/x86_64/noexec.c | 75 ++++++++++++++++++++ tests/tcg/multiarch/noexec.c.inc | 139 ++++++++++++++++++++++++++++++++++++ tests/tcg/riscv64/Makefile.target | 1 + tests/tcg/s390x/Makefile.target | 1 + tests/tcg/x86_64/Makefile.target | 3 +- 43 files changed, 966 insertions(+), 367 deletions(-) create mode 100644 tests/tcg/riscv64/noexec.c create mode 100644 tests/tcg/s390x/noexec.c create mode 100644 tests/tcg/x86_64/noexec.c create mode 100644 tests/tcg/multiarch/noexec.c.inc We're about to start validating PAGE_EXEC, which means that we've got to mark the commpage executable. We had been placing the commpage outside of reserved_va, which was incorrect and lead to an abort. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- linux-user/arm/target_cpu.h | 4 ++-- linux-user/elfload.c | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/linux-user/arm/target_cpu.h b/linux-user/arm/target_cpu.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/arm/target_cpu.h +++ b/linux-user/arm/target_cpu.h @@ -XXX,XX +XXX,XX @@ static inline unsigned long arm_max_reserved_va(CPUState *cs) } else { /* * We need to be able to map the commpage. - * See validate_guest_space in linux-user/elfload.c. + * See init_guest_commpage in linux-user/elfload.c. */ - return 0xffff0000ul; + return 0xfffffffful; } } #define MAX_RESERVED_VA arm_max_reserved_va diff --git a/linux-user/elfload.c b/linux-user/elfload.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/elfload.c +++ b/linux-user/elfload.c @@ -XXX,XX +XXX,XX @@ enum { static bool init_guest_commpage(void) { - void *want = g2h_untagged(HI_COMMPAGE & -qemu_host_page_size); + abi_ptr commpage = HI_COMMPAGE & -qemu_host_page_size; + void *want = g2h_untagged(commpage); void *addr = mmap(want, qemu_host_page_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0); @@ -XXX,XX +XXX,XX @@ static bool init_guest_commpage(void) perror("Protecting guest commpage"); exit(EXIT_FAILURE); } + + page_set_flags(commpage, commpage + qemu_host_page_size, + PAGE_READ | PAGE_EXEC | PAGE_VALID); return true; } -- 2.34.1 While there are no target-specific nonfaulting probes, generic code may grow some uses at some point. Note that the attrs argument was incorrect -- it should have been MEMTXATTRS_UNSPECIFIED. Just use the simpler interface. Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/avr/helper.c | 46 ++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/target/avr/helper.c b/target/avr/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/helper.c +++ b/target/avr/helper.c @@ -XXX,XX +XXX,XX @@ bool avr_cpu_tlb_fill(CPUState *cs, vaddr address, int size, MMUAccessType access_type, int mmu_idx, bool probe, uintptr_t retaddr) { - int prot = 0; - MemTxAttrs attrs = {}; + int prot, page_size = TARGET_PAGE_SIZE; uint32_t paddr; address &= TARGET_PAGE_MASK; if (mmu_idx == MMU_CODE_IDX) { - /* access to code in flash */ + /* Access to code in flash. */ paddr = OFFSET_CODE + address; prot = PAGE_READ | PAGE_EXEC; - if (paddr + TARGET_PAGE_SIZE > OFFSET_DATA) { + if (paddr >= OFFSET_DATA) { + /* + * This should not be possible via any architectural operations. + * There is certainly not an exception that we can deliver. + * Accept probing that might come from generic code. + */ + if (probe) { + return false; + } error_report("execution left flash memory"); abort(); } - } else if (address < NUMBER_OF_CPU_REGISTERS + NUMBER_OF_IO_REGISTERS) { - /* - * access to CPU registers, exit and rebuilt this TB to use full access - * incase it touches specially handled registers like SREG or SP - */ - AVRCPU *cpu = AVR_CPU(cs); - CPUAVRState *env = &cpu->env; - env->fullacc = 1; - cpu_loop_exit_restore(cs, retaddr); } else { - /* access to memory. nothing special */ + /* Access to memory. */ paddr = OFFSET_DATA + address; prot = PAGE_READ | PAGE_WRITE; + if (address < NUMBER_OF_CPU_REGISTERS + NUMBER_OF_IO_REGISTERS) { + /* + * Access to CPU registers, exit and rebuilt this TB to use + * full access in case it touches specially handled registers + * like SREG or SP. For probing, set page_size = 1, in order + * to force tlb_fill to be called for the next access. + */ + if (probe) { + page_size = 1; + } else { + AVRCPU *cpu = AVR_CPU(cs); + CPUAVRState *env = &cpu->env; + env->fullacc = 1; + cpu_loop_exit_restore(cs, retaddr); + } + } } - tlb_set_page_with_attrs(cs, address, paddr, attrs, prot, - mmu_idx, TARGET_PAGE_SIZE); - + tlb_set_page(cs, address, paddr, prot, mmu_idx, page_size); return true; } -- 2.34.1 There is no need to go through cc->tcg_ops when we know what value that must have. Reviewed-by: Michael Rolnik <mrolnik@gmail.com> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/avr/helper.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/target/avr/helper.c b/target/avr/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/helper.c +++ b/target/avr/helper.c @@ -XXX,XX +XXX,XX @@ bool avr_cpu_exec_interrupt(CPUState *cs, int interrupt_request) { bool ret = false; - CPUClass *cc = CPU_GET_CLASS(cs); AVRCPU *cpu = AVR_CPU(cs); CPUAVRState *env = &cpu->env; if (interrupt_request & CPU_INTERRUPT_RESET) { if (cpu_interrupts_enabled(env)) { cs->exception_index = EXCP_RESET; - cc->tcg_ops->do_interrupt(cs); + avr_cpu_do_interrupt(cs); cs->interrupt_request &= ~CPU_INTERRUPT_RESET; @@ -XXX,XX +XXX,XX @@ bool avr_cpu_exec_interrupt(CPUState *cs, int interrupt_request) if (cpu_interrupts_enabled(env) && env->intsrc != 0) { int index = ctz32(env->intsrc); cs->exception_index = EXCP_INT(index); - cc->tcg_ops->do_interrupt(cs); + avr_cpu_do_interrupt(cs); env->intsrc &= env->intsrc - 1; /* clear the interrupt */ if (!env->intsrc) { -- 2.34.1 We're about to start validating PAGE_EXEC, which means that we've got to mark page zero executable. We had been special casing this entirely within translate. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- linux-user/elfload.c | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/linux-user/elfload.c b/linux-user/elfload.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/elfload.c +++ b/linux-user/elfload.c @@ -XXX,XX +XXX,XX @@ static inline void init_thread(struct target_pt_regs *regs, regs->gr[31] = infop->entry; } +#define LO_COMMPAGE 0 + +static bool init_guest_commpage(void) +{ + void *want = g2h_untagged(LO_COMMPAGE); + void *addr = mmap(want, qemu_host_page_size, PROT_NONE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0); + + if (addr == MAP_FAILED) { + perror("Allocating guest commpage"); + exit(EXIT_FAILURE); + } + if (addr != want) { + return false; + } + + /* + * On Linux, page zero is normally marked execute only + gateway. + * Normal read or write is supposed to fail (thus PROT_NONE above), + * but specific offsets have kernel code mapped to raise permissions + * and implement syscalls. Here, simply mark the page executable. + * Special case the entry points during translation (see do_page_zero). + */ + page_set_flags(LO_COMMPAGE, LO_COMMPAGE + TARGET_PAGE_SIZE, + PAGE_EXEC | PAGE_VALID); + return true; +} + #endif /* TARGET_HPPA */ #ifdef TARGET_XTENSA @@ -XXX,XX +XXX,XX @@ static abi_ulong create_elf_tables(abi_ulong p, int argc, int envc, } #if defined(HI_COMMPAGE) -#define LO_COMMPAGE 0 +#define LO_COMMPAGE -1 #elif defined(LO_COMMPAGE) #define HI_COMMPAGE 0 #else #define HI_COMMPAGE 0 -#define LO_COMMPAGE 0 +#define LO_COMMPAGE -1 #define init_guest_commpage() true #endif @@ -XXX,XX +XXX,XX @@ static void pgb_static(const char *image_name, abi_ulong orig_loaddr, } else { offset = -(HI_COMMPAGE & -align); } - } else if (LO_COMMPAGE != 0) { + } else if (LO_COMMPAGE != -1) { loaddr = MIN(loaddr, LO_COMMPAGE & -align); } -- 2.34.1 We're about to start validating PAGE_EXEC, which means that we've got to mark the vsyscall page executable. We had been special casing this entirely within translate. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- linux-user/elfload.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/linux-user/elfload.c b/linux-user/elfload.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/elfload.c +++ b/linux-user/elfload.c @@ -XXX,XX +XXX,XX @@ static void elf_core_copy_regs(target_elf_gregset_t *regs, const CPUX86State *en (*regs)[26] = tswapreg(env->segs[R_GS].selector & 0xffff); } +#if ULONG_MAX >= TARGET_VSYSCALL_PAGE +#define INIT_GUEST_COMMPAGE +static bool init_guest_commpage(void) +{ + /* + * The vsyscall page is at a high negative address aka kernel space, + * which means that we cannot actually allocate it with target_mmap. + * We still should be able to use page_set_flags, unless the user + * has specified -R reserved_va, which would trigger an assert(). + */ + if (reserved_va != 0 && + TARGET_VSYSCALL_PAGE + TARGET_PAGE_SIZE >= reserved_va) { + error_report("Cannot allocate vsyscall page"); + exit(EXIT_FAILURE); + } + page_set_flags(TARGET_VSYSCALL_PAGE, + TARGET_VSYSCALL_PAGE + TARGET_PAGE_SIZE, + PAGE_EXEC | PAGE_VALID); + return true; +} +#endif #else #define ELF_START_MMAP 0x80000000 @@ -XXX,XX +XXX,XX @@ static abi_ulong create_elf_tables(abi_ulong p, int argc, int envc, #else #define HI_COMMPAGE 0 #define LO_COMMPAGE -1 +#ifndef INIT_GUEST_COMMPAGE #define init_guest_commpage() true #endif +#endif static void pgb_fail_in_use(const char *image_name) { -- 2.34.1 We cannot deliver two interrupts simultaneously; the first interrupt handler must execute first. Reviewed-by: Michael Rolnik <mrolnik@gmail.com> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/avr/helper.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/target/avr/helper.c b/target/avr/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/helper.c +++ b/target/avr/helper.c @@ -XXX,XX +XXX,XX @@ bool avr_cpu_exec_interrupt(CPUState *cs, int interrupt_request) { - bool ret = false; AVRCPU *cpu = AVR_CPU(cs); CPUAVRState *env = &cpu->env; @@ -XXX,XX +XXX,XX @@ bool avr_cpu_exec_interrupt(CPUState *cs, int interrupt_request) avr_cpu_do_interrupt(cs); cs->interrupt_request &= ~CPU_INTERRUPT_RESET; - - ret = true; + return true; } } if (interrupt_request & CPU_INTERRUPT_HARD) { @@ -XXX,XX +XXX,XX @@ bool avr_cpu_exec_interrupt(CPUState *cs, int interrupt_request) if (!env->intsrc) { cs->interrupt_request &= ~CPU_INTERRUPT_HARD; } - - ret = true; + return true; } } - return ret; + return false; } void avr_cpu_do_interrupt(CPUState *cs) -- 2.34.1 This bit is not saved across interrupts, so we must delay delivering the interrupt until the skip has been processed. Resolves: https://gitlab.com/qemu-project/qemu/-/issues/1118 Reviewed-by: Michael Rolnik <mrolnik@gmail.com> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/avr/helper.c | 9 +++++++++ target/avr/translate.c | 26 ++++++++++++++++++++++---- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/target/avr/helper.c b/target/avr/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/helper.c +++ b/target/avr/helper.c @@ -XXX,XX +XXX,XX @@ bool avr_cpu_exec_interrupt(CPUState *cs, int interrupt_request) AVRCPU *cpu = AVR_CPU(cs); CPUAVRState *env = &cpu->env; + /* + * We cannot separate a skip from the next instruction, + * as the skip would not be preserved across the interrupt. + * Separating the two insn normally only happens at page boundaries. + */ + if (env->skip) { + return false; + } + if (interrupt_request & CPU_INTERRUPT_RESET) { if (cpu_interrupts_enabled(env)) { cs->exception_index = EXCP_RESET; diff --git a/target/avr/translate.c b/target/avr/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/translate.c +++ b/target/avr/translate.c @@ -XXX,XX +XXX,XX @@ static void avr_tr_translate_insn(DisasContextBase *dcbase, CPUState *cs) if (skip_label) { canonicalize_skip(ctx); gen_set_label(skip_label); - if (ctx->base.is_jmp == DISAS_NORETURN) { + + switch (ctx->base.is_jmp) { + case DISAS_NORETURN: ctx->base.is_jmp = DISAS_CHAIN; + break; + case DISAS_NEXT: + if (ctx->base.tb->flags & TB_FLAGS_SKIP) { + ctx->base.is_jmp = DISAS_TOO_MANY; + } + break; + default: + break; } } @@ -XXX,XX +XXX,XX @@ static void avr_tr_tb_stop(DisasContextBase *dcbase, CPUState *cs) { DisasContext *ctx = container_of(dcbase, DisasContext, base); bool nonconst_skip = canonicalize_skip(ctx); + /* + * Because we disable interrupts while env->skip is set, + * we must return to the main loop to re-evaluate afterward. + */ + bool force_exit = ctx->base.tb->flags & TB_FLAGS_SKIP; switch (ctx->base.is_jmp) { case DISAS_NORETURN: @@ -XXX,XX +XXX,XX @@ static void avr_tr_tb_stop(DisasContextBase *dcbase, CPUState *cs) case DISAS_NEXT: case DISAS_TOO_MANY: case DISAS_CHAIN: - if (!nonconst_skip) { + if (!nonconst_skip && !force_exit) { /* Note gen_goto_tb checks singlestep. */ gen_goto_tb(ctx, 1, ctx->npc); break; @@ -XXX,XX +XXX,XX @@ static void avr_tr_tb_stop(DisasContextBase *dcbase, CPUState *cs) tcg_gen_movi_tl(cpu_pc, ctx->npc); /* fall through */ case DISAS_LOOKUP: - tcg_gen_lookup_and_goto_ptr(); - break; + if (!force_exit) { + tcg_gen_lookup_and_goto_ptr(); + break; + } + /* fall through */ case DISAS_EXIT: tcg_gen_exit_tb(NULL, 0); break; -- 2.34.1 Map the stack executable if required by default or on demand. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/elf.h | 1 + linux-user/qemu.h | 1 + linux-user/elfload.c | 19 ++++++++++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/include/elf.h b/include/elf.h index XXXXXXX..XXXXXXX 100644 --- a/include/elf.h +++ b/include/elf.h @@ -XXX,XX +XXX,XX @@ typedef int64_t Elf64_Sxword; #define PT_LOPROC 0x70000000 #define PT_HIPROC 0x7fffffff +#define PT_GNU_STACK (PT_LOOS + 0x474e551) #define PT_GNU_PROPERTY (PT_LOOS + 0x474e553) #define PT_MIPS_REGINFO 0x70000000 diff --git a/linux-user/qemu.h b/linux-user/qemu.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/qemu.h +++ b/linux-user/qemu.h @@ -XXX,XX +XXX,XX @@ struct image_info { uint32_t elf_flags; int personality; abi_ulong alignment; + bool exec_stack; /* Generic semihosting knows about these pointers. */ abi_ulong arg_strings; /* strings for argv */ diff --git a/linux-user/elfload.c b/linux-user/elfload.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/elfload.c +++ b/linux-user/elfload.c @@ -XXX,XX +XXX,XX @@ static bool init_guest_commpage(void) #define ELF_ARCH EM_386 #define ELF_PLATFORM get_elf_platform() +#define EXSTACK_DEFAULT true static const char *get_elf_platform(void) { @@ -XXX,XX +XXX,XX @@ static void elf_core_copy_regs(target_elf_gregset_t *regs, const CPUX86State *en #define ELF_ARCH EM_ARM #define ELF_CLASS ELFCLASS32 +#define EXSTACK_DEFAULT true static inline void init_thread(struct target_pt_regs *regs, struct image_info *infop) @@ -XXX,XX +XXX,XX @@ static inline void init_thread(struct target_pt_regs *regs, #else #define ELF_CLASS ELFCLASS32 +#define EXSTACK_DEFAULT true #endif @@ -XXX,XX +XXX,XX @@ static void elf_core_copy_regs(target_elf_gregset_t *regs, const CPUPPCState *en #define ELF_CLASS ELFCLASS64 #define ELF_ARCH EM_LOONGARCH +#define EXSTACK_DEFAULT true #define elf_check_arch(x) ((x) == EM_LOONGARCH) @@ -XXX,XX +XXX,XX @@ static uint32_t get_elf_hwcap(void) #define ELF_CLASS ELFCLASS32 #endif #define ELF_ARCH EM_MIPS +#define EXSTACK_DEFAULT true #ifdef TARGET_ABI_MIPSN32 #define elf_check_abi(x) ((x) & EF_MIPS_ABI2) @@ -XXX,XX +XXX,XX @@ static inline void init_thread(struct target_pt_regs *regs, #define bswaptls(ptr) bswap32s(ptr) #endif +#ifndef EXSTACK_DEFAULT +#define EXSTACK_DEFAULT false +#endif + #include "elf.h" /* We must delay the following stanzas until after "elf.h". */ @@ -XXX,XX +XXX,XX @@ static abi_ulong setup_arg_pages(struct linux_binprm *bprm, struct image_info *info) { abi_ulong size, error, guard; + int prot; size = guest_stack_size; if (size < STACK_LOWER_LIMIT) { @@ -XXX,XX +XXX,XX @@ static abi_ulong setup_arg_pages(struct linux_binprm *bprm, guard = qemu_real_host_page_size(); } - error = target_mmap(0, size + guard, PROT_READ | PROT_WRITE, + prot = PROT_READ | PROT_WRITE; + if (info->exec_stack) { + prot |= PROT_EXEC; + } + error = target_mmap(0, size + guard, prot, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (error == -1) { perror("mmap stack"); @@ -XXX,XX +XXX,XX @@ static void load_elf_image(const char *image_name, int image_fd, */ loaddr = -1, hiaddr = 0; info->alignment = 0; + info->exec_stack = EXSTACK_DEFAULT; for (i = 0; i < ehdr->e_phnum; ++i) { struct elf_phdr *eppnt = phdr + i; if (eppnt->p_type == PT_LOAD) { @@ -XXX,XX +XXX,XX @@ static void load_elf_image(const char *image_name, int image_fd, if (!parse_elf_properties(image_fd, info, eppnt, bprm_buf, &err)) { goto exit_errmsg; } + } else if (eppnt->p_type == PT_GNU_STACK) { + info->exec_stack = eppnt->p_flags & PF_X; } } -- 2.34.1 From: Ilya Leoshkevich <iii@linux.ibm.com> Currently it's possible to execute pages that do not have PAGE_EXEC if there is an existing translation block. Fix by invalidating TBs that touch the affected pages. Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com> Message-Id: <20220817150506.592862-2-iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- linux-user/mmap.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/linux-user/mmap.c b/linux-user/mmap.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/mmap.c +++ b/linux-user/mmap.c @@ -XXX,XX +XXX,XX @@ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot) goto error; } } + page_set_flags(start, start + len, page_flags); - mmap_unlock(); - return 0; + tb_invalidate_phys_range(start, start + len); + ret = 0; + error: mmap_unlock(); return ret; -- 2.34.1 We're about to start validating PAGE_EXEC, which means that we've got to put this code into a section that is both writable and executable. Note that this test did not run on hardware beforehand either. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tests/tcg/i386/test-i386.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tcg/i386/test-i386.c b/tests/tcg/i386/test-i386.c index XXXXXXX..XXXXXXX 100644 --- a/tests/tcg/i386/test-i386.c +++ b/tests/tcg/i386/test-i386.c @@ -XXX,XX +XXX,XX @@ uint8_t code[] = { 0xc3, /* ret */ }; -asm(".section \".data\"\n" +asm(".section \".data_x\",\"awx\"\n" "smc_code2:\n" "movl 4(%esp), %eax\n" "movl %eax, smc_patch_addr2 + 1\n" -- 2.34.1 From: Ilya Leoshkevich <iii@linux.ibm.com> Introduce a function that checks whether a given address is on the same page as where disassembly started. Having it improves readability of the following patches. Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com> Message-Id: <20220811095534.241224-3-iii@linux.ibm.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> [rth: Make the DisasContextBase parameter const.] Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/translator.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/exec/translator.h b/include/exec/translator.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/translator.h +++ b/include/exec/translator.h @@ -XXX,XX +XXX,XX @@ FOR_EACH_TRANSLATOR_LD(GEN_TRANSLATOR_LD) #undef GEN_TRANSLATOR_LD +/* + * Return whether addr is on the same page as where disassembly started. + * Translators can use this to enforce the rule that only single-insn + * translation blocks are allowed to cross page boundaries. + */ +static inline bool is_same_page(const DisasContextBase *db, target_ulong addr) +{ + return ((addr ^ db->pc_first) & TARGET_PAGE_MASK) == 0; +} + #endif /* EXEC__TRANSLATOR_H */ -- 2.34.1 The current implementation is a no-op, simply returning addr. This is incorrect, because we ought to be checking the page permissions for execution. Make get_page_addr_code inline for both implementations. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Acked-by: Alistair Francis <alistair.francis@wdc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/exec-all.h | 85 ++++++++++++++--------------------------- accel/tcg/cputlb.c | 5 --- accel/tcg/user-exec.c | 14 +++++++ 3 files changed, 42 insertions(+), 62 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ struct MemoryRegionSection *iotlb_to_section(CPUState *cpu, hwaddr index, MemTxAttrs attrs); #endif -#if defined(CONFIG_USER_ONLY) -void mmap_lock(void); -void mmap_unlock(void); -bool have_mmap_lock(void); - /** - * get_page_addr_code() - user-mode version + * get_page_addr_code_hostp() * @env: CPUArchState * @addr: guest virtual address of guest code * - * Returns @addr. + * See get_page_addr_code() (full-system version) for documentation on the + * return value. + * + * Sets *@hostp (when @hostp is non-NULL) as follows. + * If the return value is -1, sets *@hostp to NULL. Otherwise, sets *@hostp + * to the host address where @addr's content is kept. + * + * Note: this function can trigger an exception. + */ +tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, + void **hostp); + +/** + * get_page_addr_code() + * @env: CPUArchState + * @addr: guest virtual address of guest code + * + * If we cannot translate and execute from the entire RAM page, or if + * the region is not backed by RAM, returns -1. Otherwise, returns the + * ram_addr_t corresponding to the guest code at @addr. + * + * Note: this function can trigger an exception. */ static inline tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr) { - return addr; + return get_page_addr_code_hostp(env, addr, NULL); } -/** - * get_page_addr_code_hostp() - user-mode version - * @env: CPUArchState - * @addr: guest virtual address of guest code - * - * Returns @addr. - * - * If @hostp is non-NULL, sets *@hostp to the host address where @addr's content - * is kept. - */ -static inline tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, - target_ulong addr, - void **hostp) -{ - if (hostp) { - *hostp = g2h_untagged(addr); - } - return addr; -} +#if defined(CONFIG_USER_ONLY) +void mmap_lock(void); +void mmap_unlock(void); +bool have_mmap_lock(void); /** * adjust_signal_pc: @@ -XXX,XX +XXX,XX @@ G_NORETURN void cpu_loop_exit_sigbus(CPUState *cpu, target_ulong addr, static inline void mmap_lock(void) {} static inline void mmap_unlock(void) {} -/** - * get_page_addr_code() - full-system version - * @env: CPUArchState - * @addr: guest virtual address of guest code - * - * If we cannot translate and execute from the entire RAM page, or if - * the region is not backed by RAM, returns -1. Otherwise, returns the - * ram_addr_t corresponding to the guest code at @addr. - * - * Note: this function can trigger an exception. - */ -tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr); - -/** - * get_page_addr_code_hostp() - full-system version - * @env: CPUArchState - * @addr: guest virtual address of guest code - * - * See get_page_addr_code() (full-system version) for documentation on the - * return value. - * - * Sets *@hostp (when @hostp is non-NULL) as follows. - * If the return value is -1, sets *@hostp to NULL. Otherwise, sets *@hostp - * to the host address where @addr's content is kept. - * - * Note: this function can trigger an exception. - */ -tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, - void **hostp); - void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length); void tlb_set_dirty(CPUState *cpu, target_ulong vaddr); diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, return qemu_ram_addr_from_host_nofail(p); } -tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr) -{ - return get_page_addr_code_hostp(env, addr, NULL); -} - static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size, CPUIOTLBEntry *iotlbentry, uintptr_t retaddr) { diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -XXX,XX +XXX,XX @@ void *probe_access(CPUArchState *env, target_ulong addr, int size, return size ? g2h(env_cpu(env), addr) : NULL; } +tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, + void **hostp) +{ + int flags; + + flags = probe_access_internal(env, addr, 1, MMU_INST_FETCH, false, 0); + g_assert(flags == 0); + + if (hostp) { + *hostp = g2h_untagged(addr); + } + return addr; +} + /* The softmmu versions of these helpers are in cputlb.c. */ /* -- 2.34.1 The mmap_lock is held around tb_gen_code. While the comment is correct that the lock is dropped when tb_gen_code runs out of memory, the lock is *not* dropped when an exception is raised reading code for translation. Acked-by: Alistair Francis <alistair.francis@wdc.com> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cpu-exec.c | 12 ++++++------ accel/tcg/user-exec.c | 3 --- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ void cpu_exec_step_atomic(CPUState *cpu) cpu_tb_exec(cpu, tb, &tb_exit); cpu_exec_exit(cpu); } else { - /* - * The mmap_lock is dropped by tb_gen_code if it runs out of - * memory. - */ #ifndef CONFIG_SOFTMMU clear_helper_retaddr(); - tcg_debug_assert(!have_mmap_lock()); + if (have_mmap_lock()) { + mmap_unlock(); + } #endif if (qemu_mutex_iothread_locked()) { qemu_mutex_unlock_iothread(); @@ -XXX,XX +XXX,XX @@ int cpu_exec(CPUState *cpu) #ifndef CONFIG_SOFTMMU clear_helper_retaddr(); - tcg_debug_assert(!have_mmap_lock()); + if (have_mmap_lock()) { + mmap_unlock(); + } #endif if (qemu_mutex_iothread_locked()) { qemu_mutex_unlock_iothread(); diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -XXX,XX +XXX,XX @@ MMUAccessType adjust_signal_pc(uintptr_t *pc, bool is_write) * (and if the translator doesn't handle page boundaries correctly * there's little we can do about that here). Therefore, do not * trigger the unwinder. - * - * Like tb_gen_code, release the memory lock before cpu_loop_exit. */ - mmap_unlock(); *pc = 0; return MMU_INST_FETCH; } -- 2.34.1 The function is not used outside of cpu-exec.c. Move it and its subroutines up in the file, before the first use. Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/exec-all.h | 3 - accel/tcg/cpu-exec.c | 122 ++++++++++++++++++++-------------------- 2 files changed, 61 insertions(+), 64 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs); #endif void tb_flush(CPUState *cpu); void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr); -TranslationBlock *tb_htable_lookup(CPUState *cpu, target_ulong pc, - target_ulong cs_base, uint32_t flags, - uint32_t cflags); void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr); /* GETPC is the true target of the return instruction that we'll execute. */ diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ uint32_t curr_cflags(CPUState *cpu) return cflags; } +struct tb_desc { + target_ulong pc; + target_ulong cs_base; + CPUArchState *env; + tb_page_addr_t phys_page1; + uint32_t flags; + uint32_t cflags; + uint32_t trace_vcpu_dstate; +}; + +static bool tb_lookup_cmp(const void *p, const void *d) +{ + const TranslationBlock *tb = p; + const struct tb_desc *desc = d; + + if (tb->pc == desc->pc && + tb->page_addr[0] == desc->phys_page1 && + tb->cs_base == desc->cs_base && + tb->flags == desc->flags && + tb->trace_vcpu_dstate == desc->trace_vcpu_dstate && + tb_cflags(tb) == desc->cflags) { + /* check next page if needed */ + if (tb->page_addr[1] == -1) { + return true; + } else { + tb_page_addr_t phys_page2; + target_ulong virt_page2; + + virt_page2 = (desc->pc & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE; + phys_page2 = get_page_addr_code(desc->env, virt_page2); + if (tb->page_addr[1] == phys_page2) { + return true; + } + } + } + return false; +} + +static TranslationBlock *tb_htable_lookup(CPUState *cpu, target_ulong pc, + target_ulong cs_base, uint32_t flags, + uint32_t cflags) +{ + tb_page_addr_t phys_pc; + struct tb_desc desc; + uint32_t h; + + desc.env = cpu->env_ptr; + desc.cs_base = cs_base; + desc.flags = flags; + desc.cflags = cflags; + desc.trace_vcpu_dstate = *cpu->trace_dstate; + desc.pc = pc; + phys_pc = get_page_addr_code(desc.env, pc); + if (phys_pc == -1) { + return NULL; + } + desc.phys_page1 = phys_pc & TARGET_PAGE_MASK; + h = tb_hash_func(phys_pc, pc, flags, cflags, *cpu->trace_dstate); + return qht_lookup_custom(&tb_ctx.htable, &desc, h, tb_lookup_cmp); +} + /* Might cause an exception, so have a longjmp destination ready */ static inline TranslationBlock *tb_lookup(CPUState *cpu, target_ulong pc, target_ulong cs_base, @@ -XXX,XX +XXX,XX @@ void cpu_exec_step_atomic(CPUState *cpu) end_exclusive(); } -struct tb_desc { - target_ulong pc; - target_ulong cs_base; - CPUArchState *env; - tb_page_addr_t phys_page1; - uint32_t flags; - uint32_t cflags; - uint32_t trace_vcpu_dstate; -}; - -static bool tb_lookup_cmp(const void *p, const void *d) -{ - const TranslationBlock *tb = p; - const struct tb_desc *desc = d; - - if (tb->pc == desc->pc && - tb->page_addr[0] == desc->phys_page1 && - tb->cs_base == desc->cs_base && - tb->flags == desc->flags && - tb->trace_vcpu_dstate == desc->trace_vcpu_dstate && - tb_cflags(tb) == desc->cflags) { - /* check next page if needed */ - if (tb->page_addr[1] == -1) { - return true; - } else { - tb_page_addr_t phys_page2; - target_ulong virt_page2; - - virt_page2 = (desc->pc & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE; - phys_page2 = get_page_addr_code(desc->env, virt_page2); - if (tb->page_addr[1] == phys_page2) { - return true; - } - } - } - return false; -} - -TranslationBlock *tb_htable_lookup(CPUState *cpu, target_ulong pc, - target_ulong cs_base, uint32_t flags, - uint32_t cflags) -{ - tb_page_addr_t phys_pc; - struct tb_desc desc; - uint32_t h; - - desc.env = cpu->env_ptr; - desc.cs_base = cs_base; - desc.flags = flags; - desc.cflags = cflags; - desc.trace_vcpu_dstate = *cpu->trace_dstate; - desc.pc = pc; - phys_pc = get_page_addr_code(desc.env, pc); - if (phys_pc == -1) { - return NULL; - } - desc.phys_page1 = phys_pc & TARGET_PAGE_MASK; - h = tb_hash_func(phys_pc, pc, flags, cflags, *cpu->trace_dstate); - return qht_lookup_custom(&tb_ctx.htable, &desc, h, tb_lookup_cmp); -} - void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr) { if (TCG_TARGET_HAS_direct_jump) { -- 2.34.1 The base qemu_ram_addr_from_host function is already in softmmu/physmem.c; move the nofail version to be adjacent. Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/cpu-common.h | 1 + accel/tcg/cputlb.c | 12 ------------ softmmu/physmem.c | 12 ++++++++++++ 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu-common.h +++ b/include/exec/cpu-common.h @@ -XXX,XX +XXX,XX @@ typedef uintptr_t ram_addr_t; void qemu_ram_remap(ram_addr_t addr, ram_addr_t length); /* This should not be used by devices. */ ram_addr_t qemu_ram_addr_from_host(void *ptr); +ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr); RAMBlock *qemu_ram_block_by_name(const char *name); RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset, ram_addr_t *offset); diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ void tlb_set_page(CPUState *cpu, target_ulong vaddr, prot, mmu_idx, size); } -static inline ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr) -{ - ram_addr_t ram_addr; - - ram_addr = qemu_ram_addr_from_host(ptr); - if (ram_addr == RAM_ADDR_INVALID) { - error_report("Bad ram pointer %p", ptr); - abort(); - } - return ram_addr; -} - /* * Note: tlb_fill() can trigger a resize of the TLB. This means that all of the * caller's prior references to the TLB table (e.g. CPUTLBEntry pointers) must diff --git a/softmmu/physmem.c b/softmmu/physmem.c index XXXXXXX..XXXXXXX 100644 --- a/softmmu/physmem.c +++ b/softmmu/physmem.c @@ -XXX,XX +XXX,XX @@ ram_addr_t qemu_ram_addr_from_host(void *ptr) return block->offset + offset; } +ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr) +{ + ram_addr_t ram_addr; + + ram_addr = qemu_ram_addr_from_host(ptr); + if (ram_addr == RAM_ADDR_INVALID) { + error_report("Bad ram pointer %p", ptr); + abort(); + } + return ram_addr; +} + static MemTxResult flatview_read(FlatView *fv, hwaddr addr, MemTxAttrs attrs, void *buf, hwaddr len); static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs, -- 2.34.1 Simplify the implementation of get_page_addr_code_hostp by reusing the existing probe_access infrastructure. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cputlb.c | 76 ++++++++++++++++------------------------------ 1 file changed, 26 insertions(+), 50 deletions(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index, victim_tlb_hit(env, mmu_idx, index, offsetof(CPUTLBEntry, TY), \ (ADDR) & TARGET_PAGE_MASK) -/* - * Return a ram_addr_t for the virtual address for execution. - * - * Return -1 if we can't translate and execute from an entire page - * of RAM. This will force us to execute by loading and translating - * one insn at a time, without caching. - * - * NOTE: This function will trigger an exception if the page is - * not executable. - */ -tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, - void **hostp) -{ - uintptr_t mmu_idx = cpu_mmu_index(env, true); - uintptr_t index = tlb_index(env, mmu_idx, addr); - CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); - void *p; - - if (unlikely(!tlb_hit(entry->addr_code, addr))) { - if (!VICTIM_TLB_HIT(addr_code, addr)) { - tlb_fill(env_cpu(env), addr, 0, MMU_INST_FETCH, mmu_idx, 0); - index = tlb_index(env, mmu_idx, addr); - entry = tlb_entry(env, mmu_idx, addr); - - if (unlikely(entry->addr_code & TLB_INVALID_MASK)) { - /* - * The MMU protection covers a smaller range than a target - * page, so we must redo the MMU check for every insn. - */ - return -1; - } - } - assert(tlb_hit(entry->addr_code, addr)); - } - - if (unlikely(entry->addr_code & TLB_MMIO)) { - /* The region is not backed by RAM. */ - if (hostp) { - *hostp = NULL; - } - return -1; - } - - p = (void *)((uintptr_t)addr + entry->addend); - if (hostp) { - *hostp = p; - } - return qemu_ram_addr_from_host_nofail(p); -} - static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size, CPUIOTLBEntry *iotlbentry, uintptr_t retaddr) { @@ -XXX,XX +XXX,XX @@ void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr, return flags ? NULL : host; } +/* + * Return a ram_addr_t for the virtual address for execution. + * + * Return -1 if we can't translate and execute from an entire page + * of RAM. This will force us to execute by loading and translating + * one insn at a time, without caching. + * + * NOTE: This function will trigger an exception if the page is + * not executable. + */ +tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, + void **hostp) +{ + void *p; + + (void)probe_access_internal(env, addr, 1, MMU_INST_FETCH, + cpu_mmu_index(env, true), false, &p, 0); + if (p == NULL) { + return -1; + } + if (hostp) { + *hostp = p; + } + return qemu_ram_addr_from_host_nofail(p); +} + #ifdef CONFIG_PLUGIN /* * Perform a TLB lookup and populate the qemu_plugin_hwaddr structure. -- 2.34.1 It was non-obvious to me why we can raise an exception in the middle of a comparison function, but it works. While nearby, use TARGET_PAGE_ALIGN instead of open-coding. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cpu-exec.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ static bool tb_lookup_cmp(const void *p, const void *d) tb_page_addr_t phys_page2; target_ulong virt_page2; - virt_page2 = (desc->pc & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE; + /* + * We know that the first page matched, and an otherwise valid TB + * encountered an incomplete instruction at the end of that page, + * therefore we know that generating a new TB from the current PC + * must also require reading from the next page -- even if the + * second pages do not match, and therefore the resulting insn + * is different for the new TB. Therefore any exception raised + * here by the faulting lookup is not premature. + */ + virt_page2 = TARGET_PAGE_ALIGN(desc->pc); phys_page2 = get_page_addr_code(desc->env, virt_page2); if (tb->page_addr[1] == phys_page2) { return true; -- 2.34.1 The only user can easily use translator_lduw and adjust the type to signed during the return. Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/translator.h | 1 - target/i386/tcg/translate.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/include/exec/translator.h b/include/exec/translator.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/translator.h +++ b/include/exec/translator.h @@ -XXX,XX +XXX,XX @@ bool translator_use_goto_tb(DisasContextBase *db, target_ulong dest); #define FOR_EACH_TRANSLATOR_LD(F) \ F(translator_ldub, uint8_t, cpu_ldub_code, /* no swap */) \ - F(translator_ldsw, int16_t, cpu_ldsw_code, bswap16) \ F(translator_lduw, uint16_t, cpu_lduw_code, bswap16) \ F(translator_ldl, uint32_t, cpu_ldl_code, bswap32) \ F(translator_ldq, uint64_t, cpu_ldq_code, bswap64) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static inline uint8_t x86_ldub_code(CPUX86State *env, DisasContext *s) static inline int16_t x86_ldsw_code(CPUX86State *env, DisasContext *s) { - return translator_ldsw(env, &s->base, advance_pc(env, s, 2)); + return translator_lduw(env, &s->base, advance_pc(env, s, 2)); } static inline uint16_t x86_lduw_code(CPUX86State *env, DisasContext *s) -- 2.34.1 Pass these along to translator_loop -- pc may be used instead of tb->pc, and host_pc is currently unused. Adjust all targets at one time. Acked-by: Alistair Francis <alistair.francis@wdc.com> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/exec-all.h | 1 - include/exec/translator.h | 24 ++++++++++++++++++++---- accel/tcg/translate-all.c | 6 ++++-- accel/tcg/translator.c | 9 +++++---- target/alpha/translate.c | 5 +++-- target/arm/translate.c | 5 +++-- target/avr/translate.c | 5 +++-- target/cris/translate.c | 5 +++-- target/hexagon/translate.c | 6 ++++-- target/hppa/translate.c | 5 +++-- target/i386/tcg/translate.c | 5 +++-- target/loongarch/translate.c | 6 ++++-- target/m68k/translate.c | 5 +++-- target/microblaze/translate.c | 5 +++-- target/mips/tcg/translate.c | 5 +++-- target/nios2/translate.c | 5 +++-- target/openrisc/translate.c | 6 ++++-- target/ppc/translate.c | 5 +++-- target/riscv/translate.c | 5 +++-- target/rx/translate.c | 5 +++-- target/s390x/tcg/translate.c | 5 +++-- target/sh4/translate.c | 5 +++-- target/sparc/translate.c | 5 +++-- target/tricore/translate.c | 6 ++++-- target/xtensa/translate.c | 6 ++++-- 25 files changed, 97 insertions(+), 53 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ typedef ram_addr_t tb_page_addr_t; #define TB_PAGE_ADDR_FMT RAM_ADDR_FMT #endif -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns); void restore_state_to_opc(CPUArchState *env, TranslationBlock *tb, target_ulong *data); diff --git a/include/exec/translator.h b/include/exec/translator.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/translator.h +++ b/include/exec/translator.h @@ -XXX,XX +XXX,XX @@ #include "exec/translate-all.h" #include "tcg/tcg.h" +/** + * gen_intermediate_code + * @cpu: cpu context + * @tb: translation block + * @max_insns: max number of instructions to translate + * @pc: guest virtual program counter address + * @host_pc: host physical program counter address + * + * This function must be provided by the target, which should create + * the target-specific DisasContext, and then invoke translator_loop. + */ +void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc); /** * DisasJumpType: @@ -XXX,XX +XXX,XX @@ typedef struct TranslatorOps { /** * translator_loop: - * @ops: Target-specific operations. - * @db: Disassembly context. * @cpu: Target vCPU. * @tb: Translation block. * @max_insns: Maximum number of insns to translate. + * @pc: guest virtual program counter address + * @host_pc: host physical program counter address + * @ops: Target-specific operations. + * @db: Disassembly context. * * Generic translator loop. * @@ -XXX,XX +XXX,XX @@ typedef struct TranslatorOps { * - When single-stepping is enabled (system-wide or on the current vCPU). * - When too many instructions have been translated. */ -void translator_loop(const TranslatorOps *ops, DisasContextBase *db, - CPUState *cpu, TranslationBlock *tb, int max_insns); +void translator_loop(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc, + const TranslatorOps *ops, DisasContextBase *db); void translator_loop_temp_check(DisasContextBase *db); diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ #include "exec/cputlb.h" #include "exec/translate-all.h" +#include "exec/translator.h" #include "qemu/bitmap.h" #include "qemu/qemu-print.h" #include "qemu/timer.h" @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, TCGProfile *prof = &tcg_ctx->prof; int64_t ti; #endif + void *host_pc; assert_memory_lock(); qemu_thread_jit_write(); - phys_pc = get_page_addr_code(env, pc); + phys_pc = get_page_addr_code_hostp(env, pc, &host_pc); if (phys_pc == -1) { /* Generate a one-shot TB with 1 insn in it */ @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, tcg_func_start(tcg_ctx); tcg_ctx->cpu = env_cpu(env); - gen_intermediate_code(cpu, tb, max_insns); + gen_intermediate_code(cpu, tb, max_insns, pc, host_pc); assert(tb->size != 0); tcg_ctx->cpu = NULL; max_insns = tb->icount; diff --git a/accel/tcg/translator.c b/accel/tcg/translator.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translator.c +++ b/accel/tcg/translator.c @@ -XXX,XX +XXX,XX @@ static inline void translator_page_protect(DisasContextBase *dcbase, #endif } -void translator_loop(const TranslatorOps *ops, DisasContextBase *db, - CPUState *cpu, TranslationBlock *tb, int max_insns) +void translator_loop(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc, + const TranslatorOps *ops, DisasContextBase *db) { uint32_t cflags = tb_cflags(tb); bool plugin_enabled; /* Initialize DisasContext */ db->tb = tb; - db->pc_first = tb->pc; - db->pc_next = db->pc_first; + db->pc_first = pc; + db->pc_next = pc; db->is_jmp = DISAS_NEXT; db->num_insns = 0; db->max_insns = max_insns; diff --git a/target/alpha/translate.c b/target/alpha/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/alpha/translate.c +++ b/target/alpha/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps alpha_tr_ops = { .disas_log = alpha_tr_disas_log, }; -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc; - translator_loop(&alpha_tr_ops, &dc.base, cpu, tb, max_insns); + translator_loop(cpu, tb, max_insns, pc, host_pc, &alpha_tr_ops, &dc.base); } void restore_state_to_opc(CPUAlphaState *env, TranslationBlock *tb, diff --git a/target/arm/translate.c b/target/arm/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate.c +++ b/target/arm/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps thumb_translator_ops = { }; /* generate intermediate code for basic block 'tb'. */ -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc = { }; const TranslatorOps *ops = &arm_translator_ops; @@ -XXX,XX +XXX,XX @@ void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) } #endif - translator_loop(ops, &dc.base, cpu, tb, max_insns); + translator_loop(cpu, tb, max_insns, pc, host_pc, ops, &dc.base); } void restore_state_to_opc(CPUARMState *env, TranslationBlock *tb, diff --git a/target/avr/translate.c b/target/avr/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/translate.c +++ b/target/avr/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps avr_tr_ops = { .disas_log = avr_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc = { }; - translator_loop(&avr_tr_ops, &dc.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &avr_tr_ops, &dc.base); } void restore_state_to_opc(CPUAVRState *env, TranslationBlock *tb, diff --git a/target/cris/translate.c b/target/cris/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/cris/translate.c +++ b/target/cris/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps cris_tr_ops = { .disas_log = cris_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc; - translator_loop(&cris_tr_ops, &dc.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &cris_tr_ops, &dc.base); } void cris_cpu_dump_state(CPUState *cs, FILE *f, int flags) diff --git a/target/hexagon/translate.c b/target/hexagon/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/translate.c +++ b/target/hexagon/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps hexagon_tr_ops = { .disas_log = hexagon_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&hexagon_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, + &hexagon_tr_ops, &ctx.base); } #define NAME_LEN 64 diff --git a/target/hppa/translate.c b/target/hppa/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/hppa/translate.c +++ b/target/hppa/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps hppa_tr_ops = { .disas_log = hppa_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&hppa_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &hppa_tr_ops, &ctx.base); } void restore_state_to_opc(CPUHPPAState *env, TranslationBlock *tb, diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps i386_tr_ops = { }; /* generate intermediate code for basic block 'tb'. */ -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc; - translator_loop(&i386_tr_ops, &dc.base, cpu, tb, max_insns); + translator_loop(cpu, tb, max_insns, pc, host_pc, &i386_tr_ops, &dc.base); } void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb, diff --git a/target/loongarch/translate.c b/target/loongarch/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/loongarch/translate.c +++ b/target/loongarch/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps loongarch_tr_ops = { .disas_log = loongarch_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&loongarch_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, + &loongarch_tr_ops, &ctx.base); } void loongarch_translate_init(void) diff --git a/target/m68k/translate.c b/target/m68k/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/m68k/translate.c +++ b/target/m68k/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps m68k_tr_ops = { .disas_log = m68k_tr_disas_log, }; -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc; - translator_loop(&m68k_tr_ops, &dc.base, cpu, tb, max_insns); + translator_loop(cpu, tb, max_insns, pc, host_pc, &m68k_tr_ops, &dc.base); } static double floatx80_to_double(CPUM68KState *env, uint16_t high, uint64_t low) diff --git a/target/microblaze/translate.c b/target/microblaze/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/microblaze/translate.c +++ b/target/microblaze/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps mb_tr_ops = { .disas_log = mb_tr_disas_log, }; -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc; - translator_loop(&mb_tr_ops, &dc.base, cpu, tb, max_insns); + translator_loop(cpu, tb, max_insns, pc, host_pc, &mb_tr_ops, &dc.base); } void mb_cpu_dump_state(CPUState *cs, FILE *f, int flags) diff --git a/target/mips/tcg/translate.c b/target/mips/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/mips/tcg/translate.c +++ b/target/mips/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps mips_tr_ops = { .disas_log = mips_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&mips_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &mips_tr_ops, &ctx.base); } void mips_tcg_init(void) diff --git a/target/nios2/translate.c b/target/nios2/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/nios2/translate.c +++ b/target/nios2/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps nios2_tr_ops = { .disas_log = nios2_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc; - translator_loop(&nios2_tr_ops, &dc.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &nios2_tr_ops, &dc.base); } void nios2_cpu_dump_state(CPUState *cs, FILE *f, int flags) diff --git a/target/openrisc/translate.c b/target/openrisc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/openrisc/translate.c +++ b/target/openrisc/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps openrisc_tr_ops = { .disas_log = openrisc_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&openrisc_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, + &openrisc_tr_ops, &ctx.base); } void openrisc_cpu_dump_state(CPUState *cs, FILE *f, int flags) diff --git a/target/ppc/translate.c b/target/ppc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/translate.c +++ b/target/ppc/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps ppc_tr_ops = { .disas_log = ppc_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&ppc_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &ppc_tr_ops, &ctx.base); } void restore_state_to_opc(CPUPPCState *env, TranslationBlock *tb, diff --git a/target/riscv/translate.c b/target/riscv/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/translate.c +++ b/target/riscv/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps riscv_tr_ops = { .disas_log = riscv_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&riscv_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &riscv_tr_ops, &ctx.base); } void riscv_translate_init(void) diff --git a/target/rx/translate.c b/target/rx/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/rx/translate.c +++ b/target/rx/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps rx_tr_ops = { .disas_log = rx_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc; - translator_loop(&rx_tr_ops, &dc.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &rx_tr_ops, &dc.base); } void restore_state_to_opc(CPURXState *env, TranslationBlock *tb, diff --git a/target/s390x/tcg/translate.c b/target/s390x/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/tcg/translate.c +++ b/target/s390x/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps s390x_tr_ops = { .disas_log = s390x_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc; - translator_loop(&s390x_tr_ops, &dc.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &s390x_tr_ops, &dc.base); } void restore_state_to_opc(CPUS390XState *env, TranslationBlock *tb, diff --git a/target/sh4/translate.c b/target/sh4/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/sh4/translate.c +++ b/target/sh4/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps sh4_tr_ops = { .disas_log = sh4_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&sh4_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &sh4_tr_ops, &ctx.base); } void restore_state_to_opc(CPUSH4State *env, TranslationBlock *tb, diff --git a/target/sparc/translate.c b/target/sparc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/translate.c +++ b/target/sparc/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps sparc_tr_ops = { .disas_log = sparc_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc = {}; - translator_loop(&sparc_tr_ops, &dc.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &sparc_tr_ops, &dc.base); } void sparc_tcg_init(void) diff --git a/target/tricore/translate.c b/target/tricore/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/tricore/translate.c +++ b/target/tricore/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps tricore_tr_ops = { }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&tricore_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, + &tricore_tr_ops, &ctx.base); } void diff --git a/target/xtensa/translate.c b/target/xtensa/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/xtensa/translate.c +++ b/target/xtensa/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps xtensa_translator_ops = { .disas_log = xtensa_tr_disas_log, }; -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc = {}; - translator_loop(&xtensa_translator_ops, &dc.base, cpu, tb, max_insns); + translator_loop(cpu, tb, max_insns, pc, host_pc, + &xtensa_translator_ops, &dc.base); } void xtensa_cpu_dump_state(CPUState *cs, FILE *f, int flags) -- 2.34.1 Cache the translation from guest to host address, so we may use direct loads when we hit on the primary translation page. Look up the second translation page only once, during translation. This obviates another lookup of the second page within tb_gen_code after translation. Fixes a bug in that plugin_insn_append should be passed the bytes in the original memory order, not bswapped by pieces. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/translator.h | 63 +++++++++++-------- accel/tcg/translate-all.c | 23 +++---- accel/tcg/translator.c | 126 +++++++++++++++++++++++++++++--------- 3 files changed, 141 insertions(+), 71 deletions(-) diff --git a/include/exec/translator.h b/include/exec/translator.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/translator.h +++ b/include/exec/translator.h @@ -XXX,XX +XXX,XX @@ typedef enum DisasJumpType { * Architecture-agnostic disassembly context. */ typedef struct DisasContextBase { - const TranslationBlock *tb; + TranslationBlock *tb; target_ulong pc_first; target_ulong pc_next; DisasJumpType is_jmp; int num_insns; int max_insns; bool singlestep_enabled; -#ifdef CONFIG_USER_ONLY - /* - * Guest address of the last byte of the last protected page. - * - * Pages containing the translated instructions are made non-writable in - * order to achieve consistency in case another thread is modifying the - * code while translate_insn() fetches the instruction bytes piecemeal. - * Such writer threads are blocked on mmap_lock() in page_unprotect(). - */ - target_ulong page_protect_end; -#endif + void *host_addr[2]; } DisasContextBase; /** @@ -XXX,XX +XXX,XX @@ bool translator_use_goto_tb(DisasContextBase *db, target_ulong dest); * the relevant information at translation time. */ -#define GEN_TRANSLATOR_LD(fullname, type, load_fn, swap_fn) \ - type fullname ## _swap(CPUArchState *env, DisasContextBase *dcbase, \ - abi_ptr pc, bool do_swap); \ - static inline type fullname(CPUArchState *env, \ - DisasContextBase *dcbase, abi_ptr pc) \ - { \ - return fullname ## _swap(env, dcbase, pc, false); \ +uint8_t translator_ldub(CPUArchState *env, DisasContextBase *db, abi_ptr pc); +uint16_t translator_lduw(CPUArchState *env, DisasContextBase *db, abi_ptr pc); +uint32_t translator_ldl(CPUArchState *env, DisasContextBase *db, abi_ptr pc); +uint64_t translator_ldq(CPUArchState *env, DisasContextBase *db, abi_ptr pc); + +static inline uint16_t +translator_lduw_swap(CPUArchState *env, DisasContextBase *db, + abi_ptr pc, bool do_swap) +{ + uint16_t ret = translator_lduw(env, db, pc); + if (do_swap) { + ret = bswap16(ret); } + return ret; +} -#define FOR_EACH_TRANSLATOR_LD(F) \ - F(translator_ldub, uint8_t, cpu_ldub_code, /* no swap */) \ - F(translator_lduw, uint16_t, cpu_lduw_code, bswap16) \ - F(translator_ldl, uint32_t, cpu_ldl_code, bswap32) \ - F(translator_ldq, uint64_t, cpu_ldq_code, bswap64) +static inline uint32_t +translator_ldl_swap(CPUArchState *env, DisasContextBase *db, + abi_ptr pc, bool do_swap) +{ + uint32_t ret = translator_ldl(env, db, pc); + if (do_swap) { + ret = bswap32(ret); + } + return ret; +} -FOR_EACH_TRANSLATOR_LD(GEN_TRANSLATOR_LD) - -#undef GEN_TRANSLATOR_LD +static inline uint64_t +translator_ldq_swap(CPUArchState *env, DisasContextBase *db, + abi_ptr pc, bool do_swap) +{ + uint64_t ret = translator_ldq_swap(env, db, pc, false); + if (do_swap) { + ret = bswap64(ret); + } + return ret; +} /* * Return whether addr is on the same page as where disassembly started. diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, { CPUArchState *env = cpu->env_ptr; TranslationBlock *tb, *existing_tb; - tb_page_addr_t phys_pc, phys_page2; - target_ulong virt_page2; + tb_page_addr_t phys_pc; tcg_insn_unit *gen_code_buf; int gen_code_size, search_size, max_insns; #ifdef CONFIG_PROFILER @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, tb->flags = flags; tb->cflags = cflags; tb->trace_vcpu_dstate = *cpu->trace_dstate; + tb->page_addr[0] = phys_pc; + tb->page_addr[1] = -1; tcg_ctx->tb_cflags = cflags; tb_overflow: @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, } /* - * If the TB is not associated with a physical RAM page then - * it must be a temporary one-insn TB, and we have nothing to do - * except fill in the page_addr[] fields. Return early before - * attempting to link to other TBs or add to the lookup table. + * If the TB is not associated with a physical RAM page then it must be + * a temporary one-insn TB, and we have nothing left to do. Return early + * before attempting to link to other TBs or add to the lookup table. */ - if (phys_pc == -1) { - tb->page_addr[0] = tb->page_addr[1] = -1; + if (tb->page_addr[0] == -1) { return tb; } @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, */ tcg_tb_insert(tb); - /* check next page if needed */ - virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK; - phys_page2 = -1; - if ((pc & TARGET_PAGE_MASK) != virt_page2) { - phys_page2 = get_page_addr_code(env, virt_page2); - } /* * No explicit memory barrier is required -- tb_link_page() makes the * TB visible in a consistent state. */ - existing_tb = tb_link_page(tb, phys_pc, phys_page2); + existing_tb = tb_link_page(tb, tb->page_addr[0], tb->page_addr[1]); /* if the TB already exists, discard what we just translated */ if (unlikely(existing_tb != tb)) { uintptr_t orig_aligned = (uintptr_t)gen_code_buf; diff --git a/accel/tcg/translator.c b/accel/tcg/translator.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translator.c +++ b/accel/tcg/translator.c @@ -XXX,XX +XXX,XX @@ bool translator_use_goto_tb(DisasContextBase *db, target_ulong dest) return ((db->pc_first ^ dest) & TARGET_PAGE_MASK) == 0; } -static inline void translator_page_protect(DisasContextBase *dcbase, - target_ulong pc) -{ -#ifdef CONFIG_USER_ONLY - dcbase->page_protect_end = pc | ~TARGET_PAGE_MASK; - page_protect(pc); -#endif -} - void translator_loop(CPUState *cpu, TranslationBlock *tb, int max_insns, target_ulong pc, void *host_pc, const TranslatorOps *ops, DisasContextBase *db) @@ -XXX,XX +XXX,XX @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int max_insns, db->num_insns = 0; db->max_insns = max_insns; db->singlestep_enabled = cflags & CF_SINGLE_STEP; - translator_page_protect(db, db->pc_next); + db->host_addr[0] = host_pc; + db->host_addr[1] = NULL; + +#ifdef CONFIG_USER_ONLY + page_protect(pc); +#endif ops->init_disas_context(db, cpu); tcg_debug_assert(db->is_jmp == DISAS_NEXT); /* no early exit */ @@ -XXX,XX +XXX,XX @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int max_insns, #endif } -static inline void translator_maybe_page_protect(DisasContextBase *dcbase, - target_ulong pc, size_t len) +static void *translator_access(CPUArchState *env, DisasContextBase *db, + target_ulong pc, size_t len) { -#ifdef CONFIG_USER_ONLY - target_ulong end = pc + len - 1; + void *host; + target_ulong base, end; + TranslationBlock *tb; - if (end > dcbase->page_protect_end) { - translator_page_protect(dcbase, end); + tb = db->tb; + + /* Use slow path if first page is MMIO. */ + if (unlikely(tb->page_addr[0] == -1)) { + return NULL; } + + end = pc + len - 1; + if (likely(is_same_page(db, end))) { + host = db->host_addr[0]; + base = db->pc_first; + } else { + host = db->host_addr[1]; + base = TARGET_PAGE_ALIGN(db->pc_first); + if (host == NULL) { + tb->page_addr[1] = + get_page_addr_code_hostp(env, base, &db->host_addr[1]); +#ifdef CONFIG_USER_ONLY + page_protect(end); #endif + /* We cannot handle MMIO as second page. */ + assert(tb->page_addr[1] != -1); + host = db->host_addr[1]; + } + + /* Use slow path when crossing pages. */ + if (is_same_page(db, pc)) { + return NULL; + } + } + + tcg_debug_assert(pc >= base); + return host + (pc - base); } -#define GEN_TRANSLATOR_LD(fullname, type, load_fn, swap_fn) \ - type fullname ## _swap(CPUArchState *env, DisasContextBase *dcbase, \ - abi_ptr pc, bool do_swap) \ - { \ - translator_maybe_page_protect(dcbase, pc, sizeof(type)); \ - type ret = load_fn(env, pc); \ - if (do_swap) { \ - ret = swap_fn(ret); \ - } \ - plugin_insn_append(pc, &ret, sizeof(ret)); \ - return ret; \ +uint8_t translator_ldub(CPUArchState *env, DisasContextBase *db, abi_ptr pc) +{ + uint8_t ret; + void *p = translator_access(env, db, pc, sizeof(ret)); + + if (p) { + plugin_insn_append(pc, p, sizeof(ret)); + return ldub_p(p); } + ret = cpu_ldub_code(env, pc); + plugin_insn_append(pc, &ret, sizeof(ret)); + return ret; +} -FOR_EACH_TRANSLATOR_LD(GEN_TRANSLATOR_LD) +uint16_t translator_lduw(CPUArchState *env, DisasContextBase *db, abi_ptr pc) +{ + uint16_t ret, plug; + void *p = translator_access(env, db, pc, sizeof(ret)); -#undef GEN_TRANSLATOR_LD + if (p) { + plugin_insn_append(pc, p, sizeof(ret)); + return lduw_p(p); + } + ret = cpu_lduw_code(env, pc); + plug = tswap16(ret); + plugin_insn_append(pc, &plug, sizeof(ret)); + return ret; +} + +uint32_t translator_ldl(CPUArchState *env, DisasContextBase *db, abi_ptr pc) +{ + uint32_t ret, plug; + void *p = translator_access(env, db, pc, sizeof(ret)); + + if (p) { + plugin_insn_append(pc, p, sizeof(ret)); + return ldl_p(p); + } + ret = cpu_ldl_code(env, pc); + plug = tswap32(ret); + plugin_insn_append(pc, &plug, sizeof(ret)); + return ret; +} + +uint64_t translator_ldq(CPUArchState *env, DisasContextBase *db, abi_ptr pc) +{ + uint64_t ret, plug; + void *p = translator_access(env, db, pc, sizeof(ret)); + + if (p) { + plugin_insn_append(pc, p, sizeof(ret)); + return ldq_p(p); + } + ret = cpu_ldq_code(env, pc); + plug = tswap64(ret); + plugin_insn_append(pc, &plug, sizeof(ret)); + return ret; +} -- 2.34.1 From: Ilya Leoshkevich <iii@linux.ibm.com> Right now translator stops right *after* the end of a page, which breaks reporting of fault locations when the last instruction of a multi-insn translation block crosses a page boundary. Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20220817150506.592862-3-iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/s390x/tcg/translate.c | 15 +++- tests/tcg/s390x/noexec.c | 106 +++++++++++++++++++++++ tests/tcg/multiarch/noexec.c.inc | 139 +++++++++++++++++++++++++++++++ tests/tcg/s390x/Makefile.target | 1 + 4 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 tests/tcg/s390x/noexec.c create mode 100644 tests/tcg/multiarch/noexec.c.inc diff --git a/target/s390x/tcg/translate.c b/target/s390x/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/tcg/translate.c +++ b/target/s390x/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static void s390x_tr_insn_start(DisasContextBase *dcbase, CPUState *cs) dc->insn_start = tcg_last_op(); } +static target_ulong get_next_pc(CPUS390XState *env, DisasContext *s, + uint64_t pc) +{ + uint64_t insn = ld_code2(env, s, pc); + + return pc + get_ilen((insn >> 8) & 0xff); +} + static void s390x_tr_translate_insn(DisasContextBase *dcbase, CPUState *cs) { CPUS390XState *env = cs->env_ptr; @@ -XXX,XX +XXX,XX @@ static void s390x_tr_translate_insn(DisasContextBase *dcbase, CPUState *cs) dc->base.is_jmp = translate_one(env, dc); if (dc->base.is_jmp == DISAS_NEXT) { - uint64_t page_start; - - page_start = dc->base.pc_first & TARGET_PAGE_MASK; - if (dc->base.pc_next - page_start >= TARGET_PAGE_SIZE || dc->ex_value) { + if (!is_same_page(dcbase, dc->base.pc_next) || + !is_same_page(dcbase, get_next_pc(env, dc, dc->base.pc_next)) || + dc->ex_value) { dc->base.is_jmp = DISAS_TOO_MANY; } } diff --git a/tests/tcg/s390x/noexec.c b/tests/tcg/s390x/noexec.c new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/tcg/s390x/noexec.c @@ -XXX,XX +XXX,XX @@ +#include "../multiarch/noexec.c.inc" + +static void *arch_mcontext_pc(const mcontext_t *ctx) +{ + return (void *)ctx->psw.addr; +} + +static int arch_mcontext_arg(const mcontext_t *ctx) +{ + return ctx->gregs[2]; +} + +static void arch_flush(void *p, int len) +{ +} + +extern char noexec_1[]; +extern char noexec_2[]; +extern char noexec_end[]; + +asm("noexec_1:\n" + " lgfi %r2,1\n" /* %r2 is 0 on entry, set 1. */ + "noexec_2:\n" + " lgfi %r2,2\n" /* %r2 is 0/1; set 2. */ + " br %r14\n" /* return */ + "noexec_end:"); + +extern char exrl_1[]; +extern char exrl_2[]; +extern char exrl_end[]; + +asm("exrl_1:\n" + " exrl %r0, exrl_2\n" + " br %r14\n" + "exrl_2:\n" + " lgfi %r2,2\n" + "exrl_end:"); + +int main(void) +{ + struct noexec_test noexec_tests[] = { + { + .name = "fallthrough", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2, + .entry_ofs = noexec_1 - noexec_2, + .expected_si_ofs = 0, + .expected_pc_ofs = 0, + .expected_arg = 1, + }, + { + .name = "jump", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2, + .entry_ofs = 0, + .expected_si_ofs = 0, + .expected_pc_ofs = 0, + .expected_arg = 0, + }, + { + .name = "exrl", + .test_code = exrl_1, + .test_len = exrl_end - exrl_1, + .page_ofs = exrl_1 - exrl_2, + .entry_ofs = exrl_1 - exrl_2, + .expected_si_ofs = 0, + .expected_pc_ofs = exrl_1 - exrl_2, + .expected_arg = 0, + }, + { + .name = "fallthrough [cross]", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2 - 2, + .entry_ofs = noexec_1 - noexec_2 - 2, + .expected_si_ofs = 0, + .expected_pc_ofs = -2, + .expected_arg = 1, + }, + { + .name = "jump [cross]", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2 - 2, + .entry_ofs = -2, + .expected_si_ofs = 0, + .expected_pc_ofs = -2, + .expected_arg = 0, + }, + { + .name = "exrl [cross]", + .test_code = exrl_1, + .test_len = exrl_end - exrl_1, + .page_ofs = exrl_1 - exrl_2 - 2, + .entry_ofs = exrl_1 - exrl_2 - 2, + .expected_si_ofs = 0, + .expected_pc_ofs = exrl_1 - exrl_2 - 2, + .expected_arg = 0, + }, + }; + + return test_noexec(noexec_tests, + sizeof(noexec_tests) / sizeof(noexec_tests[0])); +} diff --git a/tests/tcg/multiarch/noexec.c.inc b/tests/tcg/multiarch/noexec.c.inc new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/tcg/multiarch/noexec.c.inc @@ -XXX,XX +XXX,XX @@ +/* + * Common code for arch-specific MMU_INST_FETCH fault testing. + */ + +#define _GNU_SOURCE + +#include <assert.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <unistd.h> +#include <sys/mman.h> +#include <sys/ucontext.h> + +/* Forward declarations. */ + +static void *arch_mcontext_pc(const mcontext_t *ctx); +static int arch_mcontext_arg(const mcontext_t *ctx); +static void arch_flush(void *p, int len); + +/* Testing infrastructure. */ + +struct noexec_test { + const char *name; + const char *test_code; + int test_len; + int page_ofs; + int entry_ofs; + int expected_si_ofs; + int expected_pc_ofs; + int expected_arg; +}; + +static void *page_base; +static int page_size; +static const struct noexec_test *current_noexec_test; + +static void handle_err(const char *syscall) +{ + printf("[ FAILED ] %s: %s\n", syscall, strerror(errno)); + exit(EXIT_FAILURE); +} + +static void handle_segv(int sig, siginfo_t *info, void *ucontext) +{ + const struct noexec_test *test = current_noexec_test; + const mcontext_t *mc = &((ucontext_t *)ucontext)->uc_mcontext; + void *expected_si; + void *expected_pc; + void *pc; + int arg; + + if (test == NULL) { + printf("[ FAILED ] unexpected SEGV\n"); + exit(EXIT_FAILURE); + } + current_noexec_test = NULL; + + expected_si = page_base + test->expected_si_ofs; + if (info->si_addr != expected_si) { + printf("[ FAILED ] wrong si_addr (%p != %p)\n", + info->si_addr, expected_si); + exit(EXIT_FAILURE); + } + + pc = arch_mcontext_pc(mc); + expected_pc = page_base + test->expected_pc_ofs; + if (pc != expected_pc) { + printf("[ FAILED ] wrong pc (%p != %p)\n", pc, expected_pc); + exit(EXIT_FAILURE); + } + + arg = arch_mcontext_arg(mc); + if (arg != test->expected_arg) { + printf("[ FAILED ] wrong arg (%d != %d)\n", arg, test->expected_arg); + exit(EXIT_FAILURE); + } + + if (mprotect(page_base, page_size, + PROT_READ | PROT_WRITE | PROT_EXEC) < 0) { + handle_err("mprotect"); + } +} + +static void test_noexec_1(const struct noexec_test *test) +{ + void *start = page_base + test->page_ofs; + void (*fn)(int arg) = page_base + test->entry_ofs; + + memcpy(start, test->test_code, test->test_len); + arch_flush(start, test->test_len); + + /* Trigger TB creation in order to test invalidation. */ + fn(0); + + if (mprotect(page_base, page_size, PROT_NONE) < 0) { + handle_err("mprotect"); + } + + /* Trigger SEGV and check that handle_segv() ran. */ + current_noexec_test = test; + fn(0); + assert(current_noexec_test == NULL); +} + +static int test_noexec(struct noexec_test *tests, size_t n_tests) +{ + struct sigaction act; + size_t i; + + memset(&act, 0, sizeof(act)); + act.sa_sigaction = handle_segv; + act.sa_flags = SA_SIGINFO; + if (sigaction(SIGSEGV, &act, NULL) < 0) { + handle_err("sigaction"); + } + + page_size = getpagesize(); + page_base = mmap(NULL, 2 * page_size, + PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (page_base == MAP_FAILED) { + handle_err("mmap"); + } + page_base += page_size; + + for (i = 0; i < n_tests; i++) { + struct noexec_test *test = &tests[i]; + + printf("[ RUN ] %s\n", test->name); + test_noexec_1(test); + printf("[ OK ]\n"); + } + + printf("[ PASSED ]\n"); + return EXIT_SUCCESS; +} diff --git a/tests/tcg/s390x/Makefile.target b/tests/tcg/s390x/Makefile.target index XXXXXXX..XXXXXXX 100644 --- a/tests/tcg/s390x/Makefile.target +++ b/tests/tcg/s390x/Makefile.target @@ -XXX,XX +XXX,XX @@ TESTS+=shift TESTS+=trap TESTS+=signals-s390x TESTS+=branch-relative-long +TESTS+=noexec Z14_TESTS=vfminmax vfminmax: LDFLAGS+=-lm -- 2.34.1 From: Ilya Leoshkevich <iii@linux.ibm.com> Right now translator stops right *after* the end of a page, which breaks reporting of fault locations when the last instruction of a multi-insn translation block crosses a page boundary. An implementation, like the one arm and s390x have, would require an i386 length disassembler, which is burdensome to maintain. Another alternative would be to single-step at the end of a guest page, but this may come with a performance impact. Fix by snapshotting disassembly state and restoring it after we figure out we crossed a page boundary. This includes rolling back cc_op updates and emitted ops. Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Resolves: https://gitlab.com/qemu-project/qemu/-/issues/1143 Message-Id: <20220817150506.592862-4-iii@linux.ibm.com> [rth: Simplify end-of-insn cross-page checks.] Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/i386/tcg/translate.c | 64 ++++++++++++++++----------- tests/tcg/x86_64/noexec.c | 75 ++++++++++++++++++++++++++++++++ tests/tcg/x86_64/Makefile.target | 3 +- 3 files changed, 116 insertions(+), 26 deletions(-) create mode 100644 tests/tcg/x86_64/noexec.c diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -XXX,XX +XXX,XX @@ typedef struct DisasContext { TCGv_i64 tmp1_i64; sigjmp_buf jmpbuf; + TCGOp *prev_insn_end; } DisasContext; /* The environment in which user-only runs is constrained. */ @@ -XXX,XX +XXX,XX @@ static uint64_t advance_pc(CPUX86State *env, DisasContext *s, int num_bytes) { uint64_t pc = s->pc; + /* This is a subsequent insn that crosses a page boundary. */ + if (s->base.num_insns > 1 && + !is_same_page(&s->base, s->pc + num_bytes - 1)) { + siglongjmp(s->jmpbuf, 2); + } + s->pc += num_bytes; if (unlikely(s->pc - s->pc_start > X86_MAX_INSN_LENGTH)) { /* If the instruction's 16th byte is on a different page than the 1st, a @@ -XXX,XX +XXX,XX @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) int modrm, reg, rm, mod, op, opreg, val; target_ulong next_eip, tval; target_ulong pc_start = s->base.pc_next; + bool orig_cc_op_dirty = s->cc_op_dirty; + CCOp orig_cc_op = s->cc_op; s->pc_start = s->pc = pc_start; s->override = -1; @@ -XXX,XX +XXX,XX @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) s->rip_offset = 0; /* for relative ip address */ s->vex_l = 0; s->vex_v = 0; - if (sigsetjmp(s->jmpbuf, 0) != 0) { + switch (sigsetjmp(s->jmpbuf, 0)) { + case 0: + break; + case 1: gen_exception_gpf(s); return s->pc; + case 2: + /* Restore state that may affect the next instruction. */ + s->cc_op_dirty = orig_cc_op_dirty; + s->cc_op = orig_cc_op; + s->base.num_insns--; + tcg_remove_ops_after(s->prev_insn_end); + s->base.is_jmp = DISAS_TOO_MANY; + return pc_start; + default: + g_assert_not_reached(); } prefixes = 0; @@ -XXX,XX +XXX,XX @@ static void i386_tr_insn_start(DisasContextBase *dcbase, CPUState *cpu) { DisasContext *dc = container_of(dcbase, DisasContext, base); + dc->prev_insn_end = tcg_last_op(); tcg_gen_insn_start(dc->base.pc_next, dc->cc_op); } @@ -XXX,XX +XXX,XX @@ static void i386_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu) #endif pc_next = disas_insn(dc, cpu); - - if (dc->flags & (HF_TF_MASK | HF_INHIBIT_IRQ_MASK)) { - /* if single step mode, we generate only one instruction and - generate an exception */ - /* if irq were inhibited with HF_INHIBIT_IRQ_MASK, we clear - the flag and abort the translation to give the irqs a - chance to happen */ - dc->base.is_jmp = DISAS_TOO_MANY; - } else if ((tb_cflags(dc->base.tb) & CF_USE_ICOUNT) - && ((pc_next & TARGET_PAGE_MASK) - != ((pc_next + TARGET_MAX_INSN_SIZE - 1) - & TARGET_PAGE_MASK) - || (pc_next & ~TARGET_PAGE_MASK) == 0)) { - /* Do not cross the boundary of the pages in icount mode, - it can cause an exception. Do it only when boundary is - crossed by the first instruction in the block. - If current instruction already crossed the bound - it's ok, - because an exception hasn't stopped this code. - */ - dc->base.is_jmp = DISAS_TOO_MANY; - } else if ((pc_next - dc->base.pc_first) >= (TARGET_PAGE_SIZE - 32)) { - dc->base.is_jmp = DISAS_TOO_MANY; - } - dc->base.pc_next = pc_next; + + if (dc->base.is_jmp == DISAS_NEXT) { + if (dc->flags & (HF_TF_MASK | HF_INHIBIT_IRQ_MASK)) { + /* + * If single step mode, we generate only one instruction and + * generate an exception. + * If irq were inhibited with HF_INHIBIT_IRQ_MASK, we clear + * the flag and abort the translation to give the irqs a + * chance to happen. + */ + dc->base.is_jmp = DISAS_TOO_MANY; + } else if (!is_same_page(&dc->base, pc_next)) { + dc->base.is_jmp = DISAS_TOO_MANY; + } + } } static void i386_tr_tb_stop(DisasContextBase *dcbase, CPUState *cpu) diff --git a/tests/tcg/x86_64/noexec.c b/tests/tcg/x86_64/noexec.c new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/tcg/x86_64/noexec.c @@ -XXX,XX +XXX,XX @@ +#include "../multiarch/noexec.c.inc" + +static void *arch_mcontext_pc(const mcontext_t *ctx) +{ + return (void *)ctx->gregs[REG_RIP]; +} + +int arch_mcontext_arg(const mcontext_t *ctx) +{ + return ctx->gregs[REG_RDI]; +} + +static void arch_flush(void *p, int len) +{ +} + +extern char noexec_1[]; +extern char noexec_2[]; +extern char noexec_end[]; + +asm("noexec_1:\n" + " movq $1,%rdi\n" /* %rdi is 0 on entry, set 1. */ + "noexec_2:\n" + " movq $2,%rdi\n" /* %rdi is 0/1; set 2. */ + " ret\n" + "noexec_end:"); + +int main(void) +{ + struct noexec_test noexec_tests[] = { + { + .name = "fallthrough", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2, + .entry_ofs = noexec_1 - noexec_2, + .expected_si_ofs = 0, + .expected_pc_ofs = 0, + .expected_arg = 1, + }, + { + .name = "jump", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2, + .entry_ofs = 0, + .expected_si_ofs = 0, + .expected_pc_ofs = 0, + .expected_arg = 0, + }, + { + .name = "fallthrough [cross]", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2 - 2, + .entry_ofs = noexec_1 - noexec_2 - 2, + .expected_si_ofs = 0, + .expected_pc_ofs = -2, + .expected_arg = 1, + }, + { + .name = "jump [cross]", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2 - 2, + .entry_ofs = -2, + .expected_si_ofs = 0, + .expected_pc_ofs = -2, + .expected_arg = 0, + }, + }; + + return test_noexec(noexec_tests, + sizeof(noexec_tests) / sizeof(noexec_tests[0])); +} diff --git a/tests/tcg/x86_64/Makefile.target b/tests/tcg/x86_64/Makefile.target index XXXXXXX..XXXXXXX 100644 --- a/tests/tcg/x86_64/Makefile.target +++ b/tests/tcg/x86_64/Makefile.target @@ -XXX,XX +XXX,XX @@ include $(SRC_PATH)/tests/tcg/i386/Makefile.target ifeq ($(filter %-linux-user, $(TARGET)),$(TARGET)) X86_64_TESTS += vsyscall +X86_64_TESTS += noexec TESTS=$(MULTIARCH_TESTS) $(X86_64_TESTS) test-x86_64 else TESTS=$(MULTIARCH_TESTS) @@ -XXX,XX +XXX,XX @@ test-x86_64: LDFLAGS+=-lm -lc test-x86_64: test-i386.c test-i386.h test-i386-shift.h test-i386-muldiv.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -vsyscall: $(SRC_PATH)/tests/tcg/x86_64/vsyscall.c +%: $(SRC_PATH)/tests/tcg/x86_64/%.c $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -- 2.34.1 These will be useful in properly ending the TB. Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/riscv/translate.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/target/riscv/translate.c b/target/riscv/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/translate.c +++ b/target/riscv/translate.c @@ -XXX,XX +XXX,XX @@ static uint32_t opcode_at(DisasContextBase *dcbase, target_ulong pc) /* Include decoders for factored-out extensions */ #include "decode-XVentanaCondOps.c.inc" +/* The specification allows for longer insns, but not supported by qemu. */ +#define MAX_INSN_LEN 4 + +static inline int insn_len(uint16_t first_word) +{ + return (first_word & 3) == 3 ? 4 : 2; +} + static void decode_opc(CPURISCVState *env, DisasContext *ctx, uint16_t opcode) { /* @@ -XXX,XX +XXX,XX @@ static void decode_opc(CPURISCVState *env, DisasContext *ctx, uint16_t opcode) }; /* Check for compressed insn */ - if (extract16(opcode, 0, 2) != 3) { + if (insn_len(opcode) == 2) { if (!has_ext(ctx, RVC)) { gen_exception_illegal(ctx); } else { -- 2.34.1 Right now the translator stops right *after* the end of a page, which breaks reporting of fault locations when the last instruction of a multi-insn translation block crosses a page boundary. Resolves: https://gitlab.com/qemu-project/qemu/-/issues/1155 Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/riscv/translate.c | 17 +++++-- tests/tcg/riscv64/noexec.c | 79 +++++++++++++++++++++++++++++++ tests/tcg/riscv64/Makefile.target | 1 + 3 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 tests/tcg/riscv64/noexec.c diff --git a/target/riscv/translate.c b/target/riscv/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/translate.c +++ b/target/riscv/translate.c @@ -XXX,XX +XXX,XX @@ static void riscv_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu) } ctx->nftemp = 0; + /* Only the first insn within a TB is allowed to cross a page boundary. */ if (ctx->base.is_jmp == DISAS_NEXT) { - target_ulong page_start; - - page_start = ctx->base.pc_first & TARGET_PAGE_MASK; - if (ctx->base.pc_next - page_start >= TARGET_PAGE_SIZE) { + if (!is_same_page(&ctx->base, ctx->base.pc_next)) { ctx->base.is_jmp = DISAS_TOO_MANY; + } else { + unsigned page_ofs = ctx->base.pc_next & ~TARGET_PAGE_MASK; + + if (page_ofs > TARGET_PAGE_SIZE - MAX_INSN_LEN) { + uint16_t next_insn = cpu_lduw_code(env, ctx->base.pc_next); + int len = insn_len(next_insn); + + if (!is_same_page(&ctx->base, ctx->base.pc_next + len)) { + ctx->base.is_jmp = DISAS_TOO_MANY; + } + } } } } diff --git a/tests/tcg/riscv64/noexec.c b/tests/tcg/riscv64/noexec.c new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/tcg/riscv64/noexec.c @@ -XXX,XX +XXX,XX @@ +#include "../multiarch/noexec.c.inc" + +static void *arch_mcontext_pc(const mcontext_t *ctx) +{ + return (void *)ctx->__gregs[REG_PC]; +} + +static int arch_mcontext_arg(const mcontext_t *ctx) +{ + return ctx->__gregs[REG_A0]; +} + +static void arch_flush(void *p, int len) +{ + __builtin___clear_cache(p, p + len); +} + +extern char noexec_1[]; +extern char noexec_2[]; +extern char noexec_end[]; + +asm(".option push\n" + ".option norvc\n" + "noexec_1:\n" + " li a0,1\n" /* a0 is 0 on entry, set 1. */ + "noexec_2:\n" + " li a0,2\n" /* a0 is 0/1; set 2. */ + " ret\n" + "noexec_end:\n" + ".option pop"); + +int main(void) +{ + struct noexec_test noexec_tests[] = { + { + .name = "fallthrough", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2, + .entry_ofs = noexec_1 - noexec_2, + .expected_si_ofs = 0, + .expected_pc_ofs = 0, + .expected_arg = 1, + }, + { + .name = "jump", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2, + .entry_ofs = 0, + .expected_si_ofs = 0, + .expected_pc_ofs = 0, + .expected_arg = 0, + }, + { + .name = "fallthrough [cross]", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2 - 2, + .entry_ofs = noexec_1 - noexec_2 - 2, + .expected_si_ofs = 0, + .expected_pc_ofs = -2, + .expected_arg = 1, + }, + { + .name = "jump [cross]", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2 - 2, + .entry_ofs = -2, + .expected_si_ofs = 0, + .expected_pc_ofs = -2, + .expected_arg = 0, + }, + }; + + return test_noexec(noexec_tests, + sizeof(noexec_tests) / sizeof(noexec_tests[0])); +} diff --git a/tests/tcg/riscv64/Makefile.target b/tests/tcg/riscv64/Makefile.target index XXXXXXX..XXXXXXX 100644 --- a/tests/tcg/riscv64/Makefile.target +++ b/tests/tcg/riscv64/Makefile.target @@ -XXX,XX +XXX,XX @@ VPATH += $(SRC_PATH)/tests/tcg/riscv64 TESTS += test-div +TESTS += noexec -- 2.34.1