The following changes since commit 470dd6bd360782f5137f7e3376af6a44658eb1d3: Merge remote-tracking branch 'remotes/stsquad/tags/pull-testing-060121-4' into staging (2021-01-06 22:18:36 +0000) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20210107 for you to fetch changes up to e5e2e4c73926f6f3c1f5da24a350e4345d5ad232: tcg: Constify TCGLabelQemuLdst.raddr (2021-01-07 05:09:42 -1000) ---------------------------------------------------------------- Build fix for ppc64 centos7. Reduce the use of scratch registers for tcg/i386. Use _aligned_malloc for Win32. Enable split w^x code gen buffers. ---------------------------------------------------------------- Philippe Mathieu-Daudé (1): util/oslib: Assert qemu_try_memalign() alignment is a power of 2 Richard Henderson (46): linux-user: Conditionalize TUNSETVNETLE tcg/i386: Adjust TCG_TARGET_HAS_MEMORY_BSWAP tcg: Introduce INDEX_op_qemu_st8_i32 util/oslib-win32: Use _aligned_malloc for qemu_try_memalign tcg: Do not flush icache for interpreter util: Enhance flush_icache_range with separate data pointer util: Specialize flush_idcache_range for aarch64 tcg: Move tcg prologue pointer out of TCGContext tcg: Move tcg epilogue pointer out of TCGContext tcg: Add in_code_gen_buffer tcg: Introduce tcg_splitwx_to_{rx,rw} tcg: Adjust TCGLabel for const tcg: Adjust tcg_out_call for const tcg: Adjust tcg_out_label for const tcg: Adjust tcg_register_jit for const tcg: Adjust tb_target_set_jmp_target for split-wx tcg: Make DisasContextBase.tb const tcg: Make tb arg to synchronize_from_tb const tcg: Use Error with alloc_code_gen_buffer tcg: Add --accel tcg,split-wx property accel/tcg: Support split-wx for linux with memfd accel/tcg: Support split-wx for darwin/iOS with vm_remap tcg: Return the TB pointer from the rx region from exit_tb tcg/i386: Support split-wx code generation tcg/aarch64: Use B not BL for tcg_out_goto_long tcg/aarch64: Support split-wx code generation disas: Push const down through host disassembly tcg/tci: Push const down through bytecode reading tcg: Introduce tcg_tbrel_diff tcg/ppc: Use tcg_tbrel_diff tcg/ppc: Use tcg_out_mem_long to reset TCG_REG_TB tcg/ppc: Support split-wx code generation tcg/sparc: Use tcg_tbrel_diff tcg/sparc: Support split-wx code generation tcg/s390: Use tcg_tbrel_diff tcg/s390: Support split-wx code generation tcg/riscv: Fix branch range checks tcg/riscv: Remove branch-over-branch fallback tcg/riscv: Support split-wx code generation accel/tcg: Add mips support to alloc_code_gen_buffer_splitwx_memfd tcg/mips: Do not assert on relocation overflow tcg/mips: Support split-wx code generation tcg/arm: Support split-wx code generation tcg: Remove TCG_TARGET_SUPPORT_MIRROR tcg: Constify tcg_code_gen_epilogue tcg: Constify TCGLabelQemuLdst.raddr accel/tcg/tcg-runtime.h | 2 +- include/disas/dis-asm.h | 4 +- include/disas/disas.h | 2 +- include/exec/exec-all.h | 2 +- include/exec/gen-icount.h | 4 +- include/exec/log.h | 2 +- include/exec/translator.h | 2 +- include/hw/core/cpu.h | 3 +- include/qemu/cacheflush.h | 15 ++- include/sysemu/tcg.h | 3 +- include/tcg/tcg-op.h | 2 +- include/tcg/tcg-opc.h | 5 + include/tcg/tcg.h | 61 +++++++-- linux-user/ioctls.h | 2 + tcg/aarch64/tcg-target.h | 3 +- tcg/arm/tcg-target.h | 3 +- tcg/i386/tcg-target.h | 12 +- tcg/mips/tcg-target.h | 3 +- tcg/ppc/tcg-target.h | 3 +- tcg/riscv/tcg-target.h | 3 +- tcg/s390/tcg-target.h | 9 +- tcg/sparc/tcg-target.h | 3 +- tcg/tci/tcg-target.h | 7 +- accel/tcg/cpu-exec.c | 41 +++--- accel/tcg/tcg-all.c | 26 +++- accel/tcg/tcg-runtime.c | 4 +- accel/tcg/translate-all.c | 311 ++++++++++++++++++++++++++++++++++--------- accel/tcg/translator.c | 4 +- bsd-user/main.c | 2 +- disas.c | 2 +- disas/capstone.c | 2 +- linux-user/main.c | 2 +- softmmu/physmem.c | 2 +- target/arm/cpu.c | 3 +- target/arm/translate-a64.c | 2 +- target/avr/cpu.c | 3 +- target/hppa/cpu.c | 3 +- target/i386/tcg/tcg-cpu.c | 3 +- target/microblaze/cpu.c | 3 +- target/mips/cpu.c | 3 +- target/riscv/cpu.c | 3 +- target/rx/cpu.c | 3 +- target/sh4/cpu.c | 3 +- target/sparc/cpu.c | 3 +- target/tricore/cpu.c | 2 +- tcg/optimize.c | 1 + tcg/tcg-op.c | 21 ++- tcg/tcg.c | 94 ++++++++++--- tcg/tci.c | 62 +++++---- util/cacheflush.c | 107 ++++++++++++--- util/cacheinfo.c | 8 +- util/oslib-posix.c | 2 + util/oslib-win32.c | 12 +- tcg/aarch64/tcg-target.c.inc | 75 ++++++----- tcg/arm/tcg-target.c.inc | 41 +++--- tcg/i386/tcg-target.c.inc | 174 +++++++++++------------- tcg/mips/tcg-target.c.inc | 97 ++++++-------- tcg/ppc/tcg-target.c.inc | 88 ++++++------ tcg/riscv/tcg-target.c.inc | 125 ++++++----------- tcg/s390/tcg-target.c.inc | 91 ++++++------- tcg/sparc/tcg-target.c.inc | 58 ++++---- tcg/tcg-ldst.c.inc | 2 +- tcg/tcg-pool.c.inc | 6 +- tcg/tci/tcg-target.c.inc | 2 +- accel/tcg/trace-events | 2 +- qemu-options.hx | 7 + tcg/README | 5 + 67 files changed, 1035 insertions(+), 630 deletions(-) This fixes the build for older ppc64 kernel headers. Fixes: 6addf06a3c4 Reviewed-by: Laurent Vivier <laurent@vivier.eu> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- linux-user/ioctls.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/linux-user/ioctls.h b/linux-user/ioctls.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/ioctls.h +++ b/linux-user/ioctls.h @@ -XXX,XX +XXX,XX @@ IOCTL(TUNSETQUEUE, IOC_W, MK_PTR(MK_STRUCT(STRUCT_short_ifreq))) IOCTL(TUNSETIFINDEX , IOC_W, MK_PTR(TYPE_INT)) /* TUNGETFILTER is not supported: see TUNATTACHFILTER. */ +#ifdef TUNSETVNETLE IOCTL(TUNSETVNETLE, IOC_W, MK_PTR(TYPE_INT)) IOCTL(TUNGETVNETLE, IOC_R, MK_PTR(TYPE_INT)) +#endif #ifdef TUNSETVNETBE IOCTL(TUNSETVNETBE, IOC_W, MK_PTR(TYPE_INT)) IOCTL(TUNGETVNETBE, IOC_R, MK_PTR(TYPE_INT)) -- 2.25.1 Always true when movbe is available, otherwise leave this to generic code. Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.h | 3 +- tcg/i386/tcg-target.c.inc | 119 ++++++++++++++------------------------ 2 files changed, 47 insertions(+), 75 deletions(-) diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_bmi1; extern bool have_popcnt; extern bool have_avx1; extern bool have_avx2; +extern bool have_movbe; /* optional instructions */ #define TCG_TARGET_HAS_div2_i32 1 @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, #define TCG_TARGET_DEFAULT_MO (TCG_MO_ALL & ~TCG_MO_ST_LD) -#define TCG_TARGET_HAS_MEMORY_BSWAP 1 +#define TCG_TARGET_HAS_MEMORY_BSWAP have_movbe #ifdef CONFIG_SOFTMMU #define TCG_TARGET_NEED_LDST_LABELS diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ bool have_bmi1; bool have_popcnt; bool have_avx1; bool have_avx2; +bool have_movbe; #ifdef CONFIG_CPUID_H -static bool have_movbe; static bool have_bmi2; static bool have_lzcnt; #else -# define have_movbe 0 # define have_bmi2 0 # define have_lzcnt 0 #endif @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, TCGReg base, int index, intptr_t ofs, int seg, bool is64, MemOp memop) { - const MemOp real_bswap = memop & MO_BSWAP; - MemOp bswap = real_bswap; + bool use_movbe = false; int rexw = is64 * P_REXW; int movop = OPC_MOVL_GvEv; - if (have_movbe && real_bswap) { - bswap = 0; + /* Do big-endian loads with movbe. */ + if (memop & MO_BSWAP) { + tcg_debug_assert(have_movbe); + use_movbe = true; movop = OPC_MOVBE_GyMy; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, base, index, 0, ofs); break; case MO_UW: - tcg_out_modrm_sib_offset(s, OPC_MOVZWL + seg, datalo, - base, index, 0, ofs); - if (real_bswap) { - tcg_out_rolw_8(s, datalo); - } - break; - case MO_SW: - if (real_bswap) { - if (have_movbe) { + if (use_movbe) { + /* There is no extending movbe; only low 16-bits are modified. */ + if (datalo != base && datalo != index) { + /* XOR breaks dependency chains. */ + tgen_arithr(s, ARITH_XOR, datalo, datalo); tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg, datalo, base, index, 0, ofs); } else { - tcg_out_modrm_sib_offset(s, OPC_MOVZWL + seg, datalo, - base, index, 0, ofs); - tcg_out_rolw_8(s, datalo); + tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg, + datalo, base, index, 0, ofs); + tcg_out_ext16u(s, datalo, datalo); } - tcg_out_modrm(s, OPC_MOVSWL + rexw, datalo, datalo); + } else { + tcg_out_modrm_sib_offset(s, OPC_MOVZWL + seg, datalo, + base, index, 0, ofs); + } + break; + case MO_SW: + if (use_movbe) { + tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg, + datalo, base, index, 0, ofs); + tcg_out_ext16s(s, datalo, datalo, rexw); } else { tcg_out_modrm_sib_offset(s, OPC_MOVSWL + rexw + seg, datalo, base, index, 0, ofs); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, break; case MO_UL: tcg_out_modrm_sib_offset(s, movop + seg, datalo, base, index, 0, ofs); - if (bswap) { - tcg_out_bswap32(s, datalo); - } break; #if TCG_TARGET_REG_BITS == 64 case MO_SL: - if (real_bswap) { - tcg_out_modrm_sib_offset(s, movop + seg, datalo, + if (use_movbe) { + tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + seg, datalo, base, index, 0, ofs); - if (bswap) { - tcg_out_bswap32(s, datalo); - } tcg_out_ext32s(s, datalo, datalo); } else { tcg_out_modrm_sib_offset(s, OPC_MOVSLQ + seg, datalo, @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, if (TCG_TARGET_REG_BITS == 64) { tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo, base, index, 0, ofs); - if (bswap) { - tcg_out_bswap64(s, datalo); - } } else { - if (real_bswap) { - int t = datalo; + if (use_movbe) { + TCGReg t = datalo; datalo = datahi; datahi = t; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, tcg_out_modrm_sib_offset(s, movop + seg, datalo, base, index, 0, ofs); } - if (bswap) { - tcg_out_bswap32(s, datalo); - tcg_out_bswap32(s, datahi); - } } break; default: - tcg_abort(); + g_assert_not_reached(); } } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, TCGReg base, int index, intptr_t ofs, int seg, MemOp memop) { - /* ??? Ideally we wouldn't need a scratch register. For user-only, - we could perform the bswap twice to restore the original value - instead of moving to the scratch. But as it is, the L constraint - means that TCG_REG_L0 is definitely free here. */ const TCGReg scratch = TCG_REG_L0; - const MemOp real_bswap = memop & MO_BSWAP; - MemOp bswap = real_bswap; + bool use_movbe = false; int movop = OPC_MOVL_EvGv; - if (have_movbe && real_bswap) { - bswap = 0; + /* + * Do big-endian stores with movbe or softmmu. + * User-only without movbe will have its swapping done generically. + */ + if (memop & MO_BSWAP) { + tcg_debug_assert(have_movbe); + use_movbe = true; movop = OPC_MOVBE_MyGy; } switch (memop & MO_SIZE) { case MO_8: - /* In 32-bit mode, 8-bit stores can only happen from [abcd]x. - Use the scratch register if necessary. */ + /* + * In 32-bit mode, 8-bit stores can only happen from [abcd]x. + * TODO: Adjust constraints such that this is is forced, + * then we won't need a scratch at all for user-only. + */ if (TCG_TARGET_REG_BITS == 32 && datalo >= 4) { tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo); datalo = scratch; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, datalo, base, index, 0, ofs); break; case MO_16: - if (bswap) { - tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo); - tcg_out_rolw_8(s, scratch); - datalo = scratch; - } tcg_out_modrm_sib_offset(s, movop + P_DATA16 + seg, datalo, base, index, 0, ofs); break; case MO_32: - if (bswap) { - tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo); - tcg_out_bswap32(s, scratch); - datalo = scratch; - } tcg_out_modrm_sib_offset(s, movop + seg, datalo, base, index, 0, ofs); break; case MO_64: if (TCG_TARGET_REG_BITS == 64) { - if (bswap) { - tcg_out_mov(s, TCG_TYPE_I64, scratch, datalo); - tcg_out_bswap64(s, scratch); - datalo = scratch; - } tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo, base, index, 0, ofs); - } else if (bswap) { - tcg_out_mov(s, TCG_TYPE_I32, scratch, datahi); - tcg_out_bswap32(s, scratch); - tcg_out_modrm_sib_offset(s, OPC_MOVL_EvGv + seg, scratch, - base, index, 0, ofs); - tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo); - tcg_out_bswap32(s, scratch); - tcg_out_modrm_sib_offset(s, OPC_MOVL_EvGv + seg, scratch, - base, index, 0, ofs + 4); } else { - if (real_bswap) { - int t = datalo; + if (use_movbe) { + TCGReg t = datalo; datalo = datahi; datahi = t; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, } break; default: - tcg_abort(); + g_assert_not_reached(); } } -- 2.25.1 Enable this on i386 to restrict the set of input registers for an 8-bit store, as required by the architecture. This removes the last use of scratch registers for user-only mode. Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg-opc.h | 5 +++++ tcg/aarch64/tcg-target.h | 1 + tcg/arm/tcg-target.h | 1 + tcg/i386/tcg-target.h | 3 +++ tcg/mips/tcg-target.h | 1 + tcg/ppc/tcg-target.h | 1 + tcg/riscv/tcg-target.h | 1 + tcg/s390/tcg-target.h | 1 + tcg/sparc/tcg-target.h | 1 + tcg/tci/tcg-target.h | 1 + tcg/optimize.c | 1 + tcg/tcg-op.c | 6 +++++- tcg/tcg.c | 4 ++++ tcg/i386/tcg-target.c.inc | 29 ++++++++++++++++++----------- tcg/README | 5 +++++ 15 files changed, 49 insertions(+), 12 deletions(-) diff --git a/include/tcg/tcg-opc.h b/include/tcg/tcg-opc.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg-opc.h +++ b/include/tcg/tcg-opc.h @@ -XXX,XX +XXX,XX @@ DEF(qemu_ld_i64, DATA64_ARGS, TLADDR_ARGS, 1, DEF(qemu_st_i64, 0, TLADDR_ARGS + DATA64_ARGS, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_64BIT) +/* Only used by i386 to cope with stupid register constraints. */ +DEF(qemu_st8_i32, 0, TLADDR_ARGS + 1, 1, + TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | + IMPL(TCG_TARGET_HAS_qemu_st8_i32)) + /* Host vector support. */ #define IMPLVEC TCG_OPF_VECTOR | IMPL(TCG_TARGET_MAYBE_vec) diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_HAS_extrl_i64_i32 0 #define TCG_TARGET_HAS_extrh_i64_i32 0 #define TCG_TARGET_HAS_goto_ptr 1 +#define TCG_TARGET_HAS_qemu_st8_i32 0 #define TCG_TARGET_HAS_div_i64 1 #define TCG_TARGET_HAS_rem_i64 1 diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_idiv_instructions; #define TCG_TARGET_HAS_rem_i32 0 #define TCG_TARGET_HAS_goto_ptr 1 #define TCG_TARGET_HAS_direct_jump 0 +#define TCG_TARGET_HAS_qemu_st8_i32 0 enum { TCG_AREG0 = TCG_REG_R6, diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_movbe; #define TCG_TARGET_HAS_muls2_i64 1 #define TCG_TARGET_HAS_muluh_i64 0 #define TCG_TARGET_HAS_mulsh_i64 0 +#define TCG_TARGET_HAS_qemu_st8_i32 0 +#else +#define TCG_TARGET_HAS_qemu_st8_i32 1 #endif /* We do not support older SSE systems, only beginning with AVX1. */ diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions; #define TCG_TARGET_HAS_clz_i32 use_mips32r2_instructions #define TCG_TARGET_HAS_ctz_i32 0 #define TCG_TARGET_HAS_ctpop_i32 0 +#define TCG_TARGET_HAS_qemu_st8_i32 0 #if TCG_TARGET_REG_BITS == 64 #define TCG_TARGET_HAS_movcond_i64 use_movnz_instructions diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_vsx; #define TCG_TARGET_HAS_mulsh_i32 1 #define TCG_TARGET_HAS_goto_ptr 1 #define TCG_TARGET_HAS_direct_jump 1 +#define TCG_TARGET_HAS_qemu_st8_i32 0 #if TCG_TARGET_REG_BITS == 64 #define TCG_TARGET_HAS_add2_i32 0 diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_HAS_direct_jump 0 #define TCG_TARGET_HAS_brcond2 1 #define TCG_TARGET_HAS_setcond2 1 +#define TCG_TARGET_HAS_qemu_st8_i32 0 #if TCG_TARGET_REG_BITS == 64 #define TCG_TARGET_HAS_movcond_i64 0 diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.h +++ b/tcg/s390/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern uint64_t s390_facilities; #define TCG_TARGET_HAS_extrh_i64_i32 0 #define TCG_TARGET_HAS_goto_ptr 1 #define TCG_TARGET_HAS_direct_jump (s390_facilities & FACILITY_GEN_INST_EXT) +#define TCG_TARGET_HAS_qemu_st8_i32 0 #define TCG_TARGET_HAS_div2_i64 1 #define TCG_TARGET_HAS_rot_i64 1 diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.h +++ b/tcg/sparc/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions; #define TCG_TARGET_HAS_mulsh_i32 0 #define TCG_TARGET_HAS_goto_ptr 1 #define TCG_TARGET_HAS_direct_jump 1 +#define TCG_TARGET_HAS_qemu_st8_i32 0 #define TCG_TARGET_HAS_extrl_i64_i32 1 #define TCG_TARGET_HAS_extrh_i64_i32 1 diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ #define TCG_TARGET_HAS_mulsh_i32 0 #define TCG_TARGET_HAS_goto_ptr 0 #define TCG_TARGET_HAS_direct_jump 1 +#define TCG_TARGET_HAS_qemu_st8_i32 0 #if TCG_TARGET_REG_BITS == 64 #define TCG_TARGET_HAS_extrl_i64_i32 0 diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_qemu_ld_i32: case INDEX_op_qemu_ld_i64: case INDEX_op_qemu_st_i32: + case INDEX_op_qemu_st8_i32: case INDEX_op_qemu_st_i64: case INDEX_op_call: /* Opcodes that touch guest memory stop the optimization. */ diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -XXX,XX +XXX,XX @@ void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) } addr = plugin_prep_mem_callbacks(addr); - gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx); + if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) { + gen_ldst_i32(INDEX_op_qemu_st8_i32, val, addr, memop, idx); + } else { + gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx); + } plugin_gen_mem_callbacks(addr, info); if (swap) { diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ bool tcg_op_supported(TCGOpcode op) case INDEX_op_qemu_st_i64: return true; + case INDEX_op_qemu_st8_i32: + return TCG_TARGET_HAS_qemu_st8_i32; + case INDEX_op_goto_ptr: return TCG_TARGET_HAS_goto_ptr; @@ -XXX,XX +XXX,XX @@ static void tcg_dump_ops(TCGContext *s, bool have_prefs) break; case INDEX_op_qemu_ld_i32: case INDEX_op_qemu_st_i32: + case INDEX_op_qemu_st8_i32: case INDEX_op_qemu_ld_i64: case INDEX_op_qemu_st_i64: { diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const char *target_parse_constraint(TCGArgConstraint *ct, ct->regs |= ALL_VECTOR_REGS; break; - /* qemu_ld/st address constraint */ case 'L': + /* qemu_ld/st data+address constraint */ ct->regs = TCG_TARGET_REG_BITS == 64 ? 0xffff : 0xff; +#ifdef CONFIG_SOFTMMU tcg_regset_reset_reg(ct->regs, TCG_REG_L0); tcg_regset_reset_reg(ct->regs, TCG_REG_L1); +#endif + break; + case 's': + /* qemu_st8_i32 data constraint */ + ct->regs = 0xf; +#ifdef CONFIG_SOFTMMU + tcg_regset_reset_reg(ct->regs, TCG_REG_L0); + tcg_regset_reset_reg(ct->regs, TCG_REG_L1); +#endif break; case 'e': @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, TCGReg base, int index, intptr_t ofs, int seg, MemOp memop) { - const TCGReg scratch = TCG_REG_L0; bool use_movbe = false; int movop = OPC_MOVL_EvGv; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, switch (memop & MO_SIZE) { case MO_8: - /* - * In 32-bit mode, 8-bit stores can only happen from [abcd]x. - * TODO: Adjust constraints such that this is is forced, - * then we won't need a scratch at all for user-only. - */ - if (TCG_TARGET_REG_BITS == 32 && datalo >= 4) { - tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo); - datalo = scratch; - } + /* This is handled with constraints on INDEX_op_qemu_st8_i32. */ + tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || datalo < 4); tcg_out_modrm_sib_offset(s, OPC_MOVB_EvGv + P_REXB_R + seg, datalo, base, index, 0, ofs); break; @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_qemu_ld(s, args, 1); break; case INDEX_op_qemu_st_i32: + case INDEX_op_qemu_st8_i32: tcg_out_qemu_st(s, args, 0); break; case INDEX_op_qemu_st_i64: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) static const TCGTargetOpDef r_0_ci = { .args_ct_str = { "r", "0", "ci" } }; static const TCGTargetOpDef r_L = { .args_ct_str = { "r", "L" } }; static const TCGTargetOpDef L_L = { .args_ct_str = { "L", "L" } }; + static const TCGTargetOpDef s_L = { .args_ct_str = { "s", "L" } }; static const TCGTargetOpDef r_L_L = { .args_ct_str = { "r", "L", "L" } }; static const TCGTargetOpDef r_r_L = { .args_ct_str = { "r", "r", "L" } }; static const TCGTargetOpDef L_L_L = { .args_ct_str = { "L", "L", "L" } }; + static const TCGTargetOpDef s_L_L = { .args_ct_str = { "s", "L", "L" } }; static const TCGTargetOpDef r_r_L_L = { .args_ct_str = { "r", "r", "L", "L" } }; static const TCGTargetOpDef L_L_L_L @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) return TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &r_L : &r_L_L; case INDEX_op_qemu_st_i32: return TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &L_L : &L_L_L; + case INDEX_op_qemu_st8_i32: + return TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &s_L : &s_L_L; case INDEX_op_qemu_ld_i64: return (TCG_TARGET_REG_BITS == 64 ? &r_L : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &r_r_L diff --git a/tcg/README b/tcg/README index XXXXXXX..XXXXXXX 100644 --- a/tcg/README +++ b/tcg/README @@ -XXX,XX +XXX,XX @@ goto_ptr opcode, emitting this op is equivalent to emitting exit_tb(0). * qemu_ld_i32/i64 t0, t1, flags, memidx * qemu_st_i32/i64 t0, t1, flags, memidx +* qemu_st8_i32 t0, t1, flags, memidx Load data at the guest address t1 into t0, or store data in t0 at guest address t1. The _i32/_i64 size applies to the size of the input/output @@ -XXX,XX +XXX,XX @@ of the memory access. For a 32-bit host, qemu_ld/st_i64 is guaranteed to only be used with a 64-bit memory access specified in flags. +For i386, qemu_st8_i32 is exactly like qemu_st_i32, except the size of +the memory operation is known to be 8-bit. This allows the backend to +provide a different set of register constraints. + ********* Host vector operations All of the vector ops have two parameters, TCGOP_VECL & TCGOP_VECE. -- 2.25.1 We do not need or want to be allocating page sized quanta. Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Stefan Weil <sw@weilnetz.de> Message-Id: <20201018164836.1149452-1-richard.henderson@linaro.org> Signed-off-by: Philippe Mathieu-Daudé <philmd@redhat.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- util/oslib-win32.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/util/oslib-win32.c b/util/oslib-win32.c index XXXXXXX..XXXXXXX 100644 --- a/util/oslib-win32.c +++ b/util/oslib-win32.c @@ -XXX,XX +XXX,XX @@ #include "trace.h" #include "qemu/sockets.h" #include "qemu/cutils.h" +#include <malloc.h> /* this must come after including "trace.h" */ #include <shlobj.h> @@ -XXX,XX +XXX,XX @@ void *qemu_try_memalign(size_t alignment, size_t size) { void *ptr; - if (!size) { - abort(); - } - ptr = VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); + g_assert(size != 0); + ptr = _aligned_malloc(alignment, size); trace_qemu_memalign(alignment, size, ptr); return ptr; } @@ -XXX,XX +XXX,XX @@ void *qemu_anon_ram_alloc(size_t size, uint64_t *align, bool shared) void qemu_vfree(void *ptr) { trace_qemu_vfree(ptr); - if (ptr) { - VirtualFree(ptr, 0, MEM_RELEASE); - } + _aligned_free(ptr); } void qemu_anon_ram_free(void *ptr, size_t size) -- 2.25.1 From: Philippe Mathieu-Daudé <philmd@redhat.com> qemu_try_memalign() expects a power of 2 alignment: - posix_memalign(3): The address of the allocated memory will be a multiple of alignment, which must be a power of two and a multiple of sizeof(void *). - _aligned_malloc() The alignment value, which must be an integer power of 2. Signed-off-by: Philippe Mathieu-Daudé <philmd@redhat.com> Message-Id: <20201021173803.2619054-3-philmd@redhat.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- util/oslib-posix.c | 2 ++ util/oslib-win32.c | 1 + 2 files changed, 3 insertions(+) diff --git a/util/oslib-posix.c b/util/oslib-posix.c index XXXXXXX..XXXXXXX 100644 --- a/util/oslib-posix.c +++ b/util/oslib-posix.c @@ -XXX,XX +XXX,XX @@ void *qemu_try_memalign(size_t alignment, size_t size) if (alignment < sizeof(void*)) { alignment = sizeof(void*); + } else { + g_assert(is_power_of_2(alignment)); } #if defined(CONFIG_POSIX_MEMALIGN) diff --git a/util/oslib-win32.c b/util/oslib-win32.c index XXXXXXX..XXXXXXX 100644 --- a/util/oslib-win32.c +++ b/util/oslib-win32.c @@ -XXX,XX +XXX,XX @@ void *qemu_try_memalign(size_t alignment, size_t size) void *ptr; g_assert(size != 0); + g_assert(is_power_of_2(alignment)); ptr = _aligned_malloc(alignment, size); trace_qemu_memalign(alignment, size, ptr); return ptr; -- 2.25.1 This is currently a no-op within tci/tcg-target.h, but is about to be moved to a more generic location. Reviewed-by: Joelle van Dyne <j@getutm.app> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ void tcg_prologue_init(TCGContext *s) #endif buf1 = s->code_ptr; +#ifndef CONFIG_TCG_INTERPRETER flush_icache_range((uintptr_t)buf0, (uintptr_t)buf1); +#endif /* Deduct the prologue from the buffer. */ prologue_size = tcg_current_code_size(s); @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb) return -2; } +#ifndef CONFIG_TCG_INTERPRETER /* flush instruction cache */ flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr); +#endif return tcg_current_code_size(s); } -- 2.25.1 We are shortly going to have a split rw/rx jit buffer. Depending on the host, we need to flush the dcache at the rw data pointer and flush the icache at the rx code pointer. For now, the two passed pointers are identical, so there is no effective change in behaviour. Reviewed-by: Joelle van Dyne <j@getutm.app> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/qemu/cacheflush.h | 15 ++++++++++++-- softmmu/physmem.c | 2 +- tcg/tcg.c | 6 ++++-- util/cacheflush.c | 38 +++++++++++++++++++++--------------- util/cacheinfo.c | 8 +++++--- tcg/aarch64/tcg-target.c.inc | 2 +- tcg/mips/tcg-target.c.inc | 2 +- tcg/ppc/tcg-target.c.inc | 4 ++-- tcg/sparc/tcg-target.c.inc | 4 ++-- 9 files changed, 51 insertions(+), 30 deletions(-) diff --git a/include/qemu/cacheflush.h b/include/qemu/cacheflush.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/cacheflush.h +++ b/include/qemu/cacheflush.h @@ -XXX,XX +XXX,XX @@ #ifndef QEMU_CACHEFLUSH_H #define QEMU_CACHEFLUSH_H +/** + * flush_idcache_range: + * @rx: instruction address + * @rw: data address + * @len: length to flush + * + * Flush @len bytes of the data cache at @rw and the icache at @rx + * to bring them in sync. The two addresses may be different virtual + * mappings of the same physical page(s). + */ + #if defined(__i386__) || defined(__x86_64__) || defined(__s390__) -static inline void flush_icache_range(uintptr_t start, uintptr_t stop) +static inline void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) { /* icache is coherent and does not require flushing. */ } #else -void flush_icache_range(uintptr_t start, uintptr_t stop); +void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len); #endif diff --git a/softmmu/physmem.c b/softmmu/physmem.c index XXXXXXX..XXXXXXX 100644 --- a/softmmu/physmem.c +++ b/softmmu/physmem.c @@ -XXX,XX +XXX,XX @@ static inline MemTxResult address_space_write_rom_internal(AddressSpace *as, invalidate_and_set_dirty(mr, addr1, l); break; case FLUSH_CACHE: - flush_icache_range((uintptr_t)ram_ptr, (uintptr_t)ram_ptr + l); + flush_idcache_range((uintptr_t)ram_ptr, (uintptr_t)ram_ptr, l); break; } } diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ void tcg_prologue_init(TCGContext *s) buf1 = s->code_ptr; #ifndef CONFIG_TCG_INTERPRETER - flush_icache_range((uintptr_t)buf0, (uintptr_t)buf1); + flush_idcache_range((uintptr_t)buf0, (uintptr_t)buf0, + tcg_ptr_byte_diff(buf1, buf0)); #endif /* Deduct the prologue from the buffer. */ @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb) #ifndef CONFIG_TCG_INTERPRETER /* flush instruction cache */ - flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr); + flush_idcache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_buf, + tcg_ptr_byte_diff(s->code_ptr, s->code_buf)); #endif return tcg_current_code_size(s); diff --git a/util/cacheflush.c b/util/cacheflush.c index XXXXXXX..XXXXXXX 100644 --- a/util/cacheflush.c +++ b/util/cacheflush.c @@ -XXX,XX +XXX,XX @@ #include <sys/cachectl.h> #endif -void flush_icache_range(uintptr_t start, uintptr_t stop) +void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) { - cacheflush((void *)start, stop - start, ICACHE); + if (rx != rw) { + cacheflush((void *)rw, len, DCACHE); + } + cacheflush((void *)rx, len, ICACHE); } #elif defined(__powerpc__) -void flush_icache_range(uintptr_t start, uintptr_t stop) +void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) { - uintptr_t p, start1, stop1; + uintptr_t p, b, e; size_t dsize = qemu_dcache_linesize; size_t isize = qemu_icache_linesize; - start1 = start & ~(dsize - 1); - stop1 = (stop + dsize - 1) & ~(dsize - 1); - for (p = start1; p < stop1; p += dsize) { + b = rw & ~(dsize - 1); + e = (rw + len + dsize - 1) & ~(dsize - 1); + for (p = b; p < e; p += dsize) { asm volatile ("dcbst 0,%0" : : "r"(p) : "memory"); } asm volatile ("sync" : : : "memory"); - start &= start & ~(isize - 1); - stop1 = (stop + isize - 1) & ~(isize - 1); - for (p = start1; p < stop1; p += isize) { + b = rx & ~(isize - 1); + e = (rx + len + isize - 1) & ~(isize - 1); + for (p = b; p < e; p += isize) { asm volatile ("icbi 0,%0" : : "r"(p) : "memory"); } asm volatile ("sync" : : : "memory"); @@ -XXX,XX +XXX,XX @@ void flush_icache_range(uintptr_t start, uintptr_t stop) #elif defined(__sparc__) -void flush_icache_range(uintptr_t start, uintptr_t stop) +void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) { - uintptr_t p; - - for (p = start & -8; p < ((stop + 7) & -8); p += 8) { + /* No additional data flush to the RW virtual address required. */ + uintptr_t p, end = (rx + len + 7) & -8; + for (p = rx & -8; p < end; p += 8) { __asm__ __volatile__("flush\t%0" : : "r" (p)); } } #else -void flush_icache_range(uintptr_t start, uintptr_t stop) +void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) { - __builtin___clear_cache((char *)start, (char *)stop); + if (rw != rx) { + __builtin___clear_cache((char *)rw, (char *)rw + len); + } + __builtin___clear_cache((char *)rx, (char *)rx + len); } #endif diff --git a/util/cacheinfo.c b/util/cacheinfo.c index XXXXXXX..XXXXXXX 100644 --- a/util/cacheinfo.c +++ b/util/cacheinfo.c @@ -XXX,XX +XXX,XX @@ static void fallback_cache_info(int *isize, int *dsize) *isize = *dsize; } else { #if defined(_ARCH_PPC) - /* For PPC, we're going to use the icache size computed for - flush_icache_range. Which means that we must use the - architecture minimum. */ + /* + * For PPC, we're going to use the cache sizes computed for + * flush_idcache_range. Which means that we must use the + * architecture minimum. + */ *isize = *dsize = 16; #else /* Otherwise, 64 bytes is not uncommon. */ diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr, } pair = (uint64_t)i2 << 32 | i1; qatomic_set((uint64_t *)jmp_addr, pair); - flush_icache_range(jmp_addr, jmp_addr + 8); + flush_idcache_range(jmp_addr, jmp_addr, 8); } static inline void tcg_out_goto_label(TCGContext *s, TCGLabel *l) diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr, uintptr_t addr) { qatomic_set((uint32_t *)jmp_addr, deposit32(OPC_J, 0, 26, addr >> 2)); - flush_icache_range(jmp_addr, jmp_addr + 4); + flush_idcache_range(jmp_addr, jmp_addr, 4); } typedef struct { diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr, /* As per the enclosing if, this is ppc64. Avoid the _Static_assert within qatomic_set that would fail to build a ppc32 host. */ qatomic_set__nocheck((uint64_t *)jmp_addr, pair); - flush_icache_range(jmp_addr, jmp_addr + 8); + flush_idcache_range(jmp_addr, jmp_addr, 8); } else { intptr_t diff = addr - jmp_addr; tcg_debug_assert(in_range_b(diff)); qatomic_set((uint32_t *)jmp_addr, B | (diff & 0x3fffffc)); - flush_icache_range(jmp_addr, jmp_addr + 4); + flush_idcache_range(jmp_addr, jmp_addr, 4); } } diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr, if (!USE_REG_TB) { qatomic_set((uint32_t *)jmp_addr, deposit32(CALL, 0, 30, br_disp >> 2)); - flush_icache_range(jmp_addr, jmp_addr + 4); + flush_idcache_range(jmp_addr, jmp_addr, 4); return; } @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr, } qatomic_set((uint64_t *)jmp_addr, deposit64(i2, 32, 32, i1)); - flush_icache_range(jmp_addr, jmp_addr + 8); + flush_idcache_range(jmp_addr, jmp_addr, 8); } -- 2.25.1 For darwin, the CTR_EL0 register is not accessible, but there are system routines that we can use. For other hosts, copy the single pointer implementation from libgcc and modify it to support the double pointer interface we require. This halves the number of cache operations required when split-rwx is enabled. Reviewed-by: Joelle van Dyne <j@getutm.app> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- util/cacheflush.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/util/cacheflush.c b/util/cacheflush.c index XXXXXXX..XXXXXXX 100644 --- a/util/cacheflush.c +++ b/util/cacheflush.c @@ -XXX,XX +XXX,XX @@ #include "qemu/osdep.h" #include "qemu/cacheflush.h" +#include "qemu/bitops.h" #if defined(__i386__) || defined(__x86_64__) || defined(__s390__) /* Caches are coherent and do not require flushing; symbol inline. */ +#elif defined(__aarch64__) + +#ifdef CONFIG_DARWIN +/* Apple does not expose CTR_EL0, so we must use system interfaces. */ +extern void sys_icache_invalidate(void *start, size_t len); +extern void sys_dcache_flush(void *start, size_t len); +void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) +{ + sys_dcache_flush((void *)rw, len); + sys_icache_invalidate((void *)rx, len); +} +#else + +/* + * TODO: unify this with cacheinfo.c. + * We want to save the whole contents of CTR_EL0, so that we + * have more than the linesize, but also IDC and DIC. + */ +static unsigned int save_ctr_el0; +static void __attribute__((constructor)) init_ctr_el0(void) +{ + asm volatile("mrs\t%0, ctr_el0" : "=r"(save_ctr_el0)); +} + +/* + * This is a copy of gcc's __aarch64_sync_cache_range, modified + * to fit this three-operand interface. + */ +void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) +{ + const unsigned CTR_IDC = 1u << 28; + const unsigned CTR_DIC = 1u << 29; + const unsigned int ctr_el0 = save_ctr_el0; + const uintptr_t icache_lsize = 4 << extract32(ctr_el0, 0, 4); + const uintptr_t dcache_lsize = 4 << extract32(ctr_el0, 16, 4); + uintptr_t p; + + /* + * If CTR_EL0.IDC is enabled, Data cache clean to the Point of Unification + * is not required for instruction to data coherence. + */ + if (!(ctr_el0 & CTR_IDC)) { + /* + * Loop over the address range, clearing one cache line at once. + * Data cache must be flushed to unification first to make sure + * the instruction cache fetches the updated data. + */ + for (p = rw & -dcache_lsize; p < rw + len; p += dcache_lsize) { + asm volatile("dc\tcvau, %0" : : "r" (p) : "memory"); + } + asm volatile("dsb\tish" : : : "memory"); + } + + /* + * If CTR_EL0.DIC is enabled, Instruction cache cleaning to the Point + * of Unification is not required for instruction to data coherence. + */ + if (!(ctr_el0 & CTR_DIC)) { + for (p = rx & -icache_lsize; p < rx + len; p += icache_lsize) { + asm volatile("ic\tivau, %0" : : "r"(p) : "memory"); + } + asm volatile ("dsb\tish" : : : "memory"); + } + + asm volatile("isb" : : : "memory"); +} +#endif /* CONFIG_DARWIN */ + #elif defined(__mips__) #ifdef __OpenBSD__ -- 2.25.1 This value is constant across all thread-local copies of TCGContext, so we might as well move it out of thread-local storage. Use the correct function pointer type, and name the variable tcg_qemu_tb_exec, which means that we are able to remove the macro that does the casting. Replace HAVE_TCG_QEMU_TB_EXEC with CONFIG_TCG_INTERPRETER, as this is somewhat clearer in intent. Reviewed-by: Joelle van Dyne <j@getutm.app> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg.h | 9 ++++----- tcg/tcg.c | 9 ++++++++- tcg/tci.c | 4 ++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ struct TCGContext { here, because there's too much arithmetic throughout that relies on addition and subtraction working on bytes. Rely on the GCC extension that allows arithmetic on void*. */ - void *code_gen_prologue; void *code_gen_epilogue; void *code_gen_buffer; size_t code_gen_buffer_size; @@ -XXX,XX +XXX,XX @@ static inline unsigned get_mmuidx(TCGMemOpIdx oi) #define TB_EXIT_IDXMAX 1 #define TB_EXIT_REQUESTED 3 -#ifdef HAVE_TCG_QEMU_TB_EXEC -uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr); +#ifdef CONFIG_TCG_INTERPRETER +uintptr_t tcg_qemu_tb_exec(CPUArchState *env, void *tb_ptr); #else -# define tcg_qemu_tb_exec(env, tb_ptr) \ - ((uintptr_t (*)(void *, void *))tcg_ctx->code_gen_prologue)(env, tb_ptr) +typedef uintptr_t tcg_prologue_fn(CPUArchState *env, void *tb_ptr); +extern tcg_prologue_fn *tcg_qemu_tb_exec; #endif void tcg_register_jit(void *buf, size_t buf_size); diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static TCGContext **tcg_ctxs; static unsigned int n_tcg_ctxs; TCGv_env cpu_env = 0; +#ifndef CONFIG_TCG_INTERPRETER +tcg_prologue_fn *tcg_qemu_tb_exec; +#endif + struct tcg_region_tree { QemuMutex lock; GTree *tree; @@ -XXX,XX +XXX,XX @@ void tcg_prologue_init(TCGContext *s) s->code_ptr = buf0; s->code_buf = buf0; s->data_gen_ptr = NULL; - s->code_gen_prologue = buf0; + +#ifndef CONFIG_TCG_INTERPRETER + tcg_qemu_tb_exec = (tcg_prologue_fn *)buf0; +#endif /* Compute a high-water mark, at which we voluntarily flush the buffer and start over. The size here is arbitrary, significantly larger diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ static bool tci_compare64(uint64_t u0, uint64_t u1, TCGCond condition) * One possible operation in the pseudo code is a call to binary code. * Therefore, disable CFI checks in the interpreter function */ -QEMU_DISABLE_CFI -uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr) +uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, void *v_tb_ptr) { + uint8_t *tb_ptr = v_tb_ptr; tcg_target_ulong regs[TCG_TARGET_NB_REGS]; long tcg_temps[CPU_TEMP_BUF_NLONGS]; uintptr_t sp_value = (uintptr_t)(tcg_temps + CPU_TEMP_BUF_NLONGS); -- 2.25.1 This value is constant across all thread-local copies of TCGContext, so we might as well move it out of thread-local storage. Reviewed-by: Joelle van Dyne <j@getutm.app> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg.h | 2 +- accel/tcg/tcg-runtime.c | 2 +- tcg/tcg.c | 3 ++- tcg/aarch64/tcg-target.c.inc | 4 ++-- tcg/arm/tcg-target.c.inc | 2 +- tcg/i386/tcg-target.c.inc | 4 ++-- tcg/mips/tcg-target.c.inc | 2 +- tcg/ppc/tcg-target.c.inc | 2 +- tcg/riscv/tcg-target.c.inc | 4 ++-- tcg/s390/tcg-target.c.inc | 4 ++-- tcg/sparc/tcg-target.c.inc | 2 +- 11 files changed, 16 insertions(+), 15 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ struct TCGContext { here, because there's too much arithmetic throughout that relies on addition and subtraction working on bytes. Rely on the GCC extension that allows arithmetic on void*. */ - void *code_gen_epilogue; void *code_gen_buffer; size_t code_gen_buffer_size; void *code_gen_ptr; @@ -XXX,XX +XXX,XX @@ struct TCGContext { extern TCGContext tcg_init_ctx; extern __thread TCGContext *tcg_ctx; +extern void *tcg_code_gen_epilogue; extern TCGv_env cpu_env; static inline size_t temp_idx(TCGTemp *ts) diff --git a/accel/tcg/tcg-runtime.c b/accel/tcg/tcg-runtime.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/tcg-runtime.c +++ b/accel/tcg/tcg-runtime.c @@ -XXX,XX +XXX,XX @@ void *HELPER(lookup_tb_ptr)(CPUArchState *env) tb = tb_lookup__cpu_state(cpu, &pc, &cs_base, &flags, curr_cflags()); if (tb == NULL) { - return tcg_ctx->code_gen_epilogue; + return tcg_code_gen_epilogue; } qemu_log_mask_and_addr(CPU_LOG_EXEC, pc, "Chain %d: %p [" diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static int tcg_out_ldst_finalize(TCGContext *s); static TCGContext **tcg_ctxs; static unsigned int n_tcg_ctxs; TCGv_env cpu_env = 0; +void *tcg_code_gen_epilogue; #ifndef CONFIG_TCG_INTERPRETER tcg_prologue_fn *tcg_qemu_tb_exec; @@ -XXX,XX +XXX,XX @@ void tcg_prologue_init(TCGContext *s) /* Assert that goto_ptr is implemented completely. */ if (TCG_TARGET_HAS_goto_ptr) { - tcg_debug_assert(s->code_gen_epilogue != NULL); + tcg_debug_assert(tcg_code_gen_epilogue != NULL); } } diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_exit_tb: /* Reuse the zeroing that exists for goto_ptr. */ if (a0 == 0) { - tcg_out_goto_long(s, s->code_gen_epilogue); + tcg_out_goto_long(s, tcg_code_gen_epilogue); } else { tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0); tcg_out_goto_long(s, tb_ret_addr); @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) * Return path for goto_ptr. Set return value to 0, a-la exit_tb, * and fall through to the rest of the epilogue. */ - s->code_gen_epilogue = s->code_ptr; + tcg_code_gen_epilogue = s->code_ptr; tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_X0, 0); /* TB epilogue */ diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) * Return path for goto_ptr. Set return value to 0, a-la exit_tb, * and fall through to the rest of the epilogue. */ - s->code_gen_epilogue = s->code_ptr; + tcg_code_gen_epilogue = s->code_ptr; tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, 0); tcg_out_epilogue(s); } diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_exit_tb: /* Reuse the zeroing that exists for goto_ptr. */ if (a0 == 0) { - tcg_out_jmp(s, s->code_gen_epilogue); + tcg_out_jmp(s, tcg_code_gen_epilogue); } else { tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0); tcg_out_jmp(s, tb_ret_addr); @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) * Return path for goto_ptr. Set return value to 0, a-la exit_tb, * and fall through to the rest of the epilogue. */ - s->code_gen_epilogue = s->code_ptr; + tcg_code_gen_epilogue = s->code_ptr; tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_EAX, 0); /* TB epilogue */ diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) * Return path for goto_ptr. Set return value to 0, a-la exit_tb, * and fall through to the rest of the epilogue. */ - s->code_gen_epilogue = s->code_ptr; + tcg_code_gen_epilogue = s->code_ptr; tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_V0, TCG_REG_ZERO); /* TB epilogue */ diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out32(s, BCCTR | BO_ALWAYS); /* Epilogue */ - s->code_gen_epilogue = tb_ret_addr = s->code_ptr; + tcg_code_gen_epilogue = tb_ret_addr = s->code_ptr; tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET); for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) { diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_exit_tb: /* Reuse the zeroing that exists for goto_ptr. */ if (a0 == 0) { - tcg_out_call_int(s, s->code_gen_epilogue, true); + tcg_out_call_int(s, tcg_code_gen_epilogue, true); } else { tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0); tcg_out_call_int(s, tb_ret_addr, true); @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, tcg_target_call_iarg_regs[1], 0); /* Return path for goto_ptr. Set return value to 0 */ - s->code_gen_epilogue = s->code_ptr; + tcg_code_gen_epilogue = s->code_ptr; tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_A0, TCG_REG_ZERO); /* TB epilogue */ diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.c.inc +++ b/tcg/s390/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, /* Reuse the zeroing that exists for goto_ptr. */ a0 = args[0]; if (a0 == 0) { - tgen_gotoi(s, S390_CC_ALWAYS, s->code_gen_epilogue); + tgen_gotoi(s, S390_CC_ALWAYS, tcg_code_gen_epilogue); } else { tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, a0); tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr); @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) * Return path for goto_ptr. Set return value to 0, a-la exit_tb, * and fall through to the rest of the epilogue. */ - s->code_gen_epilogue = s->code_ptr; + tcg_code_gen_epilogue = s->code_ptr; tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, 0); /* TB epilogue */ diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out_nop(s); /* Epilogue for goto_ptr. */ - s->code_gen_epilogue = s->code_ptr; + tcg_code_gen_epilogue = s->code_ptr; tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); /* delay slot */ tcg_out_movi_imm13(s, TCG_REG_O0, 0); -- 2.25.1 Create a function to determine if a pointer is within the buffer. Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg.h | 11 +++++++++++ accel/tcg/translate-all.c | 26 ++++++++------------------ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ extern __thread TCGContext *tcg_ctx; extern void *tcg_code_gen_epilogue; extern TCGv_env cpu_env; +static inline bool in_code_gen_buffer(const void *p) +{ + const TCGContext *s = &tcg_init_ctx; + /* + * Much like it is valid to have a pointer to the byte past the + * end of an array (so long as you don't dereference it), allow + * a pointer to the byte past the end of the code gen buffer. + */ + return (size_t)(p - s->code_gen_buffer) <= s->code_gen_buffer_size; +} + static inline size_t temp_idx(TCGTemp *ts) { ptrdiff_t n = ts - tcg_ctx->temps; diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ void tb_destroy(TranslationBlock *tb) bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit) { - TranslationBlock *tb; - bool r = false; - uintptr_t check_offset; - - /* The host_pc has to be in the region of current code buffer. If - * it is not we will not be able to resolve it here. The two cases - * where host_pc will not be correct are: + /* + * The host_pc has to be in the region of the code buffer. + * If it is not we will not be able to resolve it here. + * The two cases where host_pc will not be correct are: * * - fault during translation (instruction fetch) * - fault from helper (not using GETPC() macro) * * Either way we need return early as we can't resolve it here. - * - * We are using unsigned arithmetic so if host_pc < - * tcg_init_ctx.code_gen_buffer check_offset will wrap to way - * above the code_gen_buffer_size */ - check_offset = host_pc - (uintptr_t) tcg_init_ctx.code_gen_buffer; - - if (check_offset < tcg_init_ctx.code_gen_buffer_size) { - tb = tcg_tb_lookup(host_pc); + if (in_code_gen_buffer((const void *)host_pc)) { + TranslationBlock *tb = tcg_tb_lookup(host_pc); if (tb) { cpu_restore_state_from_tb(cpu, tb, host_pc, will_exit); if (tb_cflags(tb) & CF_NOCACHE) { @@ -XXX,XX +XXX,XX @@ bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit) tcg_tb_remove(tb); tb_destroy(tb); } - r = true; + return true; } } - - return r; + return false; } static void page_init(void) -- 2.25.1 Add two helper functions, using a global variable to hold the displacement. The displacement is currently always 0, so no change in behaviour. Begin using the functions in tcg common code only. Reviewed-by: Joelle van Dyne <j@getutm.app> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/tcg-runtime.h | 2 +- include/disas/disas.h | 2 +- include/exec/exec-all.h | 2 +- include/exec/log.h | 2 +- include/tcg/tcg.h | 26 ++++++++++++++---- accel/tcg/cpu-exec.c | 2 +- accel/tcg/tcg-runtime.c | 2 +- accel/tcg/translate-all.c | 33 +++++++++++------------ disas.c | 4 ++- tcg/tcg.c | 56 ++++++++++++++++++++++++++++++++++----- tcg/tci.c | 6 +++-- tcg/tcg-pool.c.inc | 6 ++++- accel/tcg/trace-events | 2 +- 13 files changed, 105 insertions(+), 40 deletions(-) diff --git a/accel/tcg/tcg-runtime.h b/accel/tcg/tcg-runtime.h index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/tcg-runtime.h +++ b/accel/tcg/tcg-runtime.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_1(clrsb_i64, TCG_CALL_NO_RWG_SE, i64, i64) DEF_HELPER_FLAGS_1(ctpop_i32, TCG_CALL_NO_RWG_SE, i32, i32) DEF_HELPER_FLAGS_1(ctpop_i64, TCG_CALL_NO_RWG_SE, i64, i64) -DEF_HELPER_FLAGS_1(lookup_tb_ptr, TCG_CALL_NO_WG_SE, ptr, env) +DEF_HELPER_FLAGS_1(lookup_tb_ptr, TCG_CALL_NO_WG_SE, cptr, env) DEF_HELPER_FLAGS_1(exit_atomic, TCG_CALL_NO_WG, noreturn, env) diff --git a/include/disas/disas.h b/include/disas/disas.h index XXXXXXX..XXXXXXX 100644 --- a/include/disas/disas.h +++ b/include/disas/disas.h @@ -XXX,XX +XXX,XX @@ #include "cpu.h" /* Disassemble this for me please... (debugging). */ -void disas(FILE *out, void *code, unsigned long size); +void disas(FILE *out, const void *code, unsigned long size); void target_disas(FILE *out, CPUState *cpu, target_ulong code, target_ulong size); diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ int probe_access_flags(CPUArchState *env, target_ulong addr, * Note: the address of search data can be obtained by adding @size to @ptr. */ struct tb_tc { - void *ptr; /* pointer to the translated code */ + const void *ptr; /* pointer to the translated code */ size_t size; }; diff --git a/include/exec/log.h b/include/exec/log.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/log.h +++ b/include/exec/log.h @@ -XXX,XX +XXX,XX @@ static inline void log_target_disas(CPUState *cpu, target_ulong start, rcu_read_unlock(); } -static inline void log_disas(void *code, unsigned long size) +static inline void log_disas(const void *code, unsigned long size) { QemuLogFile *logfile; rcu_read_lock(); diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ struct TCGContext { extern TCGContext tcg_init_ctx; extern __thread TCGContext *tcg_ctx; extern void *tcg_code_gen_epilogue; +extern uintptr_t tcg_splitwx_diff; extern TCGv_env cpu_env; static inline bool in_code_gen_buffer(const void *p) @@ -XXX,XX +XXX,XX @@ static inline bool in_code_gen_buffer(const void *p) return (size_t)(p - s->code_gen_buffer) <= s->code_gen_buffer_size; } +#ifdef CONFIG_DEBUG_TCG +const void *tcg_splitwx_to_rx(void *rw); +void *tcg_splitwx_to_rw(const void *rx); +#else +static inline const void *tcg_splitwx_to_rx(void *rw) +{ + return rw ? rw + tcg_splitwx_diff : NULL; +} + +static inline void *tcg_splitwx_to_rw(const void *rx) +{ + return rx ? (void *)rx - tcg_splitwx_diff : NULL; +} +#endif + static inline size_t temp_idx(TCGTemp *ts) { ptrdiff_t n = ts - tcg_ctx->temps; @@ -XXX,XX +XXX,XX @@ static inline TCGLabel *arg_label(TCGArg i) * correct result. */ -static inline ptrdiff_t tcg_ptr_byte_diff(void *a, void *b) +static inline ptrdiff_t tcg_ptr_byte_diff(const void *a, const void *b) { return a - b; } @@ -XXX,XX +XXX,XX @@ static inline ptrdiff_t tcg_ptr_byte_diff(void *a, void *b) * to the destination address. */ -static inline ptrdiff_t tcg_pcrel_diff(TCGContext *s, void *target) +static inline ptrdiff_t tcg_pcrel_diff(TCGContext *s, const void *target) { - return tcg_ptr_byte_diff(target, s->code_ptr); + return tcg_ptr_byte_diff(target, tcg_splitwx_to_rx(s->code_ptr)); } /** @@ -XXX,XX +XXX,XX @@ static inline unsigned get_mmuidx(TCGMemOpIdx oi) #define TB_EXIT_REQUESTED 3 #ifdef CONFIG_TCG_INTERPRETER -uintptr_t tcg_qemu_tb_exec(CPUArchState *env, void *tb_ptr); +uintptr_t tcg_qemu_tb_exec(CPUArchState *env, const void *tb_ptr); #else -typedef uintptr_t tcg_prologue_fn(CPUArchState *env, void *tb_ptr); +typedef uintptr_t tcg_prologue_fn(CPUArchState *env, const void *tb_ptr); extern tcg_prologue_fn *tcg_qemu_tb_exec; #endif diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, TranslationBlock *itb) uintptr_t ret; TranslationBlock *last_tb; int tb_exit; - uint8_t *tb_ptr = itb->tc.ptr; + const void *tb_ptr = itb->tc.ptr; qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc, "Trace %d: %p [" diff --git a/accel/tcg/tcg-runtime.c b/accel/tcg/tcg-runtime.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/tcg-runtime.c +++ b/accel/tcg/tcg-runtime.c @@ -XXX,XX +XXX,XX @@ uint64_t HELPER(ctpop_i64)(uint64_t arg) return ctpop64(arg); } -void *HELPER(lookup_tb_ptr)(CPUArchState *env) +const void *HELPER(lookup_tb_ptr)(CPUArchState *env) { CPUState *cpu = env_cpu(env); TranslationBlock *tb; diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ static uint8_t *encode_sleb128(uint8_t *p, target_long val) /* Decode a signed leb128 sequence at *PP; increment *PP past the decoded value. Return the decoded value. */ -static target_long decode_sleb128(uint8_t **pp) +static target_long decode_sleb128(const uint8_t **pp) { - uint8_t *p = *pp; + const uint8_t *p = *pp; target_long val = 0; int byte, shift = 0; @@ -XXX,XX +XXX,XX @@ static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc }; uintptr_t host_pc = (uintptr_t)tb->tc.ptr; CPUArchState *env = cpu->env_ptr; - uint8_t *p = tb->tc.ptr + tb->tc.size; + const uint8_t *p = tb->tc.ptr + tb->tc.size; int i, j, num_insns = tb->icount; #ifdef CONFIG_PROFILER TCGProfile *prof = &tcg_ctx->prof; @@ -XXX,XX +XXX,XX @@ void tb_destroy(TranslationBlock *tb) bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit) { /* - * The host_pc has to be in the region of the code buffer. + * The host_pc has to be in the rx region of the code buffer. * If it is not we will not be able to resolve it here. * The two cases where host_pc will not be correct are: * @@ -XXX,XX +XXX,XX @@ bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit) * * Either way we need return early as we can't resolve it here. */ - if (in_code_gen_buffer((const void *)host_pc)) { + if (in_code_gen_buffer((const void *)(host_pc - tcg_splitwx_diff))) { TranslationBlock *tb = tcg_tb_lookup(host_pc); if (tb) { cpu_restore_state_from_tb(cpu, tb, host_pc, will_exit); @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, } gen_code_buf = tcg_ctx->code_gen_ptr; - tb->tc.ptr = gen_code_buf; + tb->tc.ptr = tcg_splitwx_to_rx(gen_code_buf); tb->pc = pc; tb->cs_base = cs_base; tb->flags = flags; @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && qemu_log_in_addr_range(tb->pc)) { FILE *logfile = qemu_log_lock(); - int code_size, data_size = 0; + int code_size, data_size; + const tcg_target_ulong *rx_data_gen_ptr; size_t chunk_start; int insn = 0; if (tcg_ctx->data_gen_ptr) { - code_size = tcg_ctx->data_gen_ptr - tb->tc.ptr; + rx_data_gen_ptr = tcg_splitwx_to_rx(tcg_ctx->data_gen_ptr); + code_size = (const void *)rx_data_gen_ptr - tb->tc.ptr; data_size = gen_code_size - code_size; } else { + rx_data_gen_ptr = 0; code_size = gen_code_size; + data_size = 0; } /* Dump header and the first instruction */ @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, if (data_size) { int i; qemu_log(" data: [size=%d]\n", data_size); - for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { - if (sizeof(tcg_target_ulong) == 8) { - qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", - (uintptr_t)tcg_ctx->data_gen_ptr + i, - *(uint64_t *)(tcg_ctx->data_gen_ptr + i)); - } else { - qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n", - (uintptr_t)tcg_ctx->data_gen_ptr + i, - *(uint32_t *)(tcg_ctx->data_gen_ptr + i)); - } + for (i = 0; i < data_size / sizeof(tcg_target_ulong); i++) { + qemu_log("0x%08" PRIxPTR ": .quad 0x%" TCG_PRIlx "\n", + (uintptr_t)&rx_data_gen_ptr[i], rx_data_gen_ptr[i]); } } qemu_log("\n"); diff --git a/disas.c b/disas.c index XXXXXXX..XXXXXXX 100644 --- a/disas.c +++ b/disas.c @@ -XXX,XX +XXX,XX @@ char *plugin_disas(CPUState *cpu, uint64_t addr, size_t size) } /* Disassemble this for me please... (debugging). */ -void disas(FILE *out, void *code, unsigned long size) +void disas(FILE *out, const void *ccode, unsigned long size) { + /* TODO: Push constness through the disas backends. */ + void *code = (void *)ccode; uintptr_t pc; int count; CPUDebug s; diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static TCGContext **tcg_ctxs; static unsigned int n_tcg_ctxs; TCGv_env cpu_env = 0; void *tcg_code_gen_epilogue; +uintptr_t tcg_splitwx_diff; #ifndef CONFIG_TCG_INTERPRETER tcg_prologue_fn *tcg_qemu_tb_exec; @@ -XXX,XX +XXX,XX @@ static void tcg_region_trees_init(void) } } -static struct tcg_region_tree *tc_ptr_to_region_tree(void *p) +static struct tcg_region_tree *tc_ptr_to_region_tree(const void *cp) { + void *p = tcg_splitwx_to_rw(cp); size_t region_idx; if (p < region.start_aligned) { @@ -XXX,XX +XXX,XX @@ void tcg_region_init(void) size_t region_size; size_t n_regions; size_t i; + uintptr_t splitwx_diff; n_regions = tcg_n_regions(); @@ -XXX,XX +XXX,XX @@ void tcg_region_init(void) region.end -= page_size; /* set guard pages */ + splitwx_diff = tcg_splitwx_diff; for (i = 0; i < region.n; i++) { void *start, *end; int rc; @@ -XXX,XX +XXX,XX @@ void tcg_region_init(void) tcg_region_bounds(i, &start, &end); rc = qemu_mprotect_none(end, page_size); g_assert(!rc); + if (splitwx_diff) { + rc = qemu_mprotect_none(end + splitwx_diff, page_size); + g_assert(!rc); + } } tcg_region_trees_init(); @@ -XXX,XX +XXX,XX @@ void tcg_region_init(void) #endif } +#ifdef CONFIG_DEBUG_TCG +const void *tcg_splitwx_to_rx(void *rw) +{ + /* Pass NULL pointers unchanged. */ + if (rw) { + g_assert(in_code_gen_buffer(rw)); + rw += tcg_splitwx_diff; + } + return rw; +} + +void *tcg_splitwx_to_rw(const void *rx) +{ + /* Pass NULL pointers unchanged. */ + if (rx) { + rx -= tcg_splitwx_diff; + /* Assert that we end with a pointer in the rw region. */ + g_assert(in_code_gen_buffer(rx)); + } + return (void *)rx; +} +#endif /* CONFIG_DEBUG_TCG */ + static void alloc_tcg_plugin_context(TCGContext *s) { #ifdef CONFIG_PLUGIN @@ -XXX,XX +XXX,XX @@ void tcg_prologue_init(TCGContext *s) s->code_buf = buf0; s->data_gen_ptr = NULL; + /* + * The region trees are not yet configured, but tcg_splitwx_to_rx + * needs the bounds for an assert. + */ + region.start = buf0; + region.end = buf0 + total_size; + #ifndef CONFIG_TCG_INTERPRETER - tcg_qemu_tb_exec = (tcg_prologue_fn *)buf0; + tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(buf0); #endif /* Compute a high-water mark, at which we voluntarily flush the buffer @@ -XXX,XX +XXX,XX @@ void tcg_prologue_init(TCGContext *s) buf1 = s->code_ptr; #ifndef CONFIG_TCG_INTERPRETER - flush_idcache_range((uintptr_t)buf0, (uintptr_t)buf0, + flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(buf0), (uintptr_t)buf0, tcg_ptr_byte_diff(buf1, buf0)); #endif @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb) tcg_reg_alloc_start(s); - s->code_buf = tb->tc.ptr; - s->code_ptr = tb->tc.ptr; + /* + * Reset the buffer pointers when restarting after overflow. + * TODO: Move this into translate-all.c with the rest of the + * buffer management. Having only this done here is confusing. + */ + s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr); + s->code_ptr = s->code_buf; #ifdef TCG_TARGET_NEED_LDST_LABELS QSIMPLEQ_INIT(&s->ldst_labels); @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb) #ifndef CONFIG_TCG_INTERPRETER /* flush instruction cache */ - flush_idcache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_buf, + flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), + (uintptr_t)s->code_buf, tcg_ptr_byte_diff(s->code_ptr, s->code_buf)); #endif diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ static bool tci_compare64(uint64_t u0, uint64_t u1, TCGCond condition) * One possible operation in the pseudo code is a call to binary code. * Therefore, disable CFI checks in the interpreter function */ -uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, void *v_tb_ptr) +uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, + const void *v_tb_ptr) { - uint8_t *tb_ptr = v_tb_ptr; + /* TODO: Propagate const through this file. */ + uint8_t *tb_ptr = (uint8_t *)v_tb_ptr; tcg_target_ulong regs[TCG_TARGET_NB_REGS]; long tcg_temps[CPU_TEMP_BUF_NLONGS]; uintptr_t sp_value = (uintptr_t)(tcg_temps + CPU_TEMP_BUF_NLONGS); diff --git a/tcg/tcg-pool.c.inc b/tcg/tcg-pool.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-pool.c.inc +++ b/tcg/tcg-pool.c.inc @@ -XXX,XX +XXX,XX @@ static int tcg_out_pool_finalize(TCGContext *s) for (; p != NULL; p = p->next) { size_t size = sizeof(tcg_target_ulong) * p->nlong; + uintptr_t value; + if (!l || l->nlong != p->nlong || memcmp(l->data, p->data, size)) { if (unlikely(a > s->code_gen_highwater)) { return -1; @@ -XXX,XX +XXX,XX @@ static int tcg_out_pool_finalize(TCGContext *s) a += size; l = p; } - if (!patch_reloc(p->label, p->rtype, (intptr_t)a - size, p->addend)) { + + value = (uintptr_t)tcg_splitwx_to_rx(a) - size; + if (!patch_reloc(p->label, p->rtype, value, p->addend)) { return -2; } } diff --git a/accel/tcg/trace-events b/accel/tcg/trace-events index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/trace-events +++ b/accel/tcg/trace-events @@ -XXX,XX +XXX,XX @@ exec_tb_nocache(void *tb, uintptr_t pc) "tb:%p pc=0x%"PRIxPTR exec_tb_exit(void *last_tb, unsigned int flags) "tb:%p flags=0x%x" # translate-all.c -translate_block(void *tb, uintptr_t pc, uint8_t *tb_code) "tb:%p, pc:0x%"PRIxPTR", tb_code:%p" +translate_block(void *tb, uintptr_t pc, const void *tb_code) "tb:%p, pc:0x%"PRIxPTR", tb_code:%p" -- 2.25.1 Change TCGLabel.u.value_ptr to const, and initialize it with tcg_splitwx_to_rx. Propagate const through tcg/host/ only as far as needed to avoid errors from the value_ptr change. Reviewed-by: Joelle van Dyne <j@getutm.app> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg.h | 2 +- tcg/tcg.c | 2 +- tcg/aarch64/tcg-target.c.inc | 2 +- tcg/arm/tcg-target.c.inc | 2 +- tcg/mips/tcg-target.c.inc | 5 +++-- tcg/ppc/tcg-target.c.inc | 4 ++-- tcg/s390/tcg-target.c.inc | 2 +- 7 files changed, 10 insertions(+), 9 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ struct TCGLabel { unsigned refs : 16; union { uintptr_t value; - tcg_insn_unit *value_ptr; + const tcg_insn_unit *value_ptr; } u; QSIMPLEQ_HEAD(, TCGRelocation) relocs; QSIMPLEQ_ENTRY(TCGLabel) next; diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void tcg_out_label(TCGContext *s, TCGLabel *l, tcg_insn_unit *ptr) { tcg_debug_assert(!l->has_value); l->has_value = 1; - l->u.value_ptr = ptr; + l->u.value_ptr = tcg_splitwx_to_rx(ptr); } TCGLabel *gen_new_label(void) diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_cmp(TCGContext *s, TCGType ext, TCGReg a, } } -static inline void tcg_out_goto(TCGContext *s, tcg_insn_unit *target) +static void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target) { ptrdiff_t offset = target - s->code_ptr; tcg_debug_assert(offset == sextract64(offset, 0, 26)); diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_st8(TCGContext *s, int cond, * with the code buffer limited to 16MB we wouldn't need the long case. * But we also use it for the tail-call to the qemu_ld/st helpers, which does. */ -static void tcg_out_goto(TCGContext *s, int cond, tcg_insn_unit *addr) +static void tcg_out_goto(TCGContext *s, int cond, const tcg_insn_unit *addr) { intptr_t addri = (intptr_t)addr; ptrdiff_t disp = tcg_pcrel_diff(s, addr); diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static tcg_insn_unit *bswap32_addr; static tcg_insn_unit *bswap32u_addr; static tcg_insn_unit *bswap64_addr; -static inline uint32_t reloc_pc16_val(tcg_insn_unit *pc, tcg_insn_unit *target) +static inline uint32_t reloc_pc16_val(tcg_insn_unit *pc, + const tcg_insn_unit *target) { /* Let the compiler perform the right-shift as part of the arithmetic. */ ptrdiff_t disp = target - (pc + 1); @@ -XXX,XX +XXX,XX @@ static inline uint32_t reloc_pc16_val(tcg_insn_unit *pc, tcg_insn_unit *target) return disp & 0xffff; } -static inline void reloc_pc16(tcg_insn_unit *pc, tcg_insn_unit *target) +static inline void reloc_pc16(tcg_insn_unit *pc, const tcg_insn_unit *target) { *pc = deposit32(*pc, 0, 16, reloc_pc16_val(pc, target)); } diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline bool in_range_b(tcg_target_long target) return target == sextract64(target, 0, 26); } -static uint32_t reloc_pc24_val(tcg_insn_unit *pc, tcg_insn_unit *target) +static uint32_t reloc_pc24_val(tcg_insn_unit *pc, const tcg_insn_unit *target) { ptrdiff_t disp = tcg_ptr_byte_diff(target, pc); tcg_debug_assert(in_range_b(disp)); @@ -XXX,XX +XXX,XX @@ static bool reloc_pc24(tcg_insn_unit *pc, tcg_insn_unit *target) return false; } -static uint16_t reloc_pc14_val(tcg_insn_unit *pc, tcg_insn_unit *target) +static uint16_t reloc_pc14_val(tcg_insn_unit *pc, const tcg_insn_unit *target) { ptrdiff_t disp = tcg_ptr_byte_diff(target, pc); tcg_debug_assert(disp == (int16_t) disp); diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.c.inc +++ b/tcg/s390/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tgen_extract(TCGContext *s, TCGReg dest, TCGReg src, tcg_out_risbg(s, dest, src, 64 - len, 63, 64 - ofs, 1); } -static void tgen_gotoi(TCGContext *s, int cc, tcg_insn_unit *dest) +static void tgen_gotoi(TCGContext *s, int cc, const tcg_insn_unit *dest) { ptrdiff_t off = dest - s->code_ptr; if (off == (int16_t)off) { -- 2.25.1 We must change all targets at once, since all must match the declaration in tcg.c. Reviewed-by: Joelle van Dyne <j@getutm.app> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 2 +- tcg/aarch64/tcg-target.c.inc | 2 +- tcg/arm/tcg-target.c.inc | 2 +- tcg/i386/tcg-target.c.inc | 4 ++-- tcg/mips/tcg-target.c.inc | 6 +++--- tcg/ppc/tcg-target.c.inc | 8 ++++---- tcg/riscv/tcg-target.c.inc | 6 +++--- tcg/s390/tcg-target.c.inc | 2 +- tcg/sparc/tcg-target.c.inc | 4 ++-- tcg/tci/tcg-target.c.inc | 2 +- 10 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, intptr_t arg2); static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, TCGReg base, intptr_t ofs); -static void tcg_out_call(TCGContext *s, tcg_insn_unit *target); +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target); static int tcg_target_const_match(tcg_target_long val, TCGType type, const TCGArgConstraint *arg_ct); #ifdef TCG_TARGET_NEED_LDST_LABELS diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_callr(TCGContext *s, TCGReg reg) tcg_out_insn(s, 3207, BLR, reg); } -static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *target) +static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *target) { ptrdiff_t offset = target - s->code_ptr; if (offset == sextract64(offset, 0, 26)) { diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto(TCGContext *s, int cond, const tcg_insn_unit *addr) /* The call case is mostly used for helpers - so it's not unreasonable * for them to be beyond branch range */ -static void tcg_out_call(TCGContext *s, tcg_insn_unit *addr) +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *addr) { intptr_t addri = (intptr_t)addr; ptrdiff_t disp = tcg_pcrel_diff(s, addr); diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_clz(TCGContext *s, int rexw, TCGReg dest, TCGReg arg1, } } -static void tcg_out_branch(TCGContext *s, int call, tcg_insn_unit *dest) +static void tcg_out_branch(TCGContext *s, int call, const tcg_insn_unit *dest) { intptr_t disp = tcg_pcrel_diff(s, dest) - 5; @@ -XXX,XX +XXX,XX @@ static void tcg_out_branch(TCGContext *s, int call, tcg_insn_unit *dest) } } -static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *dest) +static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest) { tcg_out_branch(s, 1, dest); } diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_opc_sa64(TCGContext *s, MIPSInsn opc1, MIPSInsn opc2, * Type jump. * Returns true if the branch was in range and the insn was emitted. */ -static bool tcg_out_opc_jmp(TCGContext *s, MIPSInsn opc, void *target) +static bool tcg_out_opc_jmp(TCGContext *s, MIPSInsn opc, const void *target) { uintptr_t dest = (uintptr_t)target; uintptr_t from = (uintptr_t)s->code_ptr + 4; @@ -XXX,XX +XXX,XX @@ static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret, } } -static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail) +static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) { /* Note that the ABI requires the called function's address to be loaded into T9, even if a direct branch is in range. */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail) } } -static void tcg_out_call(TCGContext *s, tcg_insn_unit *arg) +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg) { tcg_out_call_int(s, arg, false); tcg_out_nop(s); diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_xori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c) tcg_out_zori32(s, dst, src, c, XORI, XORIS); } -static void tcg_out_b(TCGContext *s, int mask, tcg_insn_unit *target) +static void tcg_out_b(TCGContext *s, int mask, const tcg_insn_unit *target) { ptrdiff_t disp = tcg_pcrel_diff(s, target); if (in_range_b(disp)) { @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr, } } -static void tcg_out_call(TCGContext *s, tcg_insn_unit *target) +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target) { #ifdef _CALL_AIX /* Look through the descriptor. If the branch is in range, and we don't have to spend too much effort on building the toc. */ - void *tgt = ((void **)target)[0]; - uintptr_t toc = ((uintptr_t *)target)[1]; + const void *tgt = ((const void * const *)target)[0]; + uintptr_t toc = ((const uintptr_t *)target)[1]; intptr_t diff = tcg_pcrel_diff(s, tgt); if (in_range_b(diff) && toc == (uint32_t)toc) { diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool reloc_jimm20(tcg_insn_unit *code_ptr, tcg_insn_unit *target) return false; } -static bool reloc_call(tcg_insn_unit *code_ptr, tcg_insn_unit *target) +static bool reloc_call(tcg_insn_unit *code_ptr, const tcg_insn_unit *target) { intptr_t offset = (intptr_t)target - (intptr_t)code_ptr; int32_t lo = sextreg(offset, 0, 12); @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_goto(TCGContext *s, tcg_insn_unit *target) tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, offset); } -static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail) +static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) { TCGReg link = tail ? TCG_REG_ZERO : TCG_REG_RA; ptrdiff_t offset = tcg_pcrel_diff(s, arg); @@ -XXX,XX +XXX,XX @@ static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail) } } -static void tcg_out_call(TCGContext *s, tcg_insn_unit *arg) +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg) { tcg_out_call_int(s, arg, false); } diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.c.inc +++ b/tcg/s390/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tgen_brcond(TCGContext *s, TCGType type, TCGCond c, tgen_branch(s, cc, l); } -static void tcg_out_call(TCGContext *s, tcg_insn_unit *dest) +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest) { ptrdiff_t off = dest - s->code_ptr; if (off == (int32_t)off) { diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_addsub2_i64(TCGContext *s, TCGReg rl, TCGReg rh, tcg_out_mov(s, TCG_TYPE_I64, rl, tmp); } -static void tcg_out_call_nodelay(TCGContext *s, tcg_insn_unit *dest, +static void tcg_out_call_nodelay(TCGContext *s, const tcg_insn_unit *dest, bool in_prologue) { ptrdiff_t disp = tcg_pcrel_diff(s, dest); @@ -XXX,XX +XXX,XX @@ static void tcg_out_call_nodelay(TCGContext *s, tcg_insn_unit *dest, } } -static void tcg_out_call(TCGContext *s, tcg_insn_unit *dest) +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest) { tcg_out_call_nodelay(s, dest, false); tcg_out_nop(s); diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, old_code_ptr[1] = s->code_ptr - old_code_ptr; } -static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *arg) +static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg) { uint8_t *old_code_ptr = s->code_ptr; tcg_out_op_t(s, INDEX_op_call); -- 2.25.1 Simplify the arguments to always use s->code_ptr instead of take it as an argument. That makes it easy to ensure that the value_ptr is always the rx version. Reviewed-by: Joelle van Dyne <j@getutm.app> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 6 +++--- tcg/i386/tcg-target.c.inc | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next); } -static void tcg_out_label(TCGContext *s, TCGLabel *l, tcg_insn_unit *ptr) +static void tcg_out_label(TCGContext *s, TCGLabel *l) { tcg_debug_assert(!l->has_value); l->has_value = 1; - l->u.value_ptr = tcg_splitwx_to_rx(ptr); + l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr); } TCGLabel *gen_new_label(void) @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb) break; case INDEX_op_set_label: tcg_reg_alloc_bb_end(s, s->reserved_regs); - tcg_out_label(s, arg_label(op->args[0]), s->code_ptr); + tcg_out_label(s, arg_label(op->args[0])); break; case INDEX_op_call: tcg_reg_alloc_call(s, op); diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_brcond2(TCGContext *s, const TCGArg *args, default: tcg_abort(); } - tcg_out_label(s, label_next, s->code_ptr); + tcg_out_label(s, label_next); } #endif @@ -XXX,XX +XXX,XX @@ static void tcg_out_setcond2(TCGContext *s, const TCGArg *args, tcg_out_movi(s, TCG_TYPE_I32, args[0], 0); tcg_out_jxx(s, JCC_JMP, label_over, 1); - tcg_out_label(s, label_true, s->code_ptr); + tcg_out_label(s, label_true); tcg_out_movi(s, TCG_TYPE_I32, args[0], 1); - tcg_out_label(s, label_over, s->code_ptr); + tcg_out_label(s, label_over); } else { /* When the destination does not overlap one of the arguments, clear the destination first, jump if cond false, and emit an @@ -XXX,XX +XXX,XX @@ static void tcg_out_setcond2(TCGContext *s, const TCGArg *args, tcg_out_brcond2(s, new_args, const_args+1, 1); tgen_arithi(s, ARITH_ADD, args[0], 1, 0); - tcg_out_label(s, label_over, s->code_ptr); + tcg_out_label(s, label_over); } } #endif @@ -XXX,XX +XXX,XX @@ static void tcg_out_cmov(TCGContext *s, TCGCond cond, int rexw, TCGLabel *over = gen_new_label(); tcg_out_jxx(s, tcg_cond_to_jcc[tcg_invert_cond(cond)], over, 1); tcg_out_mov(s, TCG_TYPE_I32, dest, v1); - tcg_out_label(s, over, s->code_ptr); + tcg_out_label(s, over); } } -- 2.25.1 We must change all targets at once, since all must match the declaration in tcg.c. Reviewed-by: Joelle van Dyne <j@getutm.app> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg.h | 2 +- tcg/tcg.c | 10 +++++----- tcg/aarch64/tcg-target.c.inc | 2 +- tcg/arm/tcg-target.c.inc | 2 +- tcg/i386/tcg-target.c.inc | 2 +- tcg/mips/tcg-target.c.inc | 2 +- tcg/ppc/tcg-target.c.inc | 2 +- tcg/riscv/tcg-target.c.inc | 2 +- tcg/s390/tcg-target.c.inc | 2 +- tcg/sparc/tcg-target.c.inc | 2 +- 10 files changed, 14 insertions(+), 14 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ typedef uintptr_t tcg_prologue_fn(CPUArchState *env, const void *tb_ptr); extern tcg_prologue_fn *tcg_qemu_tb_exec; #endif -void tcg_register_jit(void *buf, size_t buf_size); +void tcg_register_jit(const void *buf, size_t buf_size); #if TCG_TARGET_MAYBE_vec /* Return zero if the tuple (opc, type, vece) is unsupportable; diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ typedef struct QEMU_PACKED { DebugFrameFDEHeader fde; } DebugFrameHeader; -static void tcg_register_jit_int(void *buf, size_t size, +static void tcg_register_jit_int(const void *buf, size_t size, const void *debug_frame, size_t debug_frame_size) __attribute__((unused)); @@ -XXX,XX +XXX,XX @@ void tcg_prologue_init(TCGContext *s) total_size -= prologue_size; s->code_gen_buffer_size = total_size; - tcg_register_jit(s->code_gen_buffer, total_size); + tcg_register_jit(tcg_splitwx_to_rx(s->code_gen_buffer), total_size); #ifdef DEBUG_DISAS if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { @@ -XXX,XX +XXX,XX @@ static int find_string(const char *strtab, const char *str) } } -static void tcg_register_jit_int(void *buf_ptr, size_t buf_size, +static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size, const void *debug_frame, size_t debug_frame_size) { @@ -XXX,XX +XXX,XX @@ static void tcg_register_jit_int(void *buf_ptr, size_t buf_size, /* No support for the feature. Provide the entry point expected by exec.c, and implement the internal function we declared earlier. */ -static void tcg_register_jit_int(void *buf, size_t size, +static void tcg_register_jit_int(const void *buf, size_t size, const void *debug_frame, size_t debug_frame_size) { } -void tcg_register_jit(void *buf, size_t buf_size) +void tcg_register_jit(const void *buf, size_t buf_size) { } #endif /* ELF_HOST_MACHINE */ diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const DebugFrame debug_frame = { } }; -void tcg_register_jit(void *buf, size_t buf_size) +void tcg_register_jit(const void *buf, size_t buf_size) { tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); } diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const DebugFrame debug_frame = { } }; -void tcg_register_jit(void *buf, size_t buf_size) +void tcg_register_jit(const void *buf, size_t buf_size) { tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); } diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const DebugFrame debug_frame = { #endif #if defined(ELF_HOST_MACHINE) -void tcg_register_jit(void *buf, size_t buf_size) +void tcg_register_jit(const void *buf, size_t buf_size) { tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); } diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const DebugFrame debug_frame = { } }; -void tcg_register_jit(void *buf, size_t buf_size) +void tcg_register_jit(const void *buf, size_t buf_size) { tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); } diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static DebugFrame debug_frame = { } }; -void tcg_register_jit(void *buf, size_t buf_size) +void tcg_register_jit(const void *buf, size_t buf_size) { uint8_t *p = &debug_frame.fde_reg_ofs[3]; int i; diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const DebugFrame debug_frame = { } }; -void tcg_register_jit(void *buf, size_t buf_size) +void tcg_register_jit(const void *buf, size_t buf_size) { tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); } diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.c.inc +++ b/tcg/s390/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const DebugFrame debug_frame = { } }; -void tcg_register_jit(void *buf, size_t buf_size) +void tcg_register_jit(const void *buf, size_t buf_size) { tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); } diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const DebugFrame debug_frame = { .fde_ret_save = { 9, 15, 31 }, /* DW_CFA_register o7, i7 */ }; -void tcg_register_jit(void *buf, size_t buf_size) +void tcg_register_jit(const void *buf, size_t buf_size) { tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); } -- 2.25.1 Pass both rx and rw addresses to tb_target_set_jmp_target. Reviewed-by: Joelle van Dyne <j@getutm.app> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target.h | 2 +- tcg/arm/tcg-target.h | 2 +- tcg/i386/tcg-target.h | 6 +++--- tcg/mips/tcg-target.h | 2 +- tcg/ppc/tcg-target.h | 2 +- tcg/riscv/tcg-target.h | 2 +- tcg/s390/tcg-target.h | 8 ++++---- tcg/sparc/tcg-target.h | 2 +- tcg/tci/tcg-target.h | 6 +++--- accel/tcg/cpu-exec.c | 4 +++- tcg/aarch64/tcg-target.c.inc | 12 ++++++------ tcg/mips/tcg-target.c.inc | 8 ++++---- tcg/ppc/tcg-target.c.inc | 16 ++++++++-------- tcg/sparc/tcg-target.c.inc | 14 +++++++------- 14 files changed, 44 insertions(+), 42 deletions(-) diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 1 -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t); +void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #ifdef CONFIG_SOFTMMU #define TCG_TARGET_NEED_LDST_LABELS diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -XXX,XX +XXX,XX @@ enum { #define TCG_TARGET_HAS_MEMORY_BSWAP 1 /* not defined -- call should be eliminated at compile time */ -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t); +void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #ifdef CONFIG_SOFTMMU #define TCG_TARGET_NEED_LDST_LABELS diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_movbe; #define TCG_TARGET_extract_i64_valid(ofs, len) \ (((ofs) == 8 && (len) == 8) || ((ofs) + (len)) == 32) -static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, - uintptr_t jmp_addr, uintptr_t addr) +static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, + uintptr_t jmp_rw, uintptr_t addr) { /* patch the branch destination */ - qatomic_set((int32_t *)jmp_addr, addr - (jmp_addr + 4)); + qatomic_set((int32_t *)jmp_rw, addr - (jmp_rx + 4)); /* no need to flush icache explicitly */ } diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions; #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 1 -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t); +void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #ifdef CONFIG_SOFTMMU #define TCG_TARGET_NEED_LDST_LABELS diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_vsx; #define TCG_TARGET_HAS_bitsel_vec have_vsx #define TCG_TARGET_HAS_cmpsel_vec 0 -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t); +void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 1 diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { #endif /* not defined -- call should be eliminated at compile time */ -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t); +void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_DEFAULT_MO (0) diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.h +++ b/tcg/s390/tcg-target.h @@ -XXX,XX +XXX,XX @@ enum { TCG_AREG0 = TCG_REG_R10, }; -static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, - uintptr_t jmp_addr, uintptr_t addr) +static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, + uintptr_t jmp_rw, uintptr_t addr) { /* patch the branch destination */ - intptr_t disp = addr - (jmp_addr - 2); - qatomic_set((int32_t *)jmp_addr, disp / 2); + intptr_t disp = addr - (jmp_rx - 2); + qatomic_set((int32_t *)jmp_rw, disp / 2); /* no need to flush icache explicitly */ } diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.h +++ b/tcg/sparc/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions; #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 1 -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t); +void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_POOL_LABELS diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tci_disas(uint8_t opc); #define TCG_TARGET_HAS_MEMORY_BSWAP 1 -static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, - uintptr_t jmp_addr, uintptr_t addr) +static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, + uintptr_t jmp_rw, uintptr_t addr) { /* patch the branch destination */ - qatomic_set((int32_t *)jmp_addr, addr - (jmp_addr + 4)); + qatomic_set((int32_t *)jmp_rw, addr - (jmp_rx + 4)); /* no need to flush icache explicitly */ } diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr) if (TCG_TARGET_HAS_direct_jump) { uintptr_t offset = tb->jmp_target_arg[n]; uintptr_t tc_ptr = (uintptr_t)tb->tc.ptr; - tb_target_set_jmp_target(tc_ptr, tc_ptr + offset, addr); + uintptr_t jmp_rx = tc_ptr + offset; + uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff; + tb_target_set_jmp_target(tc_ptr, jmp_rx, jmp_rw, addr); } else { tb->jmp_target_arg[n] = addr; } diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *target) } } -void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr, - uintptr_t addr) +void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, + uintptr_t jmp_rw, uintptr_t addr) { tcg_insn_unit i1, i2; TCGType rt = TCG_TYPE_I64; TCGReg rd = TCG_REG_TMP; uint64_t pair; - ptrdiff_t offset = addr - jmp_addr; + ptrdiff_t offset = addr - jmp_rx; if (offset == sextract64(offset, 0, 26)) { i1 = I3206_B | ((offset >> 2) & 0x3ffffff); i2 = NOP; } else { - offset = (addr >> 12) - (jmp_addr >> 12); + offset = (addr >> 12) - (jmp_rx >> 12); /* patch ADRP */ i1 = I3406_ADRP | (offset & 3) << 29 | (offset & 0x1ffffc) << (5 - 2) | rd; @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr, i2 = I3401_ADDI | rt << 31 | (addr & 0xfff) << 10 | rd << 5 | rd; } pair = (uint64_t)i2 << 32 | i1; - qatomic_set((uint64_t *)jmp_addr, pair); - flush_idcache_range(jmp_addr, jmp_addr, 8); + qatomic_set((uint64_t *)jmp_rw, pair); + flush_idcache_range(jmp_rx, jmp_rw, 8); } static inline void tcg_out_goto_label(TCGContext *s, TCGLabel *l) diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_init(TCGContext *s) tcg_regset_set_reg(s->reserved_regs, TCG_REG_GP); /* global pointer */ } -void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr, - uintptr_t addr) +void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, + uintptr_t jmp_rw, uintptr_t addr) { - qatomic_set((uint32_t *)jmp_addr, deposit32(OPC_J, 0, 26, addr >> 2)); - flush_idcache_range(jmp_addr, jmp_addr, 4); + qatomic_set((uint32_t *)jmp_rw, deposit32(OPC_J, 0, 26, addr >> 2)); + flush_idcache_range(jmp_rx, jmp_rw, 4); } typedef struct { diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_mb(TCGContext *s, TCGArg a0) tcg_out32(s, insn); } -void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr, - uintptr_t addr) +void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, + uintptr_t jmp_rw, uintptr_t addr) { if (TCG_TARGET_REG_BITS == 64) { tcg_insn_unit i1, i2; intptr_t tb_diff = addr - tc_ptr; - intptr_t br_diff = addr - (jmp_addr + 4); + intptr_t br_diff = addr - (jmp_rx + 4); uint64_t pair; /* This does not exercise the range of the branch, but we do @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr, /* As per the enclosing if, this is ppc64. Avoid the _Static_assert within qatomic_set that would fail to build a ppc32 host. */ - qatomic_set__nocheck((uint64_t *)jmp_addr, pair); - flush_idcache_range(jmp_addr, jmp_addr, 8); + qatomic_set__nocheck((uint64_t *)jmp_rw, pair); + flush_idcache_range(jmp_rx, jmp_rw, 8); } else { - intptr_t diff = addr - jmp_addr; + intptr_t diff = addr - jmp_rx; tcg_debug_assert(in_range_b(diff)); - qatomic_set((uint32_t *)jmp_addr, B | (diff & 0x3fffffc)); - flush_idcache_range(jmp_addr, jmp_addr, 4); + qatomic_set((uint32_t *)jmp_rw, B | (diff & 0x3fffffc)); + flush_idcache_range(jmp_rx, jmp_rw, 4); } } diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ void tcg_register_jit(const void *buf, size_t buf_size) tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); } -void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr, - uintptr_t addr) +void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, + uintptr_t jmp_rw, uintptr_t addr) { intptr_t tb_disp = addr - tc_ptr; - intptr_t br_disp = addr - jmp_addr; + intptr_t br_disp = addr - jmp_rx; tcg_insn_unit i1, i2; /* We can reach the entire address space for ILP32. @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr, tcg_debug_assert(br_disp == (int32_t)br_disp); if (!USE_REG_TB) { - qatomic_set((uint32_t *)jmp_addr, + qatomic_set((uint32_t *)jmp_rw, deposit32(CALL, 0, 30, br_disp >> 2)); - flush_idcache_range(jmp_addr, jmp_addr, 4); + flush_idcache_range(jmp_rx, jmp_rw, 4); return; } @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr, | INSN_IMM13((tb_disp & 0x3ff) | -0x400)); } - qatomic_set((uint64_t *)jmp_addr, deposit64(i2, 32, 32, i1)); - flush_idcache_range(jmp_addr, jmp_addr, 8); + qatomic_set((uint64_t *)jmp_rw, deposit64(i2, 32, 32, i1)); + flush_idcache_range(jmp_rx, jmp_rw, 8); } -- 2.25.1 There is nothing within the translators that ought to be changing the TranslationBlock data, so make it const. This does not actually use the read-only copy of the data structure that exists within the rx region. Reviewed-by: Joelle van Dyne <j@getutm.app> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/gen-icount.h | 4 ++-- include/exec/translator.h | 2 +- include/tcg/tcg-op.h | 2 +- accel/tcg/translator.c | 4 ++-- target/arm/translate-a64.c | 2 +- tcg/tcg-op.c | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/exec/gen-icount.h b/include/exec/gen-icount.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/gen-icount.h +++ b/include/exec/gen-icount.h @@ -XXX,XX +XXX,XX @@ static inline void gen_io_end(void) tcg_temp_free_i32(tmp); } -static inline void gen_tb_start(TranslationBlock *tb) +static inline void gen_tb_start(const TranslationBlock *tb) { TCGv_i32 count, imm; @@ -XXX,XX +XXX,XX @@ static inline void gen_tb_start(TranslationBlock *tb) tcg_temp_free_i32(count); } -static inline void gen_tb_end(TranslationBlock *tb, int num_insns) +static inline void gen_tb_end(const TranslationBlock *tb, int num_insns) { if (tb_cflags(tb) & CF_USE_ICOUNT) { /* Update the num_insn immediate parameter now that we know diff --git a/include/exec/translator.h b/include/exec/translator.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/translator.h +++ b/include/exec/translator.h @@ -XXX,XX +XXX,XX @@ typedef enum DisasJumpType { * Architecture-agnostic disassembly context. */ typedef struct DisasContextBase { - TranslationBlock *tb; + const TranslationBlock *tb; target_ulong pc_first; target_ulong pc_next; DisasJumpType is_jmp; diff --git a/include/tcg/tcg-op.h b/include/tcg/tcg-op.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg-op.h +++ b/include/tcg/tcg-op.h @@ -XXX,XX +XXX,XX @@ static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1, * be NULL and @idx should be 0. Otherwise, @tb should be valid and * @idx should be one of the TB_EXIT_ values. */ -void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx); +void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx); /** * tcg_gen_goto_tb() - output goto_tb TCG operation diff --git a/accel/tcg/translator.c b/accel/tcg/translator.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translator.c +++ b/accel/tcg/translator.c @@ -XXX,XX +XXX,XX @@ void translator_loop(const TranslatorOps *ops, DisasContextBase *db, } /* The disas_log hook may use these values rather than recompute. */ - db->tb->size = db->pc_next - db->pc_first; - db->tb->icount = db->num_insns; + tb->size = db->pc_next - db->pc_first; + tb->icount = db->num_insns; #ifdef DEBUG_DISAS if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static inline bool use_goto_tb(DisasContext *s, int n, uint64_t dest) static inline void gen_goto_tb(DisasContext *s, int n, uint64_t dest) { - TranslationBlock *tb; + const TranslationBlock *tb; tb = s->base.tb; if (use_goto_tb(s, n, dest)) { diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -XXX,XX +XXX,XX @@ void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) /* QEMU specific operations. */ -void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx) +void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx) { uintptr_t val = (uintptr_t)tb + idx; -- 2.25.1 There is nothing within the translators that ought to be changing the TranslationBlock data, so make it const. This does not actually use the read-only copy of the data structure that exists within the rx region. Reviewed-by: Joelle van Dyne <j@getutm.app> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/hw/core/cpu.h | 3 ++- target/arm/cpu.c | 3 ++- target/avr/cpu.c | 3 ++- target/hppa/cpu.c | 3 ++- target/i386/tcg/tcg-cpu.c | 3 ++- target/microblaze/cpu.c | 3 ++- target/mips/cpu.c | 3 ++- target/riscv/cpu.c | 3 ++- target/rx/cpu.c | 3 ++- target/sh4/cpu.c | 3 ++- target/sparc/cpu.c | 3 ++- target/tricore/cpu.c | 2 +- 12 files changed, 23 insertions(+), 12 deletions(-) diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/core/cpu.h +++ b/include/hw/core/cpu.h @@ -XXX,XX +XXX,XX @@ struct CPUClass { void (*get_memory_mapping)(CPUState *cpu, MemoryMappingList *list, Error **errp); void (*set_pc)(CPUState *cpu, vaddr value); - void (*synchronize_from_tb)(CPUState *cpu, struct TranslationBlock *tb); + void (*synchronize_from_tb)(CPUState *cpu, + const struct TranslationBlock *tb); bool (*tlb_fill)(CPUState *cpu, vaddr address, int size, MMUAccessType access_type, int mmu_idx, bool probe, uintptr_t retaddr); diff --git a/target/arm/cpu.c b/target/arm/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -XXX,XX +XXX,XX @@ static void arm_cpu_set_pc(CPUState *cs, vaddr value) } } -static void arm_cpu_synchronize_from_tb(CPUState *cs, TranslationBlock *tb) +static void arm_cpu_synchronize_from_tb(CPUState *cs, + const TranslationBlock *tb) { ARMCPU *cpu = ARM_CPU(cs); CPUARMState *env = &cpu->env; diff --git a/target/avr/cpu.c b/target/avr/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/cpu.c +++ b/target/avr/cpu.c @@ -XXX,XX +XXX,XX @@ static bool avr_cpu_has_work(CPUState *cs) && cpu_interrupts_enabled(env); } -static void avr_cpu_synchronize_from_tb(CPUState *cs, TranslationBlock *tb) +static void avr_cpu_synchronize_from_tb(CPUState *cs, + const TranslationBlock *tb) { AVRCPU *cpu = AVR_CPU(cs); CPUAVRState *env = &cpu->env; diff --git a/target/hppa/cpu.c b/target/hppa/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hppa/cpu.c +++ b/target/hppa/cpu.c @@ -XXX,XX +XXX,XX @@ static void hppa_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.iaoq_b = value + 4; } -static void hppa_cpu_synchronize_from_tb(CPUState *cs, TranslationBlock *tb) +static void hppa_cpu_synchronize_from_tb(CPUState *cs, + const TranslationBlock *tb) { HPPACPU *cpu = HPPA_CPU(cs); diff --git a/target/i386/tcg/tcg-cpu.c b/target/i386/tcg/tcg-cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/tcg/tcg-cpu.c +++ b/target/i386/tcg/tcg-cpu.c @@ -XXX,XX +XXX,XX @@ static void x86_cpu_exec_exit(CPUState *cs) env->eflags = cpu_compute_eflags(env); } -static void x86_cpu_synchronize_from_tb(CPUState *cs, TranslationBlock *tb) +static void x86_cpu_synchronize_from_tb(CPUState *cs, + const TranslationBlock *tb) { X86CPU *cpu = X86_CPU(cs); diff --git a/target/microblaze/cpu.c b/target/microblaze/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/microblaze/cpu.c +++ b/target/microblaze/cpu.c @@ -XXX,XX +XXX,XX @@ static void mb_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.iflags = 0; } -static void mb_cpu_synchronize_from_tb(CPUState *cs, TranslationBlock *tb) +static void mb_cpu_synchronize_from_tb(CPUState *cs, + const TranslationBlock *tb) { MicroBlazeCPU *cpu = MICROBLAZE_CPU(cs); diff --git a/target/mips/cpu.c b/target/mips/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/mips/cpu.c +++ b/target/mips/cpu.c @@ -XXX,XX +XXX,XX @@ static void mips_cpu_set_pc(CPUState *cs, vaddr value) } } -static void mips_cpu_synchronize_from_tb(CPUState *cs, TranslationBlock *tb) +static void mips_cpu_synchronize_from_tb(CPUState *cs, + const TranslationBlock *tb) { MIPSCPU *cpu = MIPS_CPU(cs); CPUMIPSState *env = &cpu->env; diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/cpu.c +++ b/target/riscv/cpu.c @@ -XXX,XX +XXX,XX @@ static void riscv_cpu_set_pc(CPUState *cs, vaddr value) env->pc = value; } -static void riscv_cpu_synchronize_from_tb(CPUState *cs, TranslationBlock *tb) +static void riscv_cpu_synchronize_from_tb(CPUState *cs, + const TranslationBlock *tb) { RISCVCPU *cpu = RISCV_CPU(cs); CPURISCVState *env = &cpu->env; diff --git a/target/rx/cpu.c b/target/rx/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/rx/cpu.c +++ b/target/rx/cpu.c @@ -XXX,XX +XXX,XX @@ static void rx_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.pc = value; } -static void rx_cpu_synchronize_from_tb(CPUState *cs, TranslationBlock *tb) +static void rx_cpu_synchronize_from_tb(CPUState *cs, + const TranslationBlock *tb) { RXCPU *cpu = RX_CPU(cs); diff --git a/target/sh4/cpu.c b/target/sh4/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/sh4/cpu.c +++ b/target/sh4/cpu.c @@ -XXX,XX +XXX,XX @@ static void superh_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.pc = value; } -static void superh_cpu_synchronize_from_tb(CPUState *cs, TranslationBlock *tb) +static void superh_cpu_synchronize_from_tb(CPUState *cs, + const TranslationBlock *tb) { SuperHCPU *cpu = SUPERH_CPU(cs); diff --git a/target/sparc/cpu.c b/target/sparc/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/cpu.c +++ b/target/sparc/cpu.c @@ -XXX,XX +XXX,XX @@ static void sparc_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.npc = value + 4; } -static void sparc_cpu_synchronize_from_tb(CPUState *cs, TranslationBlock *tb) +static void sparc_cpu_synchronize_from_tb(CPUState *cs, + const TranslationBlock *tb) { SPARCCPU *cpu = SPARC_CPU(cs); diff --git a/target/tricore/cpu.c b/target/tricore/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/tricore/cpu.c +++ b/target/tricore/cpu.c @@ -XXX,XX +XXX,XX @@ static void tricore_cpu_set_pc(CPUState *cs, vaddr value) } static void tricore_cpu_synchronize_from_tb(CPUState *cs, - TranslationBlock *tb) + const TranslationBlock *tb) { TriCoreCPU *cpu = TRICORE_CPU(cs); CPUTriCoreState *env = &cpu->env; -- 2.25.1 Report better error messages than just "could not allocate". Let alloc_code_gen_buffer set ctx->code_gen_buffer_size and ctx->code_gen_buffer, and simply return bool. Reviewed-by: Joelle van Dyne <j@getutm.app> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/translate-all.c | 60 ++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ #include "sysemu/cpus.h" #include "sysemu/cpu-timers.h" #include "sysemu/tcg.h" +#include "qapi/error.h" /* #define DEBUG_TB_INVALIDATE */ /* #define DEBUG_TB_FLUSH */ @@ -XXX,XX +XXX,XX @@ static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1, (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \ ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE) -static inline size_t size_code_gen_buffer(size_t tb_size) +static size_t size_code_gen_buffer(size_t tb_size) { /* Size the buffer. */ if (tb_size == 0) { @@ -XXX,XX +XXX,XX @@ static inline void *split_cross_256mb(void *buf1, size_t size1) static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE] __attribute__((aligned(CODE_GEN_ALIGN))); -static inline void *alloc_code_gen_buffer(void) +static bool alloc_code_gen_buffer(size_t tb_size, Error **errp) { void *buf = static_code_gen_buffer; void *end = static_code_gen_buffer + sizeof(static_code_gen_buffer); @@ -XXX,XX +XXX,XX @@ static inline void *alloc_code_gen_buffer(void) size = end - buf; /* Honor a command-line option limiting the size of the buffer. */ - if (size > tcg_ctx->code_gen_buffer_size) { - size = QEMU_ALIGN_DOWN(tcg_ctx->code_gen_buffer_size, - qemu_real_host_page_size); + if (size > tb_size) { + size = QEMU_ALIGN_DOWN(tb_size, qemu_real_host_page_size); } tcg_ctx->code_gen_buffer_size = size; @@ -XXX,XX +XXX,XX @@ static inline void *alloc_code_gen_buffer(void) #endif if (qemu_mprotect_rwx(buf, size)) { - abort(); + error_setg_errno(errp, errno, "mprotect of jit buffer"); + return false; } qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); - return buf; + tcg_ctx->code_gen_buffer = buf; + return true; } #elif defined(_WIN32) -static inline void *alloc_code_gen_buffer(void) +static bool alloc_code_gen_buffer(size_t size, Error **errp) { - size_t size = tcg_ctx->code_gen_buffer_size; - return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, - PAGE_EXECUTE_READWRITE); + void *buf = VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, + PAGE_EXECUTE_READWRITE); + if (buf == NULL) { + error_setg_win32(errp, GetLastError(), + "allocate %zu bytes for jit buffer", size); + return false; + } + + tcg_ctx->code_gen_buffer = buf; + tcg_ctx->code_gen_buffer_size = size; + return true; } #else -static inline void *alloc_code_gen_buffer(void) +static bool alloc_code_gen_buffer(size_t size, Error **errp) { int prot = PROT_WRITE | PROT_READ | PROT_EXEC; int flags = MAP_PRIVATE | MAP_ANONYMOUS; - size_t size = tcg_ctx->code_gen_buffer_size; void *buf; buf = mmap(NULL, size, prot, flags, -1, 0); if (buf == MAP_FAILED) { - return NULL; + error_setg_errno(errp, errno, + "allocate %zu bytes for jit buffer", size); + return false; } + tcg_ctx->code_gen_buffer_size = size; #ifdef __mips__ if (cross_256mb(buf, size)) { @@ -XXX,XX +XXX,XX @@ static inline void *alloc_code_gen_buffer(void) /* Request large pages for the buffer. */ qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); - return buf; + tcg_ctx->code_gen_buffer = buf; + return true; } #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */ -static inline void code_gen_alloc(size_t tb_size) -{ - tcg_ctx->code_gen_buffer_size = size_code_gen_buffer(tb_size); - tcg_ctx->code_gen_buffer = alloc_code_gen_buffer(); - if (tcg_ctx->code_gen_buffer == NULL) { - fprintf(stderr, "Could not allocate dynamic translator buffer\n"); - exit(1); - } -} - static bool tb_cmp(const void *ap, const void *bp) { const TranslationBlock *a = ap; @@ -XXX,XX +XXX,XX @@ static void tb_htable_init(void) size. */ void tcg_exec_init(unsigned long tb_size) { + bool ok; + tcg_allowed = true; cpu_gen_init(); page_init(); tb_htable_init(); - code_gen_alloc(tb_size); + + ok = alloc_code_gen_buffer(size_code_gen_buffer(tb_size), &error_fatal); + assert(ok); + #if defined(CONFIG_SOFTMMU) /* There's no guest base to take into account, so go ahead and initialize the prologue now. */ -- 2.25.1 Plumb the value through to alloc_code_gen_buffer. This is not supported by any os or tcg backend, so for now enabling it will result in an error. Reviewed-by: Joelle van Dyne <j@getutm.app> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/sysemu/tcg.h | 3 ++- tcg/aarch64/tcg-target.h | 1 + tcg/arm/tcg-target.h | 1 + tcg/i386/tcg-target.h | 1 + tcg/mips/tcg-target.h | 1 + tcg/ppc/tcg-target.h | 1 + tcg/riscv/tcg-target.h | 1 + tcg/s390/tcg-target.h | 1 + tcg/sparc/tcg-target.h | 1 + tcg/tci/tcg-target.h | 1 + accel/tcg/tcg-all.c | 26 +++++++++++++++++++++++++- accel/tcg/translate-all.c | 35 +++++++++++++++++++++++++++-------- bsd-user/main.c | 2 +- linux-user/main.c | 2 +- qemu-options.hx | 7 +++++++ 15 files changed, 72 insertions(+), 12 deletions(-) diff --git a/include/sysemu/tcg.h b/include/sysemu/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/sysemu/tcg.h +++ b/include/sysemu/tcg.h @@ -XXX,XX +XXX,XX @@ #ifndef SYSEMU_TCG_H #define SYSEMU_TCG_H -void tcg_exec_init(unsigned long tb_size); +void tcg_exec_init(unsigned long tb_size, int splitwx); + #ifdef CONFIG_TCG extern bool tcg_allowed; #define tcg_enabled() (tcg_allowed) diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_SUPPORT_MIRROR 0 #endif /* AARCH64_TCG_TARGET_H */ diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_SUPPORT_MIRROR 0 #endif diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_SUPPORT_MIRROR 0 #endif diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions; #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 1 +#define TCG_TARGET_SUPPORT_MIRROR 0 void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_SUPPORT_MIRROR 0 #endif diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_POOL_LABELS #define TCG_TARGET_HAS_MEMORY_BSWAP 0 +#define TCG_TARGET_SUPPORT_MIRROR 0 #endif diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.h +++ b/tcg/s390/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_SUPPORT_MIRROR 0 #endif diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.h +++ b/tcg/sparc/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions; void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_SUPPORT_MIRROR 0 #endif diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tci_disas(uint8_t opc); #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 1 +#define TCG_TARGET_SUPPORT_MIRROR 0 static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, uintptr_t jmp_rw, uintptr_t addr) diff --git a/accel/tcg/tcg-all.c b/accel/tcg/tcg-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/tcg-all.c +++ b/accel/tcg/tcg-all.c @@ -XXX,XX +XXX,XX @@ struct TCGState { AccelState parent_obj; bool mttcg_enabled; + int splitwx_enabled; unsigned long tb_size; }; typedef struct TCGState TCGState; @@ -XXX,XX +XXX,XX @@ static void tcg_accel_instance_init(Object *obj) TCGState *s = TCG_STATE(obj); s->mttcg_enabled = default_mttcg_enabled(); + + /* If debugging enabled, default "auto on", otherwise off. */ +#ifdef CONFIG_DEBUG_TCG + s->splitwx_enabled = -1; +#else + s->splitwx_enabled = 0; +#endif } bool mttcg_enabled; @@ -XXX,XX +XXX,XX @@ static int tcg_init(MachineState *ms) { TCGState *s = TCG_STATE(current_accel()); - tcg_exec_init(s->tb_size * 1024 * 1024); + tcg_exec_init(s->tb_size * 1024 * 1024, s->splitwx_enabled); mttcg_enabled = s->mttcg_enabled; /* @@ -XXX,XX +XXX,XX @@ static void tcg_set_tb_size(Object *obj, Visitor *v, s->tb_size = value; } +static bool tcg_get_splitwx(Object *obj, Error **errp) +{ + TCGState *s = TCG_STATE(obj); + return s->splitwx_enabled; +} + +static void tcg_set_splitwx(Object *obj, bool value, Error **errp) +{ + TCGState *s = TCG_STATE(obj); + s->splitwx_enabled = value; +} + static void tcg_accel_class_init(ObjectClass *oc, void *data) { AccelClass *ac = ACCEL_CLASS(oc); @@ -XXX,XX +XXX,XX @@ static void tcg_accel_class_init(ObjectClass *oc, void *data) object_class_property_set_description(oc, "tb-size", "TCG translation block cache size"); + object_class_property_add_bool(oc, "split-wx", + tcg_get_splitwx, tcg_set_splitwx); + object_class_property_set_description(oc, "split-wx", + "Map jit pages into separate RW and RX regions"); } static const TypeInfo tcg_accel_type = { diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ static inline void *split_cross_256mb(void *buf1, size_t size1) static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE] __attribute__((aligned(CODE_GEN_ALIGN))); -static bool alloc_code_gen_buffer(size_t tb_size, Error **errp) +static bool alloc_code_gen_buffer(size_t tb_size, int splitwx, Error **errp) { - void *buf = static_code_gen_buffer; - void *end = static_code_gen_buffer + sizeof(static_code_gen_buffer); + void *buf, *end; size_t size; + if (splitwx > 0) { + error_setg(errp, "jit split-wx not supported"); + return false; + } + /* page-align the beginning and end of the buffer */ + buf = static_code_gen_buffer; + end = static_code_gen_buffer + sizeof(static_code_gen_buffer); buf = QEMU_ALIGN_PTR_UP(buf, qemu_real_host_page_size); end = QEMU_ALIGN_PTR_DOWN(end, qemu_real_host_page_size); @@ -XXX,XX +XXX,XX @@ static bool alloc_code_gen_buffer(size_t tb_size, Error **errp) return true; } #elif defined(_WIN32) -static bool alloc_code_gen_buffer(size_t size, Error **errp) +static bool alloc_code_gen_buffer(size_t size, int splitwx, Error **errp) { - void *buf = VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, + void *buf; + + if (splitwx > 0) { + error_setg(errp, "jit split-wx not supported"); + return false; + } + + buf = VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_EXECUTE_READWRITE); if (buf == NULL) { error_setg_win32(errp, GetLastError(), @@ -XXX,XX +XXX,XX @@ static bool alloc_code_gen_buffer(size_t size, Error **errp) return true; } #else -static bool alloc_code_gen_buffer(size_t size, Error **errp) +static bool alloc_code_gen_buffer(size_t size, int splitwx, Error **errp) { int prot = PROT_WRITE | PROT_READ | PROT_EXEC; int flags = MAP_PRIVATE | MAP_ANONYMOUS; void *buf; + if (splitwx > 0) { + error_setg(errp, "jit split-wx not supported"); + return false; + } + buf = mmap(NULL, size, prot, flags, -1, 0); if (buf == MAP_FAILED) { error_setg_errno(errp, errno, @@ -XXX,XX +XXX,XX @@ static void tb_htable_init(void) /* Must be called before using the QEMU cpus. 'tb_size' is the size (in bytes) allocated to the translation buffer. Zero means default size. */ -void tcg_exec_init(unsigned long tb_size) +void tcg_exec_init(unsigned long tb_size, int splitwx) { bool ok; @@ -XXX,XX +XXX,XX @@ void tcg_exec_init(unsigned long tb_size) page_init(); tb_htable_init(); - ok = alloc_code_gen_buffer(size_code_gen_buffer(tb_size), &error_fatal); + ok = alloc_code_gen_buffer(size_code_gen_buffer(tb_size), + splitwx, &error_fatal); assert(ok); #if defined(CONFIG_SOFTMMU) diff --git a/bsd-user/main.c b/bsd-user/main.c index XXXXXXX..XXXXXXX 100644 --- a/bsd-user/main.c +++ b/bsd-user/main.c @@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv) } /* init tcg before creating CPUs and to get qemu_host_page_size */ - tcg_exec_init(0); + tcg_exec_init(0, false); cpu_type = parse_cpu_option(cpu_model); cpu = cpu_create(cpu_type); diff --git a/linux-user/main.c b/linux-user/main.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/main.c +++ b/linux-user/main.c @@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp) cpu_type = parse_cpu_option(cpu_model); /* init tcg before creating CPUs and to get qemu_host_page_size */ - tcg_exec_init(0); + tcg_exec_init(0, false); cpu = cpu_create(cpu_type); env = cpu->env_ptr; diff --git a/qemu-options.hx b/qemu-options.hx index XXXXXXX..XXXXXXX 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -XXX,XX +XXX,XX @@ DEF("accel", HAS_ARG, QEMU_OPTION_accel, " igd-passthru=on|off (enable Xen integrated Intel graphics passthrough, default=off)\n" " kernel-irqchip=on|off|split controls accelerated irqchip support (default=on)\n" " kvm-shadow-mem=size of KVM shadow MMU in bytes\n" + " split-wx=on|off (enable TCG split w^x mapping)\n" " tb-size=n (TCG translation block cache size)\n" " thread=single|multi (enable multi-threaded TCG)\n", QEMU_ARCH_ALL) SRST @@ -XXX,XX +XXX,XX @@ SRST ``kvm-shadow-mem=size`` Defines the size of the KVM shadow MMU. + ``split-wx=on|off`` + Controls the use of split w^x mapping for the TCG code generation + buffer. Some operating systems require this to be enabled, and in + such a case this will default on. On other operating systems, this + will default off, but one may enable this for testing or debugging. + ``tb-size=n`` Controls the size (in MiB) of the TCG translation block cache. -- 2.25.1 We cannot use a real temp file, because we would need to find a filesystem that does not have noexec enabled. However, a memfd is not associated with any filesystem. Reviewed-by: Joelle van Dyne <j@getutm.app> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/translate-all.c | 84 +++++++++++++++++++++++++++++++++++---- 1 file changed, 76 insertions(+), 8 deletions(-) diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ static bool alloc_code_gen_buffer(size_t size, int splitwx, Error **errp) return true; } #else -static bool alloc_code_gen_buffer(size_t size, int splitwx, Error **errp) +static bool alloc_code_gen_buffer_anon(size_t size, int prot, + int flags, Error **errp) { - int prot = PROT_WRITE | PROT_READ | PROT_EXEC; - int flags = MAP_PRIVATE | MAP_ANONYMOUS; void *buf; - if (splitwx > 0) { - error_setg(errp, "jit split-wx not supported"); - return false; - } - buf = mmap(NULL, size, prot, flags, -1, 0); if (buf == MAP_FAILED) { error_setg_errno(errp, errno, @@ -XXX,XX +XXX,XX @@ static bool alloc_code_gen_buffer(size_t size, int splitwx, Error **errp) tcg_ctx->code_gen_buffer = buf; return true; } + +#ifdef CONFIG_POSIX +#include "qemu/memfd.h" + +static bool alloc_code_gen_buffer_splitwx_memfd(size_t size, Error **errp) +{ + void *buf_rw, *buf_rx; + int fd = -1; + + buf_rw = qemu_memfd_alloc("tcg-jit", size, 0, &fd, errp); + if (buf_rw == NULL) { + return false; + } + + buf_rx = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_SHARED, fd, 0); + if (buf_rx == MAP_FAILED) { + error_setg_errno(errp, errno, + "failed to map shared memory for execute"); + munmap(buf_rw, size); + close(fd); + return false; + } + close(fd); + + tcg_ctx->code_gen_buffer = buf_rw; + tcg_ctx->code_gen_buffer_size = size; + tcg_splitwx_diff = buf_rx - buf_rw; + + /* Request large pages for the buffer and the splitwx. */ + qemu_madvise(buf_rw, size, QEMU_MADV_HUGEPAGE); + qemu_madvise(buf_rx, size, QEMU_MADV_HUGEPAGE); + return true; +} +#endif /* CONFIG_POSIX */ + +static bool alloc_code_gen_buffer_splitwx(size_t size, Error **errp) +{ + if (TCG_TARGET_SUPPORT_MIRROR) { +#ifdef CONFIG_POSIX + return alloc_code_gen_buffer_splitwx_memfd(size, errp); +#endif + } + error_setg(errp, "jit split-wx not supported"); + return false; +} + +static bool alloc_code_gen_buffer(size_t size, int splitwx, Error **errp) +{ + ERRP_GUARD(); + int prot, flags; + + if (splitwx) { + if (alloc_code_gen_buffer_splitwx(size, errp)) { + return true; + } + /* + * If splitwx force-on (1), fail; + * if splitwx default-on (-1), fall through to splitwx off. + */ + if (splitwx > 0) { + return false; + } + error_free_or_abort(errp); + } + + prot = PROT_READ | PROT_WRITE | PROT_EXEC; + flags = MAP_PRIVATE | MAP_ANONYMOUS; +#ifdef CONFIG_TCG_INTERPRETER + /* The tcg interpreter does not need execute permission. */ + prot = PROT_READ | PROT_WRITE; +#endif + + return alloc_code_gen_buffer_anon(size, prot, flags, errp); +} #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */ static bool tb_cmp(const void *ap, const void *bp) -- 2.25.1 Cribbed from code posted by Joelle van Dyne <j@getutm.app>, and rearranged to a cleaner structure. Reviewed-by: Joelle van Dyne <j@getutm.app> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/translate-all.c | 67 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ static bool alloc_code_gen_buffer_splitwx_memfd(size_t size, Error **errp) } #endif /* CONFIG_POSIX */ +#ifdef CONFIG_DARWIN +#include <mach/mach.h> + +extern kern_return_t mach_vm_remap(vm_map_t target_task, + mach_vm_address_t *target_address, + mach_vm_size_t size, + mach_vm_offset_t mask, + int flags, + vm_map_t src_task, + mach_vm_address_t src_address, + boolean_t copy, + vm_prot_t *cur_protection, + vm_prot_t *max_protection, + vm_inherit_t inheritance); + +static bool alloc_code_gen_buffer_splitwx_vmremap(size_t size, Error **errp) +{ + kern_return_t ret; + mach_vm_address_t buf_rw, buf_rx; + vm_prot_t cur_prot, max_prot; + + /* Map the read-write portion via normal anon memory. */ + if (!alloc_code_gen_buffer_anon(size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, errp)) { + return false; + } + + buf_rw = (mach_vm_address_t)tcg_ctx->code_gen_buffer; + buf_rx = 0; + ret = mach_vm_remap(mach_task_self(), + &buf_rx, + size, + 0, + VM_FLAGS_ANYWHERE, + mach_task_self(), + buf_rw, + false, + &cur_prot, + &max_prot, + VM_INHERIT_NONE); + if (ret != KERN_SUCCESS) { + /* TODO: Convert "ret" to a human readable error message. */ + error_setg(errp, "vm_remap for jit splitwx failed"); + munmap((void *)buf_rw, size); + return false; + } + + if (mprotect((void *)buf_rx, size, PROT_READ | PROT_EXEC) != 0) { + error_setg_errno(errp, errno, "mprotect for jit splitwx"); + munmap((void *)buf_rx, size); + munmap((void *)buf_rw, size); + return false; + } + + tcg_splitwx_diff = buf_rx - buf_rw; + return true; +} +#endif /* CONFIG_DARWIN */ + static bool alloc_code_gen_buffer_splitwx(size_t size, Error **errp) { if (TCG_TARGET_SUPPORT_MIRROR) { +#ifdef CONFIG_DARWIN + return alloc_code_gen_buffer_splitwx_vmremap(size, errp); +#endif #ifdef CONFIG_POSIX return alloc_code_gen_buffer_splitwx_memfd(size, errp); #endif @@ -XXX,XX +XXX,XX @@ static bool alloc_code_gen_buffer(size_t size, int splitwx, Error **errp) #ifdef CONFIG_TCG_INTERPRETER /* The tcg interpreter does not need execute permission. */ prot = PROT_READ | PROT_WRITE; +#elif defined(CONFIG_DARWIN) + /* Applicable to both iOS and macOS (Apple Silicon). */ + if (!splitwx) { + flags |= MAP_JIT; + } #endif return alloc_code_gen_buffer_anon(size, prot, flags, errp); -- 2.25.1 This produces a small pc-relative displacement within the generated code to the TB structure that preceeds it. Reviewed-by: Joelle van Dyne <j@getutm.app> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cpu-exec.c | 35 +++++++++++++++++++++-------------- tcg/tcg-op.c | 13 ++++++++++++- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ static void init_delay_params(SyncClocks *sc, const CPUState *cpu) * TCG is not considered a security-sensitive part of QEMU so this does not * affect the impact of CFI in environment with high security requirements */ -QEMU_DISABLE_CFI -static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, TranslationBlock *itb) +static inline TranslationBlock * QEMU_DISABLE_CFI +cpu_tb_exec(CPUState *cpu, TranslationBlock *itb, int *tb_exit) { CPUArchState *env = cpu->env_ptr; uintptr_t ret; TranslationBlock *last_tb; - int tb_exit; const void *tb_ptr = itb->tc.ptr; qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc, @@ -XXX,XX +XXX,XX @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, TranslationBlock *itb) ret = tcg_qemu_tb_exec(env, tb_ptr); cpu->can_do_io = 1; - last_tb = (TranslationBlock *)(ret & ~TB_EXIT_MASK); - tb_exit = ret & TB_EXIT_MASK; - trace_exec_tb_exit(last_tb, tb_exit); + /* + * TODO: Delay swapping back to the read-write region of the TB + * until we actually need to modify the TB. The read-only copy, + * coming from the rx region, shares the same host TLB entry as + * the code that executed the exit_tb opcode that arrived here. + * If we insist on touching both the RX and the RW pages, we + * double the host TLB pressure. + */ + last_tb = tcg_splitwx_to_rw((void *)(ret & ~TB_EXIT_MASK)); + *tb_exit = ret & TB_EXIT_MASK; - if (tb_exit > TB_EXIT_IDX1) { + trace_exec_tb_exit(last_tb, *tb_exit); + + if (*tb_exit > TB_EXIT_IDX1) { /* We didn't start executing this TB (eg because the instruction * counter hit zero); we must restore the guest PC to the address * of the start of the TB. @@ -XXX,XX +XXX,XX @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, TranslationBlock *itb) cc->set_pc(cpu, last_tb->pc); } } - return ret; + return last_tb; } #ifndef CONFIG_USER_ONLY @@ -XXX,XX +XXX,XX @@ static void cpu_exec_nocache(CPUState *cpu, int max_cycles, { TranslationBlock *tb; uint32_t cflags = curr_cflags() | CF_NOCACHE; + int tb_exit; if (ignore_icount) { cflags &= ~CF_USE_ICOUNT; @@ -XXX,XX +XXX,XX @@ static void cpu_exec_nocache(CPUState *cpu, int max_cycles, /* execute the generated code */ trace_exec_tb_nocache(tb, tb->pc); - cpu_tb_exec(cpu, tb); + cpu_tb_exec(cpu, tb, &tb_exit); mmap_lock(); tb_phys_invalidate(tb, -1); @@ -XXX,XX +XXX,XX @@ void cpu_exec_step_atomic(CPUState *cpu) uint32_t flags; uint32_t cflags = 1; uint32_t cf_mask = cflags & CF_HASH_MASK; + int tb_exit; if (sigsetjmp(cpu->jmp_env, 0) == 0) { start_exclusive(); @@ -XXX,XX +XXX,XX @@ void cpu_exec_step_atomic(CPUState *cpu) cpu_exec_enter(cpu); /* execute the generated code */ trace_exec_tb(tb, pc); - cpu_tb_exec(cpu, tb); + cpu_tb_exec(cpu, tb, &tb_exit); cpu_exec_exit(cpu); } else { /* @@ -XXX,XX +XXX,XX @@ static inline bool cpu_handle_interrupt(CPUState *cpu, static inline void cpu_loop_exec_tb(CPUState *cpu, TranslationBlock *tb, TranslationBlock **last_tb, int *tb_exit) { - uintptr_t ret; int32_t insns_left; trace_exec_tb(tb, tb->pc); - ret = cpu_tb_exec(cpu, tb); - tb = (TranslationBlock *)(ret & ~TB_EXIT_MASK); - *tb_exit = ret & TB_EXIT_MASK; + tb = cpu_tb_exec(cpu, tb, tb_exit); if (*tb_exit != TB_EXIT_REQUESTED) { *last_tb = tb; return; diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -XXX,XX +XXX,XX @@ void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx) { - uintptr_t val = (uintptr_t)tb + idx; + /* + * Let the jit code return the read-only version of the + * TranslationBlock, so that we minimize the pc-relative + * distance of the address of the exit_tb code to TB. + * This will improve utilization of pc-relative address loads. + * + * TODO: Move this to translator_loop, so that all const + * TranslationBlock pointers refer to read-only memory. + * This requires coordination with targets that do not use + * the translator_loop. + */ + uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx; if (tb == NULL) { tcg_debug_assert(idx == 0); -- 2.25.1 Reviewed-by: Joelle van Dyne <j@getutm.app> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.h | 2 +- tcg/i386/tcg-target.c.inc | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_SUPPORT_MIRROR 0 +#define TCG_TARGET_SUPPORT_MIRROR 1 #endif diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool have_lzcnt; # define have_lzcnt 0 #endif -static tcg_insn_unit *tb_ret_addr; +static const tcg_insn_unit *tb_ret_addr; static bool patch_reloc(tcg_insn_unit *code_ptr, int type, intptr_t value, intptr_t addend) @@ -XXX,XX +XXX,XX @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, value += addend; switch(type) { case R_386_PC32: - value -= (uintptr_t)code_ptr; + value -= (uintptr_t)tcg_splitwx_to_rx(code_ptr); if (value != (int32_t)value) { return false; } @@ -XXX,XX +XXX,XX @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, tcg_patch32(code_ptr, value); break; case R_386_PC8: - value -= (uintptr_t)code_ptr; + value -= (uintptr_t)tcg_splitwx_to_rx(code_ptr); if (value != (int8_t)value) { return false; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, } /* Try a 7 byte pc-relative lea before the 10 byte movq. */ - diff = arg - ((uintptr_t)s->code_ptr + 7); + diff = tcg_pcrel_diff(s, (const void *)arg) - 7; if (diff == (int32_t)diff) { tcg_out_opc(s, OPC_LEA | P_REXW, ret, 0, 0); tcg_out8(s, (LOWREGMASK(ret) << 3) | 5); @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest) tcg_out_branch(s, 1, dest); } -static void tcg_out_jmp(TCGContext *s, tcg_insn_unit *dest) +static void tcg_out_jmp(TCGContext *s, const tcg_insn_unit *dest) { tcg_out_branch(s, 0, dest); } @@ -XXX,XX +XXX,XX @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, bool is_64, label->datahi_reg = datahi; label->addrlo_reg = addrlo; label->addrhi_reg = addrhi; - label->raddr = raddr; + /* TODO: Cast goes away when all hosts converted */ + label->raddr = (void *)tcg_splitwx_to_rx(raddr); label->label_ptr[0] = label_ptr[0]; if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) { label->label_ptr[1] = label_ptr[1]; @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, /* jump displacement must be aligned for atomic patching; * see if we need to add extra nops before jump */ - gap = tcg_pcrel_diff(s, QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4)); + gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr; if (gap != 1) { tcg_out_nopn(s, gap - 1); } @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) * Return path for goto_ptr. Set return value to 0, a-la exit_tb, * and fall through to the rest of the epilogue. */ - tcg_code_gen_epilogue = s->code_ptr; + /* TODO: Cast goes away when all hosts converted */ + tcg_code_gen_epilogue = (void *)tcg_splitwx_to_rx(s->code_ptr); tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_EAX, 0); /* TB epilogue */ - tb_ret_addr = s->code_ptr; + tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr); tcg_out_addi(s, TCG_REG_CALL_STACK, stack_addend); -- 2.25.1 A typo generated a branch-and-link insn instead of plain branch. Reviewed-by: Joelle van Dyne <j@getutm.app> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target.c.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_goto_long(TCGContext *s, tcg_insn_unit *target) { ptrdiff_t offset = target - s->code_ptr; if (offset == sextract64(offset, 0, 26)) { - tcg_out_insn(s, 3206, BL, offset); + tcg_out_insn(s, 3206, B, offset); } else { tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, (intptr_t)target); tcg_out_insn(s, 3207, BR, TCG_REG_TMP); -- 2.25.1 Reviewed-by: Joelle van Dyne <j@getutm.app> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target.h | 2 +- tcg/aarch64/tcg-target.c.inc | 57 ++++++++++++++++++++---------------- 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_SUPPORT_MIRROR 0 +#define TCG_TARGET_SUPPORT_MIRROR 1 #endif /* AARCH64_TCG_TARGET_H */ diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const int tcg_target_call_oarg_regs[1] = { #define TCG_REG_GUEST_BASE TCG_REG_X28 #endif -static inline bool reloc_pc26(tcg_insn_unit *code_ptr, tcg_insn_unit *target) +static bool reloc_pc26(tcg_insn_unit *src_rw, const tcg_insn_unit *target) { - ptrdiff_t offset = target - code_ptr; + const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); + ptrdiff_t offset = target - src_rx; + if (offset == sextract64(offset, 0, 26)) { /* read instruction, mask away previous PC_REL26 parameter contents, set the proper offset, then write back the instruction. */ - *code_ptr = deposit32(*code_ptr, 0, 26, offset); + *src_rw = deposit32(*src_rw, 0, 26, offset); return true; } return false; } -static inline bool reloc_pc19(tcg_insn_unit *code_ptr, tcg_insn_unit *target) +static bool reloc_pc19(tcg_insn_unit *src_rw, const tcg_insn_unit *target) { - ptrdiff_t offset = target - code_ptr; + const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); + ptrdiff_t offset = target - src_rx; + if (offset == sextract64(offset, 0, 19)) { - *code_ptr = deposit32(*code_ptr, 5, 19, offset); + *src_rw = deposit32(*src_rw, 5, 19, offset); return true; } return false; } -static inline bool patch_reloc(tcg_insn_unit *code_ptr, int type, - intptr_t value, intptr_t addend) +static bool patch_reloc(tcg_insn_unit *code_ptr, int type, + intptr_t value, intptr_t addend) { tcg_debug_assert(addend == 0); switch (type) { case R_AARCH64_JUMP26: case R_AARCH64_CALL26: - return reloc_pc26(code_ptr, (tcg_insn_unit *)value); + return reloc_pc26(code_ptr, (const tcg_insn_unit *)value); case R_AARCH64_CONDBR19: - return reloc_pc19(code_ptr, (tcg_insn_unit *)value); + return reloc_pc19(code_ptr, (const tcg_insn_unit *)value); default: g_assert_not_reached(); } @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, /* Look for host pointer values within 4G of the PC. This happens often when loading pointers to QEMU's own data structures. */ if (type == TCG_TYPE_I64) { - tcg_target_long disp = value - (intptr_t)s->code_ptr; + intptr_t src_rx = (intptr_t)tcg_splitwx_to_rx(s->code_ptr); + tcg_target_long disp = value - src_rx; if (disp == sextract64(disp, 0, 21)) { tcg_out_insn(s, 3406, ADR, rd, disp); return; } - disp = (value >> 12) - ((intptr_t)s->code_ptr >> 12); + disp = (value >> 12) - (src_rx >> 12); if (disp == sextract64(disp, 0, 21)) { tcg_out_insn(s, 3406, ADRP, rd, disp); if (value & 0xfff) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_cmp(TCGContext *s, TCGType ext, TCGReg a, static void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target) { - ptrdiff_t offset = target - s->code_ptr; + ptrdiff_t offset = tcg_pcrel_diff(s, target) >> 2; tcg_debug_assert(offset == sextract64(offset, 0, 26)); tcg_out_insn(s, 3206, B, offset); } -static inline void tcg_out_goto_long(TCGContext *s, tcg_insn_unit *target) +static void tcg_out_goto_long(TCGContext *s, const tcg_insn_unit *target) { - ptrdiff_t offset = target - s->code_ptr; + ptrdiff_t offset = tcg_pcrel_diff(s, target) >> 2; if (offset == sextract64(offset, 0, 26)) { tcg_out_insn(s, 3206, B, offset); } else { @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_callr(TCGContext *s, TCGReg reg) tcg_out_insn(s, 3207, BLR, reg); } -static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *target) +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target) { - ptrdiff_t offset = target - s->code_ptr; + ptrdiff_t offset = tcg_pcrel_diff(s, target) >> 2; if (offset == sextract64(offset, 0, 26)) { tcg_out_insn(s, 3206, BL, offset); } else { @@ -XXX,XX +XXX,XX @@ static void tcg_out_brcond(TCGContext *s, TCGType ext, TCGCond c, TCGArg a, tcg_out_reloc(s, s->code_ptr, R_AARCH64_CONDBR19, l, 0); offset = tcg_in32(s) >> 5; } else { - offset = l->u.value_ptr - s->code_ptr; + offset = tcg_pcrel_diff(s, l->u.value_ptr) >> 2; tcg_debug_assert(offset == sextract64(offset, 0, 19)); } @@ -XXX,XX +XXX,XX @@ static void * const qemu_st_helpers[16] = { [MO_BEQ] = helper_be_stq_mmu, }; -static inline void tcg_out_adr(TCGContext *s, TCGReg rd, void *target) +static inline void tcg_out_adr(TCGContext *s, TCGReg rd, const void *target) { ptrdiff_t offset = tcg_pcrel_diff(s, target); tcg_debug_assert(offset == sextract64(offset, 0, 21)); @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) MemOp opc = get_memop(oi); MemOp size = opc & MO_SIZE; - if (!reloc_pc19(lb->label_ptr[0], s->code_ptr)) { + if (!reloc_pc19(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { return false; } @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) MemOp opc = get_memop(oi); MemOp size = opc & MO_SIZE; - if (!reloc_pc19(lb->label_ptr[0], s->code_ptr)) { + if (!reloc_pc19(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { return false; } @@ -XXX,XX +XXX,XX @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi, label->type = ext; label->datalo_reg = data_reg; label->addrlo_reg = addr_reg; - label->raddr = raddr; + /* TODO: Cast goes away when all hosts converted */ + label->raddr = (void *)tcg_splitwx_to_rx(raddr); label->label_ptr[0] = label_ptr; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, #endif /* CONFIG_SOFTMMU */ } -static tcg_insn_unit *tb_ret_addr; +static const tcg_insn_unit *tb_ret_addr; static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) * Return path for goto_ptr. Set return value to 0, a-la exit_tb, * and fall through to the rest of the epilogue. */ - tcg_code_gen_epilogue = s->code_ptr; + /* TODO: Cast goes away when all hosts converted */ + tcg_code_gen_epilogue = (void *)tcg_splitwx_to_rx(s->code_ptr); tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_X0, 0); /* TB epilogue */ - tb_ret_addr = s->code_ptr; + tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr); /* Remove TCG locals stack space. */ tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_SP, TCG_REG_SP, -- 2.25.1 Reviewed-by: Joelle van Dyne <j@getutm.app> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/disas/dis-asm.h | 4 ++-- disas.c | 4 +--- disas/capstone.c | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/include/disas/dis-asm.h b/include/disas/dis-asm.h index XXXXXXX..XXXXXXX 100644 --- a/include/disas/dis-asm.h +++ b/include/disas/dis-asm.h @@ -XXX,XX +XXX,XX @@ typedef struct disassemble_info { (bfd_vma addr, struct disassemble_info * info); /* These are for buffer_read_memory. */ - bfd_byte *buffer; + const bfd_byte *buffer; bfd_vma buffer_vma; int buffer_length; @@ -XXX,XX +XXX,XX @@ int print_insn_rx(bfd_vma, disassemble_info *); #ifdef CONFIG_CAPSTONE bool cap_disas_target(disassemble_info *info, uint64_t pc, size_t size); -bool cap_disas_host(disassemble_info *info, void *code, size_t size); +bool cap_disas_host(disassemble_info *info, const void *code, size_t size); bool cap_disas_monitor(disassemble_info *info, uint64_t pc, int count); bool cap_disas_plugin(disassemble_info *info, uint64_t pc, size_t size); #else diff --git a/disas.c b/disas.c index XXXXXXX..XXXXXXX 100644 --- a/disas.c +++ b/disas.c @@ -XXX,XX +XXX,XX @@ char *plugin_disas(CPUState *cpu, uint64_t addr, size_t size) } /* Disassemble this for me please... (debugging). */ -void disas(FILE *out, const void *ccode, unsigned long size) +void disas(FILE *out, const void *code, unsigned long size) { - /* TODO: Push constness through the disas backends. */ - void *code = (void *)ccode; uintptr_t pc; int count; CPUDebug s; diff --git a/disas/capstone.c b/disas/capstone.c index XXXXXXX..XXXXXXX 100644 --- a/disas/capstone.c +++ b/disas/capstone.c @@ -XXX,XX +XXX,XX @@ bool cap_disas_target(disassemble_info *info, uint64_t pc, size_t size) } /* Disassemble SIZE bytes at CODE for the host. */ -bool cap_disas_host(disassemble_info *info, void *code, size_t size) +bool cap_disas_host(disassemble_info *info, const void *code, size_t size) { csh handle; const uint8_t *cbuf; -- 2.25.1 Reviewed-by: Joelle van Dyne <j@getutm.app> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci.c | 60 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ static uint64_t tci_uint64(uint32_t high, uint32_t low) #endif /* Read constant (native size) from bytecode. */ -static tcg_target_ulong tci_read_i(uint8_t **tb_ptr) +static tcg_target_ulong tci_read_i(const uint8_t **tb_ptr) { - tcg_target_ulong value = *(tcg_target_ulong *)(*tb_ptr); + tcg_target_ulong value = *(const tcg_target_ulong *)(*tb_ptr); *tb_ptr += sizeof(value); return value; } /* Read unsigned constant (32 bit) from bytecode. */ -static uint32_t tci_read_i32(uint8_t **tb_ptr) +static uint32_t tci_read_i32(const uint8_t **tb_ptr) { - uint32_t value = *(uint32_t *)(*tb_ptr); + uint32_t value = *(const uint32_t *)(*tb_ptr); *tb_ptr += sizeof(value); return value; } /* Read signed constant (32 bit) from bytecode. */ -static int32_t tci_read_s32(uint8_t **tb_ptr) +static int32_t tci_read_s32(const uint8_t **tb_ptr) { - int32_t value = *(int32_t *)(*tb_ptr); + int32_t value = *(const int32_t *)(*tb_ptr); *tb_ptr += sizeof(value); return value; } #if TCG_TARGET_REG_BITS == 64 /* Read constant (64 bit) from bytecode. */ -static uint64_t tci_read_i64(uint8_t **tb_ptr) +static uint64_t tci_read_i64(const uint8_t **tb_ptr) { - uint64_t value = *(uint64_t *)(*tb_ptr); + uint64_t value = *(const uint64_t *)(*tb_ptr); *tb_ptr += sizeof(value); return value; } @@ -XXX,XX +XXX,XX @@ static uint64_t tci_read_i64(uint8_t **tb_ptr) /* Read indexed register (native size) from bytecode. */ static tcg_target_ulong -tci_read_r(const tcg_target_ulong *regs, uint8_t **tb_ptr) +tci_read_r(const tcg_target_ulong *regs, const uint8_t **tb_ptr) { tcg_target_ulong value = tci_read_reg(regs, **tb_ptr); *tb_ptr += 1; @@ -XXX,XX +XXX,XX @@ tci_read_r(const tcg_target_ulong *regs, uint8_t **tb_ptr) } /* Read indexed register (8 bit) from bytecode. */ -static uint8_t tci_read_r8(const tcg_target_ulong *regs, uint8_t **tb_ptr) +static uint8_t tci_read_r8(const tcg_target_ulong *regs, const uint8_t **tb_ptr) { uint8_t value = tci_read_reg8(regs, **tb_ptr); *tb_ptr += 1; @@ -XXX,XX +XXX,XX @@ static uint8_t tci_read_r8(const tcg_target_ulong *regs, uint8_t **tb_ptr) #if TCG_TARGET_HAS_ext8s_i32 || TCG_TARGET_HAS_ext8s_i64 /* Read indexed register (8 bit signed) from bytecode. */ -static int8_t tci_read_r8s(const tcg_target_ulong *regs, uint8_t **tb_ptr) +static int8_t tci_read_r8s(const tcg_target_ulong *regs, const uint8_t **tb_ptr) { int8_t value = tci_read_reg8s(regs, **tb_ptr); *tb_ptr += 1; @@ -XXX,XX +XXX,XX @@ static int8_t tci_read_r8s(const tcg_target_ulong *regs, uint8_t **tb_ptr) #endif /* Read indexed register (16 bit) from bytecode. */ -static uint16_t tci_read_r16(const tcg_target_ulong *regs, uint8_t **tb_ptr) +static uint16_t tci_read_r16(const tcg_target_ulong *regs, + const uint8_t **tb_ptr) { uint16_t value = tci_read_reg16(regs, **tb_ptr); *tb_ptr += 1; @@ -XXX,XX +XXX,XX @@ static uint16_t tci_read_r16(const tcg_target_ulong *regs, uint8_t **tb_ptr) #if TCG_TARGET_HAS_ext16s_i32 || TCG_TARGET_HAS_ext16s_i64 /* Read indexed register (16 bit signed) from bytecode. */ -static int16_t tci_read_r16s(const tcg_target_ulong *regs, uint8_t **tb_ptr) +static int16_t tci_read_r16s(const tcg_target_ulong *regs, + const uint8_t **tb_ptr) { int16_t value = tci_read_reg16s(regs, **tb_ptr); *tb_ptr += 1; @@ -XXX,XX +XXX,XX @@ static int16_t tci_read_r16s(const tcg_target_ulong *regs, uint8_t **tb_ptr) #endif /* Read indexed register (32 bit) from bytecode. */ -static uint32_t tci_read_r32(const tcg_target_ulong *regs, uint8_t **tb_ptr) +static uint32_t tci_read_r32(const tcg_target_ulong *regs, + const uint8_t **tb_ptr) { uint32_t value = tci_read_reg32(regs, **tb_ptr); *tb_ptr += 1; @@ -XXX,XX +XXX,XX @@ static uint32_t tci_read_r32(const tcg_target_ulong *regs, uint8_t **tb_ptr) #if TCG_TARGET_REG_BITS == 32 /* Read two indexed registers (2 * 32 bit) from bytecode. */ -static uint64_t tci_read_r64(const tcg_target_ulong *regs, uint8_t **tb_ptr) +static uint64_t tci_read_r64(const tcg_target_ulong *regs, + const uint8_t **tb_ptr) { uint32_t low = tci_read_r32(regs, tb_ptr); return tci_uint64(tci_read_r32(regs, tb_ptr), low); } #elif TCG_TARGET_REG_BITS == 64 /* Read indexed register (32 bit signed) from bytecode. */ -static int32_t tci_read_r32s(const tcg_target_ulong *regs, uint8_t **tb_ptr) +static int32_t tci_read_r32s(const tcg_target_ulong *regs, + const uint8_t **tb_ptr) { int32_t value = tci_read_reg32s(regs, **tb_ptr); *tb_ptr += 1; @@ -XXX,XX +XXX,XX @@ static int32_t tci_read_r32s(const tcg_target_ulong *regs, uint8_t **tb_ptr) } /* Read indexed register (64 bit) from bytecode. */ -static uint64_t tci_read_r64(const tcg_target_ulong *regs, uint8_t **tb_ptr) +static uint64_t tci_read_r64(const tcg_target_ulong *regs, + const uint8_t **tb_ptr) { uint64_t value = tci_read_reg64(regs, **tb_ptr); *tb_ptr += 1; @@ -XXX,XX +XXX,XX @@ static uint64_t tci_read_r64(const tcg_target_ulong *regs, uint8_t **tb_ptr) /* Read indexed register(s) with target address from bytecode. */ static target_ulong -tci_read_ulong(const tcg_target_ulong *regs, uint8_t **tb_ptr) +tci_read_ulong(const tcg_target_ulong *regs, const uint8_t **tb_ptr) { target_ulong taddr = tci_read_r(regs, tb_ptr); #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS @@ -XXX,XX +XXX,XX @@ tci_read_ulong(const tcg_target_ulong *regs, uint8_t **tb_ptr) /* Read indexed register or constant (native size) from bytecode. */ static tcg_target_ulong -tci_read_ri(const tcg_target_ulong *regs, uint8_t **tb_ptr) +tci_read_ri(const tcg_target_ulong *regs, const uint8_t **tb_ptr) { tcg_target_ulong value; TCGReg r = **tb_ptr; @@ -XXX,XX +XXX,XX @@ tci_read_ri(const tcg_target_ulong *regs, uint8_t **tb_ptr) } /* Read indexed register or constant (32 bit) from bytecode. */ -static uint32_t tci_read_ri32(const tcg_target_ulong *regs, uint8_t **tb_ptr) +static uint32_t tci_read_ri32(const tcg_target_ulong *regs, + const uint8_t **tb_ptr) { uint32_t value; TCGReg r = **tb_ptr; @@ -XXX,XX +XXX,XX @@ static uint32_t tci_read_ri32(const tcg_target_ulong *regs, uint8_t **tb_ptr) #if TCG_TARGET_REG_BITS == 32 /* Read two indexed registers or constants (2 * 32 bit) from bytecode. */ -static uint64_t tci_read_ri64(const tcg_target_ulong *regs, uint8_t **tb_ptr) +static uint64_t tci_read_ri64(const tcg_target_ulong *regs, + const uint8_t **tb_ptr) { uint32_t low = tci_read_ri32(regs, tb_ptr); return tci_uint64(tci_read_ri32(regs, tb_ptr), low); } #elif TCG_TARGET_REG_BITS == 64 /* Read indexed register or constant (64 bit) from bytecode. */ -static uint64_t tci_read_ri64(const tcg_target_ulong *regs, uint8_t **tb_ptr) +static uint64_t tci_read_ri64(const tcg_target_ulong *regs, + const uint8_t **tb_ptr) { uint64_t value; TCGReg r = **tb_ptr; @@ -XXX,XX +XXX,XX @@ static uint64_t tci_read_ri64(const tcg_target_ulong *regs, uint8_t **tb_ptr) } #endif -static tcg_target_ulong tci_read_label(uint8_t **tb_ptr) +static tcg_target_ulong tci_read_label(const uint8_t **tb_ptr) { tcg_target_ulong label = tci_read_i(tb_ptr); tci_assert(label != 0); @@ -XXX,XX +XXX,XX @@ static bool tci_compare64(uint64_t u0, uint64_t u1, TCGCond condition) uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, const void *v_tb_ptr) { - /* TODO: Propagate const through this file. */ - uint8_t *tb_ptr = (uint8_t *)v_tb_ptr; + const uint8_t *tb_ptr = v_tb_ptr; tcg_target_ulong regs[TCG_TARGET_NB_REGS]; long tcg_temps[CPU_TEMP_BUF_NLONGS]; uintptr_t sp_value = (uintptr_t)(tcg_temps + CPU_TEMP_BUF_NLONGS); @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, TCGOpcode opc = tb_ptr[0]; #if defined(CONFIG_DEBUG_TCG) && !defined(NDEBUG) uint8_t op_size = tb_ptr[1]; - uint8_t *old_code_ptr = tb_ptr; + const uint8_t *old_code_ptr = tb_ptr; #endif tcg_target_ulong t0; tcg_target_ulong t1; -- 2.25.1 Reviewed-by: Joelle van Dyne <j@getutm.app> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ static inline ptrdiff_t tcg_pcrel_diff(TCGContext *s, const void *target) return tcg_ptr_byte_diff(target, tcg_splitwx_to_rx(s->code_ptr)); } +/** + * tcg_tbrel_diff + * @s: the tcg context + * @target: address of the target + * + * Produce a difference, from the beginning of the current TB code + * to the destination address. + */ +static inline ptrdiff_t tcg_tbrel_diff(TCGContext *s, const void *target) +{ + return tcg_ptr_byte_diff(target, tcg_splitwx_to_rx(s->code_buf)); +} + /** * tcg_current_code_size * @s: the tcg context -- 2.25.1 Use tcg_tbrel_diff when we need a displacement to a label, and with a NULL argument when we need the normalizing addend. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/ppc/tcg-target.c.inc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, } /* Load addresses within the TB with one insn. */ - tb_diff = arg - (intptr_t)s->code_gen_ptr; + tb_diff = tcg_tbrel_diff(s, (void *)arg); if (!in_prologue && USE_REG_TB && tb_diff == (int16_t)tb_diff) { tcg_out32(s, ADDI | TAI(ret, TCG_REG_TB, tb_diff)); return; @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, /* Use the constant pool, if possible. */ if (!in_prologue && USE_REG_TB) { new_pool_label(s, arg, R_PPC_ADDR16, s->code_ptr, - -(intptr_t)s->code_gen_ptr); + tcg_tbrel_diff(s, NULL)); tcg_out32(s, LD | TAI(ret, TCG_REG_TB, 0)); return; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_dupi_vec(TCGContext *s, TCGType type, TCGReg ret, */ if (USE_REG_TB) { rel = R_PPC_ADDR16; - add = -(intptr_t)s->code_gen_ptr; + add = tcg_tbrel_diff(s, NULL); } else { rel = R_PPC_ADDR32; add = 0; -- 2.25.1 The maximum TB code gen size is UINT16_MAX, which the current code does not support. Use our utility function to optimally add an arbitrary constant. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/ppc/tcg-target.c.inc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, set_jmp_reset_offset(s, args[0]); if (USE_REG_TB) { /* For the unlinked case, need to reset TCG_REG_TB. */ - c = -tcg_current_code_size(s); - assert(c == (int16_t)c); - tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, c)); + tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB, + -tcg_current_code_size(s)); } break; case INDEX_op_goto_ptr: -- 2.25.1 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/ppc/tcg-target.h | 2 +- tcg/ppc/tcg-target.c.inc | 53 +++++++++++++++++++++++----------------- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_SUPPORT_MIRROR 0 +#define TCG_TARGET_SUPPORT_MIRROR 1 #endif diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ #define TCG_CT_CONST_MONE 0x2000 #define TCG_CT_CONST_WSZ 0x4000 -static tcg_insn_unit *tb_ret_addr; - TCGPowerISA have_isa; static bool have_isel; bool have_altivec; @@ -XXX,XX +XXX,XX @@ static inline bool in_range_b(tcg_target_long target) return target == sextract64(target, 0, 26); } -static uint32_t reloc_pc24_val(tcg_insn_unit *pc, const tcg_insn_unit *target) +static uint32_t reloc_pc24_val(const tcg_insn_unit *pc, + const tcg_insn_unit *target) { ptrdiff_t disp = tcg_ptr_byte_diff(target, pc); tcg_debug_assert(in_range_b(disp)); return disp & 0x3fffffc; } -static bool reloc_pc24(tcg_insn_unit *pc, tcg_insn_unit *target) +static bool reloc_pc24(tcg_insn_unit *src_rw, const tcg_insn_unit *target) { - ptrdiff_t disp = tcg_ptr_byte_diff(target, pc); + const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); + ptrdiff_t disp = tcg_ptr_byte_diff(target, src_rx); + if (in_range_b(disp)) { - *pc = (*pc & ~0x3fffffc) | (disp & 0x3fffffc); + *src_rw = (*src_rw & ~0x3fffffc) | (disp & 0x3fffffc); return true; } return false; } -static uint16_t reloc_pc14_val(tcg_insn_unit *pc, const tcg_insn_unit *target) +static uint16_t reloc_pc14_val(const tcg_insn_unit *pc, + const tcg_insn_unit *target) { ptrdiff_t disp = tcg_ptr_byte_diff(target, pc); tcg_debug_assert(disp == (int16_t) disp); return disp & 0xfffc; } -static bool reloc_pc14(tcg_insn_unit *pc, tcg_insn_unit *target) +static bool reloc_pc14(tcg_insn_unit *src_rw, const tcg_insn_unit *target) { - ptrdiff_t disp = tcg_ptr_byte_diff(target, pc); + const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); + ptrdiff_t disp = tcg_ptr_byte_diff(target, src_rx); + if (disp == (int16_t) disp) { - *pc = (*pc & ~0xfffc) | (disp & 0xfffc); + *src_rw = (*src_rw & ~0xfffc) | (disp & 0xfffc); return true; } return false; @@ -XXX,XX +XXX,XX @@ static const uint32_t tcg_to_isel[] = { static bool patch_reloc(tcg_insn_unit *code_ptr, int type, intptr_t value, intptr_t addend) { - tcg_insn_unit *target; + const tcg_insn_unit *target; int16_t lo; int32_t hi; value += addend; - target = (tcg_insn_unit *)value; + target = (const tcg_insn_unit *)value; switch (type) { case R_PPC_REL14: @@ -XXX,XX +XXX,XX @@ static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond, static void tcg_out_bc(TCGContext *s, int bc, TCGLabel *l) { if (l->has_value) { - bc |= reloc_pc14_val(s->code_ptr, l->u.value_ptr); + bc |= reloc_pc14_val(tcg_splitwx_to_rx(s->code_ptr), l->u.value_ptr); } else { tcg_out_reloc(s, s->code_ptr, R_PPC_REL14, l, 0); } @@ -XXX,XX +XXX,XX @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi, label->datahi_reg = datahi_reg; label->addrlo_reg = addrlo_reg; label->addrhi_reg = addrhi_reg; - label->raddr = raddr; + /* TODO: Cast goes away when all hosts converted */ + label->raddr = (void *)tcg_splitwx_to_rx(raddr); label->label_ptr[0] = lptr; } @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) MemOp opc = get_memop(oi); TCGReg hi, lo, arg = TCG_REG_R3; - if (!reloc_pc14(lb->label_ptr[0], s->code_ptr)) { + if (!reloc_pc14(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { return false; } @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) MemOp s_bits = opc & MO_SIZE; TCGReg hi, lo, arg = TCG_REG_R3; - if (!reloc_pc14(lb->label_ptr[0], s->code_ptr)) { + if (!reloc_pc14(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { return false; } @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) int i; #ifdef _CALL_AIX - void **desc = (void **)s->code_ptr; - desc[0] = desc + 2; /* entry point */ - desc[1] = 0; /* environment pointer */ - s->code_ptr = (void *)(desc + 2); /* skip over descriptor */ + const void **desc = (const void **)s->code_ptr; + desc[0] = tcg_splitwx_to_rx(desc + 2); /* entry point */ + desc[1] = 0; /* environment pointer */ + s->code_ptr = (void *)(desc + 2); /* skip over descriptor */ #endif tcg_set_frame(s, TCG_REG_CALL_STACK, REG_SAVE_BOT - CPU_TEMP_BUF_SIZE, @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out32(s, BCCTR | BO_ALWAYS); /* Epilogue */ - tcg_code_gen_epilogue = tb_ret_addr = s->code_ptr; + /* TODO: Cast goes away when all hosts converted */ + tcg_code_gen_epilogue = (void *)tcg_splitwx_to_rx(s->code_ptr); tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET); for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, switch (opc) { case INDEX_op_exit_tb: tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]); - tcg_out_b(s, 0, tb_ret_addr); + tcg_out_b(s, 0, tcg_code_gen_epilogue); break; case INDEX_op_goto_tb: if (s->tb_jmp_insn_offset) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, uint32_t insn = B; if (l->has_value) { - insn |= reloc_pc24_val(s->code_ptr, l->u.value_ptr); + insn |= reloc_pc24_val(tcg_splitwx_to_rx(s->code_ptr), + l->u.value_ptr); } else { tcg_out_reloc(s, s->code_ptr, R_PPC_REL24, l, 0); } -- 2.25.1 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc/tcg-target.c.inc | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, /* A 13-bit constant relative to the TB. */ if (!in_prologue && USE_REG_TB) { - test = arg - (uintptr_t)s->code_gen_ptr; + test = tcg_tbrel_diff(s, (void *)arg); if (check_fit_ptr(test, 13)) { tcg_out_arithi(s, ret, TCG_REG_TB, test, ARITH_ADD); return; @@ -XXX,XX +XXX,XX @@ static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, return false; } -static void tcg_out_ld_ptr(TCGContext *s, TCGReg ret, uintptr_t arg) +static void tcg_out_ld_ptr(TCGContext *s, TCGReg ret, const void *arg) { - intptr_t diff = arg - (uintptr_t)s->code_gen_ptr; + intptr_t diff = tcg_tbrel_diff(s, arg); if (USE_REG_TB && check_fit_ptr(diff, 13)) { tcg_out_ld(s, TCG_TYPE_PTR, ret, TCG_REG_TB, diff); return; } - tcg_out_movi(s, TCG_TYPE_PTR, ret, arg & ~0x3ff); - tcg_out_ld(s, TCG_TYPE_PTR, ret, ret, arg & 0x3ff); + tcg_out_movi(s, TCG_TYPE_PTR, ret, (uintptr_t)arg & ~0x3ff); + tcg_out_ld(s, TCG_TYPE_PTR, ret, ret, (uintptr_t)arg & 0x3ff); } static inline void tcg_out_sety(TCGContext *s, TCGReg rs) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_movi_imm13(s, TCG_REG_O0, a0); break; } else if (USE_REG_TB) { - intptr_t tb_diff = a0 - (uintptr_t)s->code_gen_ptr; + intptr_t tb_diff = tcg_tbrel_diff(s, (void *)a0); if (check_fit_ptr(tb_diff, 13)) { tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); /* Note that TCG_REG_TB has been unwound to O1. */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, } } else { /* indirect jump method */ - tcg_out_ld_ptr(s, TCG_REG_TB, - (uintptr_t)(s->tb_jmp_target_addr + a0)); + tcg_out_ld_ptr(s, TCG_REG_TB, s->tb_jmp_target_addr + a0); tcg_out_arithi(s, TCG_REG_G0, TCG_REG_TB, 0, JMPL); tcg_out_nop(s); } -- 2.25.1 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc/tcg-target.h | 2 +- tcg/sparc/tcg-target.c.inc | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.h +++ b/tcg/sparc/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions; void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_SUPPORT_MIRROR 0 +#define TCG_TARGET_SUPPORT_MIRROR 1 #endif diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline int check_fit_i32(int32_t val, unsigned int bits) # define check_fit_ptr check_fit_i32 #endif -static bool patch_reloc(tcg_insn_unit *code_ptr, int type, +static bool patch_reloc(tcg_insn_unit *src_rw, int type, intptr_t value, intptr_t addend) { - uint32_t insn = *code_ptr; + const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); + uint32_t insn = *src_rw; intptr_t pcrel; value += addend; - pcrel = tcg_ptr_byte_diff((tcg_insn_unit *)value, code_ptr); + pcrel = tcg_ptr_byte_diff((tcg_insn_unit *)value, src_rx); switch (type) { case R_SPARC_WDISP16: @@ -XXX,XX +XXX,XX @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, g_assert_not_reached(); } - *code_ptr = insn; + *src_rw = insn; return true; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_mb(TCGContext *s, TCGArg a0) } #ifdef CONFIG_SOFTMMU -static tcg_insn_unit *qemu_ld_trampoline[16]; -static tcg_insn_unit *qemu_st_trampoline[16]; +static const tcg_insn_unit *qemu_ld_trampoline[16]; +static const tcg_insn_unit *qemu_st_trampoline[16]; static void emit_extend(TCGContext *s, TCGReg r, int op) { @@ -XXX,XX +XXX,XX @@ static void build_trampolines(TCGContext *s) while ((uintptr_t)s->code_ptr & 15) { tcg_out_nop(s); } - qemu_ld_trampoline[i] = s->code_ptr; + qemu_ld_trampoline[i] = tcg_splitwx_to_rx(s->code_ptr); if (SPARC64 || TARGET_LONG_BITS == 32) { ra = TCG_REG_O3; @@ -XXX,XX +XXX,XX @@ static void build_trampolines(TCGContext *s) while ((uintptr_t)s->code_ptr & 15) { tcg_out_nop(s); } - qemu_st_trampoline[i] = s->code_ptr; + qemu_st_trampoline[i] = tcg_splitwx_to_rx(s->code_ptr); if (SPARC64) { emit_extend(s, TCG_REG_O2, i); @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out_nop(s); /* Epilogue for goto_ptr. */ - tcg_code_gen_epilogue = s->code_ptr; + /* TODO: Cast goes away when all hosts converted */ + tcg_code_gen_epilogue = (void *)tcg_splitwx_to_rx(s->code_ptr); tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); /* delay slot */ tcg_out_movi_imm13(s, TCG_REG_O0, 0); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, #ifdef CONFIG_SOFTMMU unsigned memi = get_mmuidx(oi); TCGReg addrz, param; - tcg_insn_unit *func; + const tcg_insn_unit *func; tcg_insn_unit *label_ptr; addrz = tcg_out_tlb_load(s, addr, memi, memop, @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr, #ifdef CONFIG_SOFTMMU unsigned memi = get_mmuidx(oi); TCGReg addrz, param; - tcg_insn_unit *func; + const tcg_insn_unit *func; tcg_insn_unit *label_ptr; addrz = tcg_out_tlb_load(s, addr, memi, memop, -- 2.25.1 Use tcg_tbrel_diff when we need a displacement to a label, and with a NULL argument when we need the normalizing addend. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/s390/tcg-target.c.inc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.c.inc +++ b/tcg/s390/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, return; } } else if (USE_REG_TB && !in_prologue) { - ptrdiff_t off = sval - (uintptr_t)s->code_gen_ptr; + ptrdiff_t off = tcg_tbrel_diff(s, (void *)sval); if (off == sextract64(off, 0, 20)) { /* This is certain to be an address within TB, and therefore OFF will be negative; don't try RX_LA. */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, } else if (USE_REG_TB && !in_prologue) { tcg_out_insn(s, RXY, LG, ret, TCG_REG_TB, TCG_REG_NONE, 0); new_pool_label(s, sval, R_390_20, s->code_ptr - 2, - -(intptr_t)s->code_gen_ptr); + tcg_tbrel_diff(s, NULL)); } else { TCGReg base = ret ? ret : TCG_TMP0; tcg_out_insn(s, RIL, LARL, base, 0); @@ -XXX,XX +XXX,XX @@ static void tcg_out_ld_abs(TCGContext *s, TCGType type, TCGReg dest, void *abs) } } if (USE_REG_TB) { - ptrdiff_t disp = abs - (void *)s->code_gen_ptr; + ptrdiff_t disp = tcg_tbrel_diff(s, abs); if (disp == sextract64(disp, 0, 20)) { tcg_out_ld(s, type, dest, TCG_REG_TB, disp); return; @@ -XXX,XX +XXX,XX @@ static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val) if (!maybe_out_small_movi(s, type, TCG_TMP0, val)) { tcg_out_insn(s, RXY, NG, dest, TCG_REG_TB, TCG_REG_NONE, 0); new_pool_label(s, val & valid, R_390_20, s->code_ptr - 2, - -(intptr_t)s->code_gen_ptr); + tcg_tbrel_diff(s, NULL)); return; } } else { @@ -XXX,XX +XXX,XX @@ static void tgen_ori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val) } else if (USE_REG_TB) { tcg_out_insn(s, RXY, OG, dest, TCG_REG_TB, TCG_REG_NONE, 0); new_pool_label(s, val, R_390_20, s->code_ptr - 2, - -(intptr_t)s->code_gen_ptr); + tcg_tbrel_diff(s, NULL)); } else { /* Perform the OR via sequential modifications to the high and low parts. Do this via recursion to handle 16-bit vs 32-bit @@ -XXX,XX +XXX,XX @@ static void tgen_xori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val) } else if (USE_REG_TB) { tcg_out_insn(s, RXY, XG, dest, TCG_REG_TB, TCG_REG_NONE, 0); new_pool_label(s, val, R_390_20, s->code_ptr - 2, - -(intptr_t)s->code_gen_ptr); + tcg_tbrel_diff(s, NULL)); } else { /* Perform the xor by parts. */ tcg_debug_assert(s390_facilities & FACILITY_EXT_IMM); @@ -XXX,XX +XXX,XX @@ static int tgen_cmp(TCGContext *s, TCGType type, TCGCond c, TCGReg r1, op = (is_unsigned ? RXY_CLY : RXY_CY); tcg_out_insn_RXY(s, op, r1, TCG_REG_TB, TCG_REG_NONE, 0); new_pool_label(s, (uint32_t)c2, R_390_20, s->code_ptr - 2, - 4 - (intptr_t)s->code_gen_ptr); + 4 - tcg_tbrel_diff(s, NULL)); } else { op = (is_unsigned ? RXY_CLG : RXY_CG); tcg_out_insn_RXY(s, op, r1, TCG_REG_TB, TCG_REG_NONE, 0); new_pool_label(s, c2, R_390_20, s->code_ptr - 2, - -(intptr_t)s->code_gen_ptr); + tcg_tbrel_diff(s, NULL)); } goto exit; } else { -- 2.25.1 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/s390/tcg-target.h | 2 +- tcg/s390/tcg-target.c.inc | 69 +++++++++++++++++---------------------- 2 files changed, 31 insertions(+), 40 deletions(-) diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.h +++ b/tcg/s390/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_SUPPORT_MIRROR 0 +#define TCG_TARGET_SUPPORT_MIRROR 1 #endif diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.c.inc +++ b/tcg/s390/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void * const qemu_st_helpers[16] = { }; #endif -static tcg_insn_unit *tb_ret_addr; +static const tcg_insn_unit *tb_ret_addr; uint64_t s390_facilities; -static bool patch_reloc(tcg_insn_unit *code_ptr, int type, +static bool patch_reloc(tcg_insn_unit *src_rw, int type, intptr_t value, intptr_t addend) { + const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); intptr_t pcrel2; uint32_t old; value += addend; - pcrel2 = (tcg_insn_unit *)value - code_ptr; + pcrel2 = (tcg_insn_unit *)value - src_rx; switch (type) { case R_390_PC16DBL: if (pcrel2 == (int16_t)pcrel2) { - tcg_patch16(code_ptr, pcrel2); + tcg_patch16(src_rw, pcrel2); return true; } break; case R_390_PC32DBL: if (pcrel2 == (int32_t)pcrel2) { - tcg_patch32(code_ptr, pcrel2); + tcg_patch32(src_rw, pcrel2); return true; } break; case R_390_20: if (value == sextract64(value, 0, 20)) { - old = *(uint32_t *)code_ptr & 0xf00000ff; + old = *(uint32_t *)src_rw & 0xf00000ff; old |= ((value & 0xfff) << 16) | ((value & 0xff000) >> 4); - tcg_patch32(code_ptr, old); + tcg_patch32(src_rw, old); return true; } break; @@ -XXX,XX +XXX,XX @@ static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, } /* load data from an absolute host address */ -static void tcg_out_ld_abs(TCGContext *s, TCGType type, TCGReg dest, void *abs) +static void tcg_out_ld_abs(TCGContext *s, TCGType type, + TCGReg dest, const void *abs) { intptr_t addr = (intptr_t)abs; @@ -XXX,XX +XXX,XX @@ static void tgen_extract(TCGContext *s, TCGReg dest, TCGReg src, static void tgen_gotoi(TCGContext *s, int cc, const tcg_insn_unit *dest) { - ptrdiff_t off = dest - s->code_ptr; + ptrdiff_t off = tcg_pcrel_diff(s, dest) >> 1; if (off == (int16_t)off) { tcg_out_insn(s, RI, BRC, cc, off); } else if (off == (int32_t)off) { @@ -XXX,XX +XXX,XX @@ static void tgen_branch(TCGContext *s, int cc, TCGLabel *l) static void tgen_compare_branch(TCGContext *s, S390Opcode opc, int cc, TCGReg r1, TCGReg r2, TCGLabel *l) { - intptr_t off = 0; - - if (l->has_value) { - off = l->u.value_ptr - s->code_ptr; - tcg_debug_assert(off == (int16_t)off); - } else { - tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2); - } - + tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2); tcg_out16(s, (opc & 0xff00) | (r1 << 4) | r2); - tcg_out16(s, off); + tcg_out16(s, 0); tcg_out16(s, cc << 12 | (opc & 0xff)); } static void tgen_compare_imm_branch(TCGContext *s, S390Opcode opc, int cc, TCGReg r1, int i2, TCGLabel *l) { - tcg_target_long off = 0; - - if (l->has_value) { - off = l->u.value_ptr - s->code_ptr; - tcg_debug_assert(off == (int16_t)off); - } else { - tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2); - } - + tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2); tcg_out16(s, (opc & 0xff00) | (r1 << 4) | cc); - tcg_out16(s, off); + tcg_out16(s, 0); tcg_out16(s, (i2 << 8) | (opc & 0xff)); } @@ -XXX,XX +XXX,XX @@ static void tgen_brcond(TCGContext *s, TCGType type, TCGCond c, static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest) { - ptrdiff_t off = dest - s->code_ptr; + ptrdiff_t off = tcg_pcrel_diff(s, dest) >> 1; if (off == (int32_t)off) { tcg_out_insn(s, RIL, BRASL, TCG_REG_R14, off); } else { @@ -XXX,XX +XXX,XX @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi, label->oi = oi; label->datalo_reg = data; label->addrlo_reg = addr; - label->raddr = raddr; + /* TODO: Cast goes away when all hosts converted */ + label->raddr = (void *)tcg_splitwx_to_rx(raddr); label->label_ptr[0] = label_ptr; } @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) MemOp opc = get_memop(oi); if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL, - (intptr_t)s->code_ptr, 2)) { + (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) { return false; } @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) MemOp opc = get_memop(oi); if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL, - (intptr_t)s->code_ptr, 2)) { + (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) { return false; } @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_goto_tb: a0 = args[0]; if (s->tb_jmp_insn_offset) { - /* branch displacement must be aligned for atomic patching; + /* + * branch displacement must be aligned for atomic patching; * see if we need to add extra nop before branch */ if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) { @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, } else { /* load address stored at s->tb_jmp_target_addr + a0 */ tcg_out_ld_abs(s, TCG_TYPE_PTR, TCG_REG_TB, - s->tb_jmp_target_addr + a0); + tcg_splitwx_to_rx(s->tb_jmp_target_addr + a0)); /* and go there */ tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, TCG_REG_TB); } @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, TCG_REG_TB to the beginning of this TB. */ if (USE_REG_TB) { int ofs = -tcg_current_code_size(s); - assert(ofs == (int16_t)ofs); - tcg_out_insn(s, RI, AGHI, TCG_REG_TB, ofs); + /* All TB are restricted to 64KiB by unwind info. */ + tcg_debug_assert(ofs == sextract64(ofs, 0, 20)); + tcg_out_insn(s, RXY, LAY, TCG_REG_TB, + TCG_REG_TB, TCG_REG_NONE, ofs); } break; @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) * Return path for goto_ptr. Set return value to 0, a-la exit_tb, * and fall through to the rest of the epilogue. */ - tcg_code_gen_epilogue = s->code_ptr; + /* TODO: Cast goes away when all hosts converted */ + tcg_code_gen_epilogue = (void *)tcg_splitwx_to_rx(s->code_ptr); tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, 0); /* TB epilogue */ - tb_ret_addr = s->code_ptr; + tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr); /* lmg %r6,%r15,fs+48(%r15) (restore registers) */ tcg_out_insn(s, RXY, LMG, TCG_REG_R6, TCG_REG_R15, TCG_REG_R15, -- 2.25.1 The offset even checks were folded into the range check incorrectly. By offsetting by 1, and not decrementing the width, we silently allowed out of range branches. Assert that the offset is always even instead. Move tcg_out_goto down into the CONFIG_SOFTMMU block so that it is not unused. Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target.c.inc | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool reloc_sbimm12(tcg_insn_unit *code_ptr, tcg_insn_unit *target) { intptr_t offset = (intptr_t)target - (intptr_t)code_ptr; - if (offset == sextreg(offset, 1, 12) << 1) { + tcg_debug_assert((offset & 1) == 0); + if (offset == sextreg(offset, 0, 12)) { code_ptr[0] |= encode_sbimm12(offset); return true; } @@ -XXX,XX +XXX,XX @@ static bool reloc_jimm20(tcg_insn_unit *code_ptr, tcg_insn_unit *target) { intptr_t offset = (intptr_t)target - (intptr_t)code_ptr; - if (offset == sextreg(offset, 1, 20) << 1) { + tcg_debug_assert((offset & 1) == 0); + if (offset == sextreg(offset, 0, 20)) { code_ptr[0] |= encode_ujimm20(offset); return true; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret, g_assert_not_reached(); } -static inline void tcg_out_goto(TCGContext *s, tcg_insn_unit *target) -{ - ptrdiff_t offset = tcg_pcrel_diff(s, target); - tcg_debug_assert(offset == sextreg(offset, 1, 20) << 1); - tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, offset); -} - static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) { TCGReg link = tail ? TCG_REG_ZERO : TCG_REG_RA; ptrdiff_t offset = tcg_pcrel_diff(s, arg); int ret; - if (offset == sextreg(offset, 1, 20) << 1) { + tcg_debug_assert((offset & 1) == 0); + if (offset == sextreg(offset, 0, 20)) { /* short jump: -2097150 to 2097152 */ tcg_out_opc_jump(s, OPC_JAL, link, offset); - } else if (TCG_TARGET_REG_BITS == 32 || - offset == sextreg(offset, 1, 31) << 1) { + } else if (TCG_TARGET_REG_BITS == 32 || offset == (int32_t)offset) { /* long jump: -2147483646 to 2147483648 */ tcg_out_opc_upper(s, OPC_AUIPC, TCG_REG_TMP0, 0); tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, 0); - ret = reloc_call(s->code_ptr - 2, arg);\ + ret = reloc_call(s->code_ptr - 2, arg); tcg_debug_assert(ret == true); } else if (TCG_TARGET_REG_BITS == 64) { /* far jump: 64-bit */ @@ -XXX,XX +XXX,XX @@ QEMU_BUILD_BUG_ON(TCG_TARGET_REG_BITS < TARGET_LONG_BITS); QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 11)); +static void tcg_out_goto(TCGContext *s, tcg_insn_unit *target) +{ + tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, 0); + bool ok = reloc_jimm20(s->code_ptr - 1, target); + tcg_debug_assert(ok); +} + static void tcg_out_tlb_load(TCGContext *s, TCGReg addrl, TCGReg addrh, TCGMemOpIdx oi, tcg_insn_unit **label_ptr, bool is_load) -- 2.25.1 Since 7ecd02a06f8, we are prepared to re-start code generation with a smaller TB if a relocation is out of range. We no longer need to leave a nop in the stream Just In Case. Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target.c.inc | 56 ++++---------------------------------- 1 file changed, 6 insertions(+), 50 deletions(-) diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool reloc_call(tcg_insn_unit *code_ptr, const tcg_insn_unit *target) static bool patch_reloc(tcg_insn_unit *code_ptr, int type, intptr_t value, intptr_t addend) { - uint32_t insn = *code_ptr; - intptr_t diff; - bool short_jmp; - tcg_debug_assert(addend == 0); - switch (type) { case R_RISCV_BRANCH: - diff = value - (uintptr_t)code_ptr; - short_jmp = diff == sextreg(diff, 0, 12); - if (short_jmp) { - return reloc_sbimm12(code_ptr, (tcg_insn_unit *)value); - } else { - /* Invert the condition */ - insn = insn ^ (1 << 12); - /* Clear the offset */ - insn &= 0x01fff07f; - /* Set the offset to the PC + 8 */ - insn |= encode_sbimm12(8); - - /* Move forward */ - code_ptr[0] = insn; - - /* Overwrite the NOP with jal x0,value */ - diff = value - (uintptr_t)(code_ptr + 1); - insn = encode_uj(OPC_JAL, TCG_REG_ZERO, diff); - code_ptr[1] = insn; - - return true; - } - break; + return reloc_sbimm12(code_ptr, (tcg_insn_unit *)value); case R_RISCV_JAL: return reloc_jimm20(code_ptr, (tcg_insn_unit *)value); case R_RISCV_CALL: return reloc_call(code_ptr, (tcg_insn_unit *)value); default: - tcg_abort(); + g_assert_not_reached(); } } @@ -XXX,XX +XXX,XX @@ static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1, arg2 = t; } - if (l->has_value) { - intptr_t diff = tcg_pcrel_diff(s, l->u.value_ptr); - if (diff == sextreg(diff, 0, 12)) { - tcg_out_opc_branch(s, op, arg1, arg2, diff); - } else { - /* Invert the conditional branch. */ - tcg_out_opc_branch(s, op ^ (1 << 12), arg1, arg2, 8); - tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, diff - 4); - } - } else { - tcg_out_reloc(s, s->code_ptr, R_RISCV_BRANCH, l, 0); - tcg_out_opc_branch(s, op, arg1, arg2, 0); - /* NOP to allow patching later */ - tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_ZERO, TCG_REG_ZERO, 0); - } + tcg_out_reloc(s, s->code_ptr, R_RISCV_BRANCH, l, 0); + tcg_out_opc_branch(s, op, arg1, arg2, 0); } static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret, @@ -XXX,XX +XXX,XX @@ static void tcg_out_tlb_load(TCGContext *s, TCGReg addrl, /* Compare masked address with the TLB entry. */ label_ptr[0] = s->code_ptr; tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP0, TCG_REG_TMP1, 0); - /* NOP to allow patching later */ - tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_ZERO, TCG_REG_ZERO, 0); /* TLB Hit - translate address using addend. */ if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) } /* resolve label address */ - if (!patch_reloc(l->label_ptr[0], R_RISCV_BRANCH, - (intptr_t) s->code_ptr, 0)) { + if (!reloc_sbimm12(l->label_ptr[0], s->code_ptr)) { return false; } @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) } /* resolve label address */ - if (!patch_reloc(l->label_ptr[0], R_RISCV_BRANCH, - (intptr_t) s->code_ptr, 0)) { + if (!reloc_sbimm12(l->label_ptr[0], s->code_ptr)) { return false; } -- 2.25.1 Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target.h | 2 +- tcg/riscv/tcg-target.c.inc | 41 +++++++++++++++++++++----------------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_POOL_LABELS #define TCG_TARGET_HAS_MEMORY_BSWAP 0 -#define TCG_TARGET_SUPPORT_MIRROR 0 +#define TCG_TARGET_SUPPORT_MIRROR 1 #endif diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_nop_fill(tcg_insn_unit *p, int count) * Relocations */ -static bool reloc_sbimm12(tcg_insn_unit *code_ptr, tcg_insn_unit *target) +static bool reloc_sbimm12(tcg_insn_unit *src_rw, const tcg_insn_unit *target) { - intptr_t offset = (intptr_t)target - (intptr_t)code_ptr; + const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); + intptr_t offset = (intptr_t)target - (intptr_t)src_rx; tcg_debug_assert((offset & 1) == 0); if (offset == sextreg(offset, 0, 12)) { - code_ptr[0] |= encode_sbimm12(offset); + *src_rw |= encode_sbimm12(offset); return true; } return false; } -static bool reloc_jimm20(tcg_insn_unit *code_ptr, tcg_insn_unit *target) +static bool reloc_jimm20(tcg_insn_unit *src_rw, const tcg_insn_unit *target) { - intptr_t offset = (intptr_t)target - (intptr_t)code_ptr; + const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); + intptr_t offset = (intptr_t)target - (intptr_t)src_rx; tcg_debug_assert((offset & 1) == 0); if (offset == sextreg(offset, 0, 20)) { - code_ptr[0] |= encode_ujimm20(offset); + *src_rw |= encode_ujimm20(offset); return true; } return false; } -static bool reloc_call(tcg_insn_unit *code_ptr, const tcg_insn_unit *target) +static bool reloc_call(tcg_insn_unit *src_rw, const tcg_insn_unit *target) { - intptr_t offset = (intptr_t)target - (intptr_t)code_ptr; + const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); + intptr_t offset = (intptr_t)target - (intptr_t)src_rx; int32_t lo = sextreg(offset, 0, 12); int32_t hi = offset - lo; if (offset == hi + lo) { - code_ptr[0] |= encode_uimm20(hi); - code_ptr[1] |= encode_imm12(lo); + src_rw[0] |= encode_uimm20(hi); + src_rw[1] |= encode_imm12(lo); return true; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, if (tmp == (int32_t)tmp) { tcg_out_opc_upper(s, OPC_AUIPC, rd, 0); tcg_out_opc_imm(s, OPC_ADDI, rd, rd, 0); - ret = reloc_call(s->code_ptr - 2, (tcg_insn_unit *)val); + ret = reloc_call(s->code_ptr - 2, (const tcg_insn_unit *)val); tcg_debug_assert(ret == true); return; } @@ -XXX,XX +XXX,XX @@ QEMU_BUILD_BUG_ON(TCG_TARGET_REG_BITS < TARGET_LONG_BITS); QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 11)); -static void tcg_out_goto(TCGContext *s, tcg_insn_unit *target) +static void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target) { tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, 0); bool ok = reloc_jimm20(s->code_ptr - 1, target); @@ -XXX,XX +XXX,XX @@ static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi, label->datahi_reg = datahi; label->addrlo_reg = addrlo; label->addrhi_reg = addrhi; - label->raddr = raddr; + /* TODO: Cast goes away when all hosts converted */ + label->raddr = (void *)tcg_splitwx_to_rx(raddr); label->label_ptr[0] = label_ptr[0]; } @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) } /* resolve label address */ - if (!reloc_sbimm12(l->label_ptr[0], s->code_ptr)) { + if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { return false; } @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) } /* resolve label address */ - if (!reloc_sbimm12(l->label_ptr[0], s->code_ptr)) { + if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { return false; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) #endif } -static tcg_insn_unit *tb_ret_addr; +static const tcg_insn_unit *tb_ret_addr; static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, const int *const_args) @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, tcg_target_call_iarg_regs[1], 0); /* Return path for goto_ptr. Set return value to 0 */ - tcg_code_gen_epilogue = s->code_ptr; + /* TODO: Cast goes away when all hosts converted */ + tcg_code_gen_epilogue = (void *)tcg_splitwx_to_rx(s->code_ptr); tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_A0, TCG_REG_ZERO); /* TB epilogue */ - tb_ret_addr = s->code_ptr; + tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr); for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) { tcg_out_ld(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i], TCG_REG_SP, SAVE_OFS + i * REG_SIZE); -- 2.25.1 Re-use the 256MiB region handling from alloc_code_gen_buffer_anon, and replace that with the shared file mapping. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/translate-all.c | 46 ++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ static bool alloc_code_gen_buffer_anon(size_t size, int prot, static bool alloc_code_gen_buffer_splitwx_memfd(size_t size, Error **errp) { - void *buf_rw, *buf_rx; + void *buf_rw = NULL, *buf_rx = MAP_FAILED; int fd = -1; +#ifdef __mips__ + /* Find space for the RX mapping, vs the 256MiB regions. */ + if (!alloc_code_gen_buffer_anon(size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | + MAP_NORESERVE, errp)) { + return false; + } + /* The size of the mapping may have been adjusted. */ + size = tcg_ctx->code_gen_buffer_size; + buf_rx = tcg_ctx->code_gen_buffer; +#endif + buf_rw = qemu_memfd_alloc("tcg-jit", size, 0, &fd, errp); if (buf_rw == NULL) { - return false; + goto fail; } +#ifdef __mips__ + void *tmp = mmap(buf_rx, size, PROT_READ | PROT_EXEC, + MAP_SHARED | MAP_FIXED, fd, 0); + if (tmp != buf_rx) { + goto fail_rx; + } +#else buf_rx = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_SHARED, fd, 0); if (buf_rx == MAP_FAILED) { - error_setg_errno(errp, errno, - "failed to map shared memory for execute"); - munmap(buf_rw, size); - close(fd); - return false; + goto fail_rx; } - close(fd); +#endif + close(fd); tcg_ctx->code_gen_buffer = buf_rw; tcg_ctx->code_gen_buffer_size = size; tcg_splitwx_diff = buf_rx - buf_rw; @@ -XXX,XX +XXX,XX @@ static bool alloc_code_gen_buffer_splitwx_memfd(size_t size, Error **errp) qemu_madvise(buf_rw, size, QEMU_MADV_HUGEPAGE); qemu_madvise(buf_rx, size, QEMU_MADV_HUGEPAGE); return true; + + fail_rx: + error_setg_errno(errp, errno, "failed to map shared memory for execute"); + fail: + if (buf_rx != MAP_FAILED) { + munmap(buf_rx, size); + } + if (buf_rw) { + munmap(buf_rw, size); + } + if (fd >= 0) { + close(fd); + } + return false; } #endif /* CONFIG_POSIX */ -- 2.25.1 This target was not updated with 7ecd02a06f8, and so did not allow re-compilation with relocation overflow. Remove reloc_26 and reloc_26_val as unused. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target.c.inc | 53 ++++++++++++++------------------------- 1 file changed, 19 insertions(+), 34 deletions(-) diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static tcg_insn_unit *bswap32_addr; static tcg_insn_unit *bswap32u_addr; static tcg_insn_unit *bswap64_addr; -static inline uint32_t reloc_pc16_val(tcg_insn_unit *pc, - const tcg_insn_unit *target) +static bool reloc_pc16(tcg_insn_unit *pc, const tcg_insn_unit *target) { /* Let the compiler perform the right-shift as part of the arithmetic. */ ptrdiff_t disp = target - (pc + 1); - tcg_debug_assert(disp == (int16_t)disp); - return disp & 0xffff; -} - -static inline void reloc_pc16(tcg_insn_unit *pc, const tcg_insn_unit *target) -{ - *pc = deposit32(*pc, 0, 16, reloc_pc16_val(pc, target)); -} - -static inline uint32_t reloc_26_val(tcg_insn_unit *pc, tcg_insn_unit *target) -{ - tcg_debug_assert((((uintptr_t)pc ^ (uintptr_t)target) & 0xf0000000) == 0); - return ((uintptr_t)target >> 2) & 0x3ffffff; -} - -static inline void reloc_26(tcg_insn_unit *pc, tcg_insn_unit *target) -{ - *pc = deposit32(*pc, 0, 26, reloc_26_val(pc, target)); + if (disp == (int16_t)disp) { + *pc = deposit32(*pc, 0, 16, disp); + return true; + } + return false; } static bool patch_reloc(tcg_insn_unit *code_ptr, int type, @@ -XXX,XX +XXX,XX @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, { tcg_debug_assert(type == R_MIPS_PC16); tcg_debug_assert(addend == 0); - reloc_pc16(code_ptr, (tcg_insn_unit *)value); - return true; + return reloc_pc16(code_ptr, (const tcg_insn_unit *)value); } #define TCG_CT_CONST_ZERO 0x100 @@ -XXX,XX +XXX,XX @@ static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1, } tcg_out_opc_br(s, b_opc, arg1, arg2); - if (l->has_value) { - reloc_pc16(s->code_ptr - 1, l->u.value_ptr); - } else { - tcg_out_reloc(s, s->code_ptr - 1, R_MIPS_PC16, l, 0); - } + tcg_out_reloc(s, s->code_ptr - 1, R_MIPS_PC16, l, 0); tcg_out_nop(s); } @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) int i; /* resolve label address */ - reloc_pc16(l->label_ptr[0], s->code_ptr); - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { - reloc_pc16(l->label_ptr[1], s->code_ptr); + if (!reloc_pc16(l->label_ptr[0], s->code_ptr) + || (TCG_TARGET_REG_BITS < TARGET_LONG_BITS + && !reloc_pc16(l->label_ptr[1], s->code_ptr))) { + return false; } i = 1; @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) } tcg_out_opc_br(s, OPC_BEQ, TCG_REG_ZERO, TCG_REG_ZERO); - reloc_pc16(s->code_ptr - 1, l->raddr); + if (!reloc_pc16(s->code_ptr - 1, l->raddr)) { + return false; + } /* delay slot */ if (TCG_TARGET_REG_BITS == 64 && l->type == TCG_TYPE_I32) { @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) int i; /* resolve label address */ - reloc_pc16(l->label_ptr[0], s->code_ptr); - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { - reloc_pc16(l->label_ptr[1], s->code_ptr); + if (!reloc_pc16(l->label_ptr[0], s->code_ptr) + || (TCG_TARGET_REG_BITS < TARGET_LONG_BITS + && !reloc_pc16(l->label_ptr[1], s->code_ptr))) { + return false; } i = 1; -- 2.25.1 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target.h | 2 +- tcg/mips/tcg-target.c.inc | 43 ++++++++++++++++++++++----------------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions; #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 1 -#define TCG_TARGET_SUPPORT_MIRROR 0 +#define TCG_TARGET_SUPPORT_MIRROR 1 void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const TCGReg tcg_target_call_oarg_regs[2] = { TCG_REG_V1 }; -static tcg_insn_unit *tb_ret_addr; -static tcg_insn_unit *bswap32_addr; -static tcg_insn_unit *bswap32u_addr; -static tcg_insn_unit *bswap64_addr; +static const tcg_insn_unit *tb_ret_addr; +static const tcg_insn_unit *bswap32_addr; +static const tcg_insn_unit *bswap32u_addr; +static const tcg_insn_unit *bswap64_addr; -static bool reloc_pc16(tcg_insn_unit *pc, const tcg_insn_unit *target) +static bool reloc_pc16(tcg_insn_unit *src_rw, const tcg_insn_unit *target) { /* Let the compiler perform the right-shift as part of the arithmetic. */ - ptrdiff_t disp = target - (pc + 1); + const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); + ptrdiff_t disp = target - (src_rx + 1); if (disp == (int16_t)disp) { - *pc = deposit32(*pc, 0, 16, disp); + *src_rw = deposit32(*src_rw, 0, 16, disp); return true; } return false; @@ -XXX,XX +XXX,XX @@ static void tcg_out_opc_sa64(TCGContext *s, MIPSInsn opc1, MIPSInsn opc2, static bool tcg_out_opc_jmp(TCGContext *s, MIPSInsn opc, const void *target) { uintptr_t dest = (uintptr_t)target; - uintptr_t from = (uintptr_t)s->code_ptr + 4; + uintptr_t from = (uintptr_t)tcg_splitwx_to_rx(s->code_ptr) + 4; int32_t inst; /* The pc-region branch happens within the 256MB region of @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_bswap16s(TCGContext *s, TCGReg ret, TCGReg arg) } } -static void tcg_out_bswap_subr(TCGContext *s, tcg_insn_unit *sub) +static void tcg_out_bswap_subr(TCGContext *s, const tcg_insn_unit *sub) { bool ok = tcg_out_opc_jmp(s, OPC_JAL, sub); tcg_debug_assert(ok); @@ -XXX,XX +XXX,XX @@ static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi, label->datahi_reg = datahi; label->addrlo_reg = addrlo; label->addrhi_reg = addrhi; - label->raddr = raddr; + /* TODO: Cast goes away when all hosts converted */ + label->raddr = (void *)tcg_splitwx_to_rx(raddr); label->label_ptr[0] = label_ptr[0]; if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { label->label_ptr[1] = label_ptr[1]; @@ -XXX,XX +XXX,XX @@ static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi, static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) { + const tcg_insn_unit *tgt_rx = tcg_splitwx_to_rx(s->code_ptr); TCGMemOpIdx oi = l->oi; MemOp opc = get_memop(oi); TCGReg v0; int i; /* resolve label address */ - if (!reloc_pc16(l->label_ptr[0], s->code_ptr) + if (!reloc_pc16(l->label_ptr[0], tgt_rx) || (TCG_TARGET_REG_BITS < TARGET_LONG_BITS - && !reloc_pc16(l->label_ptr[1], s->code_ptr))) { + && !reloc_pc16(l->label_ptr[1], tgt_rx))) { return false; } @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) { + const tcg_insn_unit *tgt_rx = tcg_splitwx_to_rx(s->code_ptr); TCGMemOpIdx oi = l->oi; MemOp opc = get_memop(oi); MemOp s_bits = opc & MO_SIZE; int i; /* resolve label address */ - if (!reloc_pc16(l->label_ptr[0], s->code_ptr) + if (!reloc_pc16(l->label_ptr[0], tgt_rx) || (TCG_TARGET_REG_BITS < TARGET_LONG_BITS - && !reloc_pc16(l->label_ptr[1], s->code_ptr))) { + && !reloc_pc16(l->label_ptr[1], tgt_rx))) { return false; } @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) * Return path for goto_ptr. Set return value to 0, a-la exit_tb, * and fall through to the rest of the epilogue. */ - tcg_code_gen_epilogue = s->code_ptr; + /* TODO: Cast goes away when all hosts converted */ + tcg_code_gen_epilogue = (void *)tcg_splitwx_to_rx(s->code_ptr); tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_V0, TCG_REG_ZERO); /* TB epilogue */ - tb_ret_addr = s->code_ptr; + tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr); for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) { tcg_out_ld(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i], TCG_REG_SP, SAVE_OFS + i * REG_SIZE); @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) /* * bswap32 -- 32-bit swap (signed result for mips64). a0 = abcd. */ - bswap32_addr = align_code_ptr(s); + bswap32_addr = tcg_splitwx_to_rx(align_code_ptr(s)); /* t3 = (ssss)d000 */ tcg_out_opc_sa(s, OPC_SLL, TCG_TMP3, TCG_TMP0, 24); /* t1 = 000a */ @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) /* * bswap32u -- unsigned 32-bit swap. a0 = ....abcd. */ - bswap32u_addr = align_code_ptr(s); + bswap32u_addr = tcg_splitwx_to_rx(align_code_ptr(s)); /* t1 = (0000)000d */ tcg_out_opc_imm(s, OPC_ANDI, TCG_TMP1, TCG_TMP0, 0xff); /* t3 = 000a */ @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) /* * bswap64 -- 64-bit swap. a0 = abcdefgh */ - bswap64_addr = align_code_ptr(s); + bswap64_addr = tcg_splitwx_to_rx(align_code_ptr(s)); /* t3 = h0000000 */ tcg_out_dsll(s, TCG_TMP3, TCG_TMP0, 56); /* t1 = 0000000a */ -- 2.25.1 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/arm/tcg-target.h | 2 +- tcg/arm/tcg-target.c.inc | 37 +++++++++++++++++++++---------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_SUPPORT_MIRROR 0 +#define TCG_TARGET_SUPPORT_MIRROR 1 #endif diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const uint8_t tcg_cond_to_arm_cond[] = { [TCG_COND_GTU] = COND_HI, }; -static inline bool reloc_pc24(tcg_insn_unit *code_ptr, tcg_insn_unit *target) +static bool reloc_pc24(tcg_insn_unit *src_rw, const tcg_insn_unit *target) { - ptrdiff_t offset = (tcg_ptr_byte_diff(target, code_ptr) - 8) >> 2; + const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); + ptrdiff_t offset = (tcg_ptr_byte_diff(target, src_rx) - 8) >> 2; + if (offset == sextract32(offset, 0, 24)) { - *code_ptr = (*code_ptr & ~0xffffff) | (offset & 0xffffff); + *src_rw = deposit32(*src_rw, 0, 24, offset); return true; } return false; } -static inline bool reloc_pc13(tcg_insn_unit *code_ptr, tcg_insn_unit *target) +static bool reloc_pc13(tcg_insn_unit *src_rw, const tcg_insn_unit *target) { - ptrdiff_t offset = tcg_ptr_byte_diff(target, code_ptr) - 8; + const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); + ptrdiff_t offset = tcg_ptr_byte_diff(target, src_rx) - 8; if (offset >= -0xfff && offset <= 0xfff) { - tcg_insn_unit insn = *code_ptr; + tcg_insn_unit insn = *src_rw; bool u = (offset >= 0); if (!u) { offset = -offset; } insn = deposit32(insn, 23, 1, u); insn = deposit32(insn, 0, 12, offset); - *code_ptr = insn; + *src_rw = insn; return true; } return false; @@ -XXX,XX +XXX,XX @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, tcg_debug_assert(addend == 0); if (type == R_ARM_PC24) { - return reloc_pc24(code_ptr, (tcg_insn_unit *)value); + return reloc_pc24(code_ptr, (const tcg_insn_unit *)value); } else if (type == R_ARM_PC13) { - return reloc_pc13(code_ptr, (tcg_insn_unit *)value); + return reloc_pc13(code_ptr, (const tcg_insn_unit *)value); } else { g_assert_not_reached(); } @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi32(TCGContext *s, int cond, int rd, uint32_t arg) /* Check for a pc-relative address. This will usually be the TB, or within the TB, which is immediately before the code block. */ - diff = arg - ((intptr_t)s->code_ptr + 8); + diff = tcg_pcrel_diff(s, (void *)arg) - 8; if (diff >= 0) { rot = encode_imm(diff); if (rot >= 0) { @@ -XXX,XX +XXX,XX @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi, label->datahi_reg = datahi; label->addrlo_reg = addrlo; label->addrhi_reg = addrhi; - label->raddr = raddr; + /* TODO: Cast goes away when all hosts converted */ + label->raddr = (void *)tcg_splitwx_to_rx(raddr); label->label_ptr[0] = label_ptr; } @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) MemOp opc = get_memop(oi); void *func; - if (!reloc_pc24(lb->label_ptr[0], s->code_ptr)) { + if (!reloc_pc24(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { return false; } @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) TCGMemOpIdx oi = lb->oi; MemOp opc = get_memop(oi); - if (!reloc_pc24(lb->label_ptr[0], s->code_ptr)) { + if (!reloc_pc24(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { return false; } @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGReg base = TCG_REG_PC; tcg_debug_assert(s->tb_jmp_insn_offset == 0); - ptr = (intptr_t)(s->tb_jmp_target_addr + args[0]); - dif = ptr - ((intptr_t)s->code_ptr + 8); + ptr = (intptr_t)tcg_splitwx_to_rx(s->tb_jmp_target_addr + args[0]); + dif = tcg_pcrel_diff(s, (void *)ptr) - 8; dil = sextract32(dif, 0, 12); if (dif != dil) { /* The TB is close, but outside the 12 bits addressable by @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) * Return path for goto_ptr. Set return value to 0, a-la exit_tb, * and fall through to the rest of the epilogue. */ - tcg_code_gen_epilogue = s->code_ptr; + /* TODO: Cast goes away when all hosts converted */ + tcg_code_gen_epilogue = (void *)tcg_splitwx_to_rx(s->code_ptr); tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, 0); tcg_out_epilogue(s); } -- 2.25.1 Now that all native tcg hosts support splitwx, remove the define. Replace the one use with a test for CONFIG_TCG_INTERPRETER. Reviewed-by: Joelle van Dyne <j@getutm.app> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target.h | 1 - tcg/arm/tcg-target.h | 1 - tcg/i386/tcg-target.h | 1 - tcg/mips/tcg-target.h | 1 - tcg/ppc/tcg-target.h | 1 - tcg/riscv/tcg-target.h | 1 - tcg/s390/tcg-target.h | 1 - tcg/sparc/tcg-target.h | 1 - tcg/tci/tcg-target.h | 1 - accel/tcg/translate-all.c | 16 +++++++++------- 10 files changed, 9 insertions(+), 16 deletions(-) diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_SUPPORT_MIRROR 1 #endif /* AARCH64_TCG_TARGET_H */ diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_SUPPORT_MIRROR 1 #endif diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_SUPPORT_MIRROR 1 #endif diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions; #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 1 -#define TCG_TARGET_SUPPORT_MIRROR 1 void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_SUPPORT_MIRROR 1 #endif diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_POOL_LABELS #define TCG_TARGET_HAS_MEMORY_BSWAP 0 -#define TCG_TARGET_SUPPORT_MIRROR 1 #endif diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.h +++ b/tcg/s390/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_SUPPORT_MIRROR 1 #endif diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.h +++ b/tcg/sparc/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions; void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_SUPPORT_MIRROR 1 #endif diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tci_disas(uint8_t opc); #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 1 -#define TCG_TARGET_SUPPORT_MIRROR 0 static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, uintptr_t jmp_rw, uintptr_t addr) diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ static bool alloc_code_gen_buffer_anon(size_t size, int prot, return true; } +#ifndef CONFIG_TCG_INTERPRETER #ifdef CONFIG_POSIX #include "qemu/memfd.h" @@ -XXX,XX +XXX,XX @@ static bool alloc_code_gen_buffer_splitwx_vmremap(size_t size, Error **errp) return true; } #endif /* CONFIG_DARWIN */ +#endif /* CONFIG_TCG_INTERPRETER */ static bool alloc_code_gen_buffer_splitwx(size_t size, Error **errp) { - if (TCG_TARGET_SUPPORT_MIRROR) { -#ifdef CONFIG_DARWIN - return alloc_code_gen_buffer_splitwx_vmremap(size, errp); +#ifndef CONFIG_TCG_INTERPRETER +# ifdef CONFIG_DARWIN + return alloc_code_gen_buffer_splitwx_vmremap(size, errp); +# endif +# ifdef CONFIG_POSIX + return alloc_code_gen_buffer_splitwx_memfd(size, errp); +# endif #endif -#ifdef CONFIG_POSIX - return alloc_code_gen_buffer_splitwx_memfd(size, errp); -#endif - } error_setg(errp, "jit split-wx not supported"); return false; } -- 2.25.1 Now that all native tcg hosts support splitwx, make this pointer const. Reviewed-by: Joelle van Dyne <j@getutm.app> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg.h | 2 +- tcg/tcg.c | 2 +- tcg/aarch64/tcg-target.c.inc | 3 +-- tcg/arm/tcg-target.c.inc | 3 +-- tcg/i386/tcg-target.c.inc | 3 +-- tcg/mips/tcg-target.c.inc | 3 +-- tcg/ppc/tcg-target.c.inc | 3 +-- tcg/riscv/tcg-target.c.inc | 3 +-- tcg/s390/tcg-target.c.inc | 3 +-- tcg/sparc/tcg-target.c.inc | 3 +-- 10 files changed, 10 insertions(+), 18 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ struct TCGContext { extern TCGContext tcg_init_ctx; extern __thread TCGContext *tcg_ctx; -extern void *tcg_code_gen_epilogue; +extern const void *tcg_code_gen_epilogue; extern uintptr_t tcg_splitwx_diff; extern TCGv_env cpu_env; diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static int tcg_out_ldst_finalize(TCGContext *s); static TCGContext **tcg_ctxs; static unsigned int n_tcg_ctxs; TCGv_env cpu_env = 0; -void *tcg_code_gen_epilogue; +const void *tcg_code_gen_epilogue; uintptr_t tcg_splitwx_diff; #ifndef CONFIG_TCG_INTERPRETER diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) * Return path for goto_ptr. Set return value to 0, a-la exit_tb, * and fall through to the rest of the epilogue. */ - /* TODO: Cast goes away when all hosts converted */ - tcg_code_gen_epilogue = (void *)tcg_splitwx_to_rx(s->code_ptr); + tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr); tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_X0, 0); /* TB epilogue */ diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) * Return path for goto_ptr. Set return value to 0, a-la exit_tb, * and fall through to the rest of the epilogue. */ - /* TODO: Cast goes away when all hosts converted */ - tcg_code_gen_epilogue = (void *)tcg_splitwx_to_rx(s->code_ptr); + tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr); tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, 0); tcg_out_epilogue(s); } diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) * Return path for goto_ptr. Set return value to 0, a-la exit_tb, * and fall through to the rest of the epilogue. */ - /* TODO: Cast goes away when all hosts converted */ - tcg_code_gen_epilogue = (void *)tcg_splitwx_to_rx(s->code_ptr); + tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr); tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_EAX, 0); /* TB epilogue */ diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) * Return path for goto_ptr. Set return value to 0, a-la exit_tb, * and fall through to the rest of the epilogue. */ - /* TODO: Cast goes away when all hosts converted */ - tcg_code_gen_epilogue = (void *)tcg_splitwx_to_rx(s->code_ptr); + tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr); tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_V0, TCG_REG_ZERO); /* TB epilogue */ diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out32(s, BCCTR | BO_ALWAYS); /* Epilogue */ - /* TODO: Cast goes away when all hosts converted */ - tcg_code_gen_epilogue = (void *)tcg_splitwx_to_rx(s->code_ptr); + tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr); tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET); for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) { diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, tcg_target_call_iarg_regs[1], 0); /* Return path for goto_ptr. Set return value to 0 */ - /* TODO: Cast goes away when all hosts converted */ - tcg_code_gen_epilogue = (void *)tcg_splitwx_to_rx(s->code_ptr); + tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr); tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_A0, TCG_REG_ZERO); /* TB epilogue */ diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.c.inc +++ b/tcg/s390/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) * Return path for goto_ptr. Set return value to 0, a-la exit_tb, * and fall through to the rest of the epilogue. */ - /* TODO: Cast goes away when all hosts converted */ - tcg_code_gen_epilogue = (void *)tcg_splitwx_to_rx(s->code_ptr); + tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr); tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, 0); /* TB epilogue */ diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out_nop(s); /* Epilogue for goto_ptr. */ - /* TODO: Cast goes away when all hosts converted */ - tcg_code_gen_epilogue = (void *)tcg_splitwx_to_rx(s->code_ptr); + tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr); tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); /* delay slot */ tcg_out_movi_imm13(s, TCG_REG_O0, 0); -- 2.25.1 Now that all native tcg hosts support splitwx, make this pointer const. Reviewed-by: Joelle van Dyne <j@getutm.app> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target.c.inc | 3 +-- tcg/arm/tcg-target.c.inc | 3 +-- tcg/i386/tcg-target.c.inc | 3 +-- tcg/mips/tcg-target.c.inc | 3 +-- tcg/ppc/tcg-target.c.inc | 3 +-- tcg/riscv/tcg-target.c.inc | 3 +-- tcg/s390/tcg-target.c.inc | 3 +-- tcg/tcg-ldst.c.inc | 2 +- 8 files changed, 8 insertions(+), 15 deletions(-) diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi, label->type = ext; label->datalo_reg = data_reg; label->addrlo_reg = addr_reg; - /* TODO: Cast goes away when all hosts converted */ - label->raddr = (void *)tcg_splitwx_to_rx(raddr); + label->raddr = tcg_splitwx_to_rx(raddr); label->label_ptr[0] = label_ptr; } diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi, label->datahi_reg = datahi; label->addrlo_reg = addrlo; label->addrhi_reg = addrhi; - /* TODO: Cast goes away when all hosts converted */ - label->raddr = (void *)tcg_splitwx_to_rx(raddr); + label->raddr = tcg_splitwx_to_rx(raddr); label->label_ptr[0] = label_ptr; } diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, bool is_64, label->datahi_reg = datahi; label->addrlo_reg = addrlo; label->addrhi_reg = addrhi; - /* TODO: Cast goes away when all hosts converted */ - label->raddr = (void *)tcg_splitwx_to_rx(raddr); + label->raddr = tcg_splitwx_to_rx(raddr); label->label_ptr[0] = label_ptr[0]; if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) { label->label_ptr[1] = label_ptr[1]; diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi, label->datahi_reg = datahi; label->addrlo_reg = addrlo; label->addrhi_reg = addrhi; - /* TODO: Cast goes away when all hosts converted */ - label->raddr = (void *)tcg_splitwx_to_rx(raddr); + label->raddr = tcg_splitwx_to_rx(raddr); label->label_ptr[0] = label_ptr[0]; if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { label->label_ptr[1] = label_ptr[1]; diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi, label->datahi_reg = datahi_reg; label->addrlo_reg = addrlo_reg; label->addrhi_reg = addrhi_reg; - /* TODO: Cast goes away when all hosts converted */ - label->raddr = (void *)tcg_splitwx_to_rx(raddr); + label->raddr = tcg_splitwx_to_rx(raddr); label->label_ptr[0] = lptr; } diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi, label->datahi_reg = datahi; label->addrlo_reg = addrlo; label->addrhi_reg = addrhi; - /* TODO: Cast goes away when all hosts converted */ - label->raddr = (void *)tcg_splitwx_to_rx(raddr); + label->raddr = tcg_splitwx_to_rx(raddr); label->label_ptr[0] = label_ptr[0]; } diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.c.inc +++ b/tcg/s390/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi, label->oi = oi; label->datalo_reg = data; label->addrlo_reg = addr; - /* TODO: Cast goes away when all hosts converted */ - label->raddr = (void *)tcg_splitwx_to_rx(raddr); + label->raddr = tcg_splitwx_to_rx(raddr); label->label_ptr[0] = label_ptr; } diff --git a/tcg/tcg-ldst.c.inc b/tcg/tcg-ldst.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-ldst.c.inc +++ b/tcg/tcg-ldst.c.inc @@ -XXX,XX +XXX,XX @@ typedef struct TCGLabelQemuLdst { TCGReg addrhi_reg; /* reg index for high word of guest virtual addr */ TCGReg datalo_reg; /* reg index for low word to be loaded or stored */ TCGReg datahi_reg; /* reg index for high word to be loaded or stored */ - tcg_insn_unit *raddr; /* gen code addr of the next IR of qemu_ld/st IR */ + const tcg_insn_unit *raddr; /* addr of the next IR of qemu_ld/st IR */ tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */ QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next; } TCGLabelQemuLdst; -- 2.25.1 The following changes since commit aa3a285b5bc56a4208b3b57d4a55291e9c260107: Merge tag 'mem-2024-12-21' of https://github.com/davidhildenbrand/qemu into staging (2024-12-22 14:33:27 -0500) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20241224 for you to fetch changes up to e4a8e093dc74be049f4829831dce76e5edab0003: accel/tcg: Move gen_intermediate_code to TCGCPUOps.translate_core (2024-12-24 08:32:15 -0800) ---------------------------------------------------------------- tcg/optimize: Remove in-flight mask data from OptContext fpu: Add float*_muladd_scalbn fpu: Remove float_muladd_halve_result fpu: Add float_round_nearest_even_max fpu: Add float_muladd_suppress_add_product_zero target/hexagon: Use float32_muladd accel/tcg: Move gen_intermediate_code to TCGCPUOps.translate_core ---------------------------------------------------------------- Ilya Leoshkevich (1): tests/tcg: Do not use inttypes.h in multiarch/system/memory.c Pierrick Bouvier (1): plugins: optimize cpu_index code generation Richard Henderson (70): tcg/optimize: Split out finish_bb, finish_ebb tcg/optimize: Split out fold_affected_mask tcg/optimize: Copy mask writeback to fold_masks tcg/optimize: Split out fold_masks_zs tcg/optimize: Augment s_mask from z_mask in fold_masks_zs tcg/optimize: Change representation of s_mask tcg/optimize: Use finish_folding in fold_add, fold_add_vec, fold_addsub2 tcg/optimize: Introduce const value accessors for TempOptInfo tcg/optimize: Use fold_masks_zs in fold_and tcg/optimize: Use fold_masks_zs in fold_andc tcg/optimize: Use fold_masks_zs in fold_bswap tcg/optimize: Use fold_masks_zs in fold_count_zeros tcg/optimize: Use fold_masks_z in fold_ctpop tcg/optimize: Use fold_and and fold_masks_z in fold_deposit tcg/optimize: Compute sign mask in fold_deposit tcg/optimize: Use finish_folding in fold_divide tcg/optimize: Use finish_folding in fold_dup, fold_dup2 tcg/optimize: Use fold_masks_s in fold_eqv tcg/optimize: Use fold_masks_z in fold_extract tcg/optimize: Use finish_folding in fold_extract2 tcg/optimize: Use fold_masks_zs in fold_exts tcg/optimize: Use fold_masks_z in fold_extu tcg/optimize: Use fold_masks_zs in fold_movcond tcg/optimize: Use finish_folding in fold_mul* tcg/optimize: Use fold_masks_s in fold_nand tcg/optimize: Use fold_masks_z in fold_neg_no_const tcg/optimize: Use fold_masks_s in fold_nor tcg/optimize: Use fold_masks_s in fold_not tcg/optimize: Use fold_masks_zs in fold_or tcg/optimize: Use fold_masks_zs in fold_orc tcg/optimize: Use fold_masks_zs in fold_qemu_ld tcg/optimize: Return true from fold_qemu_st, fold_tcg_st tcg/optimize: Use finish_folding in fold_remainder tcg/optimize: Distinguish simplification in fold_setcond_zmask tcg/optimize: Use fold_masks_z in fold_setcond tcg/optimize: Use fold_masks_s in fold_negsetcond tcg/optimize: Use fold_masks_z in fold_setcond2 tcg/optimize: Use finish_folding in fold_cmp_vec tcg/optimize: Use finish_folding in fold_cmpsel_vec tcg/optimize: Use fold_masks_zs in fold_sextract tcg/optimize: Use fold_masks_zs, fold_masks_s in fold_shift tcg/optimize: Simplify sign bit test in fold_shift tcg/optimize: Use finish_folding in fold_sub, fold_sub_vec tcg/optimize: Use fold_masks_zs in fold_tcg_ld tcg/optimize: Use finish_folding in fold_tcg_ld_memcopy tcg/optimize: Use fold_masks_zs in fold_xor tcg/optimize: Use finish_folding in fold_bitsel_vec tcg/optimize: Use finish_folding as default in tcg_optimize tcg/optimize: Remove z_mask, s_mask from OptContext tcg/optimize: Re-enable sign-mask optimizations tcg/optimize: Move fold_bitsel_vec into alphabetic sort tcg/optimize: Move fold_cmp_vec, fold_cmpsel_vec into alphabetic sort softfloat: Add float{16,32,64}_muladd_scalbn target/arm: Use float*_muladd_scalbn target/sparc: Use float*_muladd_scalbn softfloat: Remove float_muladd_halve_result softfloat: Add float_round_nearest_even_max softfloat: Add float_muladd_suppress_add_product_zero target/hexagon: Use float32_mul in helper_sfmpy target/hexagon: Use float32_muladd for helper_sffma target/hexagon: Use float32_muladd for helper_sffms target/hexagon: Use float32_muladd_scalbn for helper_sffma_sc target/hexagon: Use float32_muladd for helper_sffm[as]_lib target/hexagon: Remove internal_fmafx target/hexagon: Expand GEN_XF_ROUND target/hexagon: Remove Float target/hexagon: Remove Double target/hexagon: Use mulu64 for int128_mul_6464 target/hexagon: Simplify internal_mpyhh setup accel/tcg: Move gen_intermediate_code to TCGCPUOps.translate_core include/exec/translator.h | 14 - include/fpu/softfloat-types.h | 2 + include/fpu/softfloat.h | 14 +- include/hw/core/tcg-cpu-ops.h | 13 + target/alpha/cpu.h | 2 + target/arm/internals.h | 2 + target/avr/cpu.h | 2 + target/hexagon/cpu.h | 2 + target/hexagon/fma_emu.h | 3 - target/hppa/cpu.h | 2 + target/i386/tcg/helper-tcg.h | 2 + target/loongarch/internals.h | 2 + target/m68k/cpu.h | 2 + target/microblaze/cpu.h | 2 + target/mips/tcg/tcg-internal.h | 2 + target/openrisc/cpu.h | 2 + target/ppc/cpu.h | 2 + target/riscv/cpu.h | 3 + target/rx/cpu.h | 2 + target/s390x/s390x-internal.h | 2 + target/sh4/cpu.h | 2 + target/sparc/cpu.h | 2 + target/sparc/helper.h | 4 +- target/tricore/cpu.h | 2 + target/xtensa/cpu.h | 2 + accel/tcg/cpu-exec.c | 8 +- accel/tcg/plugin-gen.c | 9 + accel/tcg/translate-all.c | 8 +- fpu/softfloat.c | 63 +-- target/alpha/cpu.c | 1 + target/alpha/translate.c | 4 +- target/arm/cpu.c | 1 + target/arm/tcg/cpu-v7m.c | 1 + target/arm/tcg/helper-a64.c | 6 +- target/arm/tcg/translate.c | 5 +- target/avr/cpu.c | 1 + target/avr/translate.c | 6 +- target/hexagon/cpu.c | 1 + target/hexagon/fma_emu.c | 496 ++++++--------------- target/hexagon/op_helper.c | 125 ++---- target/hexagon/translate.c | 4 +- target/hppa/cpu.c | 1 + target/hppa/translate.c | 4 +- target/i386/tcg/tcg-cpu.c | 1 + target/i386/tcg/translate.c | 5 +- target/loongarch/cpu.c | 1 + target/loongarch/tcg/translate.c | 4 +- target/m68k/cpu.c | 1 + target/m68k/translate.c | 4 +- target/microblaze/cpu.c | 1 + target/microblaze/translate.c | 4 +- target/mips/cpu.c | 1 + target/mips/tcg/translate.c | 4 +- target/openrisc/cpu.c | 1 + target/openrisc/translate.c | 4 +- target/ppc/cpu_init.c | 1 + target/ppc/translate.c | 4 +- target/riscv/tcg/tcg-cpu.c | 1 + target/riscv/translate.c | 4 +- target/rx/cpu.c | 1 + target/rx/translate.c | 4 +- target/s390x/cpu.c | 1 + target/s390x/tcg/translate.c | 4 +- target/sh4/cpu.c | 1 + target/sh4/translate.c | 4 +- target/sparc/cpu.c | 1 + target/sparc/fop_helper.c | 8 +- target/sparc/translate.c | 84 ++-- target/tricore/cpu.c | 1 + target/tricore/translate.c | 5 +- target/xtensa/cpu.c | 1 + target/xtensa/translate.c | 4 +- tcg/optimize.c | 857 +++++++++++++++++++----------------- tests/tcg/multiarch/system/memory.c | 9 +- fpu/softfloat-parts.c.inc | 16 +- 75 files changed, 866 insertions(+), 1009 deletions(-) From: Ilya Leoshkevich <iii@linux.ibm.com> make check-tcg fails on Fedora with the following error message: alpha-linux-gnu-gcc [...] qemu/tests/tcg/multiarch/system/memory.c -o memory [...] qemu/tests/tcg/multiarch/system/memory.c:17:10: fatal error: inttypes.h: No such file or directory 17 | #include <inttypes.h> | ^~~~~~~~~~~~ compilation terminated. The reason is that Fedora has cross-compilers, but no cross-glibc headers. Fix by hardcoding the format specifiers and dropping the include. An alternative fix would be to introduce a configure check for inttypes.h. But this would make it impossible to use Fedora cross-compilers for softmmu tests, which used to work so far. Fixes: ecbcc9ead2f8 ("tests/tcg: add a system test to check memory instrumentation") Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com> Reviewed-by: Paolo Bonzini <pbonzini@redhat.com> Message-ID: <20241010085906.226249-1-iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tests/tcg/multiarch/system/memory.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/tcg/multiarch/system/memory.c b/tests/tcg/multiarch/system/memory.c index XXXXXXX..XXXXXXX 100644 --- a/tests/tcg/multiarch/system/memory.c +++ b/tests/tcg/multiarch/system/memory.c @@ -XXX,XX +XXX,XX @@ #include <stdint.h> #include <stdbool.h> -#include <inttypes.h> #include <minilib.h> #ifndef CHECK_UNALIGNED @@ -XXX,XX +XXX,XX @@ int main(void) int i; bool ok = true; - ml_printf("Test data start: 0x%"PRIxPTR"\n", &test_data[0]); - ml_printf("Test data end: 0x%"PRIxPTR"\n", &test_data[TEST_SIZE]); + ml_printf("Test data start: 0x%lx\n", (unsigned long)&test_data[0]); + ml_printf("Test data end: 0x%lx\n", (unsigned long)&test_data[TEST_SIZE]); /* Run through the unsigned tests first */ for (i = 0; i < ARRAY_SIZE(init_ufns) && ok; i++) { @@ -XXX,XX +XXX,XX @@ int main(void) ok = do_signed_reads(true); } - ml_printf("Test data read: %"PRId32"\n", test_read_count); - ml_printf("Test data write: %"PRId32"\n", test_write_count); + ml_printf("Test data read: %lu\n", (unsigned long)test_read_count); + ml_printf("Test data write: %lu\n", (unsigned long)test_write_count); ml_printf("Test complete: %s\n", ok ? "PASSED" : "FAILED"); return ok ? 0 : -1; } -- 2.43.0 From: Pierrick Bouvier <pierrick.bouvier@linaro.org> When running with a single vcpu, we can return a constant instead of a load when accessing cpu_index. A side effect is that all tcg operations using it are optimized, most notably scoreboard access. When running a simple loop in user-mode, the speedup is around 20%. Signed-off-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-ID: <20241128213843.1023080-1-pierrick.bouvier@linaro.org> --- accel/tcg/plugin-gen.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -XXX,XX +XXX,XX @@ static void gen_disable_mem_helper(void) static TCGv_i32 gen_cpu_index(void) { + /* + * Optimize when we run with a single vcpu. All values using cpu_index, + * including scoreboard index, will be optimized out. + * User-mode calls tb_flush when setting this flag. In system-mode, all + * vcpus are created before generating code. + */ + if (!tcg_cflags_has(current_cpu, CF_PARALLEL)) { + return tcg_constant_i32(current_cpu->cpu_index); + } TCGv_i32 cpu_index = tcg_temp_ebb_new_i32(); tcg_gen_ld_i32(cpu_index, tcg_env, -offsetof(ArchCPU, env) + offsetof(CPUState, cpu_index)); -- 2.43.0 Call them directly from the opcode switch statement in tcg_optimize, rather than in finish_folding based on opcode flags. Adjust folding of conditional branches to match. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 47 +++++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void copy_propagate(OptContext *ctx, TCGOp *op, } } +static void finish_bb(OptContext *ctx) +{ + /* We only optimize memory barriers across basic blocks. */ + ctx->prev_mb = NULL; +} + +static void finish_ebb(OptContext *ctx) +{ + finish_bb(ctx); + /* We only optimize across extended basic blocks. */ + memset(&ctx->temps_used, 0, sizeof(ctx->temps_used)); + remove_mem_copy_all(ctx); +} + static void finish_folding(OptContext *ctx, TCGOp *op) { const TCGOpDef *def = &tcg_op_defs[op->opc]; int i, nb_oargs; - /* - * We only optimize extended basic blocks. If the opcode ends a BB - * and is not a conditional branch, reset all temp data. - */ - if (def->flags & TCG_OPF_BB_END) { - ctx->prev_mb = NULL; - if (!(def->flags & TCG_OPF_COND_BRANCH)) { - memset(&ctx->temps_used, 0, sizeof(ctx->temps_used)); - remove_mem_copy_all(ctx); - } - return; - } - nb_oargs = def->nb_oargs; for (i = 0; i < nb_oargs; i++) { TCGTemp *ts = arg_temp(op->args[i]); @@ -XXX,XX +XXX,XX @@ static bool fold_brcond(OptContext *ctx, TCGOp *op) if (i > 0) { op->opc = INDEX_op_br; op->args[0] = op->args[3]; + finish_ebb(ctx); + } else { + finish_bb(ctx); } - return false; + return true; } static bool fold_brcond2(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) } op->opc = INDEX_op_br; op->args[0] = label; - break; + finish_ebb(ctx); + return true; } - return false; + + finish_bb(ctx); + return true; } static bool fold_bswap(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(xor): done = fold_xor(&ctx, op); break; + case INDEX_op_set_label: + case INDEX_op_br: + case INDEX_op_exit_tb: + case INDEX_op_goto_tb: + case INDEX_op_goto_ptr: + finish_ebb(&ctx); + done = true; + break; default: break; } -- 2.43.0 There are only a few logical operations which can compute an "affected" mask. Split out handling of this optimization to a separate function, only to be called when applicable. Remove the a_mask field from OptContext, as the mask is no longer stored anywhere. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { QSIMPLEQ_HEAD(, MemCopyInfo) mem_free; /* In flight values from optimization. */ - uint64_t a_mask; /* mask bit is 0 iff value identical to first input */ uint64_t z_mask; /* mask bit is 0 iff value bit is 0 */ uint64_t s_mask; /* mask of clrsb(value) bits */ TCGType type; @@ -XXX,XX +XXX,XX @@ static bool fold_const2_commutative(OptContext *ctx, TCGOp *op) static bool fold_masks(OptContext *ctx, TCGOp *op) { - uint64_t a_mask = ctx->a_mask; uint64_t z_mask = ctx->z_mask; uint64_t s_mask = ctx->s_mask; @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) * type changing opcodes. */ if (ctx->type == TCG_TYPE_I32) { - a_mask = (int32_t)a_mask; z_mask = (int32_t)z_mask; s_mask |= MAKE_64BIT_MASK(32, 32); ctx->z_mask = z_mask; @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) if (z_mask == 0) { return tcg_opt_gen_movi(ctx, op, op->args[0], 0); } + return false; +} + +/* + * An "affected" mask bit is 0 if and only if the result is identical + * to the first input. Thus if the entire mask is 0, the operation + * is equivalent to a copy. + */ +static bool fold_affected_mask(OptContext *ctx, TCGOp *op, uint64_t a_mask) +{ + if (ctx->type == TCG_TYPE_I32) { + a_mask = (uint32_t)a_mask; + } if (a_mask == 0) { return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); } @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) * Known-zeros does not imply known-ones. Therefore unless * arg2 is constant, we can't infer affected bits from it. */ - if (arg_is_const(op->args[2])) { - ctx->a_mask = z1 & ~z2; + if (arg_is_const(op->args[2]) && + fold_affected_mask(ctx, op, z1 & ~z2)) { + return true; } return fold_masks(ctx, op); @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) */ if (arg_is_const(op->args[2])) { uint64_t z2 = ~arg_info(op->args[2])->z_mask; - ctx->a_mask = z1 & ~z2; + if (fold_affected_mask(ctx, op, z1 & ~z2)) { + return true; + } z1 &= z2; } ctx->z_mask = z1; @@ -XXX,XX +XXX,XX @@ static bool fold_extract(OptContext *ctx, TCGOp *op) z_mask_old = arg_info(op->args[1])->z_mask; z_mask = extract64(z_mask_old, pos, len); - if (pos == 0) { - ctx->a_mask = z_mask_old ^ z_mask; + if (pos == 0 && fold_affected_mask(ctx, op, z_mask_old ^ z_mask)) { + return true; } ctx->z_mask = z_mask; ctx->s_mask = smask_from_zmask(z_mask); @@ -XXX,XX +XXX,XX @@ static bool fold_exts(OptContext *ctx, TCGOp *op) ctx->z_mask = z_mask; ctx->s_mask = s_mask; - if (!type_change) { - ctx->a_mask = s_mask & ~s_mask_old; + if (!type_change && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { + return true; } return fold_masks(ctx, op); @@ -XXX,XX +XXX,XX @@ static bool fold_extu(OptContext *ctx, TCGOp *op) ctx->z_mask = z_mask; ctx->s_mask = smask_from_zmask(z_mask); - if (!type_change) { - ctx->a_mask = z_mask_old ^ z_mask; + if (!type_change && fold_affected_mask(ctx, op, z_mask_old ^ z_mask)) { + return true; } return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) s_mask |= MAKE_64BIT_MASK(len, 64 - len); ctx->s_mask = s_mask; - if (pos == 0) { - ctx->a_mask = s_mask & ~s_mask_old; + if (pos == 0 && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { + return true; } return fold_masks(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } /* Assume all bits affected, no bits known zero, no sign reps. */ - ctx.a_mask = -1; ctx.z_mask = -1; ctx.s_mask = 0; -- 2.43.0 Use of fold_masks should be restricted to those opcodes that can reliably make use of it -- those with a single output, and from higher-level folders that set up the masks. Prepare for conversion of each folder in turn. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) { uint64_t z_mask = ctx->z_mask; uint64_t s_mask = ctx->s_mask; + const TCGOpDef *def = &tcg_op_defs[op->opc]; + TCGTemp *ts; + TempOptInfo *ti; + + /* Only single-output opcodes are supported here. */ + tcg_debug_assert(def->nb_oargs == 1); /* * 32-bit ops generate 32-bit results, which for the purpose of @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) if (ctx->type == TCG_TYPE_I32) { z_mask = (int32_t)z_mask; s_mask |= MAKE_64BIT_MASK(32, 32); - ctx->z_mask = z_mask; - ctx->s_mask = s_mask; } if (z_mask == 0) { return tcg_opt_gen_movi(ctx, op, op->args[0], 0); } - return false; + + ts = arg_temp(op->args[0]); + reset_ts(ctx, ts); + + ti = ts_info(ts); + ti->z_mask = z_mask; + ti->s_mask = s_mask; + return true; } /* -- 2.43.0 Add a routine to which masks can be passed directly, rather than storing them into OptContext. To be used in upcoming patches. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_const2_commutative(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } -static bool fold_masks(OptContext *ctx, TCGOp *op) +/* + * Record "zero" and "sign" masks for the single output of @op. + * See TempOptInfo definition of z_mask and s_mask. + * If z_mask allows, fold the output to constant zero. + */ +static bool fold_masks_zs(OptContext *ctx, TCGOp *op, + uint64_t z_mask, uint64_t s_mask) { - uint64_t z_mask = ctx->z_mask; - uint64_t s_mask = ctx->s_mask; const TCGOpDef *def = &tcg_op_defs[op->opc]; TCGTemp *ts; TempOptInfo *ti; @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) return true; } +static bool fold_masks(OptContext *ctx, TCGOp *op) +{ + return fold_masks_zs(ctx, op, ctx->z_mask, ctx->s_mask); +} + /* * An "affected" mask bit is 0 if and only if the result is identical * to the first input. Thus if the entire mask is 0, the operation -- 2.43.0 Consider the passed s_mask to be a minimum deduced from either existing s_mask or from a sign-extension operation. We may be able to deduce more from the set of known zeros. Remove identical logic from several opcode folders. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_const2_commutative(OptContext *ctx, TCGOp *op) * Record "zero" and "sign" masks for the single output of @op. * See TempOptInfo definition of z_mask and s_mask. * If z_mask allows, fold the output to constant zero. + * The passed s_mask may be augmented by z_mask. */ static bool fold_masks_zs(OptContext *ctx, TCGOp *op, uint64_t z_mask, uint64_t s_mask) @@ -XXX,XX +XXX,XX @@ static bool fold_masks_zs(OptContext *ctx, TCGOp *op, ti = ts_info(ts); ti->z_mask = z_mask; - ti->s_mask = s_mask; + ti->s_mask = s_mask | smask_from_zmask(z_mask); return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) default: g_assert_not_reached(); } - s_mask = smask_from_zmask(z_mask); + s_mask = 0; switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { case TCG_BSWAP_OZ: break; @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) default: /* The high bits are undefined: force all bits above the sign to 1. */ z_mask |= sign << 1; - s_mask = 0; break; } ctx->z_mask = z_mask; @@ -XXX,XX +XXX,XX @@ static bool fold_count_zeros(OptContext *ctx, TCGOp *op) g_assert_not_reached(); } ctx->z_mask = arg_info(op->args[2])->z_mask | z_mask; - ctx->s_mask = smask_from_zmask(ctx->z_mask); return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_ctpop(OptContext *ctx, TCGOp *op) default: g_assert_not_reached(); } - ctx->s_mask = smask_from_zmask(ctx->z_mask); return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_extract(OptContext *ctx, TCGOp *op) return true; } ctx->z_mask = z_mask; - ctx->s_mask = smask_from_zmask(z_mask); return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_extu(OptContext *ctx, TCGOp *op) } ctx->z_mask = z_mask; - ctx->s_mask = smask_from_zmask(z_mask); if (!type_change && fold_affected_mask(ctx, op, z_mask_old ^ z_mask)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) int width = 8 * memop_size(mop); if (width < 64) { - ctx->s_mask = MAKE_64BIT_MASK(width, 64 - width); - if (!(mop & MO_SIGN)) { + if (mop & MO_SIGN) { + ctx->s_mask = MAKE_64BIT_MASK(width, 64 - width); + } else { ctx->z_mask = MAKE_64BIT_MASK(0, width); - ctx->s_mask <<= 1; } } @@ -XXX,XX +XXX,XX @@ static bool fold_setcond(OptContext *ctx, TCGOp *op) fold_setcond_tst_pow2(ctx, op, false); ctx->z_mask = 1; - ctx->s_mask = smask_from_zmask(1); return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) } ctx->z_mask = 1; - ctx->s_mask = smask_from_zmask(1); return false; do_setcond_const: @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) break; CASE_OP_32_64(ld8u): ctx->z_mask = MAKE_64BIT_MASK(0, 8); - ctx->s_mask = MAKE_64BIT_MASK(9, 55); break; CASE_OP_32_64(ld16s): ctx->s_mask = MAKE_64BIT_MASK(16, 48); break; CASE_OP_32_64(ld16u): ctx->z_mask = MAKE_64BIT_MASK(0, 16); - ctx->s_mask = MAKE_64BIT_MASK(17, 47); break; case INDEX_op_ld32s_i64: ctx->s_mask = MAKE_64BIT_MASK(32, 32); break; case INDEX_op_ld32u_i64: ctx->z_mask = MAKE_64BIT_MASK(0, 32); - ctx->s_mask = MAKE_64BIT_MASK(33, 31); break; default: g_assert_not_reached(); -- 2.43.0 Change the representation from sign bit repetitions to all bits equal to the sign bit, including the sign bit itself. The previous format has a problem in that it is difficult to recreate a valid sign mask after a shift operation: the "repetitions" part of the previous format meant that applying the same shift as for the value lead to an off-by-one value. The new format, including the sign bit itself, means that the sign mask can be manipulated in exactly the same way as the value, canonicalization is easier. Canonicalize the s_mask in fold_masks_zs, rather than requiring callers to do so. Treat 0 as a non-canonical but typeless input for no sign information, which will be reset as appropriate for the data type. We can easily fold in the data from z_mask while canonicalizing. Temporarily disable optimizations using s_mask while each operation is converted to use fold_masks_zs and to the new form. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 64 ++++++++++++-------------------------------------- 1 file changed, 15 insertions(+), 49 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { QSIMPLEQ_HEAD(, MemCopyInfo) mem_copy; uint64_t val; uint64_t z_mask; /* mask bit is 0 if and only if value bit is 0 */ - uint64_t s_mask; /* a left-aligned mask of clrsb(value) bits. */ + uint64_t s_mask; /* mask bit is 1 if value bit matches msb */ } TempOptInfo; typedef struct OptContext { @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { /* In flight values from optimization. */ uint64_t z_mask; /* mask bit is 0 iff value bit is 0 */ - uint64_t s_mask; /* mask of clrsb(value) bits */ + uint64_t s_mask; /* mask bit is 1 if value bit matches msb */ TCGType type; } OptContext; -/* Calculate the smask for a specific value. */ -static uint64_t smask_from_value(uint64_t value) -{ - int rep = clrsb64(value); - return ~(~0ull >> rep); -} - -/* - * Calculate the smask for a given set of known-zeros. - * If there are lots of zeros on the left, we can consider the remainder - * an unsigned field, and thus the corresponding signed field is one bit - * larger. - */ -static uint64_t smask_from_zmask(uint64_t zmask) -{ - /* - * Only the 0 bits are significant for zmask, thus the msb itself - * must be zero, else we have no sign information. - */ - int rep = clz64(zmask); - if (rep == 0) { - return 0; - } - rep -= 1; - return ~(~0ull >> rep); -} - -/* - * Recreate a properly left-aligned smask after manipulation. - * Some bit-shuffling, particularly shifts and rotates, may - * retain sign bits on the left, but may scatter disconnected - * sign bits on the right. Retain only what remains to the left. - */ -static uint64_t smask_from_smask(int64_t smask) -{ - /* Only the 1 bits are significant for smask */ - return smask_from_zmask(~smask); -} - static inline TempOptInfo *ts_info(TCGTemp *ts) { return ts->state_ptr; @@ -XXX,XX +XXX,XX @@ static void init_ts_info(OptContext *ctx, TCGTemp *ts) ti->is_const = true; ti->val = ts->val; ti->z_mask = ts->val; - ti->s_mask = smask_from_value(ts->val); + ti->s_mask = INT64_MIN >> clrsb64(ts->val); } else { ti->is_const = false; ti->z_mask = -1; @@ -XXX,XX +XXX,XX @@ static void finish_folding(OptContext *ctx, TCGOp *op) */ if (i == 0) { ts_info(ts)->z_mask = ctx->z_mask; - ts_info(ts)->s_mask = ctx->s_mask; } } } @@ -XXX,XX +XXX,XX @@ static bool fold_const2_commutative(OptContext *ctx, TCGOp *op) * The passed s_mask may be augmented by z_mask. */ static bool fold_masks_zs(OptContext *ctx, TCGOp *op, - uint64_t z_mask, uint64_t s_mask) + uint64_t z_mask, int64_t s_mask) { const TCGOpDef *def = &tcg_op_defs[op->opc]; TCGTemp *ts; TempOptInfo *ti; + int rep; /* Only single-output opcodes are supported here. */ tcg_debug_assert(def->nb_oargs == 1); @@ -XXX,XX +XXX,XX @@ static bool fold_masks_zs(OptContext *ctx, TCGOp *op, */ if (ctx->type == TCG_TYPE_I32) { z_mask = (int32_t)z_mask; - s_mask |= MAKE_64BIT_MASK(32, 32); + s_mask |= INT32_MIN; } if (z_mask == 0) { @@ -XXX,XX +XXX,XX @@ static bool fold_masks_zs(OptContext *ctx, TCGOp *op, ti = ts_info(ts); ti->z_mask = z_mask; - ti->s_mask = s_mask | smask_from_zmask(z_mask); + + /* Canonicalize s_mask and incorporate data from z_mask. */ + rep = clz64(~s_mask); + rep = MAX(rep, clz64(z_mask)); + rep = MAX(rep - 1, 0); + ti->s_mask = INT64_MIN >> rep; + return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_exts(OptContext *ctx, TCGOp *op) ctx->z_mask = z_mask; ctx->s_mask = s_mask; - if (!type_change && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { + if (0 && !type_change && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) s_mask |= MAKE_64BIT_MASK(len, 64 - len); ctx->s_mask = s_mask; - if (pos == 0 && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { + if (0 && pos == 0 && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_shift(OptContext *ctx, TCGOp *op) ctx->z_mask = do_constant_folding(op->opc, ctx->type, z_mask, sh); s_mask = do_constant_folding(op->opc, ctx->type, s_mask, sh); - ctx->s_mask = smask_from_smask(s_mask); return fold_masks(ctx, op); } -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void finish_ebb(OptContext *ctx) remove_mem_copy_all(ctx); } -static void finish_folding(OptContext *ctx, TCGOp *op) +static bool finish_folding(OptContext *ctx, TCGOp *op) { const TCGOpDef *def = &tcg_op_defs[op->opc]; int i, nb_oargs; @@ -XXX,XX +XXX,XX @@ static void finish_folding(OptContext *ctx, TCGOp *op) ts_info(ts)->z_mask = ctx->z_mask; } } + return true; } /* @@ -XXX,XX +XXX,XX @@ static bool fold_add(OptContext *ctx, TCGOp *op) fold_xi_to_x(ctx, op, 0)) { return true; } - return false; + return finish_folding(ctx, op); } /* We cannot as yet do_constant_folding with vectors. */ @@ -XXX,XX +XXX,XX @@ static bool fold_add_vec(OptContext *ctx, TCGOp *op) fold_xi_to_x(ctx, op, 0)) { return true; } - return false; + return finish_folding(ctx, op); } static bool fold_addsub2(OptContext *ctx, TCGOp *op, bool add) @@ -XXX,XX +XXX,XX @@ static bool fold_addsub2(OptContext *ctx, TCGOp *op, bool add) op->args[4] = arg_new_constant(ctx, bl); op->args[5] = arg_new_constant(ctx, bh); } - return false; + return finish_folding(ctx, op); } static bool fold_add2(OptContext *ctx, TCGOp *op) -- 2.43.0 Introduce ti_is_const, ti_const_val, ti_is_const_val. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static inline TempOptInfo *arg_info(TCGArg arg) return ts_info(arg_temp(arg)); } +static inline bool ti_is_const(TempOptInfo *ti) +{ + return ti->is_const; +} + +static inline uint64_t ti_const_val(TempOptInfo *ti) +{ + return ti->val; +} + +static inline bool ti_is_const_val(TempOptInfo *ti, uint64_t val) +{ + return ti_is_const(ti) && ti_const_val(ti) == val; +} + static inline bool ts_is_const(TCGTemp *ts) { - return ts_info(ts)->is_const; + return ti_is_const(ts_info(ts)); } static inline bool ts_is_const_val(TCGTemp *ts, uint64_t val) { - TempOptInfo *ti = ts_info(ts); - return ti->is_const && ti->val == val; + return ti_is_const_val(ts_info(ts), val); } static inline bool arg_is_const(TCGArg arg) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Sink mask computation below fold_affected_mask early exit. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_add2(OptContext *ctx, TCGOp *op) static bool fold_and(OptContext *ctx, TCGOp *op) { - uint64_t z1, z2; + uint64_t z1, z2, z_mask, s_mask; + TempOptInfo *t1, *t2; if (fold_const2_commutative(ctx, op) || fold_xi_to_i(ctx, op, 0) || @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) return true; } - z1 = arg_info(op->args[1])->z_mask; - z2 = arg_info(op->args[2])->z_mask; - ctx->z_mask = z1 & z2; - - /* - * Sign repetitions are perforce all identical, whether they are 1 or 0. - * Bitwise operations preserve the relative quantity of the repetitions. - */ - ctx->s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; + t1 = arg_info(op->args[1]); + t2 = arg_info(op->args[2]); + z1 = t1->z_mask; + z2 = t2->z_mask; /* * Known-zeros does not imply known-ones. Therefore unless * arg2 is constant, we can't infer affected bits from it. */ - if (arg_is_const(op->args[2]) && - fold_affected_mask(ctx, op, z1 & ~z2)) { + if (ti_is_const(t2) && fold_affected_mask(ctx, op, z1 & ~z2)) { return true; } - return fold_masks(ctx, op); + z_mask = z1 & z2; + + /* + * Sign repetitions are perforce all identical, whether they are 1 or 0. + * Bitwise operations preserve the relative quantity of the repetitions. + */ + s_mask = t1->s_mask & t2->s_mask; + + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_andc(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Avoid double inversion of the value of second const operand. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) static bool fold_andc(OptContext *ctx, TCGOp *op) { - uint64_t z1; + uint64_t z_mask, s_mask; + TempOptInfo *t1, *t2; if (fold_const2(ctx, op) || fold_xx_to_i(ctx, op, 0) || @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) return true; } - z1 = arg_info(op->args[1])->z_mask; + t1 = arg_info(op->args[1]); + t2 = arg_info(op->args[2]); + z_mask = t1->z_mask; /* * Known-zeros does not imply known-ones. Therefore unless * arg2 is constant, we can't infer anything from it. */ - if (arg_is_const(op->args[2])) { - uint64_t z2 = ~arg_info(op->args[2])->z_mask; - if (fold_affected_mask(ctx, op, z1 & ~z2)) { + if (ti_is_const(t2)) { + uint64_t v2 = ti_const_val(t2); + if (fold_affected_mask(ctx, op, z_mask & v2)) { return true; } - z1 &= z2; + z_mask &= ~v2; } - ctx->z_mask = z1; - ctx->s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; - return fold_masks(ctx, op); + s_mask = t1->s_mask & t2->s_mask; + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_brcond(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Always set s_mask along the BSWAP_OS path, since the result is being explicitly sign-extended. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) static bool fold_bswap(OptContext *ctx, TCGOp *op) { uint64_t z_mask, s_mask, sign; + TempOptInfo *t1 = arg_info(op->args[1]); - if (arg_is_const(op->args[1])) { - uint64_t t = arg_info(op->args[1])->val; - - t = do_constant_folding(op->opc, ctx->type, t, op->args[2]); - return tcg_opt_gen_movi(ctx, op, op->args[0], t); + if (ti_is_const(t1)) { + return tcg_opt_gen_movi(ctx, op, op->args[0], + do_constant_folding(op->opc, ctx->type, + ti_const_val(t1), + op->args[2])); } - z_mask = arg_info(op->args[1])->z_mask; - + z_mask = t1->z_mask; switch (op->opc) { case INDEX_op_bswap16_i32: case INDEX_op_bswap16_i64: @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) /* If the sign bit may be 1, force all the bits above to 1. */ if (z_mask & sign) { z_mask |= sign; - s_mask = sign << 1; } + /* The value and therefore s_mask is explicitly sign-extended. */ + s_mask = sign; break; default: /* The high bits are undefined: force all bits above the sign to 1. */ z_mask |= sign << 1; break; } - ctx->z_mask = z_mask; - ctx->s_mask = s_mask; - return fold_masks(ctx, op); + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_call(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Compute s_mask from the union of the maximum count and the op2 fallback for op1 being zero. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_call(OptContext *ctx, TCGOp *op) static bool fold_count_zeros(OptContext *ctx, TCGOp *op) { - uint64_t z_mask; + uint64_t z_mask, s_mask; + TempOptInfo *t1 = arg_info(op->args[1]); + TempOptInfo *t2 = arg_info(op->args[2]); - if (arg_is_const(op->args[1])) { - uint64_t t = arg_info(op->args[1])->val; + if (ti_is_const(t1)) { + uint64_t t = ti_const_val(t1); if (t != 0) { t = do_constant_folding(op->opc, ctx->type, t, 0); @@ -XXX,XX +XXX,XX @@ static bool fold_count_zeros(OptContext *ctx, TCGOp *op) default: g_assert_not_reached(); } - ctx->z_mask = arg_info(op->args[2])->z_mask | z_mask; - return false; + s_mask = ~z_mask; + z_mask |= t2->z_mask; + s_mask &= t2->s_mask; + + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_ctpop(OptContext *ctx, TCGOp *op) -- 2.43.0 Add fold_masks_z as a trivial wrapper around fold_masks_zs. Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_masks_zs(OptContext *ctx, TCGOp *op, return true; } +static bool fold_masks_z(OptContext *ctx, TCGOp *op, uint64_t z_mask) +{ + return fold_masks_zs(ctx, op, z_mask, 0); +} + static bool fold_masks(OptContext *ctx, TCGOp *op) { return fold_masks_zs(ctx, op, ctx->z_mask, ctx->s_mask); @@ -XXX,XX +XXX,XX @@ static bool fold_count_zeros(OptContext *ctx, TCGOp *op) static bool fold_ctpop(OptContext *ctx, TCGOp *op) { + uint64_t z_mask; + if (fold_const1(ctx, op)) { return true; } switch (ctx->type) { case TCG_TYPE_I32: - ctx->z_mask = 32 | 31; + z_mask = 32 | 31; break; case TCG_TYPE_I64: - ctx->z_mask = 64 | 63; + z_mask = 64 | 63; break; default: g_assert_not_reached(); } - return false; + return fold_masks_z(ctx, op, z_mask); } static bool fold_deposit(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. When we fold to and, use fold_and. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_ctpop(OptContext *ctx, TCGOp *op) static bool fold_deposit(OptContext *ctx, TCGOp *op) { + TempOptInfo *t1 = arg_info(op->args[1]); + TempOptInfo *t2 = arg_info(op->args[2]); + int ofs = op->args[3]; + int len = op->args[4]; TCGOpcode and_opc; + uint64_t z_mask; - if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - uint64_t t1 = arg_info(op->args[1])->val; - uint64_t t2 = arg_info(op->args[2])->val; - - t1 = deposit64(t1, op->args[3], op->args[4], t2); - return tcg_opt_gen_movi(ctx, op, op->args[0], t1); + if (ti_is_const(t1) && ti_is_const(t2)) { + return tcg_opt_gen_movi(ctx, op, op->args[0], + deposit64(ti_const_val(t1), ofs, len, + ti_const_val(t2))); } switch (ctx->type) { @@ -XXX,XX +XXX,XX @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) } /* Inserting a value into zero at offset 0. */ - if (arg_is_const_val(op->args[1], 0) && op->args[3] == 0) { - uint64_t mask = MAKE_64BIT_MASK(0, op->args[4]); + if (ti_is_const_val(t1, 0) && ofs == 0) { + uint64_t mask = MAKE_64BIT_MASK(0, len); op->opc = and_opc; op->args[1] = op->args[2]; op->args[2] = arg_new_constant(ctx, mask); - ctx->z_mask = mask & arg_info(op->args[1])->z_mask; - return false; + return fold_and(ctx, op); } /* Inserting zero into a value. */ - if (arg_is_const_val(op->args[2], 0)) { - uint64_t mask = deposit64(-1, op->args[3], op->args[4], 0); + if (ti_is_const_val(t2, 0)) { + uint64_t mask = deposit64(-1, ofs, len, 0); op->opc = and_opc; op->args[2] = arg_new_constant(ctx, mask); - ctx->z_mask = mask & arg_info(op->args[1])->z_mask; - return false; + return fold_and(ctx, op); } - ctx->z_mask = deposit64(arg_info(op->args[1])->z_mask, - op->args[3], op->args[4], - arg_info(op->args[2])->z_mask); - return false; + z_mask = deposit64(t1->z_mask, ofs, len, t2->z_mask); + return fold_masks_z(ctx, op, z_mask); } static bool fold_divide(OptContext *ctx, TCGOp *op) -- 2.43.0 The input which overlaps the sign bit of the output can have its input s_mask propagated to the output s_mask. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) TempOptInfo *t2 = arg_info(op->args[2]); int ofs = op->args[3]; int len = op->args[4]; + int width; TCGOpcode and_opc; - uint64_t z_mask; + uint64_t z_mask, s_mask; if (ti_is_const(t1) && ti_is_const(t2)) { return tcg_opt_gen_movi(ctx, op, op->args[0], @@ -XXX,XX +XXX,XX @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) switch (ctx->type) { case TCG_TYPE_I32: and_opc = INDEX_op_and_i32; + width = 32; break; case TCG_TYPE_I64: and_opc = INDEX_op_and_i64; + width = 64; break; default: g_assert_not_reached(); @@ -XXX,XX +XXX,XX @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) return fold_and(ctx, op); } + /* The s_mask from the top portion of the deposit is still valid. */ + if (ofs + len == width) { + s_mask = t2->s_mask << ofs; + } else { + s_mask = t1->s_mask & ~MAKE_64BIT_MASK(0, ofs + len); + } + z_mask = deposit64(t1->z_mask, ofs, len, t2->z_mask); - return fold_masks_z(ctx, op, z_mask); + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_divide(OptContext *ctx, TCGOp *op) -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_divide(OptContext *ctx, TCGOp *op) fold_xi_to_x(ctx, op, 1)) { return true; } - return false; + return finish_folding(ctx, op); } static bool fold_dup(OptContext *ctx, TCGOp *op) -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_dup(OptContext *ctx, TCGOp *op) t = dup_const(TCGOP_VECE(op), t); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } - return false; + return finish_folding(ctx, op); } static bool fold_dup2(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_dup2(OptContext *ctx, TCGOp *op) op->opc = INDEX_op_dup_vec; TCGOP_VECE(op) = MO_32; } - return false; + return finish_folding(ctx, op); } static bool fold_eqv(OptContext *ctx, TCGOp *op) -- 2.43.0 Add fold_masks_s as a trivial wrapper around fold_masks_zs. Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_masks_z(OptContext *ctx, TCGOp *op, uint64_t z_mask) return fold_masks_zs(ctx, op, z_mask, 0); } +static bool fold_masks_s(OptContext *ctx, TCGOp *op, uint64_t s_mask) +{ + return fold_masks_zs(ctx, op, -1, s_mask); +} + static bool fold_masks(OptContext *ctx, TCGOp *op) { return fold_masks_zs(ctx, op, ctx->z_mask, ctx->s_mask); @@ -XXX,XX +XXX,XX @@ static bool fold_dup2(OptContext *ctx, TCGOp *op) static bool fold_eqv(OptContext *ctx, TCGOp *op) { + uint64_t s_mask; + if (fold_const2_commutative(ctx, op) || fold_xi_to_x(ctx, op, -1) || fold_xi_to_not(ctx, op, 0)) { return true; } - ctx->s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; - return false; + s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; + return fold_masks_s(ctx, op, s_mask); } static bool fold_extract(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) static bool fold_extract(OptContext *ctx, TCGOp *op) { uint64_t z_mask_old, z_mask; + TempOptInfo *t1 = arg_info(op->args[1]); int pos = op->args[2]; int len = op->args[3]; - if (arg_is_const(op->args[1])) { - uint64_t t; - - t = arg_info(op->args[1])->val; - t = extract64(t, pos, len); - return tcg_opt_gen_movi(ctx, op, op->args[0], t); + if (ti_is_const(t1)) { + return tcg_opt_gen_movi(ctx, op, op->args[0], + extract64(ti_const_val(t1), pos, len)); } - z_mask_old = arg_info(op->args[1])->z_mask; + z_mask_old = t1->z_mask; z_mask = extract64(z_mask_old, pos, len); if (pos == 0 && fold_affected_mask(ctx, op, z_mask_old ^ z_mask)) { return true; } - ctx->z_mask = z_mask; - return fold_masks(ctx, op); + return fold_masks_z(ctx, op, z_mask); } static bool fold_extract2(OptContext *ctx, TCGOp *op) -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_extract2(OptContext *ctx, TCGOp *op) } return tcg_opt_gen_movi(ctx, op, op->args[0], v1 | v2); } - return false; + return finish_folding(ctx, op); } static bool fold_exts(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Explicitly sign-extend z_mask instead of doing that manually. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_extract2(OptContext *ctx, TCGOp *op) static bool fold_exts(OptContext *ctx, TCGOp *op) { - uint64_t s_mask_old, s_mask, z_mask, sign; + uint64_t s_mask_old, s_mask, z_mask; bool type_change = false; + TempOptInfo *t1; if (fold_const1(ctx, op)) { return true; } - z_mask = arg_info(op->args[1])->z_mask; - s_mask = arg_info(op->args[1])->s_mask; + t1 = arg_info(op->args[1]); + z_mask = t1->z_mask; + s_mask = t1->s_mask; s_mask_old = s_mask; switch (op->opc) { CASE_OP_32_64(ext8s): - sign = INT8_MIN; - z_mask = (uint8_t)z_mask; + s_mask |= INT8_MIN; + z_mask = (int8_t)z_mask; break; CASE_OP_32_64(ext16s): - sign = INT16_MIN; - z_mask = (uint16_t)z_mask; + s_mask |= INT16_MIN; + z_mask = (int16_t)z_mask; break; case INDEX_op_ext_i32_i64: type_change = true; QEMU_FALLTHROUGH; case INDEX_op_ext32s_i64: - sign = INT32_MIN; - z_mask = (uint32_t)z_mask; + s_mask |= INT32_MIN; + z_mask = (int32_t)z_mask; break; default: g_assert_not_reached(); } - if (z_mask & sign) { - z_mask |= sign; - } - s_mask |= sign << 1; - - ctx->z_mask = z_mask; - ctx->s_mask = s_mask; if (0 && !type_change && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { return true; } - return fold_masks(ctx, op); + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_extu(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_extu(OptContext *ctx, TCGOp *op) g_assert_not_reached(); } - ctx->z_mask = z_mask; if (!type_change && fold_affected_mask(ctx, op, z_mask_old ^ z_mask)) { return true; } - return fold_masks(ctx, op); + + return fold_masks_z(ctx, op, z_mask); } static bool fold_mb(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_mov(OptContext *ctx, TCGOp *op) static bool fold_movcond(OptContext *ctx, TCGOp *op) { + uint64_t z_mask, s_mask; + TempOptInfo *tt, *ft; int i; /* If true and false values are the same, eliminate the cmp. */ @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]); } - ctx->z_mask = arg_info(op->args[3])->z_mask - | arg_info(op->args[4])->z_mask; - ctx->s_mask = arg_info(op->args[3])->s_mask - & arg_info(op->args[4])->s_mask; + tt = arg_info(op->args[3]); + ft = arg_info(op->args[4]); + z_mask = tt->z_mask | ft->z_mask; + s_mask = tt->s_mask & ft->s_mask; - if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { - uint64_t tv = arg_info(op->args[3])->val; - uint64_t fv = arg_info(op->args[4])->val; + if (ti_is_const(tt) && ti_is_const(ft)) { + uint64_t tv = ti_const_val(tt); + uint64_t fv = ti_const_val(ft); TCGOpcode opc, negopc = 0; TCGCond cond = op->args[5]; @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) } } } - return false; + + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_mul(OptContext *ctx, TCGOp *op) -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_mul(OptContext *ctx, TCGOp *op) fold_xi_to_x(ctx, op, 1)) { return true; } - return false; + return finish_folding(ctx, op); } static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) fold_xi_to_i(ctx, op, 0)) { return true; } - return false; + return finish_folding(ctx, op); } static bool fold_multiply2(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_multiply2(OptContext *ctx, TCGOp *op) tcg_opt_gen_movi(ctx, op2, rh, h); return true; } - return false; + return finish_folding(ctx, op); } static bool fold_nand(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_multiply2(OptContext *ctx, TCGOp *op) static bool fold_nand(OptContext *ctx, TCGOp *op) { + uint64_t s_mask; + if (fold_const2_commutative(ctx, op) || fold_xi_to_not(ctx, op, -1)) { return true; } - ctx->s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; - return false; + s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; + return fold_masks_s(ctx, op, s_mask); } static bool fold_neg_no_const(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_neg_no_const(OptContext *ctx, TCGOp *op) { /* Set to 1 all bits to the left of the rightmost. */ uint64_t z_mask = arg_info(op->args[1])->z_mask; - ctx->z_mask = -(z_mask & -z_mask); + z_mask = -(z_mask & -z_mask); - /* - * Because of fold_sub_to_neg, we want to always return true, - * via finish_folding. - */ - finish_folding(ctx, op); - return true; + return fold_masks_z(ctx, op, z_mask); } static bool fold_neg(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_neg(OptContext *ctx, TCGOp *op) static bool fold_nor(OptContext *ctx, TCGOp *op) { + uint64_t s_mask; + if (fold_const2_commutative(ctx, op) || fold_xi_to_not(ctx, op, 0)) { return true; } - ctx->s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; - return false; + s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; + return fold_masks_s(ctx, op, s_mask); } static bool fold_not(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_not(OptContext *ctx, TCGOp *op) if (fold_const1(ctx, op)) { return true; } - - ctx->s_mask = arg_info(op->args[1])->s_mask; - - /* Because of fold_to_not, we want to always return true, via finish. */ - finish_folding(ctx, op); - return true; + return fold_masks_s(ctx, op, arg_info(op->args[1])->s_mask); } static bool fold_or(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_not(OptContext *ctx, TCGOp *op) static bool fold_or(OptContext *ctx, TCGOp *op) { + uint64_t z_mask, s_mask; + TempOptInfo *t1, *t2; + if (fold_const2_commutative(ctx, op) || fold_xi_to_x(ctx, op, 0) || fold_xx_to_x(ctx, op)) { return true; } - ctx->z_mask = arg_info(op->args[1])->z_mask - | arg_info(op->args[2])->z_mask; - ctx->s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; - return fold_masks(ctx, op); + t1 = arg_info(op->args[1]); + t2 = arg_info(op->args[2]); + z_mask = t1->z_mask | t2->z_mask; + s_mask = t1->s_mask & t2->s_mask; + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_orc(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_or(OptContext *ctx, TCGOp *op) static bool fold_orc(OptContext *ctx, TCGOp *op) { + uint64_t s_mask; + if (fold_const2(ctx, op) || fold_xx_to_i(ctx, op, -1) || fold_xi_to_x(ctx, op, -1) || @@ -XXX,XX +XXX,XX @@ static bool fold_orc(OptContext *ctx, TCGOp *op) return true; } - ctx->s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; - return false; + s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; + return fold_masks_s(ctx, op, s_mask); } static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Be careful not to call fold_masks_zs when the memory operation is wide enough to require multiple outputs, so split into two functions: fold_qemu_ld_1reg and fold_qemu_ld_2reg. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_orc(OptContext *ctx, TCGOp *op) return fold_masks_s(ctx, op, s_mask); } -static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) +static bool fold_qemu_ld_1reg(OptContext *ctx, TCGOp *op) { const TCGOpDef *def = &tcg_op_defs[op->opc]; MemOpIdx oi = op->args[def->nb_oargs + def->nb_iargs]; MemOp mop = get_memop(oi); int width = 8 * memop_size(mop); + uint64_t z_mask = -1, s_mask = 0; if (width < 64) { if (mop & MO_SIGN) { - ctx->s_mask = MAKE_64BIT_MASK(width, 64 - width); + s_mask = MAKE_64BIT_MASK(width - 1, 64 - (width - 1)); } else { - ctx->z_mask = MAKE_64BIT_MASK(0, width); + z_mask = MAKE_64BIT_MASK(0, width); } } /* Opcodes that touch guest memory stop the mb optimization. */ ctx->prev_mb = NULL; - return false; + + return fold_masks_zs(ctx, op, z_mask, s_mask); +} + +static bool fold_qemu_ld_2reg(OptContext *ctx, TCGOp *op) +{ + /* Opcodes that touch guest memory stop the mb optimization. */ + ctx->prev_mb = NULL; + return finish_folding(ctx, op); } static bool fold_qemu_st(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; case INDEX_op_qemu_ld_a32_i32: case INDEX_op_qemu_ld_a64_i32: + done = fold_qemu_ld_1reg(&ctx, op); + break; case INDEX_op_qemu_ld_a32_i64: case INDEX_op_qemu_ld_a64_i64: + if (TCG_TARGET_REG_BITS == 64) { + done = fold_qemu_ld_1reg(&ctx, op); + break; + } + QEMU_FALLTHROUGH; case INDEX_op_qemu_ld_a32_i128: case INDEX_op_qemu_ld_a64_i128: - done = fold_qemu_ld(&ctx, op); + done = fold_qemu_ld_2reg(&ctx, op); break; case INDEX_op_qemu_st8_a32_i32: case INDEX_op_qemu_st8_a64_i32: -- 2.43.0 Stores have no output operands, and so need no further work. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_qemu_st(OptContext *ctx, TCGOp *op) { /* Opcodes that touch guest memory stop the mb optimization. */ ctx->prev_mb = NULL; - return false; + return true; } static bool fold_remainder(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_st(OptContext *ctx, TCGOp *op) if (op->args[1] != tcgv_ptr_arg(tcg_env)) { remove_mem_copy_all(ctx); - return false; + return true; } switch (op->opc) { @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_st(OptContext *ctx, TCGOp *op) g_assert_not_reached(); } remove_mem_copy_in(ctx, ofs, ofs + lm1); - return false; + return true; } static bool fold_tcg_st_memcopy(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_st_memcopy(OptContext *ctx, TCGOp *op) TCGType type; if (op->args[1] != tcgv_ptr_arg(tcg_env)) { - fold_tcg_st(ctx, op); - return false; + return fold_tcg_st(ctx, op); } src = arg_temp(op->args[0]); @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_st_memcopy(OptContext *ctx, TCGOp *op) last = ofs + tcg_type_size(type) - 1; remove_mem_copy_in(ctx, ofs, last); record_mem_copy(ctx, type, src, ofs, last); - return false; + return true; } static bool fold_xor(OptContext *ctx, TCGOp *op) -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_remainder(OptContext *ctx, TCGOp *op) fold_xx_to_i(ctx, op, 0)) { return true; } - return false; + return finish_folding(ctx, op); } static bool fold_setcond_zmask(OptContext *ctx, TCGOp *op, bool neg) -- 2.43.0 Change return from bool to int; distinguish between complete folding, simplification, and no change. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_remainder(OptContext *ctx, TCGOp *op) return finish_folding(ctx, op); } -static bool fold_setcond_zmask(OptContext *ctx, TCGOp *op, bool neg) +/* Return 1 if finished, -1 if simplified, 0 if unchanged. */ +static int fold_setcond_zmask(OptContext *ctx, TCGOp *op, bool neg) { uint64_t a_zmask, b_val; TCGCond cond; @@ -XXX,XX +XXX,XX @@ static bool fold_setcond_zmask(OptContext *ctx, TCGOp *op, bool neg) op->opc = xor_opc; op->args[2] = arg_new_constant(ctx, 1); } - return false; + return -1; } } - - return false; + return 0; } static void fold_setcond_tst_pow2(OptContext *ctx, TCGOp *op, bool neg) @@ -XXX,XX +XXX,XX @@ static bool fold_setcond(OptContext *ctx, TCGOp *op) return tcg_opt_gen_movi(ctx, op, op->args[0], i); } - if (fold_setcond_zmask(ctx, op, false)) { + i = fold_setcond_zmask(ctx, op, false); + if (i > 0) { return true; } - fold_setcond_tst_pow2(ctx, op, false); + if (i == 0) { + fold_setcond_tst_pow2(ctx, op, false); + } ctx->z_mask = 1; return false; @@ -XXX,XX +XXX,XX @@ static bool fold_negsetcond(OptContext *ctx, TCGOp *op) return tcg_opt_gen_movi(ctx, op, op->args[0], -i); } - if (fold_setcond_zmask(ctx, op, true)) { + i = fold_setcond_zmask(ctx, op, true); + if (i > 0) { return true; } - fold_setcond_tst_pow2(ctx, op, true); + if (i == 0) { + fold_setcond_tst_pow2(ctx, op, true); + } /* Value is {0,-1} so all bits are repetitions of the sign. */ ctx->s_mask = -1; -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_setcond(OptContext *ctx, TCGOp *op) fold_setcond_tst_pow2(ctx, op, false); } - ctx->z_mask = 1; - return false; + return fold_masks_z(ctx, op, 1); } static bool fold_negsetcond(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_negsetcond(OptContext *ctx, TCGOp *op) } /* Value is {0,-1} so all bits are repetitions of the sign. */ - ctx->s_mask = -1; - return false; + return fold_masks_s(ctx, op, -1); } static bool fold_setcond2(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) return fold_setcond(ctx, op); } - ctx->z_mask = 1; - return false; + return fold_masks_z(ctx, op, 1); do_setcond_const: return tcg_opt_gen_movi(ctx, op, op->args[0], i); -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_cmp_vec(OptContext *ctx, TCGOp *op) if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) { op->args[3] = tcg_swap_cond(op->args[3]); } - return false; + return finish_folding(ctx, op); } static bool fold_cmpsel_vec(OptContext *ctx, TCGOp *op) -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_cmpsel_vec(OptContext *ctx, TCGOp *op) if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) { op->args[5] = tcg_invert_cond(op->args[5]); } - return false; + return finish_folding(ctx, op); } static bool fold_sextract(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_cmpsel_vec(OptContext *ctx, TCGOp *op) static bool fold_sextract(OptContext *ctx, TCGOp *op) { uint64_t z_mask, s_mask, s_mask_old; + TempOptInfo *t1 = arg_info(op->args[1]); int pos = op->args[2]; int len = op->args[3]; - if (arg_is_const(op->args[1])) { - uint64_t t; - - t = arg_info(op->args[1])->val; - t = sextract64(t, pos, len); - return tcg_opt_gen_movi(ctx, op, op->args[0], t); + if (ti_is_const(t1)) { + return tcg_opt_gen_movi(ctx, op, op->args[0], + sextract64(ti_const_val(t1), pos, len)); } - z_mask = arg_info(op->args[1])->z_mask; - z_mask = sextract64(z_mask, pos, len); - ctx->z_mask = z_mask; - - s_mask_old = arg_info(op->args[1])->s_mask; - s_mask = sextract64(s_mask_old, pos, len); - s_mask |= MAKE_64BIT_MASK(len, 64 - len); - ctx->s_mask = s_mask; + s_mask_old = t1->s_mask; + s_mask = s_mask_old >> pos; + s_mask |= -1ull << (len - 1); if (0 && pos == 0 && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { return true; } - return fold_masks(ctx, op); + z_mask = sextract64(t1->z_mask, pos, len); + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_shift(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) static bool fold_shift(OptContext *ctx, TCGOp *op) { uint64_t s_mask, z_mask, sign; + TempOptInfo *t1, *t2; if (fold_const2(ctx, op) || fold_ix_to_i(ctx, op, 0) || @@ -XXX,XX +XXX,XX @@ static bool fold_shift(OptContext *ctx, TCGOp *op) return true; } - s_mask = arg_info(op->args[1])->s_mask; - z_mask = arg_info(op->args[1])->z_mask; + t1 = arg_info(op->args[1]); + t2 = arg_info(op->args[2]); + s_mask = t1->s_mask; + z_mask = t1->z_mask; - if (arg_is_const(op->args[2])) { - int sh = arg_info(op->args[2])->val; - - ctx->z_mask = do_constant_folding(op->opc, ctx->type, z_mask, sh); + if (ti_is_const(t2)) { + int sh = ti_const_val(t2); + z_mask = do_constant_folding(op->opc, ctx->type, z_mask, sh); s_mask = do_constant_folding(op->opc, ctx->type, s_mask, sh); - return fold_masks(ctx, op); + return fold_masks_zs(ctx, op, z_mask, s_mask); } switch (op->opc) { @@ -XXX,XX +XXX,XX @@ static bool fold_shift(OptContext *ctx, TCGOp *op) * Arithmetic right shift will not reduce the number of * input sign repetitions. */ - ctx->s_mask = s_mask; - break; + return fold_masks_s(ctx, op, s_mask); CASE_OP_32_64(shr): /* * If the sign bit is known zero, then logical right shift - * will not reduced the number of input sign repetitions. + * will not reduce the number of input sign repetitions. */ - sign = (s_mask & -s_mask) >> 1; + sign = -s_mask; if (sign && !(z_mask & sign)) { - ctx->s_mask = s_mask; + return fold_masks_s(ctx, op, s_mask); } break; default: break; } - return false; + return finish_folding(ctx, op); } static bool fold_sub_to_neg(OptContext *ctx, TCGOp *op) -- 2.43.0 Merge the two conditions, sign != 0 && !(z_mask & sign), by testing ~z_mask & sign. If sign == 0, the logical and will produce false. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) static bool fold_shift(OptContext *ctx, TCGOp *op) { - uint64_t s_mask, z_mask, sign; + uint64_t s_mask, z_mask; TempOptInfo *t1, *t2; if (fold_const2(ctx, op) || @@ -XXX,XX +XXX,XX @@ static bool fold_shift(OptContext *ctx, TCGOp *op) * If the sign bit is known zero, then logical right shift * will not reduce the number of input sign repetitions. */ - sign = -s_mask; - if (sign && !(z_mask & sign)) { + if (~z_mask & -s_mask) { return fold_masks_s(ctx, op, s_mask); } break; -- 2.43.0 Duplicate fold_sub_vec into fold_sub instead of calling it, now that fold_sub_vec always returns true. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_sub_vec(OptContext *ctx, TCGOp *op) fold_sub_to_neg(ctx, op)) { return true; } - return false; + return finish_folding(ctx, op); } static bool fold_sub(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || fold_sub_vec(ctx, op)) { + if (fold_const2(ctx, op) || + fold_xx_to_i(ctx, op, 0) || + fold_xi_to_x(ctx, op, 0) || + fold_sub_to_neg(ctx, op)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_sub(OptContext *ctx, TCGOp *op) ? INDEX_op_add_i32 : INDEX_op_add_i64); op->args[2] = arg_new_constant(ctx, -val); } - return false; + return finish_folding(ctx, op); } static bool fold_sub2(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_sub2(OptContext *ctx, TCGOp *op) static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) { + uint64_t z_mask = -1, s_mask = 0; + /* We can't do any folding with a load, but we can record bits. */ switch (op->opc) { CASE_OP_32_64(ld8s): - ctx->s_mask = MAKE_64BIT_MASK(8, 56); + s_mask = INT8_MIN; break; CASE_OP_32_64(ld8u): - ctx->z_mask = MAKE_64BIT_MASK(0, 8); + z_mask = MAKE_64BIT_MASK(0, 8); break; CASE_OP_32_64(ld16s): - ctx->s_mask = MAKE_64BIT_MASK(16, 48); + s_mask = INT16_MIN; break; CASE_OP_32_64(ld16u): - ctx->z_mask = MAKE_64BIT_MASK(0, 16); + z_mask = MAKE_64BIT_MASK(0, 16); break; case INDEX_op_ld32s_i64: - ctx->s_mask = MAKE_64BIT_MASK(32, 32); + s_mask = INT32_MIN; break; case INDEX_op_ld32u_i64: - ctx->z_mask = MAKE_64BIT_MASK(0, 32); + z_mask = MAKE_64BIT_MASK(0, 32); break; default: g_assert_not_reached(); } - return false; + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_tcg_ld_memcopy(OptContext *ctx, TCGOp *op) -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_ld_memcopy(OptContext *ctx, TCGOp *op) TCGType type; if (op->args[1] != tcgv_ptr_arg(tcg_env)) { - return false; + return finish_folding(ctx, op); } type = ctx->type; -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Remove fold_masks as the function becomes unused. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_masks_s(OptContext *ctx, TCGOp *op, uint64_t s_mask) return fold_masks_zs(ctx, op, -1, s_mask); } -static bool fold_masks(OptContext *ctx, TCGOp *op) -{ - return fold_masks_zs(ctx, op, ctx->z_mask, ctx->s_mask); -} - /* * An "affected" mask bit is 0 if and only if the result is identical * to the first input. Thus if the entire mask is 0, the operation @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_st_memcopy(OptContext *ctx, TCGOp *op) static bool fold_xor(OptContext *ctx, TCGOp *op) { + uint64_t z_mask, s_mask; + TempOptInfo *t1, *t2; + if (fold_const2_commutative(ctx, op) || fold_xx_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, 0) || @@ -XXX,XX +XXX,XX @@ static bool fold_xor(OptContext *ctx, TCGOp *op) return true; } - ctx->z_mask = arg_info(op->args[1])->z_mask - | arg_info(op->args[2])->z_mask; - ctx->s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; - return fold_masks(ctx, op); + t1 = arg_info(op->args[1]); + t2 = arg_info(op->args[2]); + z_mask = t1->z_mask | t2->z_mask; + s_mask = t1->s_mask & t2->s_mask; + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_bitsel_vec(OptContext *ctx, TCGOp *op) -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_bitsel_vec(OptContext *ctx, TCGOp *op) return fold_orc(ctx, op); } } - return false; + return finish_folding(ctx, op); } /* Propagate constants and copies, fold constant expressions. */ -- 2.43.0 All non-default cases now finish folding within each function. Do the same with the default case and assert it is done after. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) done = true; break; default: + done = finish_folding(&ctx, op); break; } - - if (!done) { - finish_folding(&ctx, op); - } + tcg_debug_assert(done); } } -- 2.43.0 All mask setting is now done with parameters via fold_masks_*. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { QSIMPLEQ_HEAD(, MemCopyInfo) mem_free; /* In flight values from optimization. */ - uint64_t z_mask; /* mask bit is 0 iff value bit is 0 */ - uint64_t s_mask; /* mask bit is 1 if value bit matches msb */ TCGType type; } OptContext; @@ -XXX,XX +XXX,XX @@ static bool finish_folding(OptContext *ctx, TCGOp *op) for (i = 0; i < nb_oargs; i++) { TCGTemp *ts = arg_temp(op->args[i]); reset_ts(ctx, ts); - /* - * Save the corresponding known-zero/sign bits mask for the - * first output argument (only one supported so far). - */ - if (i == 0) { - ts_info(ts)->z_mask = ctx->z_mask; - } } return true; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) ctx.type = TCG_TYPE_I32; } - /* Assume all bits affected, no bits known zero, no sign reps. */ - ctx.z_mask = -1; - ctx.s_mask = 0; - /* * Process each opcode. * Sorted alphabetically by opcode as much as possible. -- 2.43.0 All instances of s_mask have been converted to the new representation. We can now re-enable usage. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_exts(OptContext *ctx, TCGOp *op) g_assert_not_reached(); } - if (0 && !type_change && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { + if (!type_change && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) s_mask = s_mask_old >> pos; s_mask |= -1ull << (len - 1); - if (0 && pos == 0 && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { + if (pos == 0 && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { return true; } -- 2.43.0 The big comment just above says functions should be sorted. Add forward declarations as needed. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 114 +++++++++++++++++++++++++------------------------ 1 file changed, 59 insertions(+), 55 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_xx_to_x(OptContext *ctx, TCGOp *op) * 3) those that produce information about the result value. */ +static bool fold_or(OptContext *ctx, TCGOp *op); +static bool fold_orc(OptContext *ctx, TCGOp *op); +static bool fold_xor(OptContext *ctx, TCGOp *op); + static bool fold_add(OptContext *ctx, TCGOp *op) { if (fold_const2_commutative(ctx, op) || @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) return fold_masks_zs(ctx, op, z_mask, s_mask); } +static bool fold_bitsel_vec(OptContext *ctx, TCGOp *op) +{ + /* If true and false values are the same, eliminate the cmp. */ + if (args_are_copies(op->args[2], op->args[3])) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]); + } + + if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { + uint64_t tv = arg_info(op->args[2])->val; + uint64_t fv = arg_info(op->args[3])->val; + + if (tv == -1 && fv == 0) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); + } + if (tv == 0 && fv == -1) { + if (TCG_TARGET_HAS_not_vec) { + op->opc = INDEX_op_not_vec; + return fold_not(ctx, op); + } else { + op->opc = INDEX_op_xor_vec; + op->args[2] = arg_new_constant(ctx, -1); + return fold_xor(ctx, op); + } + } + } + if (arg_is_const(op->args[2])) { + uint64_t tv = arg_info(op->args[2])->val; + if (tv == -1) { + op->opc = INDEX_op_or_vec; + op->args[2] = op->args[3]; + return fold_or(ctx, op); + } + if (tv == 0 && TCG_TARGET_HAS_andc_vec) { + op->opc = INDEX_op_andc_vec; + op->args[2] = op->args[1]; + op->args[1] = op->args[3]; + return fold_andc(ctx, op); + } + } + if (arg_is_const(op->args[3])) { + uint64_t fv = arg_info(op->args[3])->val; + if (fv == 0) { + op->opc = INDEX_op_and_vec; + return fold_and(ctx, op); + } + if (fv == -1 && TCG_TARGET_HAS_orc_vec) { + op->opc = INDEX_op_orc_vec; + op->args[2] = op->args[1]; + op->args[1] = op->args[3]; + return fold_orc(ctx, op); + } + } + return finish_folding(ctx, op); +} + static bool fold_brcond(OptContext *ctx, TCGOp *op) { int i = do_constant_folding_cond1(ctx, op, NO_DEST, &op->args[0], @@ -XXX,XX +XXX,XX @@ static bool fold_xor(OptContext *ctx, TCGOp *op) return fold_masks_zs(ctx, op, z_mask, s_mask); } -static bool fold_bitsel_vec(OptContext *ctx, TCGOp *op) -{ - /* If true and false values are the same, eliminate the cmp. */ - if (args_are_copies(op->args[2], op->args[3])) { - return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]); - } - - if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { - uint64_t tv = arg_info(op->args[2])->val; - uint64_t fv = arg_info(op->args[3])->val; - - if (tv == -1 && fv == 0) { - return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); - } - if (tv == 0 && fv == -1) { - if (TCG_TARGET_HAS_not_vec) { - op->opc = INDEX_op_not_vec; - return fold_not(ctx, op); - } else { - op->opc = INDEX_op_xor_vec; - op->args[2] = arg_new_constant(ctx, -1); - return fold_xor(ctx, op); - } - } - } - if (arg_is_const(op->args[2])) { - uint64_t tv = arg_info(op->args[2])->val; - if (tv == -1) { - op->opc = INDEX_op_or_vec; - op->args[2] = op->args[3]; - return fold_or(ctx, op); - } - if (tv == 0 && TCG_TARGET_HAS_andc_vec) { - op->opc = INDEX_op_andc_vec; - op->args[2] = op->args[1]; - op->args[1] = op->args[3]; - return fold_andc(ctx, op); - } - } - if (arg_is_const(op->args[3])) { - uint64_t fv = arg_info(op->args[3])->val; - if (fv == 0) { - op->opc = INDEX_op_and_vec; - return fold_and(ctx, op); - } - if (fv == -1 && TCG_TARGET_HAS_orc_vec) { - op->opc = INDEX_op_orc_vec; - op->args[2] = op->args[1]; - op->args[1] = op->args[3]; - return fold_orc(ctx, op); - } - } - return finish_folding(ctx, op); -} - /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { -- 2.43.0 The big comment just above says functions should be sorted. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 60 +++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_call(OptContext *ctx, TCGOp *op) return true; } +static bool fold_cmp_vec(OptContext *ctx, TCGOp *op) +{ + /* Canonicalize the comparison to put immediate second. */ + if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) { + op->args[3] = tcg_swap_cond(op->args[3]); + } + return finish_folding(ctx, op); +} + +static bool fold_cmpsel_vec(OptContext *ctx, TCGOp *op) +{ + /* If true and false values are the same, eliminate the cmp. */ + if (args_are_copies(op->args[3], op->args[4])) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[3]); + } + + /* Canonicalize the comparison to put immediate second. */ + if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) { + op->args[5] = tcg_swap_cond(op->args[5]); + } + /* + * Canonicalize the "false" input reg to match the destination, + * so that the tcg backend can implement "move if true". + */ + if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) { + op->args[5] = tcg_invert_cond(op->args[5]); + } + return finish_folding(ctx, op); +} + static bool fold_count_zeros(OptContext *ctx, TCGOp *op) { uint64_t z_mask, s_mask; @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) return tcg_opt_gen_movi(ctx, op, op->args[0], i); } -static bool fold_cmp_vec(OptContext *ctx, TCGOp *op) -{ - /* Canonicalize the comparison to put immediate second. */ - if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) { - op->args[3] = tcg_swap_cond(op->args[3]); - } - return finish_folding(ctx, op); -} - -static bool fold_cmpsel_vec(OptContext *ctx, TCGOp *op) -{ - /* If true and false values are the same, eliminate the cmp. */ - if (args_are_copies(op->args[3], op->args[4])) { - return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[3]); - } - - /* Canonicalize the comparison to put immediate second. */ - if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) { - op->args[5] = tcg_swap_cond(op->args[5]); - } - /* - * Canonicalize the "false" input reg to match the destination, - * so that the tcg backend can implement "move if true". - */ - if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) { - op->args[5] = tcg_invert_cond(op->args[5]); - } - return finish_folding(ctx, op); -} - static bool fold_sextract(OptContext *ctx, TCGOp *op) { uint64_t z_mask, s_mask, s_mask_old; -- 2.43.0 We currently have a flag, float_muladd_halve_result, to scale the result by 2**-1. Extend this to handle arbitrary scaling. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat.h | 6 ++++ fpu/softfloat.c | 58 ++++++++++++++++++++++----------------- fpu/softfloat-parts.c.inc | 7 +++-- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/include/fpu/softfloat.h b/include/fpu/softfloat.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat.h +++ b/include/fpu/softfloat.h @@ -XXX,XX +XXX,XX @@ float16 float16_add(float16, float16, float_status *status); float16 float16_sub(float16, float16, float_status *status); float16 float16_mul(float16, float16, float_status *status); float16 float16_muladd(float16, float16, float16, int, float_status *status); +float16 float16_muladd_scalbn(float16, float16, float16, + int, int, float_status *status); float16 float16_div(float16, float16, float_status *status); float16 float16_scalbn(float16, int, float_status *status); float16 float16_min(float16, float16, float_status *status); @@ -XXX,XX +XXX,XX @@ float32 float32_mul(float32, float32, float_status *status); float32 float32_div(float32, float32, float_status *status); float32 float32_rem(float32, float32, float_status *status); float32 float32_muladd(float32, float32, float32, int, float_status *status); +float32 float32_muladd_scalbn(float32, float32, float32, + int, int, float_status *status); float32 float32_sqrt(float32, float_status *status); float32 float32_exp2(float32, float_status *status); float32 float32_log2(float32, float_status *status); @@ -XXX,XX +XXX,XX @@ float64 float64_mul(float64, float64, float_status *status); float64 float64_div(float64, float64, float_status *status); float64 float64_rem(float64, float64, float_status *status); float64 float64_muladd(float64, float64, float64, int, float_status *status); +float64 float64_muladd_scalbn(float64, float64, float64, + int, int, float_status *status); float64 float64_sqrt(float64, float_status *status); float64 float64_log2(float64, float_status *status); FloatRelation float64_compare(float64, float64, float_status *status); diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static FloatParts128 *parts128_mul(FloatParts128 *a, FloatParts128 *b, #define parts_mul(A, B, S) \ PARTS_GENERIC_64_128(mul, A)(A, B, S) -static FloatParts64 *parts64_muladd(FloatParts64 *a, FloatParts64 *b, - FloatParts64 *c, int flags, - float_status *s); -static FloatParts128 *parts128_muladd(FloatParts128 *a, FloatParts128 *b, - FloatParts128 *c, int flags, - float_status *s); +static FloatParts64 *parts64_muladd_scalbn(FloatParts64 *a, FloatParts64 *b, + FloatParts64 *c, int scale, + int flags, float_status *s); +static FloatParts128 *parts128_muladd_scalbn(FloatParts128 *a, FloatParts128 *b, + FloatParts128 *c, int scale, + int flags, float_status *s); -#define parts_muladd(A, B, C, Z, S) \ - PARTS_GENERIC_64_128(muladd, A)(A, B, C, Z, S) +#define parts_muladd_scalbn(A, B, C, Z, Y, S) \ + PARTS_GENERIC_64_128(muladd_scalbn, A)(A, B, C, Z, Y, S) static FloatParts64 *parts64_div(FloatParts64 *a, FloatParts64 *b, float_status *s); @@ -XXX,XX +XXX,XX @@ floatx80_mul(floatx80 a, floatx80 b, float_status *status) * Fused multiply-add */ -float16 QEMU_FLATTEN float16_muladd(float16 a, float16 b, float16 c, - int flags, float_status *status) +float16 QEMU_FLATTEN +float16_muladd_scalbn(float16 a, float16 b, float16 c, + int scale, int flags, float_status *status) { FloatParts64 pa, pb, pc, *pr; float16_unpack_canonical(&pa, a, status); float16_unpack_canonical(&pb, b, status); float16_unpack_canonical(&pc, c, status); - pr = parts_muladd(&pa, &pb, &pc, flags, status); + pr = parts_muladd_scalbn(&pa, &pb, &pc, scale, flags, status); return float16_round_pack_canonical(pr, status); } -static float32 QEMU_SOFTFLOAT_ATTR -soft_f32_muladd(float32 a, float32 b, float32 c, int flags, - float_status *status) +float16 float16_muladd(float16 a, float16 b, float16 c, + int flags, float_status *status) +{ + return float16_muladd_scalbn(a, b, c, 0, flags, status); +} + +float32 QEMU_SOFTFLOAT_ATTR +float32_muladd_scalbn(float32 a, float32 b, float32 c, + int scale, int flags, float_status *status) { FloatParts64 pa, pb, pc, *pr; float32_unpack_canonical(&pa, a, status); float32_unpack_canonical(&pb, b, status); float32_unpack_canonical(&pc, c, status); - pr = parts_muladd(&pa, &pb, &pc, flags, status); + pr = parts_muladd_scalbn(&pa, &pb, &pc, scale, flags, status); return float32_round_pack_canonical(pr, status); } -static float64 QEMU_SOFTFLOAT_ATTR -soft_f64_muladd(float64 a, float64 b, float64 c, int flags, - float_status *status) +float64 QEMU_SOFTFLOAT_ATTR +float64_muladd_scalbn(float64 a, float64 b, float64 c, + int scale, int flags, float_status *status) { FloatParts64 pa, pb, pc, *pr; float64_unpack_canonical(&pa, a, status); float64_unpack_canonical(&pb, b, status); float64_unpack_canonical(&pc, c, status); - pr = parts_muladd(&pa, &pb, &pc, flags, status); + pr = parts_muladd_scalbn(&pa, &pb, &pc, scale, flags, status); return float64_round_pack_canonical(pr, status); } @@ -XXX,XX +XXX,XX @@ float32_muladd(float32 xa, float32 xb, float32 xc, int flags, float_status *s) return ur.s; soft: - return soft_f32_muladd(ua.s, ub.s, uc.s, flags, s); + return float32_muladd_scalbn(ua.s, ub.s, uc.s, 0, flags, s); } float64 QEMU_FLATTEN @@ -XXX,XX +XXX,XX @@ float64_muladd(float64 xa, float64 xb, float64 xc, int flags, float_status *s) return ur.s; soft: - return soft_f64_muladd(ua.s, ub.s, uc.s, flags, s); + return float64_muladd_scalbn(ua.s, ub.s, uc.s, 0, flags, s); } float64 float64r32_muladd(float64 a, float64 b, float64 c, @@ -XXX,XX +XXX,XX @@ float64 float64r32_muladd(float64 a, float64 b, float64 c, float64_unpack_canonical(&pa, a, status); float64_unpack_canonical(&pb, b, status); float64_unpack_canonical(&pc, c, status); - pr = parts_muladd(&pa, &pb, &pc, flags, status); + pr = parts_muladd_scalbn(&pa, &pb, &pc, 0, flags, status); return float64r32_round_pack_canonical(pr, status); } @@ -XXX,XX +XXX,XX @@ bfloat16 QEMU_FLATTEN bfloat16_muladd(bfloat16 a, bfloat16 b, bfloat16 c, bfloat16_unpack_canonical(&pa, a, status); bfloat16_unpack_canonical(&pb, b, status); bfloat16_unpack_canonical(&pc, c, status); - pr = parts_muladd(&pa, &pb, &pc, flags, status); + pr = parts_muladd_scalbn(&pa, &pb, &pc, 0, flags, status); return bfloat16_round_pack_canonical(pr, status); } @@ -XXX,XX +XXX,XX @@ float128 QEMU_FLATTEN float128_muladd(float128 a, float128 b, float128 c, float128_unpack_canonical(&pa, a, status); float128_unpack_canonical(&pb, b, status); float128_unpack_canonical(&pc, c, status); - pr = parts_muladd(&pa, &pb, &pc, flags, status); + pr = parts_muladd_scalbn(&pa, &pb, &pc, 0, flags, status); return float128_round_pack_canonical(pr, status); } @@ -XXX,XX +XXX,XX @@ float32 float32_exp2(float32 a, float_status *status) float64_unpack_canonical(&rp, float64_one, status); for (i = 0 ; i < 15 ; i++) { + float64_unpack_canonical(&tp, float32_exp2_coefficients[i], status); - rp = *parts_muladd(&tp, &xnp, &rp, 0, status); + rp = *parts_muladd_scalbn(&tp, &xnp, &rp, 0, 0, status); xnp = *parts_mul(&xnp, &xp, status); } diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(mul)(FloatPartsN *a, FloatPartsN *b, * Requires A and C extracted into a double-sized structure to provide the * extra space for the widening multiply. */ -static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, - FloatPartsN *c, int flags, float_status *s) +static FloatPartsN *partsN(muladd_scalbn)(FloatPartsN *a, FloatPartsN *b, + FloatPartsN *c, int scale, + int flags, float_status *s) { int ab_mask, abc_mask; FloatPartsW p_widen, c_widen; @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, a->exp = p_widen.exp; return_normal: + /* TODO: Replace all use of float_muladd_halve_result with scale. */ if (flags & float_muladd_halve_result) { a->exp -= 1; } + a->exp += scale; finish_sign: if (flags & float_muladd_negate_result) { a->sign ^= 1; -- 2.43.0 Use the scalbn interface instead of float_muladd_halve_result. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/arm/tcg/helper-a64.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/target/arm/tcg/helper-a64.c b/target/arm/tcg/helper-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/tcg/helper-a64.c +++ b/target/arm/tcg/helper-a64.c @@ -XXX,XX +XXX,XX @@ uint32_t HELPER(rsqrtsf_f16)(uint32_t a, uint32_t b, float_status *fpst) (float16_is_infinity(b) && float16_is_zero(a))) { return float16_one_point_five; } - return float16_muladd(a, b, float16_three, float_muladd_halve_result, fpst); + return float16_muladd_scalbn(a, b, float16_three, -1, 0, fpst); } float32 HELPER(rsqrtsf_f32)(float32 a, float32 b, float_status *fpst) @@ -XXX,XX +XXX,XX @@ float32 HELPER(rsqrtsf_f32)(float32 a, float32 b, float_status *fpst) (float32_is_infinity(b) && float32_is_zero(a))) { return float32_one_point_five; } - return float32_muladd(a, b, float32_three, float_muladd_halve_result, fpst); + return float32_muladd_scalbn(a, b, float32_three, -1, 0, fpst); } float64 HELPER(rsqrtsf_f64)(float64 a, float64 b, float_status *fpst) @@ -XXX,XX +XXX,XX @@ float64 HELPER(rsqrtsf_f64)(float64 a, float64 b, float_status *fpst) (float64_is_infinity(b) && float64_is_zero(a))) { return float64_one_point_five; } - return float64_muladd(a, b, float64_three, float_muladd_halve_result, fpst); + return float64_muladd_scalbn(a, b, float64_three, -1, 0, fpst); } /* Floating-point reciprocal exponent - see FPRecpX in ARM ARM */ -- 2.43.0 Use the scalbn interface instead of float_muladd_halve_result. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/sparc/helper.h | 4 +- target/sparc/fop_helper.c | 8 ++-- target/sparc/translate.c | 80 +++++++++++++++++++++++---------------- 3 files changed, 54 insertions(+), 38 deletions(-) diff --git a/target/sparc/helper.h b/target/sparc/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/helper.h +++ b/target/sparc/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_3(faddd, TCG_CALL_NO_WG, f64, env, f64, f64) DEF_HELPER_FLAGS_3(fsubd, TCG_CALL_NO_WG, f64, env, f64, f64) DEF_HELPER_FLAGS_3(fmuld, TCG_CALL_NO_WG, f64, env, f64, f64) DEF_HELPER_FLAGS_3(fdivd, TCG_CALL_NO_WG, f64, env, f64, f64) -DEF_HELPER_FLAGS_5(fmaddd, TCG_CALL_NO_WG, f64, env, f64, f64, f64, i32) +DEF_HELPER_FLAGS_6(fmaddd, TCG_CALL_NO_WG, f64, env, f64, f64, f64, s32, i32) DEF_HELPER_FLAGS_3(fnaddd, TCG_CALL_NO_WG, f64, env, f64, f64) DEF_HELPER_FLAGS_3(fnmuld, TCG_CALL_NO_WG, f64, env, f64, f64) @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_3(fadds, TCG_CALL_NO_WG, f32, env, f32, f32) DEF_HELPER_FLAGS_3(fsubs, TCG_CALL_NO_WG, f32, env, f32, f32) DEF_HELPER_FLAGS_3(fmuls, TCG_CALL_NO_WG, f32, env, f32, f32) DEF_HELPER_FLAGS_3(fdivs, TCG_CALL_NO_WG, f32, env, f32, f32) -DEF_HELPER_FLAGS_5(fmadds, TCG_CALL_NO_WG, f32, env, f32, f32, f32, i32) +DEF_HELPER_FLAGS_6(fmadds, TCG_CALL_NO_WG, f32, env, f32, f32, f32, s32, i32) DEF_HELPER_FLAGS_3(fnadds, TCG_CALL_NO_WG, f32, env, f32, f32) DEF_HELPER_FLAGS_3(fnmuls, TCG_CALL_NO_WG, f32, env, f32, f32) diff --git a/target/sparc/fop_helper.c b/target/sparc/fop_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/fop_helper.c +++ b/target/sparc/fop_helper.c @@ -XXX,XX +XXX,XX @@ Int128 helper_fsqrtq(CPUSPARCState *env, Int128 src) } float32 helper_fmadds(CPUSPARCState *env, float32 s1, - float32 s2, float32 s3, uint32_t op) + float32 s2, float32 s3, int32_t sc, uint32_t op) { - float32 ret = float32_muladd(s1, s2, s3, op, &env->fp_status); + float32 ret = float32_muladd_scalbn(s1, s2, s3, sc, op, &env->fp_status); check_ieee_exceptions(env, GETPC()); return ret; } float64 helper_fmaddd(CPUSPARCState *env, float64 s1, - float64 s2, float64 s3, uint32_t op) + float64 s2, float64 s3, int32_t sc, uint32_t op) { - float64 ret = float64_muladd(s1, s2, s3, op, &env->fp_status); + float64 ret = float64_muladd_scalbn(s1, s2, s3, sc, op, &env->fp_status); check_ieee_exceptions(env, GETPC()); return ret; } diff --git a/target/sparc/translate.c b/target/sparc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/translate.c +++ b/target/sparc/translate.c @@ -XXX,XX +XXX,XX @@ static void gen_op_fabsq(TCGv_i128 dst, TCGv_i128 src) static void gen_op_fmadds(TCGv_i32 d, TCGv_i32 s1, TCGv_i32 s2, TCGv_i32 s3) { - gen_helper_fmadds(d, tcg_env, s1, s2, s3, tcg_constant_i32(0)); + TCGv_i32 z = tcg_constant_i32(0); + gen_helper_fmadds(d, tcg_env, s1, s2, s3, z, z); } static void gen_op_fmaddd(TCGv_i64 d, TCGv_i64 s1, TCGv_i64 s2, TCGv_i64 s3) { - gen_helper_fmaddd(d, tcg_env, s1, s2, s3, tcg_constant_i32(0)); + TCGv_i32 z = tcg_constant_i32(0); + gen_helper_fmaddd(d, tcg_env, s1, s2, s3, z, z); } static void gen_op_fmsubs(TCGv_i32 d, TCGv_i32 s1, TCGv_i32 s2, TCGv_i32 s3) { - int op = float_muladd_negate_c; - gen_helper_fmadds(d, tcg_env, s1, s2, s3, tcg_constant_i32(op)); + TCGv_i32 z = tcg_constant_i32(0); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_c); + gen_helper_fmadds(d, tcg_env, s1, s2, s3, z, op); } static void gen_op_fmsubd(TCGv_i64 d, TCGv_i64 s1, TCGv_i64 s2, TCGv_i64 s3) { - int op = float_muladd_negate_c; - gen_helper_fmaddd(d, tcg_env, s1, s2, s3, tcg_constant_i32(op)); + TCGv_i32 z = tcg_constant_i32(0); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_c); + gen_helper_fmaddd(d, tcg_env, s1, s2, s3, z, op); } static void gen_op_fnmsubs(TCGv_i32 d, TCGv_i32 s1, TCGv_i32 s2, TCGv_i32 s3) { - int op = float_muladd_negate_c | float_muladd_negate_result; - gen_helper_fmadds(d, tcg_env, s1, s2, s3, tcg_constant_i32(op)); + TCGv_i32 z = tcg_constant_i32(0); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_c | + float_muladd_negate_result); + gen_helper_fmadds(d, tcg_env, s1, s2, s3, z, op); } static void gen_op_fnmsubd(TCGv_i64 d, TCGv_i64 s1, TCGv_i64 s2, TCGv_i64 s3) { - int op = float_muladd_negate_c | float_muladd_negate_result; - gen_helper_fmaddd(d, tcg_env, s1, s2, s3, tcg_constant_i32(op)); + TCGv_i32 z = tcg_constant_i32(0); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_c | + float_muladd_negate_result); + gen_helper_fmaddd(d, tcg_env, s1, s2, s3, z, op); } static void gen_op_fnmadds(TCGv_i32 d, TCGv_i32 s1, TCGv_i32 s2, TCGv_i32 s3) { - int op = float_muladd_negate_result; - gen_helper_fmadds(d, tcg_env, s1, s2, s3, tcg_constant_i32(op)); + TCGv_i32 z = tcg_constant_i32(0); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_result); + gen_helper_fmadds(d, tcg_env, s1, s2, s3, z, op); } static void gen_op_fnmaddd(TCGv_i64 d, TCGv_i64 s1, TCGv_i64 s2, TCGv_i64 s3) { - int op = float_muladd_negate_result; - gen_helper_fmaddd(d, tcg_env, s1, s2, s3, tcg_constant_i32(op)); + TCGv_i32 z = tcg_constant_i32(0); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_result); + gen_helper_fmaddd(d, tcg_env, s1, s2, s3, z, op); } /* Use muladd to compute (1 * src1) + src2 / 2 with one rounding. */ static void gen_op_fhadds(TCGv_i32 d, TCGv_i32 s1, TCGv_i32 s2) { - TCGv_i32 one = tcg_constant_i32(float32_one); - int op = float_muladd_halve_result; - gen_helper_fmadds(d, tcg_env, one, s1, s2, tcg_constant_i32(op)); + TCGv_i32 fone = tcg_constant_i32(float32_one); + TCGv_i32 mone = tcg_constant_i32(-1); + TCGv_i32 op = tcg_constant_i32(0); + gen_helper_fmadds(d, tcg_env, fone, s1, s2, mone, op); } static void gen_op_fhaddd(TCGv_i64 d, TCGv_i64 s1, TCGv_i64 s2) { - TCGv_i64 one = tcg_constant_i64(float64_one); - int op = float_muladd_halve_result; - gen_helper_fmaddd(d, tcg_env, one, s1, s2, tcg_constant_i32(op)); + TCGv_i64 fone = tcg_constant_i64(float64_one); + TCGv_i32 mone = tcg_constant_i32(-1); + TCGv_i32 op = tcg_constant_i32(0); + gen_helper_fmaddd(d, tcg_env, fone, s1, s2, mone, op); } /* Use muladd to compute (1 * src1) - src2 / 2 with one rounding. */ static void gen_op_fhsubs(TCGv_i32 d, TCGv_i32 s1, TCGv_i32 s2) { - TCGv_i32 one = tcg_constant_i32(float32_one); - int op = float_muladd_negate_c | float_muladd_halve_result; - gen_helper_fmadds(d, tcg_env, one, s1, s2, tcg_constant_i32(op)); + TCGv_i32 fone = tcg_constant_i32(float32_one); + TCGv_i32 mone = tcg_constant_i32(-1); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_c); + gen_helper_fmadds(d, tcg_env, fone, s1, s2, mone, op); } static void gen_op_fhsubd(TCGv_i64 d, TCGv_i64 s1, TCGv_i64 s2) { - TCGv_i64 one = tcg_constant_i64(float64_one); - int op = float_muladd_negate_c | float_muladd_halve_result; - gen_helper_fmaddd(d, tcg_env, one, s1, s2, tcg_constant_i32(op)); + TCGv_i64 fone = tcg_constant_i64(float64_one); + TCGv_i32 mone = tcg_constant_i32(-1); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_c); + gen_helper_fmaddd(d, tcg_env, fone, s1, s2, mone, op); } /* Use muladd to compute -((1 * src1) + src2 / 2) with one rounding. */ static void gen_op_fnhadds(TCGv_i32 d, TCGv_i32 s1, TCGv_i32 s2) { - TCGv_i32 one = tcg_constant_i32(float32_one); - int op = float_muladd_negate_result | float_muladd_halve_result; - gen_helper_fmadds(d, tcg_env, one, s1, s2, tcg_constant_i32(op)); + TCGv_i32 fone = tcg_constant_i32(float32_one); + TCGv_i32 mone = tcg_constant_i32(-1); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_result); + gen_helper_fmadds(d, tcg_env, fone, s1, s2, mone, op); } static void gen_op_fnhaddd(TCGv_i64 d, TCGv_i64 s1, TCGv_i64 s2) { - TCGv_i64 one = tcg_constant_i64(float64_one); - int op = float_muladd_negate_result | float_muladd_halve_result; - gen_helper_fmaddd(d, tcg_env, one, s1, s2, tcg_constant_i32(op)); + TCGv_i64 fone = tcg_constant_i64(float64_one); + TCGv_i32 mone = tcg_constant_i32(-1); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_result); + gen_helper_fmaddd(d, tcg_env, fone, s1, s2, mone, op); } static void gen_op_fpexception_im(DisasContext *dc, int ftt) -- 2.43.0 All uses have been convered to float*_muladd_scalbn. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat.h | 3 --- fpu/softfloat.c | 6 ------ fpu/softfloat-parts.c.inc | 4 ---- 3 files changed, 13 deletions(-) diff --git a/include/fpu/softfloat.h b/include/fpu/softfloat.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat.h +++ b/include/fpu/softfloat.h @@ -XXX,XX +XXX,XX @@ bfloat16 bfloat16_squash_input_denormal(bfloat16 a, float_status *status); | Using these differs from negating an input or output before calling | the muladd function in that this means that a NaN doesn't have its | sign bit inverted before it is propagated. -| We also support halving the result before rounding, as a special -| case to support the ARM fused-sqrt-step instruction FRSQRTS. *----------------------------------------------------------------------------*/ enum { float_muladd_negate_c = 1, float_muladd_negate_product = 2, float_muladd_negate_result = 4, - float_muladd_halve_result = 8, }; /*---------------------------------------------------------------------------- diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ float32_muladd(float32 xa, float32 xb, float32 xc, int flags, float_status *s) if (unlikely(!can_use_fpu(s))) { goto soft; } - if (unlikely(flags & float_muladd_halve_result)) { - goto soft; - } float32_input_flush3(&ua.s, &ub.s, &uc.s, s); if (unlikely(!f32_is_zon3(ua, ub, uc))) { @@ -XXX,XX +XXX,XX @@ float64_muladd(float64 xa, float64 xb, float64 xc, int flags, float_status *s) if (unlikely(!can_use_fpu(s))) { goto soft; } - if (unlikely(flags & float_muladd_halve_result)) { - goto soft; - } float64_input_flush3(&ua.s, &ub.s, &uc.s, s); if (unlikely(!f64_is_zon3(ua, ub, uc))) { diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd_scalbn)(FloatPartsN *a, FloatPartsN *b, a->exp = p_widen.exp; return_normal: - /* TODO: Replace all use of float_muladd_halve_result with scale. */ - if (flags & float_muladd_halve_result) { - a->exp -= 1; - } a->exp += scale; finish_sign: if (flags & float_muladd_negate_result) { -- 2.43.0 This rounding mode is used by Hexagon. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-types.h | 2 ++ fpu/softfloat-parts.c.inc | 3 +++ 2 files changed, 5 insertions(+) diff --git a/include/fpu/softfloat-types.h b/include/fpu/softfloat-types.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-types.h +++ b/include/fpu/softfloat-types.h @@ -XXX,XX +XXX,XX @@ typedef enum __attribute__((__packed__)) { float_round_to_odd = 5, /* Not an IEEE rounding mode: round to closest odd, overflow to inf */ float_round_to_odd_inf = 6, + /* Not an IEEE rounding mode: round to nearest even, overflow to max */ + float_round_nearest_even_max = 7, } FloatRoundMode; /* diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static void partsN(uncanon_normal)(FloatPartsN *p, float_status *s, int exp, flags = 0; switch (s->float_rounding_mode) { + case float_round_nearest_even_max: + overflow_norm = true; + /* fall through */ case float_round_nearest_even: if (N > 64 && frac_lsb == 0) { inc = ((p->frac_hi & 1) || (p->frac_lo & round_mask) != frac_lsbm1 -- 2.43.0 Certain Hexagon instructions suppress changes to the result when the product of fma() is a true zero. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat.h | 5 +++++ fpu/softfloat.c | 3 +++ fpu/softfloat-parts.c.inc | 4 +++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/include/fpu/softfloat.h b/include/fpu/softfloat.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat.h +++ b/include/fpu/softfloat.h @@ -XXX,XX +XXX,XX @@ bfloat16 bfloat16_squash_input_denormal(bfloat16 a, float_status *status); | Using these differs from negating an input or output before calling | the muladd function in that this means that a NaN doesn't have its | sign bit inverted before it is propagated. +| +| With float_muladd_suppress_add_product_zero, if A or B is zero +| such that the product is a true zero, then return C without addition. +| This preserves the sign of C when C is +/- 0. Used for Hexagon. *----------------------------------------------------------------------------*/ enum { float_muladd_negate_c = 1, float_muladd_negate_product = 2, float_muladd_negate_result = 4, + float_muladd_suppress_add_product_zero = 8, }; /*---------------------------------------------------------------------------- diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ float32_muladd(float32 xa, float32 xb, float32 xc, int flags, float_status *s) if (unlikely(!can_use_fpu(s))) { goto soft; } + if (unlikely(flags & float_muladd_suppress_add_product_zero)) { + goto soft; + } float32_input_flush3(&ua.s, &ub.s, &uc.s, s); if (unlikely(!f32_is_zon3(ua, ub, uc))) { diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd_scalbn)(FloatPartsN *a, FloatPartsN *b, goto return_normal; } if (c->cls == float_class_zero) { - if (a->sign != c->sign) { + if (flags & float_muladd_suppress_add_product_zero) { + a->sign = c->sign; + } else if (a->sign != c->sign) { goto return_sub_zero; } goto return_zero; -- 2.43.0 There are no special cases for this instruction. Remove internal_mpyf as unused. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/fma_emu.h | 1 - target/hexagon/fma_emu.c | 8 -------- target/hexagon/op_helper.c | 2 +- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/target/hexagon/fma_emu.h b/target/hexagon/fma_emu.h index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.h +++ b/target/hexagon/fma_emu.h @@ -XXX,XX +XXX,XX @@ int32_t float32_getexp(float32 f32); float32 infinite_float32(uint8_t sign); float32 internal_fmafx(float32 a, float32 b, float32 c, int scale, float_status *fp_status); -float32 internal_mpyf(float32 a, float32 b, float_status *fp_status); float64 internal_mpyhh(float64 a, float64 b, unsigned long long int accumulated, float_status *fp_status); diff --git a/target/hexagon/fma_emu.c b/target/hexagon/fma_emu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.c +++ b/target/hexagon/fma_emu.c @@ -XXX,XX +XXX,XX @@ float32 internal_fmafx(float32 a, float32 b, float32 c, int scale, return accum_round_float32(result, fp_status); } -float32 internal_mpyf(float32 a, float32 b, float_status *fp_status) -{ - if (float32_is_zero(a) || float32_is_zero(b)) { - return float32_mul(a, b, fp_status); - } - return internal_fmafx(a, b, float32_zero, 0, fp_status); -} - float64 internal_mpyhh(float64 a, float64 b, unsigned long long int accumulated, float_status *fp_status) diff --git a/target/hexagon/op_helper.c b/target/hexagon/op_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/op_helper.c +++ b/target/hexagon/op_helper.c @@ -XXX,XX +XXX,XX @@ float32 HELPER(sfmpy)(CPUHexagonState *env, float32 RsV, float32 RtV) { float32 RdV; arch_fpop_start(env); - RdV = internal_mpyf(RsV, RtV, &env->fp_status); + RdV = float32_mul(RsV, RtV, &env->fp_status); arch_fpop_end(env); return RdV; } -- 2.43.0 There are no special cases for this instruction. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/op_helper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/hexagon/op_helper.c b/target/hexagon/op_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/op_helper.c +++ b/target/hexagon/op_helper.c @@ -XXX,XX +XXX,XX @@ float32 HELPER(sffma)(CPUHexagonState *env, float32 RxV, float32 RsV, float32 RtV) { arch_fpop_start(env); - RxV = internal_fmafx(RsV, RtV, RxV, 0, &env->fp_status); + RxV = float32_muladd(RsV, RtV, RxV, 0, &env->fp_status); arch_fpop_end(env); return RxV; } -- 2.43.0 There are no special cases for this instruction. Since hexagon always uses default-nan mode, explicitly negating the first input is unnecessary. Use float_muladd_negate_product instead. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/op_helper.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/target/hexagon/op_helper.c b/target/hexagon/op_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/op_helper.c +++ b/target/hexagon/op_helper.c @@ -XXX,XX +XXX,XX @@ float32 HELPER(sffma_sc)(CPUHexagonState *env, float32 RxV, float32 HELPER(sffms)(CPUHexagonState *env, float32 RxV, float32 RsV, float32 RtV) { - float32 neg_RsV; arch_fpop_start(env); - neg_RsV = float32_set_sign(RsV, float32_is_neg(RsV) ? 0 : 1); - RxV = internal_fmafx(neg_RsV, RtV, RxV, 0, &env->fp_status); + RxV = float32_muladd(RsV, RtV, RxV, float_muladd_negate_product, + &env->fp_status); arch_fpop_end(env); return RxV; } -- 2.43.0 This instruction has a special case that 0 * x + c returns c without the normal sign folding that comes with 0 + -0. Use the new float_muladd_suppress_add_product_zero to describe this. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/op_helper.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/target/hexagon/op_helper.c b/target/hexagon/op_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/op_helper.c +++ b/target/hexagon/op_helper.c @@ -XXX,XX +XXX,XX @@ static float32 check_nan(float32 dst, float32 x, float_status *fp_status) float32 HELPER(sffma_sc)(CPUHexagonState *env, float32 RxV, float32 RsV, float32 RtV, float32 PuV) { - size4s_t tmp; arch_fpop_start(env); - RxV = check_nan(RxV, RxV, &env->fp_status); - RxV = check_nan(RxV, RsV, &env->fp_status); - RxV = check_nan(RxV, RtV, &env->fp_status); - tmp = internal_fmafx(RsV, RtV, RxV, fSXTN(8, 64, PuV), &env->fp_status); - if (!(float32_is_zero(RxV) && is_zero_prod(RsV, RtV))) { - RxV = tmp; - } + RxV = float32_muladd_scalbn(RsV, RtV, RxV, fSXTN(8, 64, PuV), + float_muladd_suppress_add_product_zero, + &env->fp_status); arch_fpop_end(env); return RxV; } -- 2.43.0 There are multiple special cases for this instruction. (1) The saturate to normal maximum instead of overflow to infinity is handled by the new float_round_nearest_even_max rounding mode. (2) The 0 * n + c special case is handled by the new float_muladd_suppress_add_product_zero flag. (3) The Inf - Inf -> 0 special case can be detected after the fact by examining float_flag_invalid_isi. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/op_helper.c | 105 +++++++++---------------------------- 1 file changed, 26 insertions(+), 79 deletions(-) diff --git a/target/hexagon/op_helper.c b/target/hexagon/op_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/op_helper.c +++ b/target/hexagon/op_helper.c @@ -XXX,XX +XXX,XX @@ float32 HELPER(sffma)(CPUHexagonState *env, float32 RxV, return RxV; } -static bool is_zero_prod(float32 a, float32 b) -{ - return ((float32_is_zero(a) && is_finite(b)) || - (float32_is_zero(b) && is_finite(a))); -} - -static float32 check_nan(float32 dst, float32 x, float_status *fp_status) -{ - float32 ret = dst; - if (float32_is_any_nan(x)) { - if (extract32(x, 22, 1) == 0) { - float_raise(float_flag_invalid, fp_status); - } - ret = make_float32(0xffffffff); /* nan */ - } - return ret; -} - float32 HELPER(sffma_sc)(CPUHexagonState *env, float32 RxV, float32 RsV, float32 RtV, float32 PuV) { @@ -XXX,XX +XXX,XX @@ float32 HELPER(sffms)(CPUHexagonState *env, float32 RxV, return RxV; } -static bool is_inf_prod(int32_t a, int32_t b) +static float32 do_sffma_lib(CPUHexagonState *env, float32 RxV, + float32 RsV, float32 RtV, int negate) { - return (float32_is_infinity(a) && float32_is_infinity(b)) || - (float32_is_infinity(a) && is_finite(b) && !float32_is_zero(b)) || - (float32_is_infinity(b) && is_finite(a) && !float32_is_zero(a)); + int flags; + + arch_fpop_start(env); + + set_float_rounding_mode(float_round_nearest_even_max, &env->fp_status); + RxV = float32_muladd(RsV, RtV, RxV, + negate | float_muladd_suppress_add_product_zero, + &env->fp_status); + + flags = get_float_exception_flags(&env->fp_status); + if (flags) { + /* Flags are suppressed by this instruction. */ + set_float_exception_flags(0, &env->fp_status); + + /* Return 0 for Inf - Inf. */ + if (flags & float_flag_invalid_isi) { + RxV = 0; + } + } + + arch_fpop_end(env); + return RxV; } float32 HELPER(sffma_lib)(CPUHexagonState *env, float32 RxV, float32 RsV, float32 RtV) { - bool infinp; - bool infminusinf; - float32 tmp; - - arch_fpop_start(env); - set_float_rounding_mode(float_round_nearest_even, &env->fp_status); - infminusinf = float32_is_infinity(RxV) && - is_inf_prod(RsV, RtV) && - (fGETBIT(31, RsV ^ RxV ^ RtV) != 0); - infinp = float32_is_infinity(RxV) || - float32_is_infinity(RtV) || - float32_is_infinity(RsV); - RxV = check_nan(RxV, RxV, &env->fp_status); - RxV = check_nan(RxV, RsV, &env->fp_status); - RxV = check_nan(RxV, RtV, &env->fp_status); - tmp = internal_fmafx(RsV, RtV, RxV, 0, &env->fp_status); - if (!(float32_is_zero(RxV) && is_zero_prod(RsV, RtV))) { - RxV = tmp; - } - set_float_exception_flags(0, &env->fp_status); - if (float32_is_infinity(RxV) && !infinp) { - RxV = RxV - 1; - } - if (infminusinf) { - RxV = 0; - } - arch_fpop_end(env); - return RxV; + return do_sffma_lib(env, RxV, RsV, RtV, 0); } float32 HELPER(sffms_lib)(CPUHexagonState *env, float32 RxV, float32 RsV, float32 RtV) { - bool infinp; - bool infminusinf; - float32 tmp; - - arch_fpop_start(env); - set_float_rounding_mode(float_round_nearest_even, &env->fp_status); - infminusinf = float32_is_infinity(RxV) && - is_inf_prod(RsV, RtV) && - (fGETBIT(31, RsV ^ RxV ^ RtV) == 0); - infinp = float32_is_infinity(RxV) || - float32_is_infinity(RtV) || - float32_is_infinity(RsV); - RxV = check_nan(RxV, RxV, &env->fp_status); - RxV = check_nan(RxV, RsV, &env->fp_status); - RxV = check_nan(RxV, RtV, &env->fp_status); - float32 minus_RsV = float32_sub(float32_zero, RsV, &env->fp_status); - tmp = internal_fmafx(minus_RsV, RtV, RxV, 0, &env->fp_status); - if (!(float32_is_zero(RxV) && is_zero_prod(RsV, RtV))) { - RxV = tmp; - } - set_float_exception_flags(0, &env->fp_status); - if (float32_is_infinity(RxV) && !infinp) { - RxV = RxV - 1; - } - if (infminusinf) { - RxV = 0; - } - arch_fpop_end(env); - return RxV; + return do_sffma_lib(env, RxV, RsV, RtV, float_muladd_negate_product); } float64 HELPER(dfmpyfix)(CPUHexagonState *env, float64 RssV, float64 RttV) -- 2.43.0 The function is now unused. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/fma_emu.h | 2 - target/hexagon/fma_emu.c | 171 --------------------------------------- 2 files changed, 173 deletions(-) diff --git a/target/hexagon/fma_emu.h b/target/hexagon/fma_emu.h index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.h +++ b/target/hexagon/fma_emu.h @@ -XXX,XX +XXX,XX @@ static inline uint32_t float32_getexp_raw(float32 f32) } int32_t float32_getexp(float32 f32); float32 infinite_float32(uint8_t sign); -float32 internal_fmafx(float32 a, float32 b, float32 c, - int scale, float_status *fp_status); float64 internal_mpyhh(float64 a, float64 b, unsigned long long int accumulated, float_status *fp_status); diff --git a/target/hexagon/fma_emu.c b/target/hexagon/fma_emu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.c +++ b/target/hexagon/fma_emu.c @@ -XXX,XX +XXX,XX @@ int32_t float64_getexp(float64 f64) return -1; } -static uint64_t float32_getmant(float32 f32) -{ - Float a = { .i = f32 }; - if (float32_is_normal(f32)) { - return a.mant | 1ULL << 23; - } - if (float32_is_zero(f32)) { - return 0; - } - if (float32_is_denormal(f32)) { - return a.mant; - } - return ~0ULL; -} - int32_t float32_getexp(float32 f32) { Float a = { .i = f32 }; @@ -XXX,XX +XXX,XX @@ float32 infinite_float32(uint8_t sign) } /* Return a maximum finite value with the requested sign */ -static float32 maxfinite_float32(uint8_t sign) -{ - if (sign) { - return make_float32(SF_MINUS_MAXF); - } else { - return make_float32(SF_MAXF); - } -} - -/* Return a zero value with requested sign */ -static float32 zero_float32(uint8_t sign) -{ - if (sign) { - return make_float32(0x80000000); - } else { - return float32_zero; - } -} - #define GEN_XF_ROUND(SUFFIX, MANTBITS, INF_EXP, INTERNAL_TYPE) \ static SUFFIX accum_round_##SUFFIX(Accum a, float_status * fp_status) \ { \ @@ -XXX,XX +XXX,XX @@ static SUFFIX accum_round_##SUFFIX(Accum a, float_status * fp_status) \ } GEN_XF_ROUND(float64, DF_MANTBITS, DF_INF_EXP, Double) -GEN_XF_ROUND(float32, SF_MANTBITS, SF_INF_EXP, Float) - -static bool is_inf_prod(float64 a, float64 b) -{ - return ((float64_is_infinity(a) && float64_is_infinity(b)) || - (float64_is_infinity(a) && is_finite(b) && (!float64_is_zero(b))) || - (float64_is_infinity(b) && is_finite(a) && (!float64_is_zero(a)))); -} - -static float64 special_fma(float64 a, float64 b, float64 c, - float_status *fp_status) -{ - float64 ret = make_float64(0); - - /* - * If A multiplied by B is an exact infinity and C is also an infinity - * but with the opposite sign, FMA returns NaN and raises invalid. - */ - uint8_t a_sign = float64_is_neg(a); - uint8_t b_sign = float64_is_neg(b); - uint8_t c_sign = float64_is_neg(c); - if (is_inf_prod(a, b) && float64_is_infinity(c)) { - if ((a_sign ^ b_sign) != c_sign) { - ret = make_float64(DF_NAN); - float_raise(float_flag_invalid, fp_status); - return ret; - } - } - if ((float64_is_infinity(a) && float64_is_zero(b)) || - (float64_is_zero(a) && float64_is_infinity(b))) { - ret = make_float64(DF_NAN); - float_raise(float_flag_invalid, fp_status); - return ret; - } - /* - * If none of the above checks are true and C is a NaN, - * a NaN shall be returned - * If A or B are NaN, a NAN shall be returned. - */ - if (float64_is_any_nan(a) || - float64_is_any_nan(b) || - float64_is_any_nan(c)) { - if (float64_is_any_nan(a) && (fGETBIT(51, a) == 0)) { - float_raise(float_flag_invalid, fp_status); - } - if (float64_is_any_nan(b) && (fGETBIT(51, b) == 0)) { - float_raise(float_flag_invalid, fp_status); - } - if (float64_is_any_nan(c) && (fGETBIT(51, c) == 0)) { - float_raise(float_flag_invalid, fp_status); - } - ret = make_float64(DF_NAN); - return ret; - } - /* - * We have checked for adding opposite-signed infinities. - * Other infinities return infinity with the correct sign - */ - if (float64_is_infinity(c)) { - ret = infinite_float64(c_sign); - return ret; - } - if (float64_is_infinity(a) || float64_is_infinity(b)) { - ret = infinite_float64(a_sign ^ b_sign); - return ret; - } - g_assert_not_reached(); -} - -static float32 special_fmaf(float32 a, float32 b, float32 c, - float_status *fp_status) -{ - float64 aa, bb, cc; - aa = float32_to_float64(a, fp_status); - bb = float32_to_float64(b, fp_status); - cc = float32_to_float64(c, fp_status); - return float64_to_float32(special_fma(aa, bb, cc, fp_status), fp_status); -} - -float32 internal_fmafx(float32 a, float32 b, float32 c, int scale, - float_status *fp_status) -{ - Accum prod; - Accum acc; - Accum result; - accum_init(&prod); - accum_init(&acc); - accum_init(&result); - - uint8_t a_sign = float32_is_neg(a); - uint8_t b_sign = float32_is_neg(b); - uint8_t c_sign = float32_is_neg(c); - if (float32_is_infinity(a) || - float32_is_infinity(b) || - float32_is_infinity(c)) { - return special_fmaf(a, b, c, fp_status); - } - if (float32_is_any_nan(a) || - float32_is_any_nan(b) || - float32_is_any_nan(c)) { - return special_fmaf(a, b, c, fp_status); - } - if ((scale == 0) && (float32_is_zero(a) || float32_is_zero(b))) { - float32 tmp = float32_mul(a, b, fp_status); - tmp = float32_add(tmp, c, fp_status); - return tmp; - } - - /* (a * 2**b) * (c * 2**d) == a*c * 2**(b+d) */ - prod.mant = int128_mul_6464(float32_getmant(a), float32_getmant(b)); - - /* - * Note: extracting the mantissa into an int is multiplying by - * 2**23, so adjust here - */ - prod.exp = float32_getexp(a) + float32_getexp(b) - SF_BIAS - 23; - prod.sign = a_sign ^ b_sign; - if (float32_is_zero(a) || float32_is_zero(b)) { - prod.exp = -2 * WAY_BIG_EXP; - } - if ((scale > 0) && float32_is_denormal(c)) { - acc.mant = int128_mul_6464(0, 0); - acc.exp = -WAY_BIG_EXP; - acc.sign = c_sign; - acc.sticky = 1; - result = accum_add(prod, acc); - } else if (!float32_is_zero(c)) { - acc.mant = int128_mul_6464(float32_getmant(c), 1); - acc.exp = float32_getexp(c); - acc.sign = c_sign; - result = accum_add(prod, acc); - } else { - result = prod; - } - result.exp += scale; - return accum_round_float32(result, fp_status); -} float64 internal_mpyhh(float64 a, float64 b, unsigned long long int accumulated, -- 2.43.0 This massive macro is now only used once. Expand it for use only by float64. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/fma_emu.c | 255 +++++++++++++++++++-------------------- 1 file changed, 127 insertions(+), 128 deletions(-) diff --git a/target/hexagon/fma_emu.c b/target/hexagon/fma_emu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.c +++ b/target/hexagon/fma_emu.c @@ -XXX,XX +XXX,XX @@ float32 infinite_float32(uint8_t sign) } /* Return a maximum finite value with the requested sign */ -#define GEN_XF_ROUND(SUFFIX, MANTBITS, INF_EXP, INTERNAL_TYPE) \ -static SUFFIX accum_round_##SUFFIX(Accum a, float_status * fp_status) \ -{ \ - if ((int128_gethi(a.mant) == 0) && (int128_getlo(a.mant) == 0) \ - && ((a.guard | a.round | a.sticky) == 0)) { \ - /* result zero */ \ - switch (fp_status->float_rounding_mode) { \ - case float_round_down: \ - return zero_##SUFFIX(1); \ - default: \ - return zero_##SUFFIX(0); \ - } \ - } \ - /* Normalize right */ \ - /* We want MANTBITS bits of mantissa plus the leading one. */ \ - /* That means that we want MANTBITS+1 bits, or 0x000000000000FF_FFFF */ \ - /* So we need to normalize right while the high word is non-zero and \ - * while the low word is nonzero when masked with 0xffe0_0000_0000_0000 */ \ - while ((int128_gethi(a.mant) != 0) || \ - ((int128_getlo(a.mant) >> (MANTBITS + 1)) != 0)) { \ - a = accum_norm_right(a, 1); \ - } \ - /* \ - * OK, now normalize left \ - * We want to normalize left until we have a leading one in bit 24 \ - * Theoretically, we only need to shift a maximum of one to the left if we \ - * shifted out lots of bits from B, or if we had no shift / 1 shift sticky \ - * should be 0 \ - */ \ - while ((int128_getlo(a.mant) & (1ULL << MANTBITS)) == 0) { \ - a = accum_norm_left(a); \ - } \ - /* \ - * OK, now we might need to denormalize because of potential underflow. \ - * We need to do this before rounding, and rounding might make us normal \ - * again \ - */ \ - while (a.exp <= 0) { \ - a = accum_norm_right(a, 1 - a.exp); \ - /* \ - * Do we have underflow? \ - * That's when we get an inexact answer because we ran out of bits \ - * in a denormal. \ - */ \ - if (a.guard || a.round || a.sticky) { \ - float_raise(float_flag_underflow, fp_status); \ - } \ - } \ - /* OK, we're relatively canonical... now we need to round */ \ - if (a.guard || a.round || a.sticky) { \ - float_raise(float_flag_inexact, fp_status); \ - switch (fp_status->float_rounding_mode) { \ - case float_round_to_zero: \ - /* Chop and we're done */ \ - break; \ - case float_round_up: \ - if (a.sign == 0) { \ - a.mant = int128_add(a.mant, int128_one()); \ - } \ - break; \ - case float_round_down: \ - if (a.sign != 0) { \ - a.mant = int128_add(a.mant, int128_one()); \ - } \ - break; \ - default: \ - if (a.round || a.sticky) { \ - /* round up if guard is 1, down if guard is zero */ \ - a.mant = int128_add(a.mant, int128_make64(a.guard)); \ - } else if (a.guard) { \ - /* exactly .5, round up if odd */ \ - a.mant = int128_add(a.mant, int128_and(a.mant, int128_one())); \ - } \ - break; \ - } \ - } \ - /* \ - * OK, now we might have carried all the way up. \ - * So we might need to shr once \ - * at least we know that the lsb should be zero if we rounded and \ - * got a carry out... \ - */ \ - if ((int128_getlo(a.mant) >> (MANTBITS + 1)) != 0) { \ - a = accum_norm_right(a, 1); \ - } \ - /* Overflow? */ \ - if (a.exp >= INF_EXP) { \ - /* Yep, inf result */ \ - float_raise(float_flag_overflow, fp_status); \ - float_raise(float_flag_inexact, fp_status); \ - switch (fp_status->float_rounding_mode) { \ - case float_round_to_zero: \ - return maxfinite_##SUFFIX(a.sign); \ - case float_round_up: \ - if (a.sign == 0) { \ - return infinite_##SUFFIX(a.sign); \ - } else { \ - return maxfinite_##SUFFIX(a.sign); \ - } \ - case float_round_down: \ - if (a.sign != 0) { \ - return infinite_##SUFFIX(a.sign); \ - } else { \ - return maxfinite_##SUFFIX(a.sign); \ - } \ - default: \ - return infinite_##SUFFIX(a.sign); \ - } \ - } \ - /* Underflow? */ \ - if (int128_getlo(a.mant) & (1ULL << MANTBITS)) { \ - /* Leading one means: No, we're normal. So, we should be done... */ \ - INTERNAL_TYPE ret; \ - ret.i = 0; \ - ret.sign = a.sign; \ - ret.exp = a.exp; \ - ret.mant = int128_getlo(a.mant); \ - return ret.i; \ - } \ - assert(a.exp == 1); \ - INTERNAL_TYPE ret; \ - ret.i = 0; \ - ret.sign = a.sign; \ - ret.exp = 0; \ - ret.mant = int128_getlo(a.mant); \ - return ret.i; \ +static float64 accum_round_float64(Accum a, float_status *fp_status) +{ + if ((int128_gethi(a.mant) == 0) && (int128_getlo(a.mant) == 0) + && ((a.guard | a.round | a.sticky) == 0)) { + /* result zero */ + switch (fp_status->float_rounding_mode) { + case float_round_down: + return zero_float64(1); + default: + return zero_float64(0); + } + } + /* + * Normalize right + * We want DF_MANTBITS bits of mantissa plus the leading one. + * That means that we want DF_MANTBITS+1 bits, or 0x000000000000FF_FFFF + * So we need to normalize right while the high word is non-zero and + * while the low word is nonzero when masked with 0xffe0_0000_0000_0000 + */ + while ((int128_gethi(a.mant) != 0) || + ((int128_getlo(a.mant) >> (DF_MANTBITS + 1)) != 0)) { + a = accum_norm_right(a, 1); + } + /* + * OK, now normalize left + * We want to normalize left until we have a leading one in bit 24 + * Theoretically, we only need to shift a maximum of one to the left if we + * shifted out lots of bits from B, or if we had no shift / 1 shift sticky + * should be 0 + */ + while ((int128_getlo(a.mant) & (1ULL << DF_MANTBITS)) == 0) { + a = accum_norm_left(a); + } + /* + * OK, now we might need to denormalize because of potential underflow. + * We need to do this before rounding, and rounding might make us normal + * again + */ + while (a.exp <= 0) { + a = accum_norm_right(a, 1 - a.exp); + /* + * Do we have underflow? + * That's when we get an inexact answer because we ran out of bits + * in a denormal. + */ + if (a.guard || a.round || a.sticky) { + float_raise(float_flag_underflow, fp_status); + } + } + /* OK, we're relatively canonical... now we need to round */ + if (a.guard || a.round || a.sticky) { + float_raise(float_flag_inexact, fp_status); + switch (fp_status->float_rounding_mode) { + case float_round_to_zero: + /* Chop and we're done */ + break; + case float_round_up: + if (a.sign == 0) { + a.mant = int128_add(a.mant, int128_one()); + } + break; + case float_round_down: + if (a.sign != 0) { + a.mant = int128_add(a.mant, int128_one()); + } + break; + default: + if (a.round || a.sticky) { + /* round up if guard is 1, down if guard is zero */ + a.mant = int128_add(a.mant, int128_make64(a.guard)); + } else if (a.guard) { + /* exactly .5, round up if odd */ + a.mant = int128_add(a.mant, int128_and(a.mant, int128_one())); + } + break; + } + } + /* + * OK, now we might have carried all the way up. + * So we might need to shr once + * at least we know that the lsb should be zero if we rounded and + * got a carry out... + */ + if ((int128_getlo(a.mant) >> (DF_MANTBITS + 1)) != 0) { + a = accum_norm_right(a, 1); + } + /* Overflow? */ + if (a.exp >= DF_INF_EXP) { + /* Yep, inf result */ + float_raise(float_flag_overflow, fp_status); + float_raise(float_flag_inexact, fp_status); + switch (fp_status->float_rounding_mode) { + case float_round_to_zero: + return maxfinite_float64(a.sign); + case float_round_up: + if (a.sign == 0) { + return infinite_float64(a.sign); + } else { + return maxfinite_float64(a.sign); + } + case float_round_down: + if (a.sign != 0) { + return infinite_float64(a.sign); + } else { + return maxfinite_float64(a.sign); + } + default: + return infinite_float64(a.sign); + } + } + /* Underflow? */ + if (int128_getlo(a.mant) & (1ULL << DF_MANTBITS)) { + /* Leading one means: No, we're normal. So, we should be done... */ + Double ret; + ret.i = 0; + ret.sign = a.sign; + ret.exp = a.exp; + ret.mant = int128_getlo(a.mant); + return ret.i; + } + assert(a.exp == 1); + Double ret; + ret.i = 0; + ret.sign = a.sign; + ret.exp = 0; + ret.mant = int128_getlo(a.mant); + return ret.i; } -GEN_XF_ROUND(float64, DF_MANTBITS, DF_INF_EXP, Double) - float64 internal_mpyhh(float64 a, float64 b, unsigned long long int accumulated, float_status *fp_status) -- 2.43.0 This structure, with bitfields, is incorrect for big-endian. Use the existing float32_getexp_raw which uses extract32. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/fma_emu.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/target/hexagon/fma_emu.c b/target/hexagon/fma_emu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.c +++ b/target/hexagon/fma_emu.c @@ -XXX,XX +XXX,XX @@ typedef union { }; } Double; -typedef union { - float f; - uint32_t i; - struct { - uint32_t mant:23; - uint32_t exp:8; - uint32_t sign:1; - }; -} Float; - static uint64_t float64_getmant(float64 f64) { Double a = { .i = f64 }; @@ -XXX,XX +XXX,XX @@ int32_t float64_getexp(float64 f64) int32_t float32_getexp(float32 f32) { - Float a = { .i = f32 }; + int exp = float32_getexp_raw(f32); if (float32_is_normal(f32)) { - return a.exp; + return exp; } if (float32_is_denormal(f32)) { - return a.exp + 1; + return exp + 1; } return -1; } -- 2.43.0 This structure, with bitfields, is incorrect for big-endian. Use extract64 and deposit64 instead. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/fma_emu.c | 46 ++++++++++++++-------------------------- 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/target/hexagon/fma_emu.c b/target/hexagon/fma_emu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.c +++ b/target/hexagon/fma_emu.c @@ -XXX,XX +XXX,XX @@ #define WAY_BIG_EXP 4096 -typedef union { - double f; - uint64_t i; - struct { - uint64_t mant:52; - uint64_t exp:11; - uint64_t sign:1; - }; -} Double; - static uint64_t float64_getmant(float64 f64) { - Double a = { .i = f64 }; + uint64_t mant = extract64(f64, 0, 52); if (float64_is_normal(f64)) { - return a.mant | 1ULL << 52; + return mant | 1ULL << 52; } if (float64_is_zero(f64)) { return 0; } if (float64_is_denormal(f64)) { - return a.mant; + return mant; } return ~0ULL; } int32_t float64_getexp(float64 f64) { - Double a = { .i = f64 }; + int exp = extract64(f64, 52, 11); if (float64_is_normal(f64)) { - return a.exp; + return exp; } if (float64_is_denormal(f64)) { - return a.exp + 1; + return exp + 1; } return -1; } @@ -XXX,XX +XXX,XX @@ float32 infinite_float32(uint8_t sign) /* Return a maximum finite value with the requested sign */ static float64 accum_round_float64(Accum a, float_status *fp_status) { + uint64_t ret; + if ((int128_gethi(a.mant) == 0) && (int128_getlo(a.mant) == 0) && ((a.guard | a.round | a.sticky) == 0)) { /* result zero */ @@ -XXX,XX +XXX,XX @@ static float64 accum_round_float64(Accum a, float_status *fp_status) } } /* Underflow? */ - if (int128_getlo(a.mant) & (1ULL << DF_MANTBITS)) { + ret = int128_getlo(a.mant); + if (ret & (1ULL << DF_MANTBITS)) { /* Leading one means: No, we're normal. So, we should be done... */ - Double ret; - ret.i = 0; - ret.sign = a.sign; - ret.exp = a.exp; - ret.mant = int128_getlo(a.mant); - return ret.i; + ret = deposit64(ret, 52, 11, a.exp); + } else { + assert(a.exp == 1); + ret = deposit64(ret, 52, 11, 0); } - assert(a.exp == 1); - Double ret; - ret.i = 0; - ret.sign = a.sign; - ret.exp = 0; - ret.mant = int128_getlo(a.mant); - return ret.i; + ret = deposit64(ret, 63, 1, a.sign); + return ret; } float64 internal_mpyhh(float64 a, float64 b, -- 2.43.0 No need to open-code 64x64->128-bit multiplication. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/fma_emu.c | 32 +++----------------------------- 1 file changed, 3 insertions(+), 29 deletions(-) diff --git a/target/hexagon/fma_emu.c b/target/hexagon/fma_emu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.c +++ b/target/hexagon/fma_emu.c @@ -XXX,XX +XXX,XX @@ int32_t float32_getexp(float32 f32) return -1; } -static uint32_t int128_getw0(Int128 x) -{ - return int128_getlo(x); -} - -static uint32_t int128_getw1(Int128 x) -{ - return int128_getlo(x) >> 32; -} - static Int128 int128_mul_6464(uint64_t ai, uint64_t bi) { - Int128 a, b; - uint64_t pp0, pp1a, pp1b, pp1s, pp2; + uint64_t l, h; - a = int128_make64(ai); - b = int128_make64(bi); - pp0 = (uint64_t)int128_getw0(a) * (uint64_t)int128_getw0(b); - pp1a = (uint64_t)int128_getw1(a) * (uint64_t)int128_getw0(b); - pp1b = (uint64_t)int128_getw1(b) * (uint64_t)int128_getw0(a); - pp2 = (uint64_t)int128_getw1(a) * (uint64_t)int128_getw1(b); - - pp1s = pp1a + pp1b; - if ((pp1s < pp1a) || (pp1s < pp1b)) { - pp2 += (1ULL << 32); - } - uint64_t ret_low = pp0 + (pp1s << 32); - if ((ret_low < pp0) || (ret_low < (pp1s << 32))) { - pp2 += 1; - } - - return int128_make128(ret_low, pp2 + (pp1s >> 32)); + mulu64(&l, &h, ai, bi); + return int128_make128(l, h); } static Int128 int128_sub_borrow(Int128 a, Int128 b, int borrow) -- 2.43.0 Initialize x with accumulated via direct assignment, rather than multiplying by 1. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/fma_emu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/hexagon/fma_emu.c b/target/hexagon/fma_emu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.c +++ b/target/hexagon/fma_emu.c @@ -XXX,XX +XXX,XX @@ float64 internal_mpyhh(float64 a, float64 b, float64_is_infinity(b)) { return float64_mul(a, b, fp_status); } - x.mant = int128_mul_6464(accumulated, 1); + x.mant = int128_make64(accumulated); x.sticky = sticky; prod = fGETUWORD(1, float64_getmant(a)) * fGETUWORD(1, float64_getmant(b)); x.mant = int128_add(x.mant, int128_mul_6464(prod, 0x100000000ULL)); -- 2.43.0 Convert all targets simultaneously, as the gen_intermediate_code function disappears from the target. While there are possible workarounds, they're larger than simply performing the conversion. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/translator.h | 14 -------------- include/hw/core/tcg-cpu-ops.h | 13 +++++++++++++ target/alpha/cpu.h | 2 ++ target/arm/internals.h | 2 ++ target/avr/cpu.h | 2 ++ target/hexagon/cpu.h | 2 ++ target/hppa/cpu.h | 2 ++ target/i386/tcg/helper-tcg.h | 2 ++ target/loongarch/internals.h | 2 ++ target/m68k/cpu.h | 2 ++ target/microblaze/cpu.h | 2 ++ target/mips/tcg/tcg-internal.h | 2 ++ target/openrisc/cpu.h | 2 ++ target/ppc/cpu.h | 2 ++ target/riscv/cpu.h | 3 +++ target/rx/cpu.h | 2 ++ target/s390x/s390x-internal.h | 2 ++ target/sh4/cpu.h | 2 ++ target/sparc/cpu.h | 2 ++ target/tricore/cpu.h | 2 ++ target/xtensa/cpu.h | 2 ++ accel/tcg/cpu-exec.c | 8 +++++--- accel/tcg/translate-all.c | 8 +++++--- target/alpha/cpu.c | 1 + target/alpha/translate.c | 4 ++-- target/arm/cpu.c | 1 + target/arm/tcg/cpu-v7m.c | 1 + target/arm/tcg/translate.c | 5 ++--- target/avr/cpu.c | 1 + target/avr/translate.c | 6 +++--- target/hexagon/cpu.c | 1 + target/hexagon/translate.c | 4 ++-- target/hppa/cpu.c | 1 + target/hppa/translate.c | 4 ++-- target/i386/tcg/tcg-cpu.c | 1 + target/i386/tcg/translate.c | 5 ++--- target/loongarch/cpu.c | 1 + target/loongarch/tcg/translate.c | 4 ++-- target/m68k/cpu.c | 1 + target/m68k/translate.c | 4 ++-- target/microblaze/cpu.c | 1 + target/microblaze/translate.c | 4 ++-- target/mips/cpu.c | 1 + target/mips/tcg/translate.c | 4 ++-- target/openrisc/cpu.c | 1 + target/openrisc/translate.c | 4 ++-- target/ppc/cpu_init.c | 1 + target/ppc/translate.c | 4 ++-- target/riscv/tcg/tcg-cpu.c | 1 + target/riscv/translate.c | 4 ++-- target/rx/cpu.c | 1 + target/rx/translate.c | 4 ++-- target/s390x/cpu.c | 1 + target/s390x/tcg/translate.c | 4 ++-- target/sh4/cpu.c | 1 + target/sh4/translate.c | 4 ++-- target/sparc/cpu.c | 1 + target/sparc/translate.c | 4 ++-- target/tricore/cpu.c | 1 + target/tricore/translate.c | 5 ++--- target/xtensa/cpu.c | 1 + target/xtensa/translate.c | 4 ++-- 62 files changed, 121 insertions(+), 62 deletions(-) diff --git a/include/exec/translator.h b/include/exec/translator.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/translator.h +++ b/include/exec/translator.h @@ -XXX,XX +XXX,XX @@ #include "qemu/bswap.h" #include "exec/vaddr.h" -/** - * gen_intermediate_code - * @cpu: cpu context - * @tb: translation block - * @max_insns: max number of instructions to translate - * @pc: guest virtual program counter address - * @host_pc: host physical program counter address - * - * This function must be provided by the target, which should create - * the target-specific DisasContext, and then invoke translator_loop. - */ -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc); - /** * DisasJumpType: * @DISAS_NEXT: Next instruction in program order. diff --git a/include/hw/core/tcg-cpu-ops.h b/include/hw/core/tcg-cpu-ops.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/core/tcg-cpu-ops.h +++ b/include/hw/core/tcg-cpu-ops.h @@ -XXX,XX +XXX,XX @@ struct TCGCPUOps { * Called when the first CPU is realized. */ void (*initialize)(void); + /** + * @translate_code: Translate guest instructions to TCGOps + * @cpu: cpu context + * @tb: translation block + * @max_insns: max number of instructions to translate + * @pc: guest virtual program counter address + * @host_pc: host physical program counter address + * + * This function must be provided by the target, which should create + * the target-specific DisasContext, and then invoke translator_loop. + */ + void (*translate_code)(CPUState *cpu, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); /** * @synchronize_from_tb: Synchronize state from a TCG #TranslationBlock * diff --git a/target/alpha/cpu.h b/target/alpha/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/alpha/cpu.h +++ b/target/alpha/cpu.h @@ -XXX,XX +XXX,XX @@ enum { }; void alpha_translate_init(void); +void alpha_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); #define CPU_RESOLVING_TYPE TYPE_ALPHA_CPU diff --git a/target/arm/internals.h b/target/arm/internals.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -XXX,XX +XXX,XX @@ void init_cpreg_list(ARMCPU *cpu); void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu); void arm_translate_init(void); +void arm_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); void arm_cpu_register_gdb_commands(ARMCPU *cpu); void aarch64_cpu_register_gdb_commands(ARMCPU *cpu, GString *, diff --git a/target/avr/cpu.h b/target/avr/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/avr/cpu.h +++ b/target/avr/cpu.h @@ -XXX,XX +XXX,XX @@ static inline void set_avr_feature(CPUAVRState *env, int feature) } void avr_cpu_tcg_init(void); +void avr_cpu_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); int cpu_avr_exec(CPUState *cpu); diff --git a/target/hexagon/cpu.h b/target/hexagon/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/cpu.h +++ b/target/hexagon/cpu.h @@ -XXX,XX +XXX,XX @@ static inline void cpu_get_tb_cpu_state(CPUHexagonState *env, vaddr *pc, typedef HexagonCPU ArchCPU; void hexagon_translate_init(void); +void hexagon_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); #include "exec/cpu-all.h" diff --git a/target/hppa/cpu.h b/target/hppa/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/hppa/cpu.h +++ b/target/hppa/cpu.h @@ -XXX,XX +XXX,XX @@ static inline int HPPA_BTLB_ENTRIES(CPUHPPAState *env) } void hppa_translate_init(void); +void hppa_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); #define CPU_RESOLVING_TYPE TYPE_HPPA_CPU diff --git a/target/i386/tcg/helper-tcg.h b/target/i386/tcg/helper-tcg.h index XXXXXXX..XXXXXXX 100644 --- a/target/i386/tcg/helper-tcg.h +++ b/target/i386/tcg/helper-tcg.h @@ -XXX,XX +XXX,XX @@ static inline target_long lshift(target_long x, int n) /* translate.c */ void tcg_x86_init(void); +void x86_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); /* excp_helper.c */ G_NORETURN void raise_exception(CPUX86State *env, int exception_index); diff --git a/target/loongarch/internals.h b/target/loongarch/internals.h index XXXXXXX..XXXXXXX 100644 --- a/target/loongarch/internals.h +++ b/target/loongarch/internals.h @@ -XXX,XX +XXX,XX @@ #define TARGET_VIRT_MASK MAKE_64BIT_MASK(0, TARGET_VIRT_ADDR_SPACE_BITS) void loongarch_translate_init(void); +void loongarch_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); void G_NORETURN do_raise_exception(CPULoongArchState *env, uint32_t exception, diff --git a/target/m68k/cpu.h b/target/m68k/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/m68k/cpu.h +++ b/target/m68k/cpu.h @@ -XXX,XX +XXX,XX @@ int m68k_cpu_gdb_read_register(CPUState *cpu, GByteArray *buf, int reg); int m68k_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg); void m68k_tcg_init(void); +void m68k_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); void m68k_cpu_init_gdb(M68kCPU *cpu); uint32_t cpu_m68k_get_ccr(CPUM68KState *env); void cpu_m68k_set_ccr(CPUM68KState *env, uint32_t); diff --git a/target/microblaze/cpu.h b/target/microblaze/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/microblaze/cpu.h +++ b/target/microblaze/cpu.h @@ -XXX,XX +XXX,XX @@ static inline void mb_cpu_write_msr(CPUMBState *env, uint32_t val) } void mb_tcg_init(void); +void mb_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); #define CPU_RESOLVING_TYPE TYPE_MICROBLAZE_CPU diff --git a/target/mips/tcg/tcg-internal.h b/target/mips/tcg/tcg-internal.h index XXXXXXX..XXXXXXX 100644 --- a/target/mips/tcg/tcg-internal.h +++ b/target/mips/tcg/tcg-internal.h @@ -XXX,XX +XXX,XX @@ #include "cpu.h" void mips_tcg_init(void); +void mips_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); void mips_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb); G_NORETURN void mips_cpu_do_unaligned_access(CPUState *cpu, vaddr addr, diff --git a/target/openrisc/cpu.h b/target/openrisc/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/openrisc/cpu.h +++ b/target/openrisc/cpu.h @@ -XXX,XX +XXX,XX @@ void openrisc_cpu_dump_state(CPUState *cpu, FILE *f, int flags); int openrisc_cpu_gdb_read_register(CPUState *cpu, GByteArray *buf, int reg); int openrisc_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg); void openrisc_translate_init(void); +void openrisc_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); int print_insn_or1k(bfd_vma addr, disassemble_info *info); #ifndef CONFIG_USER_ONLY diff --git a/target/ppc/cpu.h b/target/ppc/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/cpu.h +++ b/target/ppc/cpu.h @@ -XXX,XX +XXX,XX @@ extern const VMStateDescription vmstate_ppc_cpu; /*****************************************************************************/ void ppc_translate_init(void); +void ppc_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); #if !defined(CONFIG_USER_ONLY) void ppc_store_sdr1(CPUPPCState *env, target_ulong value); diff --git a/target/riscv/cpu.h b/target/riscv/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/cpu.h +++ b/target/riscv/cpu.h @@ -XXX,XX +XXX,XX @@ RISCVException smstateen_acc_ok(CPURISCVState *env, int index, uint64_t bit); void riscv_cpu_set_mode(CPURISCVState *env, target_ulong newpriv, bool virt_en); void riscv_translate_init(void); +void riscv_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); + G_NORETURN void riscv_raise_exception(CPURISCVState *env, uint32_t exception, uintptr_t pc); diff --git a/target/rx/cpu.h b/target/rx/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/rx/cpu.h +++ b/target/rx/cpu.h @@ -XXX,XX +XXX,XX @@ int rx_cpu_gdb_read_register(CPUState *cpu, GByteArray *buf, int reg); int rx_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg); void rx_translate_init(void); +void rx_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); void rx_cpu_unpack_psw(CPURXState *env, uint32_t psw, int rte); #include "exec/cpu-all.h" diff --git a/target/s390x/s390x-internal.h b/target/s390x/s390x-internal.h index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/s390x-internal.h +++ b/target/s390x/s390x-internal.h @@ -XXX,XX +XXX,XX @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, /* translate.c */ void s390x_translate_init(void); +void s390x_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); void s390x_restore_state_to_opc(CPUState *cs, const TranslationBlock *tb, const uint64_t *data); diff --git a/target/sh4/cpu.h b/target/sh4/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/sh4/cpu.h +++ b/target/sh4/cpu.h @@ -XXX,XX +XXX,XX @@ G_NORETURN void superh_cpu_do_unaligned_access(CPUState *cpu, vaddr addr, uintptr_t retaddr); void sh4_translate_init(void); +void sh4_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); #if !defined(CONFIG_USER_ONLY) hwaddr superh_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr); diff --git a/target/sparc/cpu.h b/target/sparc/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/cpu.h +++ b/target/sparc/cpu.h @@ -XXX,XX +XXX,XX @@ int sparc_cpu_memory_rw_debug(CPUState *cpu, vaddr addr, /* translate.c */ void sparc_tcg_init(void); +void sparc_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); /* fop_helper.c */ target_ulong cpu_get_fsr(CPUSPARCState *); diff --git a/target/tricore/cpu.h b/target/tricore/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/tricore/cpu.h +++ b/target/tricore/cpu.h @@ -XXX,XX +XXX,XX @@ FIELD(TB_FLAGS, PRIV, 0, 2) void cpu_state_reset(CPUTriCoreState *s); void tricore_tcg_init(void); +void tricore_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); static inline void cpu_get_tb_cpu_state(CPUTriCoreState *env, vaddr *pc, uint64_t *cs_base, uint32_t *flags) diff --git a/target/xtensa/cpu.h b/target/xtensa/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/xtensa/cpu.h +++ b/target/xtensa/cpu.h @@ -XXX,XX +XXX,XX @@ G_NORETURN void xtensa_cpu_do_unaligned_access(CPUState *cpu, vaddr addr, void xtensa_collect_sr_names(const XtensaConfig *config); void xtensa_translate_init(void); +void xtensa_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); void **xtensa_get_regfile_by_name(const char *name, int entries, int bits); void xtensa_breakpoint_handler(CPUState *cs); void xtensa_register_core(XtensaConfigList *node); diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ bool tcg_exec_realizefn(CPUState *cpu, Error **errp) if (!tcg_target_initialized) { /* Check mandatory TCGCPUOps handlers */ + const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops; #ifndef CONFIG_USER_ONLY - assert(cpu->cc->tcg_ops->cpu_exec_halt); - assert(cpu->cc->tcg_ops->cpu_exec_interrupt); + assert(tcg_ops->cpu_exec_halt); + assert(tcg_ops->cpu_exec_interrupt); #endif /* !CONFIG_USER_ONLY */ - cpu->cc->tcg_ops->initialize(); + assert(tcg_ops->translate_code); + tcg_ops->initialize(); tcg_target_initialized = true; } diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ static int setjmp_gen_code(CPUArchState *env, TranslationBlock *tb, tcg_func_start(tcg_ctx); - tcg_ctx->cpu = env_cpu(env); - gen_intermediate_code(env_cpu(env), tb, max_insns, pc, host_pc); + CPUState *cs = env_cpu(env); + tcg_ctx->cpu = cs; + cs->cc->tcg_ops->translate_code(cs, tb, max_insns, pc, host_pc); + assert(tb->size != 0); tcg_ctx->cpu = NULL; *max_insns = tb->icount; @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, /* * Overflow of code_gen_buffer, or the current slice of it. * - * TODO: We don't need to re-do gen_intermediate_code, nor + * TODO: We don't need to re-do tcg_ops->translate_code, nor * should we re-do the tcg optimization currently hidden * inside tcg_gen_code. All that should be required is to * flush the TBs, allocate a new TB, re-initialize it per diff --git a/target/alpha/cpu.c b/target/alpha/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/alpha/cpu.c +++ b/target/alpha/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps alpha_sysemu_ops = { static const TCGCPUOps alpha_tcg_ops = { .initialize = alpha_translate_init, + .translate_code = alpha_translate_code, .synchronize_from_tb = alpha_cpu_synchronize_from_tb, .restore_state_to_opc = alpha_restore_state_to_opc, diff --git a/target/alpha/translate.c b/target/alpha/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/alpha/translate.c +++ b/target/alpha/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps alpha_tr_ops = { .tb_stop = alpha_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void alpha_translate_code(CPUState *cpu, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc; translator_loop(cpu, tb, max_insns, pc, host_pc, &alpha_tr_ops, &dc.base); diff --git a/target/arm/cpu.c b/target/arm/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps arm_sysemu_ops = { #ifdef CONFIG_TCG static const TCGCPUOps arm_tcg_ops = { .initialize = arm_translate_init, + .translate_code = arm_translate_code, .synchronize_from_tb = arm_cpu_synchronize_from_tb, .debug_excp_handler = arm_debug_excp_handler, .restore_state_to_opc = arm_restore_state_to_opc, diff --git a/target/arm/tcg/cpu-v7m.c b/target/arm/tcg/cpu-v7m.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/tcg/cpu-v7m.c +++ b/target/arm/tcg/cpu-v7m.c @@ -XXX,XX +XXX,XX @@ static void cortex_m55_initfn(Object *obj) static const TCGCPUOps arm_v7m_tcg_ops = { .initialize = arm_translate_init, + .translate_code = arm_translate_code, .synchronize_from_tb = arm_cpu_synchronize_from_tb, .debug_excp_handler = arm_debug_excp_handler, .restore_state_to_opc = arm_restore_state_to_opc, diff --git a/target/arm/tcg/translate.c b/target/arm/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/tcg/translate.c +++ b/target/arm/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps thumb_translator_ops = { .tb_stop = arm_tr_tb_stop, }; -/* generate intermediate code for basic block 'tb'. */ -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void arm_translate_code(CPUState *cpu, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc = { }; const TranslatorOps *ops = &arm_translator_ops; diff --git a/target/avr/cpu.c b/target/avr/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/cpu.c +++ b/target/avr/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps avr_sysemu_ops = { static const TCGCPUOps avr_tcg_ops = { .initialize = avr_cpu_tcg_init, + .translate_code = avr_cpu_translate_code, .synchronize_from_tb = avr_cpu_synchronize_from_tb, .restore_state_to_opc = avr_restore_state_to_opc, .cpu_exec_interrupt = avr_cpu_exec_interrupt, diff --git a/target/avr/translate.c b/target/avr/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/translate.c +++ b/target/avr/translate.c @@ -XXX,XX +XXX,XX @@ static bool trans_WDR(DisasContext *ctx, arg_WDR *a) * * - translate() * - canonicalize_skip() - * - gen_intermediate_code() + * - translate_code() * - restore_state_to_opc() * */ @@ -XXX,XX +XXX,XX @@ static const TranslatorOps avr_tr_ops = { .tb_stop = avr_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void avr_cpu_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc = { }; translator_loop(cs, tb, max_insns, pc, host_pc, &avr_tr_ops, &dc.base); diff --git a/target/hexagon/cpu.c b/target/hexagon/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/cpu.c +++ b/target/hexagon/cpu.c @@ -XXX,XX +XXX,XX @@ static void hexagon_cpu_init(Object *obj) static const TCGCPUOps hexagon_tcg_ops = { .initialize = hexagon_translate_init, + .translate_code = hexagon_translate_code, .synchronize_from_tb = hexagon_cpu_synchronize_from_tb, .restore_state_to_opc = hexagon_restore_state_to_opc, }; diff --git a/target/hexagon/translate.c b/target/hexagon/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/translate.c +++ b/target/hexagon/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps hexagon_tr_ops = { .tb_stop = hexagon_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void hexagon_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx; diff --git a/target/hppa/cpu.c b/target/hppa/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hppa/cpu.c +++ b/target/hppa/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps hppa_sysemu_ops = { static const TCGCPUOps hppa_tcg_ops = { .initialize = hppa_translate_init, + .translate_code = hppa_translate_code, .synchronize_from_tb = hppa_cpu_synchronize_from_tb, .restore_state_to_opc = hppa_restore_state_to_opc, diff --git a/target/hppa/translate.c b/target/hppa/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/hppa/translate.c +++ b/target/hppa/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps hppa_tr_ops = { #endif }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void hppa_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx = { }; translator_loop(cs, tb, max_insns, pc, host_pc, &hppa_tr_ops, &ctx.base); diff --git a/target/i386/tcg/tcg-cpu.c b/target/i386/tcg/tcg-cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/tcg/tcg-cpu.c +++ b/target/i386/tcg/tcg-cpu.c @@ -XXX,XX +XXX,XX @@ static bool x86_debug_check_breakpoint(CPUState *cs) static const TCGCPUOps x86_tcg_ops = { .initialize = tcg_x86_init, + .translate_code = x86_translate_code, .synchronize_from_tb = x86_cpu_synchronize_from_tb, .restore_state_to_opc = x86_restore_state_to_opc, .cpu_exec_enter = x86_cpu_exec_enter, diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps i386_tr_ops = { .tb_stop = i386_tr_tb_stop, }; -/* generate intermediate code for basic block 'tb'. */ -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void x86_translate_code(CPUState *cpu, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc; diff --git a/target/loongarch/cpu.c b/target/loongarch/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/loongarch/cpu.c +++ b/target/loongarch/cpu.c @@ -XXX,XX +XXX,XX @@ static void loongarch_cpu_dump_state(CPUState *cs, FILE *f, int flags) static const TCGCPUOps loongarch_tcg_ops = { .initialize = loongarch_translate_init, + .translate_code = loongarch_translate_code, .synchronize_from_tb = loongarch_cpu_synchronize_from_tb, .restore_state_to_opc = loongarch_restore_state_to_opc, diff --git a/target/loongarch/tcg/translate.c b/target/loongarch/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/loongarch/tcg/translate.c +++ b/target/loongarch/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps loongarch_tr_ops = { .tb_stop = loongarch_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void loongarch_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx; diff --git a/target/m68k/cpu.c b/target/m68k/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/m68k/cpu.c +++ b/target/m68k/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps m68k_sysemu_ops = { static const TCGCPUOps m68k_tcg_ops = { .initialize = m68k_tcg_init, + .translate_code = m68k_translate_code, .restore_state_to_opc = m68k_restore_state_to_opc, #ifndef CONFIG_USER_ONLY diff --git a/target/m68k/translate.c b/target/m68k/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/m68k/translate.c +++ b/target/m68k/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps m68k_tr_ops = { .tb_stop = m68k_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void m68k_translate_code(CPUState *cpu, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc; translator_loop(cpu, tb, max_insns, pc, host_pc, &m68k_tr_ops, &dc.base); diff --git a/target/microblaze/cpu.c b/target/microblaze/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/microblaze/cpu.c +++ b/target/microblaze/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps mb_sysemu_ops = { static const TCGCPUOps mb_tcg_ops = { .initialize = mb_tcg_init, + .translate_code = mb_translate_code, .synchronize_from_tb = mb_cpu_synchronize_from_tb, .restore_state_to_opc = mb_restore_state_to_opc, diff --git a/target/microblaze/translate.c b/target/microblaze/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/microblaze/translate.c +++ b/target/microblaze/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps mb_tr_ops = { .tb_stop = mb_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void mb_translate_code(CPUState *cpu, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc; translator_loop(cpu, tb, max_insns, pc, host_pc, &mb_tr_ops, &dc.base); diff --git a/target/mips/cpu.c b/target/mips/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/mips/cpu.c +++ b/target/mips/cpu.c @@ -XXX,XX +XXX,XX @@ static const Property mips_cpu_properties[] = { #include "hw/core/tcg-cpu-ops.h" static const TCGCPUOps mips_tcg_ops = { .initialize = mips_tcg_init, + .translate_code = mips_translate_code, .synchronize_from_tb = mips_cpu_synchronize_from_tb, .restore_state_to_opc = mips_restore_state_to_opc, diff --git a/target/mips/tcg/translate.c b/target/mips/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/mips/tcg/translate.c +++ b/target/mips/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps mips_tr_ops = { .tb_stop = mips_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void mips_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx; diff --git a/target/openrisc/cpu.c b/target/openrisc/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/openrisc/cpu.c +++ b/target/openrisc/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps openrisc_sysemu_ops = { static const TCGCPUOps openrisc_tcg_ops = { .initialize = openrisc_translate_init, + .translate_code = openrisc_translate_code, .synchronize_from_tb = openrisc_cpu_synchronize_from_tb, .restore_state_to_opc = openrisc_restore_state_to_opc, diff --git a/target/openrisc/translate.c b/target/openrisc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/openrisc/translate.c +++ b/target/openrisc/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps openrisc_tr_ops = { .tb_stop = openrisc_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void openrisc_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx; diff --git a/target/ppc/cpu_init.c b/target/ppc/cpu_init.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/cpu_init.c +++ b/target/ppc/cpu_init.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps ppc_sysemu_ops = { static const TCGCPUOps ppc_tcg_ops = { .initialize = ppc_translate_init, + .translate_code = ppc_translate_code, .restore_state_to_opc = ppc_restore_state_to_opc, #ifdef CONFIG_USER_ONLY diff --git a/target/ppc/translate.c b/target/ppc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/translate.c +++ b/target/ppc/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps ppc_tr_ops = { .tb_stop = ppc_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void ppc_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx; diff --git a/target/riscv/tcg/tcg-cpu.c b/target/riscv/tcg/tcg-cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/tcg/tcg-cpu.c +++ b/target/riscv/tcg/tcg-cpu.c @@ -XXX,XX +XXX,XX @@ static void riscv_restore_state_to_opc(CPUState *cs, static const TCGCPUOps riscv_tcg_ops = { .initialize = riscv_translate_init, + .translate_code = riscv_translate_code, .synchronize_from_tb = riscv_cpu_synchronize_from_tb, .restore_state_to_opc = riscv_restore_state_to_opc, diff --git a/target/riscv/translate.c b/target/riscv/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/translate.c +++ b/target/riscv/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps riscv_tr_ops = { .tb_stop = riscv_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void riscv_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx; diff --git a/target/rx/cpu.c b/target/rx/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/rx/cpu.c +++ b/target/rx/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps rx_sysemu_ops = { static const TCGCPUOps rx_tcg_ops = { .initialize = rx_translate_init, + .translate_code = rx_translate_code, .synchronize_from_tb = rx_cpu_synchronize_from_tb, .restore_state_to_opc = rx_restore_state_to_opc, .tlb_fill = rx_cpu_tlb_fill, diff --git a/target/rx/translate.c b/target/rx/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/rx/translate.c +++ b/target/rx/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps rx_tr_ops = { .tb_stop = rx_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void rx_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc; diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/cpu.c +++ b/target/s390x/cpu.c @@ -XXX,XX +XXX,XX @@ void cpu_get_tb_cpu_state(CPUS390XState *env, vaddr *pc, static const TCGCPUOps s390_tcg_ops = { .initialize = s390x_translate_init, + .translate_code = s390x_translate_code, .restore_state_to_opc = s390x_restore_state_to_opc, #ifdef CONFIG_USER_ONLY diff --git a/target/s390x/tcg/translate.c b/target/s390x/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/tcg/translate.c +++ b/target/s390x/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps s390x_tr_ops = { .disas_log = s390x_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void s390x_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc; diff --git a/target/sh4/cpu.c b/target/sh4/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/sh4/cpu.c +++ b/target/sh4/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps sh4_sysemu_ops = { static const TCGCPUOps superh_tcg_ops = { .initialize = sh4_translate_init, + .translate_code = sh4_translate_code, .synchronize_from_tb = superh_cpu_synchronize_from_tb, .restore_state_to_opc = superh_restore_state_to_opc, diff --git a/target/sh4/translate.c b/target/sh4/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/sh4/translate.c +++ b/target/sh4/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps sh4_tr_ops = { .tb_stop = sh4_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void sh4_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx; diff --git a/target/sparc/cpu.c b/target/sparc/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/cpu.c +++ b/target/sparc/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps sparc_sysemu_ops = { static const TCGCPUOps sparc_tcg_ops = { .initialize = sparc_tcg_init, + .translate_code = sparc_translate_code, .synchronize_from_tb = sparc_cpu_synchronize_from_tb, .restore_state_to_opc = sparc_restore_state_to_opc, diff --git a/target/sparc/translate.c b/target/sparc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/translate.c +++ b/target/sparc/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps sparc_tr_ops = { .tb_stop = sparc_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void sparc_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc = {}; diff --git a/target/tricore/cpu.c b/target/tricore/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/tricore/cpu.c +++ b/target/tricore/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps tricore_sysemu_ops = { static const TCGCPUOps tricore_tcg_ops = { .initialize = tricore_tcg_init, + .translate_code = tricore_translate_code, .synchronize_from_tb = tricore_cpu_synchronize_from_tb, .restore_state_to_opc = tricore_restore_state_to_opc, .tlb_fill = tricore_cpu_tlb_fill, diff --git a/target/tricore/translate.c b/target/tricore/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/tricore/translate.c +++ b/target/tricore/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps tricore_tr_ops = { .tb_stop = tricore_tr_tb_stop, }; - -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void tricore_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx; translator_loop(cs, tb, max_insns, pc, host_pc, diff --git a/target/xtensa/cpu.c b/target/xtensa/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/xtensa/cpu.c +++ b/target/xtensa/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps xtensa_sysemu_ops = { static const TCGCPUOps xtensa_tcg_ops = { .initialize = xtensa_translate_init, + .translate_code = xtensa_translate_code, .debug_excp_handler = xtensa_breakpoint_handler, .restore_state_to_opc = xtensa_restore_state_to_opc, diff --git a/target/xtensa/translate.c b/target/xtensa/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/xtensa/translate.c +++ b/target/xtensa/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps xtensa_translator_ops = { .tb_stop = xtensa_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void xtensa_translate_code(CPUState *cpu, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc = {}; translator_loop(cpu, tb, max_insns, pc, host_pc, -- 2.43.0