Second pull for this week, since this set is large enough by itself. r~ The following changes since commit 7c9236d6d61f30583d5d860097d88dbf0fe487bf: Merge tag 'pull-tcg-20230116' of https://gitlab.com/rth7680/qemu into staging (2023-01-17 10:24:16 +0000) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20230117 for you to fetch changes up to 493c9b19a7fb7f387c4fcf57d3836504d5242bf5: tcg/riscv: Implement direct branch for goto_tb (2023-01-17 22:36:17 +0000) ---------------------------------------------------------------- tcg: Fix race conditions in (most) goto_tb implementations ---------------------------------------------------------------- Richard Henderson (22): tcg: Split out tcg_out_exit_tb tcg/i386: Remove unused goto_tb code for indirect jump tcg/ppc: Remove unused goto_tb code for indirect jump tcg/sparc64: Remove unused goto_tb code for indirect jump tcg: Replace asserts on tcg_jmp_insn_offset tcg: Introduce set_jmp_insn_offset tcg: Introduce get_jmp_target_addr tcg: Split out tcg_out_goto_tb tcg: Rename TB_JMP_RESET_OFFSET_INVALID to TB_JMP_OFFSET_INVALID tcg: Add gen_tb to TCGContext tcg: Add TranslationBlock.jmp_insn_offset tcg: Change tb_target_set_jmp_target arguments tcg: Move tb_target_set_jmp_target declaration to tcg.h tcg: Always define tb_target_set_jmp_target tcg: Remove TCG_TARGET_HAS_direct_jump tcg/aarch64: Reorg goto_tb implementation tcg/ppc: Reorg goto_tb implementation tcg/sparc64: Remove USE_REG_TB tcg/sparc64: Reorg goto_tb implementation tcg/arm: Implement direct branch for goto_tb tcg/riscv: Introduce OPC_NOP tcg/riscv: Implement direct branch for goto_tb include/exec/exec-all.h | 5 +- include/tcg/tcg.h | 14 ++- tcg/aarch64/tcg-target.h | 6 +- tcg/arm/tcg-target.h | 5 - tcg/i386/tcg-target.h | 9 -- tcg/loongarch64/tcg-target.h | 3 - tcg/mips/tcg-target.h | 5 - tcg/ppc/tcg-target.h | 7 +- tcg/riscv/tcg-target.h | 4 - tcg/s390x/tcg-target.h | 11 --- tcg/sparc64/tcg-target.h | 4 - tcg/tci/tcg-target.h | 4 - accel/tcg/cpu-exec.c | 21 ++-- accel/tcg/translate-all.c | 10 +- tcg/tcg-op.c | 14 +-- tcg/tcg.c | 42 +++++--- tcg/aarch64/tcg-target.c.inc | 106 ++++++++++----------- tcg/arm/tcg-target.c.inc | 89 +++++++++++------ tcg/i386/tcg-target.c.inc | 68 +++++++------ tcg/loongarch64/tcg-target.c.inc | 66 +++++++------ tcg/mips/tcg-target.c.inc | 59 +++++++----- tcg/ppc/tcg-target.c.inc | 193 ++++++++++++------------------------- tcg/riscv/tcg-target.c.inc | 65 +++++++++---- tcg/s390x/tcg-target.c.inc | 67 ++++++++----- tcg/sparc64/tcg-target.c.inc | 201 +++++++++++++++------------------------ tcg/tci/tcg-target.c.inc | 31 +++--- 26 files changed, 528 insertions(+), 581 deletions(-) The INDEX_op_exit_tb opcode needs no register allocation. Split out a dedicated helper function for it. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 4 ++++ tcg/aarch64/tcg-target.c.inc | 22 ++++++++++-------- tcg/arm/tcg-target.c.inc | 11 +++++---- tcg/i386/tcg-target.c.inc | 21 +++++++++-------- tcg/loongarch64/tcg-target.c.inc | 22 ++++++++++-------- tcg/mips/tcg-target.c.inc | 33 +++++++++++++-------------- tcg/ppc/tcg-target.c.inc | 11 +++++---- tcg/riscv/tcg-target.c.inc | 22 ++++++++++-------- tcg/s390x/tcg-target.c.inc | 23 ++++++++++--------- tcg/sparc64/tcg-target.c.inc | 39 +++++++++++++++++--------------- tcg/tci/tcg-target.c.inc | 10 ++++---- 11 files changed, 121 insertions(+), 97 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg ret, tcg_target_long arg); +static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg); static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]); @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) case INDEX_op_call: tcg_reg_alloc_call(s, op); break; + case INDEX_op_exit_tb: + tcg_out_exit_tb(s, op->args[0]); + break; case INDEX_op_dup2_vec: if (tcg_reg_alloc_dup2(s, op)) { break; diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, static const tcg_insn_unit *tb_ret_addr; +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) +{ + /* Reuse the zeroing that exists for goto_ptr. */ + if (a0 == 0) { + tcg_out_goto_long(s, tcg_code_gen_epilogue); + } else { + tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0); + tcg_out_goto_long(s, tb_ret_addr); + } +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, #define REG0(I) (const_args[I] ? TCG_REG_XZR : (TCGReg)args[I]) switch (opc) { - case INDEX_op_exit_tb: - /* Reuse the zeroing that exists for goto_ptr. */ - if (a0 == 0) { - tcg_out_goto_long(s, tcg_code_gen_epilogue); - } else { - tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0); - tcg_out_goto_long(s, tb_ret_addr); - } - break; - case INDEX_op_goto_tb: tcg_debug_assert(s->tb_jmp_insn_offset != NULL); /* @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ default: g_assert_not_reached(); } diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) static void tcg_out_epilogue(TCGContext *s); +static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) +{ + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, arg); + tcg_out_epilogue(s); +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, int c; switch (opc) { - case INDEX_op_exit_tb: - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, args[0]); - tcg_out_epilogue(s); - break; case INDEX_op_goto_tb: { /* Indirect jump method */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_call: /* Always emitted via tcg_out_call. */ + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ default: tcg_abort(); } diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) #endif } +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) +{ + /* Reuse the zeroing that exists for goto_ptr. */ + if (a0 == 0) { + tcg_out_jmp(s, tcg_code_gen_epilogue); + } else { + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0); + tcg_out_jmp(s, tb_ret_addr); + } +} + static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const_a2 = const_args[2]; switch (opc) { - case INDEX_op_exit_tb: - /* Reuse the zeroing that exists for goto_ptr. */ - if (a0 == 0) { - tcg_out_jmp(s, tcg_code_gen_epilogue); - } else { - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0); - tcg_out_jmp(s, tb_ret_addr); - } - break; case INDEX_op_goto_tb: if (s->tb_jmp_insn_offset) { /* direct jump method */ @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ default: tcg_abort(); } diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/loongarch64/tcg-target.c.inc +++ b/tcg/loongarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, static const tcg_insn_unit *tb_ret_addr; +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) +{ + /* Reuse the zeroing that exists for goto_ptr. */ + if (a0 == 0) { + tcg_out_call_int(s, tcg_code_gen_epilogue, true); + } else { + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0); + tcg_out_call_int(s, tb_ret_addr, true); + } +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, int c2 = const_args[2]; switch (opc) { - case INDEX_op_exit_tb: - /* Reuse the zeroing that exists for goto_ptr. */ - if (a0 == 0) { - tcg_out_call_int(s, tcg_code_gen_epilogue, true); - } else { - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0); - tcg_out_call_int(s, tb_ret_addr, true); - } - break; - case INDEX_op_goto_tb: tcg_debug_assert(s->tb_jmp_insn_offset != NULL); /* @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ default: g_assert_not_reached(); } diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_clz(TCGContext *s, MIPSInsn opcv2, MIPSInsn opcv6, } } +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) +{ + TCGReg b0 = TCG_REG_ZERO; + + if (a0 & ~0xffff) { + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_V0, a0 & ~0xffff); + b0 = TCG_REG_V0; + } + if (!tcg_out_opc_jmp(s, OPC_J, tb_ret_addr)) { + tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)tb_ret_addr); + tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0); + } + tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff); +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, c2 = const_args[2]; switch (opc) { - case INDEX_op_exit_tb: - { - TCGReg b0 = TCG_REG_ZERO; - - a0 = (intptr_t)a0; - if (a0 & ~0xffff) { - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_V0, a0 & ~0xffff); - b0 = TCG_REG_V0; - } - if (!tcg_out_opc_jmp(s, OPC_J, tb_ret_addr)) { - tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, - (uintptr_t)tb_ret_addr); - tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0); - } - tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff); - } - break; case INDEX_op_goto_tb: /* indirect jump method */ tcg_debug_assert(s->tb_jmp_insn_offset == 0); @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ default: tcg_abort(); } diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out32(s, BCLR | BO_ALWAYS); } +static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) +{ + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, arg); + tcg_out_b(s, 0, tcg_code_gen_epilogue); +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGArg a0, a1, a2; switch (opc) { - case INDEX_op_exit_tb: - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]); - tcg_out_b(s, 0, tcg_code_gen_epilogue); - break; case INDEX_op_goto_tb: if (s->tb_jmp_insn_offset) { /* Direct jump. */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ default: tcg_abort(); } diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) static const tcg_insn_unit *tb_ret_addr; +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) +{ + /* Reuse the zeroing that exists for goto_ptr. */ + if (a0 == 0) { + tcg_out_call_int(s, tcg_code_gen_epilogue, true); + } else { + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0); + tcg_out_call_int(s, tb_ret_addr, true); + } +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, int c2 = const_args[2]; switch (opc) { - case INDEX_op_exit_tb: - /* Reuse the zeroing that exists for goto_ptr. */ - if (a0 == 0) { - tcg_out_call_int(s, tcg_code_gen_epilogue, true); - } else { - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0); - tcg_out_call_int(s, tb_ret_addr, true); - } - break; - case INDEX_op_goto_tb: assert(s->tb_jmp_insn_offset == 0); /* indirect jump method */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ default: g_assert_not_reached(); } diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, #endif } +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) +{ + /* Reuse the zeroing that exists for goto_ptr. */ + if (a0 == 0) { + tgen_gotoi(s, S390_CC_ALWAYS, tcg_code_gen_epilogue); + } else { + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, a0); + tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr); + } +} + # define OP_32_64(x) \ case glue(glue(INDEX_op_,x),_i32): \ case glue(glue(INDEX_op_,x),_i64) @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGArg a0, a1, a2; switch (opc) { - case INDEX_op_exit_tb: - /* Reuse the zeroing that exists for goto_ptr. */ - a0 = args[0]; - if (a0 == 0) { - tgen_gotoi(s, S390_CC_ALWAYS, tcg_code_gen_epilogue); - } else { - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, a0); - tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr); - } - break; - case INDEX_op_goto_tb: a0 = args[0]; /* @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ default: tcg_abort(); } diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc64/tcg-target.c.inc +++ b/tcg/sparc64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr, #endif /* CONFIG_SOFTMMU */ } +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) +{ + if (check_fit_ptr(a0, 13)) { + tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); + tcg_out_movi_imm13(s, TCG_REG_O0, a0); + return; + } else if (USE_REG_TB) { + intptr_t tb_diff = tcg_tbrel_diff(s, (void *)a0); + if (check_fit_ptr(tb_diff, 13)) { + tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); + /* Note that TCG_REG_TB has been unwound to O1. */ + tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O1, tb_diff, ARITH_ADD); + return; + } + } + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_I0, a0 & ~0x3ff); + tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); + tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, a0 & 0x3ff, ARITH_OR); +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, c2 = const_args[2]; switch (opc) { - case INDEX_op_exit_tb: - if (check_fit_ptr(a0, 13)) { - tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); - tcg_out_movi_imm13(s, TCG_REG_O0, a0); - break; - } else if (USE_REG_TB) { - intptr_t tb_diff = tcg_tbrel_diff(s, (void *)a0); - if (check_fit_ptr(tb_diff, 13)) { - tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); - /* Note that TCG_REG_TB has been unwound to O1. */ - tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O1, tb_diff, ARITH_ADD); - break; - } - } - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_I0, a0 & ~0x3ff); - tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); - tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, a0 & 0x3ff, ARITH_OR); - break; case INDEX_op_goto_tb: if (s->tb_jmp_insn_offset) { /* direct jump method */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ default: tcg_abort(); } diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *func, # define CASE_64(x) #endif +static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) +{ + tcg_out_op_p(s, INDEX_op_exit_tb, (void *)arg); +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGOpcode exts; switch (opc) { - case INDEX_op_exit_tb: - tcg_out_op_p(s, opc, (void *)args[0]); - break; - case INDEX_op_goto_tb: tcg_debug_assert(s->tb_jmp_insn_offset == 0); /* indirect jump method. */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ default: tcg_abort(); } -- 2.34.1 Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, switch (opc) { case INDEX_op_goto_tb: - if (s->tb_jmp_insn_offset) { - /* direct jump method */ - int gap; - /* jump displacement must be aligned for atomic patching; + qemu_build_assert(TCG_TARGET_HAS_direct_jump); + { + /* + * Jump displacement must be aligned for atomic patching; * see if we need to add extra nops before jump */ - gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr; + int gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr; if (gap != 1) { tcg_out_nopn(s, gap - 1); } tcg_out8(s, OPC_JMP_long); /* jmp im */ s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); tcg_out32(s, 0); - } else { - /* indirect jump method */ - tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, -1, - (intptr_t)(s->tb_jmp_target_addr + a0)); } set_jmp_reset_offset(s, a0); break; -- 2.34.1 Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/ppc/tcg-target.c.inc | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, switch (opc) { case INDEX_op_goto_tb: - if (s->tb_jmp_insn_offset) { - /* Direct jump. */ - if (TCG_TARGET_REG_BITS == 64) { - /* Ensure the next insns are 8 or 16-byte aligned. */ - while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) { - tcg_out32(s, NOP); - } - s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); - tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0)); - tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0)); - } else { - s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); - tcg_out32(s, B); - s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s); - break; + qemu_build_assert(TCG_TARGET_HAS_direct_jump); + /* Direct jump. */ + if (TCG_TARGET_REG_BITS == 64) { + /* Ensure the next insns are 8 or 16-byte aligned. */ + while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) { + tcg_out32(s, NOP); } + s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); + tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0)); + tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0)); } else { - /* Indirect jump. */ - tcg_debug_assert(s->tb_jmp_insn_offset == NULL); - tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TB, 0, - (intptr_t)(s->tb_jmp_insn_offset + args[0])); + s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); + tcg_out32(s, B); + s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s); + break; } tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR); tcg_out32(s, BCCTR | BO_ALWAYS); -- 2.34.1 Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc64/tcg-target.c.inc | 41 +++++++++++------------------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc64/tcg-target.c.inc +++ b/tcg/sparc64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, return false; } -static void tcg_out_ld_ptr(TCGContext *s, TCGReg ret, const void *arg) -{ - intptr_t diff = tcg_tbrel_diff(s, arg); - if (USE_REG_TB && check_fit_ptr(diff, 13)) { - tcg_out_ld(s, TCG_TYPE_PTR, ret, TCG_REG_TB, diff); - return; - } - tcg_out_movi(s, TCG_TYPE_PTR, ret, (uintptr_t)arg & ~0x3ff); - tcg_out_ld(s, TCG_TYPE_PTR, ret, ret, (uintptr_t)arg & 0x3ff); -} - static void tcg_out_sety(TCGContext *s, TCGReg rs) { tcg_out32(s, WRY | INSN_RS1(TCG_REG_G0) | INSN_RS2(rs)); @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, switch (opc) { case INDEX_op_goto_tb: - if (s->tb_jmp_insn_offset) { - /* direct jump method */ - if (USE_REG_TB) { - /* make sure the patch is 8-byte aligned. */ - if ((intptr_t)s->code_ptr & 4) { - tcg_out_nop(s); - } - s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); - tcg_out_sethi(s, TCG_REG_T1, 0); - tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR); - tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL); - tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); - } else { - s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); - tcg_out32(s, CALL); + qemu_build_assert(TCG_TARGET_HAS_direct_jump); + /* Direct jump. */ + if (USE_REG_TB) { + /* make sure the patch is 8-byte aligned. */ + if ((intptr_t)s->code_ptr & 4) { tcg_out_nop(s); } + s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); + tcg_out_sethi(s, TCG_REG_T1, 0); + tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR); + tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL); + tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); } else { - /* indirect jump method */ - tcg_out_ld_ptr(s, TCG_REG_TB, s->tb_jmp_target_addr + a0); - tcg_out_arithi(s, TCG_REG_G0, TCG_REG_TB, 0, JMPL); + s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); + tcg_out32(s, CALL); tcg_out_nop(s); } set_jmp_reset_offset(s, a0); -- 2.34.1 Test TCG_TARGET_HAS_direct_jump instead of testing an implementation pointer. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target.c.inc | 2 +- tcg/arm/tcg-target.c.inc | 2 +- tcg/loongarch64/tcg-target.c.inc | 2 +- tcg/mips/tcg-target.c.inc | 2 +- tcg/riscv/tcg-target.c.inc | 2 +- tcg/tci/tcg-target.c.inc | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, switch (opc) { case INDEX_op_goto_tb: - tcg_debug_assert(s->tb_jmp_insn_offset != NULL); + qemu_build_assert(TCG_TARGET_HAS_direct_jump); /* * Ensure that ADRP+ADD are 8-byte aligned so that an atomic * write can be used to patch the target address. diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, intptr_t ptr, dif, dil; TCGReg base = TCG_REG_PC; - tcg_debug_assert(s->tb_jmp_insn_offset == 0); + qemu_build_assert(!TCG_TARGET_HAS_direct_jump); ptr = (intptr_t)tcg_splitwx_to_rx(s->tb_jmp_target_addr + args[0]); dif = tcg_pcrel_diff(s, (void *)ptr) - 8; dil = sextract32(dif, 0, 12); diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/loongarch64/tcg-target.c.inc +++ b/tcg/loongarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, switch (opc) { case INDEX_op_goto_tb: - tcg_debug_assert(s->tb_jmp_insn_offset != NULL); + qemu_build_assert(TCG_TARGET_HAS_direct_jump); /* * Ensure that patch area is 8-byte aligned so that an * atomic write can be used to patch the target address. diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, switch (opc) { case INDEX_op_goto_tb: /* indirect jump method */ - tcg_debug_assert(s->tb_jmp_insn_offset == 0); + qemu_build_assert(!TCG_TARGET_HAS_direct_jump); tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO, (uintptr_t)(s->tb_jmp_target_addr + a0)); tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0); diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, switch (opc) { case INDEX_op_goto_tb: - assert(s->tb_jmp_insn_offset == 0); + qemu_build_assert(!TCG_TARGET_HAS_direct_jump); /* indirect jump method */ tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO, (uintptr_t)(s->tb_jmp_target_addr + a0)); diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, switch (opc) { case INDEX_op_goto_tb: - tcg_debug_assert(s->tb_jmp_insn_offset == 0); + qemu_build_assert(!TCG_TARGET_HAS_direct_jump); /* indirect jump method. */ tcg_out_op_p(s, opc, s->tb_jmp_target_addr + args[0]); set_jmp_reset_offset(s, args[0]); -- 2.34.1 Similar to the existing set_jmp_reset_offset. Move any assert for TCG_TARGET_HAS_direct_jump into the new function (which now cannot be build-time). Will be unused if TCG_TARGET_HAS_direct_jump is constant 0, but we can't test for constant in the preprocessor, so just mark it G_GNUC_UNUSED. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 10 ++++++++++ tcg/aarch64/tcg-target.c.inc | 3 +-- tcg/i386/tcg-target.c.inc | 3 +-- tcg/loongarch64/tcg-target.c.inc | 3 +-- tcg/ppc/tcg-target.c.inc | 7 +++---- tcg/s390x/tcg-target.c.inc | 2 +- tcg/sparc64/tcg-target.c.inc | 5 ++--- 7 files changed, 19 insertions(+), 14 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void set_jmp_reset_offset(TCGContext *s, int which) s->tb_jmp_reset_offset[which] = tcg_current_code_size(s); } +static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) +{ + /* + * We will check for overflow at the end of the opcode loop in + * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. + */ + tcg_debug_assert(TCG_TARGET_HAS_direct_jump); + s->tb_jmp_insn_offset[which] = tcg_current_code_size(s); +} + /* Signal overflow, starting over with fewer guest insns. */ static G_NORETURN void tcg_raise_tb_overflow(TCGContext *s) diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, switch (opc) { case INDEX_op_goto_tb: - qemu_build_assert(TCG_TARGET_HAS_direct_jump); /* * Ensure that ADRP+ADD are 8-byte aligned so that an atomic * write can be used to patch the target address. @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, if ((uintptr_t)s->code_ptr & 7) { tcg_out32(s, NOP); } - s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); + set_jmp_insn_offset(s, a0); /* * actual branch destination will be patched by * tb_target_set_jmp_target later diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, switch (opc) { case INDEX_op_goto_tb: - qemu_build_assert(TCG_TARGET_HAS_direct_jump); { /* * Jump displacement must be aligned for atomic patching; @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_nopn(s, gap - 1); } tcg_out8(s, OPC_JMP_long); /* jmp im */ - s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); + set_jmp_insn_offset(s, a0); tcg_out32(s, 0); } set_jmp_reset_offset(s, a0); diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/loongarch64/tcg-target.c.inc +++ b/tcg/loongarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, switch (opc) { case INDEX_op_goto_tb: - qemu_build_assert(TCG_TARGET_HAS_direct_jump); /* * Ensure that patch area is 8-byte aligned so that an * atomic write can be used to patch the target address. @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, if ((uintptr_t)s->code_ptr & 7) { tcg_out_nop(s); } - s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); + set_jmp_insn_offset(s, a0); /* * actual branch destination will be patched by * tb_target_set_jmp_target later diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, switch (opc) { case INDEX_op_goto_tb: - qemu_build_assert(TCG_TARGET_HAS_direct_jump); /* Direct jump. */ if (TCG_TARGET_REG_BITS == 64) { /* Ensure the next insns are 8 or 16-byte aligned. */ while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) { tcg_out32(s, NOP); } - s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); + set_jmp_insn_offset(s, args[0]); tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0)); tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0)); } else { - s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); + set_jmp_insn_offset(s, args[0]); tcg_out32(s, B); - s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s); + set_jmp_reset_offset(s, args[0]); break; } tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR); diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out16(s, NOP); } tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4)); - s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); + set_jmp_insn_offset(s, a0); s->code_ptr += 2; set_jmp_reset_offset(s, a0); break; diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc64/tcg-target.c.inc +++ b/tcg/sparc64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, switch (opc) { case INDEX_op_goto_tb: - qemu_build_assert(TCG_TARGET_HAS_direct_jump); /* Direct jump. */ if (USE_REG_TB) { /* make sure the patch is 8-byte aligned. */ if ((intptr_t)s->code_ptr & 4) { tcg_out_nop(s); } - s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); + set_jmp_insn_offset(s, a0); tcg_out_sethi(s, TCG_REG_T1, 0); tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR); tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL); tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); } else { - s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); + set_jmp_insn_offset(s, a0); tcg_out32(s, CALL); tcg_out_nop(s); } -- 2.34.1 Similar to the existing set_jmp_reset_offset. Include the rw->rx address space conversion done by arm and s390x, and forgotten by mips and riscv. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 9 +++++++++ tcg/arm/tcg-target.c.inc | 2 +- tcg/mips/tcg-target.c.inc | 2 +- tcg/riscv/tcg-target.c.inc | 2 +- tcg/tci/tcg-target.c.inc | 2 +- 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) s->tb_jmp_insn_offset[which] = tcg_current_code_size(s); } +static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) +{ + /* + * Return the read-execute version of the pointer, for the benefit + * of any pc-relative addressing mode. + */ + return (uintptr_t)tcg_splitwx_to_rx(&s->tb_jmp_target_addr[which]); +} + /* Signal overflow, starting over with fewer guest insns. */ static G_NORETURN void tcg_raise_tb_overflow(TCGContext *s) diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGReg base = TCG_REG_PC; qemu_build_assert(!TCG_TARGET_HAS_direct_jump); - ptr = (intptr_t)tcg_splitwx_to_rx(s->tb_jmp_target_addr + args[0]); + ptr = get_jmp_target_addr(s, args[0]); dif = tcg_pcrel_diff(s, (void *)ptr) - 8; dil = sextract32(dif, 0, 12); if (dif != dil) { diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, /* indirect jump method */ qemu_build_assert(!TCG_TARGET_HAS_direct_jump); tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO, - (uintptr_t)(s->tb_jmp_target_addr + a0)); + get_jmp_target_addr(s, a0)); tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0); tcg_out_nop(s); set_jmp_reset_offset(s, a0); diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, qemu_build_assert(!TCG_TARGET_HAS_direct_jump); /* indirect jump method */ tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO, - (uintptr_t)(s->tb_jmp_target_addr + a0)); + get_jmp_target_addr(s, a0)); tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0); set_jmp_reset_offset(s, a0); break; diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_goto_tb: qemu_build_assert(!TCG_TARGET_HAS_direct_jump); /* indirect jump method. */ - tcg_out_op_p(s, opc, s->tb_jmp_target_addr + args[0]); + tcg_out_op_p(s, opc, (void *)get_jmp_target_addr(s, args[0])); set_jmp_reset_offset(s, args[0]); break; -- 2.34.1 The INDEX_op_goto_tb opcode needs no register allocation. Split out a dedicated helper function for it. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 4 ++ tcg/aarch64/tcg-target.c.inc | 40 ++++++++++--------- tcg/arm/tcg-target.c.inc | 49 ++++++++++++----------- tcg/i386/tcg-target.c.inc | 33 ++++++++-------- tcg/loongarch64/tcg-target.c.inc | 38 +++++++++--------- tcg/mips/tcg-target.c.inc | 21 +++++----- tcg/ppc/tcg-target.c.inc | 52 ++++++++++++------------ tcg/riscv/tcg-target.c.inc | 20 +++++----- tcg/s390x/tcg-target.c.inc | 31 ++++++++------- tcg/sparc64/tcg-target.c.inc | 68 +++++++++++++++++--------------- tcg/tci/tcg-target.c.inc | 16 ++++---- 11 files changed, 199 insertions(+), 173 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg ret, tcg_target_long arg); static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg); +static void tcg_out_goto_tb(TCGContext *s, int which); static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]); @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) case INDEX_op_exit_tb: tcg_out_exit_tb(s, op->args[0]); break; + case INDEX_op_goto_tb: + tcg_out_goto_tb(s, op->args[0]); + break; case INDEX_op_dup2_vec: if (tcg_reg_alloc_dup2(s, op)) { break; diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) } } +static void tcg_out_goto_tb(TCGContext *s, int which) +{ + /* + * Ensure that ADRP+ADD are 8-byte aligned so that an atomic + * write can be used to patch the target address. + */ + if ((uintptr_t)s->code_ptr & 7) { + tcg_out32(s, NOP); + } + set_jmp_insn_offset(s, which); + /* + * actual branch destination will be patched by + * tb_target_set_jmp_target later + */ + tcg_out_insn(s, 3406, ADRP, TCG_REG_TMP, 0); + tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_TMP, TCG_REG_TMP, 0); + tcg_out_insn(s, 3207, BR, TCG_REG_TMP); + set_jmp_reset_offset(s, which); +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, #define REG0(I) (const_args[I] ? TCG_REG_XZR : (TCGReg)args[I]) switch (opc) { - case INDEX_op_goto_tb: - /* - * Ensure that ADRP+ADD are 8-byte aligned so that an atomic - * write can be used to patch the target address. - */ - if ((uintptr_t)s->code_ptr & 7) { - tcg_out32(s, NOP); - } - set_jmp_insn_offset(s, a0); - /* - * actual branch destination will be patched by - * tb_target_set_jmp_target later - */ - tcg_out_insn(s, 3406, ADRP, TCG_REG_TMP, 0); - tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_TMP, TCG_REG_TMP, 0); - tcg_out_insn(s, 3207, BR, TCG_REG_TMP); - set_jmp_reset_offset(s, a0); - break; - case INDEX_op_goto_ptr: tcg_out_insn(s, 3207, BR, a0); break; @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ default: g_assert_not_reached(); } diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) tcg_out_epilogue(s); } +static void tcg_out_goto_tb(TCGContext *s, int which) +{ + /* Indirect jump method */ + intptr_t ptr, dif, dil; + TCGReg base = TCG_REG_PC; + + qemu_build_assert(!TCG_TARGET_HAS_direct_jump); + ptr = get_jmp_target_addr(s, which); + dif = tcg_pcrel_diff(s, (void *)ptr) - 8; + dil = sextract32(dif, 0, 12); + if (dif != dil) { + /* + * The TB is close, but outside the 12 bits addressable by + * the load. We can extend this to 20 bits with a sub of a + * shifted immediate from pc. In the vastly unlikely event + * the code requires more than 1MB, we'll use 2 insns and + * be no worse off. + */ + base = TCG_REG_R0; + tcg_out_movi32(s, COND_AL, base, ptr - dil); + } + tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, base, dil); + set_jmp_reset_offset(s, which); +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, int c; switch (opc) { - case INDEX_op_goto_tb: - { - /* Indirect jump method */ - intptr_t ptr, dif, dil; - TCGReg base = TCG_REG_PC; - - qemu_build_assert(!TCG_TARGET_HAS_direct_jump); - ptr = get_jmp_target_addr(s, args[0]); - dif = tcg_pcrel_diff(s, (void *)ptr) - 8; - dil = sextract32(dif, 0, 12); - if (dif != dil) { - /* The TB is close, but outside the 12 bits addressable by - the load. We can extend this to 20 bits with a sub of a - shifted immediate from pc. In the vastly unlikely event - the code requires more than 1MB, we'll use 2 insns and - be no worse off. */ - base = TCG_REG_R0; - tcg_out_movi32(s, COND_AL, base, ptr - dil); - } - tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, base, dil); - set_jmp_reset_offset(s, args[0]); - } - break; case INDEX_op_goto_ptr: tcg_out_b_reg(s, COND_AL, args[0]); break; @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_call: /* Always emitted via tcg_out_call. */ case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ default: tcg_abort(); } diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) } } +static void tcg_out_goto_tb(TCGContext *s, int which) +{ + /* + * Jump displacement must be aligned for atomic patching; + * see if we need to add extra nops before jump + */ + int gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr; + if (gap != 1) { + tcg_out_nopn(s, gap - 1); + } + tcg_out8(s, OPC_JMP_long); /* jmp im */ + set_jmp_insn_offset(s, which); + tcg_out32(s, 0); + set_jmp_reset_offset(s, which); +} + static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const_a2 = const_args[2]; switch (opc) { - case INDEX_op_goto_tb: - { - /* - * Jump displacement must be aligned for atomic patching; - * see if we need to add extra nops before jump - */ - int gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr; - if (gap != 1) { - tcg_out_nopn(s, gap - 1); - } - tcg_out8(s, OPC_JMP_long); /* jmp im */ - set_jmp_insn_offset(s, a0); - tcg_out32(s, 0); - } - set_jmp_reset_offset(s, a0); - break; case INDEX_op_goto_ptr: /* jmp to the given host address (could be epilogue) */ tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, a0); @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ default: tcg_abort(); } diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/loongarch64/tcg-target.c.inc +++ b/tcg/loongarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) } } +static void tcg_out_goto_tb(TCGContext *s, int which) +{ + /* + * Ensure that patch area is 8-byte aligned so that an + * atomic write can be used to patch the target address. + */ + if ((uintptr_t)s->code_ptr & 7) { + tcg_out_nop(s); + } + set_jmp_insn_offset(s, which); + /* + * actual branch destination will be patched by + * tb_target_set_jmp_target later + */ + tcg_out_opc_pcaddu18i(s, TCG_REG_TMP0, 0); + tcg_out_opc_jirl(s, TCG_REG_ZERO, TCG_REG_TMP0, 0); + set_jmp_reset_offset(s, which); +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, int c2 = const_args[2]; switch (opc) { - case INDEX_op_goto_tb: - /* - * Ensure that patch area is 8-byte aligned so that an - * atomic write can be used to patch the target address. - */ - if ((uintptr_t)s->code_ptr & 7) { - tcg_out_nop(s); - } - set_jmp_insn_offset(s, a0); - /* - * actual branch destination will be patched by - * tb_target_set_jmp_target later - */ - tcg_out_opc_pcaddu18i(s, TCG_REG_TMP0, 0); - tcg_out_opc_jirl(s, TCG_REG_ZERO, TCG_REG_TMP0, 0); - set_jmp_reset_offset(s, a0); - break; - case INDEX_op_mb: tcg_out_mb(s, a0); break; @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ default: g_assert_not_reached(); } diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff); } +static void tcg_out_goto_tb(TCGContext *s, int which) +{ + /* indirect jump method */ + qemu_build_assert(!TCG_TARGET_HAS_direct_jump); + tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO, + get_jmp_target_addr(s, which)); + tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0); + tcg_out_nop(s); + set_jmp_reset_offset(s, which); +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, c2 = const_args[2]; switch (opc) { - case INDEX_op_goto_tb: - /* indirect jump method */ - qemu_build_assert(!TCG_TARGET_HAS_direct_jump); - tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO, - get_jmp_target_addr(s, a0)); - tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0); - tcg_out_nop(s); - set_jmp_reset_offset(s, a0); - break; case INDEX_op_goto_ptr: /* jmp to the given host address (could be epilogue) */ tcg_out_opc_reg(s, OPC_JR, 0, a0, 0); @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ default: tcg_abort(); } diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) tcg_out_b(s, 0, tcg_code_gen_epilogue); } +static void tcg_out_goto_tb(TCGContext *s, int which) +{ + /* Direct jump. */ + if (TCG_TARGET_REG_BITS == 64) { + /* Ensure the next insns are 8 or 16-byte aligned. */ + while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) { + tcg_out32(s, NOP); + } + set_jmp_insn_offset(s, which); + tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0)); + tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0)); + tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR); + tcg_out32(s, BCCTR | BO_ALWAYS); + set_jmp_reset_offset(s, which); + if (USE_REG_TB) { + /* For the unlinked case, need to reset TCG_REG_TB. */ + tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB, + -tcg_current_code_size(s)); + } + } else { + set_jmp_insn_offset(s, which); + tcg_out32(s, B); + set_jmp_reset_offset(s, which); + } +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGArg a0, a1, a2; switch (opc) { - case INDEX_op_goto_tb: - /* Direct jump. */ - if (TCG_TARGET_REG_BITS == 64) { - /* Ensure the next insns are 8 or 16-byte aligned. */ - while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) { - tcg_out32(s, NOP); - } - set_jmp_insn_offset(s, args[0]); - tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0)); - tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0)); - } else { - set_jmp_insn_offset(s, args[0]); - tcg_out32(s, B); - set_jmp_reset_offset(s, args[0]); - break; - } - tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR); - tcg_out32(s, BCCTR | BO_ALWAYS); - set_jmp_reset_offset(s, args[0]); - if (USE_REG_TB) { - /* For the unlinked case, need to reset TCG_REG_TB. */ - tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB, - -tcg_current_code_size(s)); - } - break; case INDEX_op_goto_ptr: tcg_out32(s, MTSPR | RS(args[0]) | CTR); if (USE_REG_TB) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ default: tcg_abort(); } diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) } } +static void tcg_out_goto_tb(TCGContext *s, int which) +{ + qemu_build_assert(!TCG_TARGET_HAS_direct_jump); + /* indirect jump method */ + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO, + get_jmp_target_addr(s, which)); + tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0); + set_jmp_reset_offset(s, which); +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, int c2 = const_args[2]; switch (opc) { - case INDEX_op_goto_tb: - qemu_build_assert(!TCG_TARGET_HAS_direct_jump); - /* indirect jump method */ - tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO, - get_jmp_target_addr(s, a0)); - tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0); - set_jmp_reset_offset(s, a0); - break; - case INDEX_op_goto_ptr: tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, a0, 0); break; @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ default: g_assert_not_reached(); } diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) } } +static void tcg_out_goto_tb(TCGContext *s, int which) +{ + /* + * Branch displacement must be aligned for atomic patching; + * see if we need to add extra nop before branch + */ + if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) { + tcg_out16(s, NOP); + } + tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4)); + set_jmp_insn_offset(s, which); + s->code_ptr += 2; + set_jmp_reset_offset(s, which); +} + # define OP_32_64(x) \ case glue(glue(INDEX_op_,x),_i32): \ case glue(glue(INDEX_op_,x),_i64) @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGArg a0, a1, a2; switch (opc) { - case INDEX_op_goto_tb: - a0 = args[0]; - /* - * branch displacement must be aligned for atomic patching; - * see if we need to add extra nop before branch - */ - if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) { - tcg_out16(s, NOP); - } - tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4)); - set_jmp_insn_offset(s, a0); - s->code_ptr += 2; - set_jmp_reset_offset(s, a0); - break; - case INDEX_op_goto_ptr: a0 = args[0]; tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, a0); @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ default: tcg_abort(); } diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc64/tcg-target.c.inc +++ b/tcg/sparc64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, a0 & 0x3ff, ARITH_OR); } +static void tcg_out_goto_tb(TCGContext *s, int which) +{ + /* Direct jump. */ + if (USE_REG_TB) { + /* make sure the patch is 8-byte aligned. */ + if ((intptr_t)s->code_ptr & 4) { + tcg_out_nop(s); + } + set_jmp_insn_offset(s, which); + tcg_out_sethi(s, TCG_REG_T1, 0); + tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR); + tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL); + tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); + } else { + set_jmp_insn_offset(s, which); + tcg_out32(s, CALL); + tcg_out_nop(s); + } + set_jmp_reset_offset(s, which); + + /* + * For the unlinked path of goto_tb, we need to reset TCG_REG_TB + * to the beginning of this TB. + */ + if (USE_REG_TB) { + int c = -tcg_current_code_size(s); + if (check_fit_i32(c, 13)) { + tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD); + } else { + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c); + tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); + } + } +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, c2 = const_args[2]; switch (opc) { - case INDEX_op_goto_tb: - /* Direct jump. */ - if (USE_REG_TB) { - /* make sure the patch is 8-byte aligned. */ - if ((intptr_t)s->code_ptr & 4) { - tcg_out_nop(s); - } - set_jmp_insn_offset(s, a0); - tcg_out_sethi(s, TCG_REG_T1, 0); - tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR); - tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL); - tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); - } else { - set_jmp_insn_offset(s, a0); - tcg_out32(s, CALL); - tcg_out_nop(s); - } - set_jmp_reset_offset(s, a0); - - /* For the unlinked path of goto_tb, we need to reset - TCG_REG_TB to the beginning of this TB. */ - if (USE_REG_TB) { - c = -tcg_current_code_size(s); - if (check_fit_i32(c, 13)) { - tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD); - } else { - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c); - tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, - TCG_REG_T1, ARITH_ADD); - } - } - break; case INDEX_op_goto_ptr: tcg_out_arithi(s, TCG_REG_G0, a0, 0, JMPL); if (USE_REG_TB) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ default: tcg_abort(); } diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) tcg_out_op_p(s, INDEX_op_exit_tb, (void *)arg); } +static void tcg_out_goto_tb(TCGContext *s, int which) +{ + qemu_build_assert(!TCG_TARGET_HAS_direct_jump); + /* indirect jump method. */ + tcg_out_op_p(s, INDEX_op_goto_tb, (void *)get_jmp_target_addr(s, which)); + set_jmp_reset_offset(s, which); +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGOpcode exts; switch (opc) { - case INDEX_op_goto_tb: - qemu_build_assert(!TCG_TARGET_HAS_direct_jump); - /* indirect jump method. */ - tcg_out_op_p(s, opc, (void *)get_jmp_target_addr(s, args[0])); - set_jmp_reset_offset(s, args[0]); - break; - case INDEX_op_goto_ptr: tcg_out_op_r(s, opc, args[0]); break; @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ default: tcg_abort(); } -- 2.34.1 This will shortly be used for more than reset. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/exec-all.h | 2 +- accel/tcg/translate-all.c | 8 ++++---- tcg/tcg.c | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ struct TranslationBlock { * setting one of the jump targets (or patching the jump instruction). Only * two of such jumps are supported. */ +#define TB_JMP_OFFSET_INVALID 0xffff /* indicates no jump generated */ uint16_t jmp_reset_offset[2]; /* offset of original jump target */ -#define TB_JMP_RESET_OFFSET_INVALID 0xffff /* indicates no jump generated */ uintptr_t jmp_target_arg[2]; /* target address or offset */ /* diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, tb->jmp_dest[1] = (uintptr_t)NULL; /* init original jump addresses which have been set during tcg_gen_code() */ - if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { + if (tb->jmp_reset_offset[0] != TB_JMP_OFFSET_INVALID) { tb_reset_jump(tb, 0); } - if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { + if (tb->jmp_reset_offset[1] != TB_JMP_OFFSET_INVALID) { tb_reset_jump(tb, 1); } @@ -XXX,XX +XXX,XX @@ static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data) if (tb_page_addr1(tb) != -1) { tst->cross_page++; } - if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { + if (tb->jmp_reset_offset[0] != TB_JMP_OFFSET_INVALID) { tst->direct_jmp_count++; - if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { + if (tb->jmp_reset_offset[1] != TB_JMP_OFFSET_INVALID) { tst->direct_jmp2_count++; } } diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) #endif /* Initialize goto_tb jump offsets. */ - tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; - tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID; + tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID; + tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID; tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset; if (TCG_TARGET_HAS_direct_jump) { tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg; -- 2.34.1 This can replace four other variables that are references into the TranslationBlock structure. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg.h | 11 +++-------- accel/tcg/translate-all.c | 2 +- tcg/tcg-op.c | 14 +++++++------- tcg/tcg.c | 14 +++----------- 4 files changed, 14 insertions(+), 27 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ struct TCGContext { int nb_indirects; int nb_ops; - /* goto_tb support */ - tcg_insn_unit *code_buf; - uint16_t *tb_jmp_reset_offset; /* tb->jmp_reset_offset */ - uintptr_t *tb_jmp_insn_offset; /* tb->jmp_target_arg if direct_jump */ - uintptr_t *tb_jmp_target_addr; /* tb->jmp_target_arg if !direct_jump */ - TCGRegSet reserved_regs; - uint32_t tb_cflags; /* cflags of the current TB */ intptr_t current_frame_offset; intptr_t frame_start; intptr_t frame_end; TCGTemp *frame_temp; - tcg_insn_unit *code_ptr; + TranslationBlock *gen_tb; /* tb for which code is being generated */ + tcg_insn_unit *code_buf; /* pointer for start of tb */ + tcg_insn_unit *code_ptr; /* pointer for running end of tb */ #ifdef CONFIG_PROFILER TCGProfile prof; diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, tb->trace_vcpu_dstate = *cpu->trace_dstate; tb_set_page_addr0(tb, phys_pc); tb_set_page_addr1(tb, -1); - tcg_ctx->tb_cflags = cflags; + tcg_ctx->gen_tb = tb; tb_overflow: #ifdef CONFIG_PROFILER diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -XXX,XX +XXX,XX @@ void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, void tcg_gen_mb(TCGBar mb_type) { - if (tcg_ctx->tb_cflags & CF_PARALLEL) { + if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { tcg_gen_op1(INDEX_op_mb, mb_type); } } @@ -XXX,XX +XXX,XX @@ void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx) void tcg_gen_goto_tb(unsigned idx) { /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */ - tcg_debug_assert(!(tcg_ctx->tb_cflags & CF_NO_GOTO_TB)); + tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB)); /* We only support two chained exits. */ tcg_debug_assert(idx <= TB_EXIT_IDXMAX); #ifdef CONFIG_DEBUG_TCG @@ -XXX,XX +XXX,XX @@ void tcg_gen_lookup_and_goto_ptr(void) { TCGv_ptr ptr; - if (tcg_ctx->tb_cflags & CF_NO_GOTO_PTR) { + if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) { tcg_gen_exit_tb(NULL, 0); return; } @@ -XXX,XX +XXX,XX @@ void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, { memop = tcg_canonicalize_memop(memop, 0, 0); - if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) { + if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { TCGv_i32 t1 = tcg_temp_new_i32(); TCGv_i32 t2 = tcg_temp_new_i32(); @@ -XXX,XX +XXX,XX @@ void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, { memop = tcg_canonicalize_memop(memop, 1, 0); - if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) { + if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { TCGv_i64 t1 = tcg_temp_new_i64(); TCGv_i64 t2 = tcg_temp_new_i64(); @@ -XXX,XX +XXX,XX @@ static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \ void tcg_gen_atomic_##NAME##_i32 \ (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop) \ { \ - if (tcg_ctx->tb_cflags & CF_PARALLEL) { \ + if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ } else { \ do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \ @@ -XXX,XX +XXX,XX @@ void tcg_gen_atomic_##NAME##_i32 \ void tcg_gen_atomic_##NAME##_i64 \ (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop) \ { \ - if (tcg_ctx->tb_cflags & CF_PARALLEL) { \ + if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ } else { \ do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \ diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void set_jmp_reset_offset(TCGContext *s, int which) * We will check for overflow at the end of the opcode loop in * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. */ - s->tb_jmp_reset_offset[which] = tcg_current_code_size(s); + s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s); } static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) @@ -XXX,XX +XXX,XX @@ static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. */ tcg_debug_assert(TCG_TARGET_HAS_direct_jump); - s->tb_jmp_insn_offset[which] = tcg_current_code_size(s); + s->gen_tb->jmp_target_arg[which] = tcg_current_code_size(s); } static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) @@ -XXX,XX +XXX,XX @@ static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) * Return the read-execute version of the pointer, for the benefit * of any pc-relative addressing mode. */ - return (uintptr_t)tcg_splitwx_to_rx(&s->tb_jmp_target_addr[which]); + return (uintptr_t)tcg_splitwx_to_rx(s->gen_tb->jmp_target_arg + which); } /* Signal overflow, starting over with fewer guest insns. */ @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) /* Initialize goto_tb jump offsets. */ tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID; tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID; - tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset; - if (TCG_TARGET_HAS_direct_jump) { - tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg; - tcg_ctx->tb_jmp_target_addr = NULL; - } else { - tcg_ctx->tb_jmp_insn_offset = NULL; - tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg; - } tcg_reg_alloc_start(s); -- 2.34.1 Stop overloading jmp_target_arg for both offset and address, depending on TCG_TARGET_HAS_direct_jump. Instead, add a new field to hold the jump insn offset and always set the target address in jmp_target_addr[]. This will allow a tcg backend to use either direct or indirect depending on displacement. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/exec-all.h | 3 ++- accel/tcg/cpu-exec.c | 5 ++--- tcg/tcg.c | 6 ++++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ struct TranslationBlock { */ #define TB_JMP_OFFSET_INVALID 0xffff /* indicates no jump generated */ uint16_t jmp_reset_offset[2]; /* offset of original jump target */ - uintptr_t jmp_target_arg[2]; /* target address or offset */ + uint16_t jmp_insn_offset[2]; /* offset of direct jump insn */ + uintptr_t jmp_target_addr[2]; /* target address */ /* * Each TB has a NULL-terminated list (jmp_list_head) of incoming jumps. diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ void cpu_exec_step_atomic(CPUState *cpu) void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr) { + tb->jmp_target_addr[n] = addr; if (TCG_TARGET_HAS_direct_jump) { - uintptr_t offset = tb->jmp_target_arg[n]; + uintptr_t offset = tb->jmp_insn_offset[n]; uintptr_t tc_ptr = (uintptr_t)tb->tc.ptr; uintptr_t jmp_rx = tc_ptr + offset; uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff; tb_target_set_jmp_target(tc_ptr, jmp_rx, jmp_rw, addr); - } else { - tb->jmp_target_arg[n] = addr; } } diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. */ tcg_debug_assert(TCG_TARGET_HAS_direct_jump); - s->gen_tb->jmp_target_arg[which] = tcg_current_code_size(s); + s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s); } static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) @@ -XXX,XX +XXX,XX @@ static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) * Return the read-execute version of the pointer, for the benefit * of any pc-relative addressing mode. */ - return (uintptr_t)tcg_splitwx_to_rx(s->gen_tb->jmp_target_arg + which); + return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]); } /* Signal overflow, starting over with fewer guest insns. */ @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) /* Initialize goto_tb jump offsets. */ tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID; tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID; + tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID; + tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID; tcg_reg_alloc_start(s); -- 2.34.1 Replace 'tc_ptr' and 'addr' with 'tb' and 'n'. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target.h | 3 ++- tcg/arm/tcg-target.h | 3 ++- tcg/i386/tcg-target.h | 9 ++------- tcg/loongarch64/tcg-target.h | 3 ++- tcg/mips/tcg-target.h | 3 ++- tcg/ppc/tcg-target.h | 3 ++- tcg/riscv/tcg-target.h | 3 ++- tcg/s390x/tcg-target.h | 10 ++-------- tcg/sparc64/tcg-target.h | 3 ++- tcg/tci/tcg-target.h | 3 ++- accel/tcg/cpu-exec.c | 11 ++++++++--- tcg/aarch64/tcg-target.c.inc | 5 +++-- tcg/i386/tcg-target.c.inc | 9 +++++++++ tcg/loongarch64/tcg-target.c.inc | 5 +++-- tcg/ppc/tcg-target.c.inc | 7 ++++--- tcg/s390x/tcg-target.c.inc | 10 ++++++++++ tcg/sparc64/tcg-target.c.inc | 7 ++++--- 17 files changed, 61 insertions(+), 36 deletions(-) diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 0 -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); +void tb_target_set_jmp_target(const TranslationBlock *, int, + uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #define TCG_TARGET_NEED_POOL_LABELS diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_neon_instructions; #define TCG_TARGET_HAS_MEMORY_BSWAP 0 /* not defined -- call should be eliminated at compile time */ -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #define TCG_TARGET_NEED_POOL_LABELS diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_movbe; #define TCG_TARGET_extract_i64_valid(ofs, len) \ (((ofs) == 8 && (len) == 8) || ((ofs) + (len)) == 32) -static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, - uintptr_t jmp_rw, uintptr_t addr) -{ - /* patch the branch destination */ - qatomic_set((int32_t *)jmp_rw, addr - (jmp_rx + 4)); - /* no need to flush icache explicitly */ -} +void tb_target_set_jmp_target(const TranslationBlock *, int, + uintptr_t, uintptr_t); /* This defines the natural memory order supported by this * architecture before guarantees made by various barrier diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/loongarch64/tcg-target.h +++ b/tcg/loongarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_HAS_muluh_i64 1 #define TCG_TARGET_HAS_mulsh_i64 1 -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t, uintptr_t); #define TCG_TARGET_DEFAULT_MO (0) diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions; #define TCG_TARGET_HAS_MEMORY_BSWAP 1 /* not defined -- call should be eliminated at compile time */ -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t) +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t, uintptr_t) QEMU_ERROR("code path is reachable"); #define TCG_TARGET_NEED_LDST_LABELS diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_vsx; #define TCG_TARGET_HAS_bitsel_vec have_vsx #define TCG_TARGET_HAS_cmpsel_vec 0 -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t, uintptr_t); #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 1 diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { #endif /* not defined -- call should be eliminated at compile time */ -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t, uintptr_t); #define TCG_TARGET_DEFAULT_MO (0) diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern uint64_t s390_facilities[3]; #define TCG_TARGET_DEFAULT_MO (TCG_MO_ALL & ~TCG_MO_ST_LD) -static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, - uintptr_t jmp_rw, uintptr_t addr) -{ - /* patch the branch destination */ - intptr_t disp = addr - (jmp_rx - 2); - qatomic_set((int32_t *)jmp_rw, disp / 2); - /* no need to flush icache explicitly */ -} +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t jmp_rx, uintptr_t jmp_rw); #define TCG_TARGET_NEED_LDST_LABELS #define TCG_TARGET_NEED_POOL_LABELS diff --git a/tcg/sparc64/tcg-target.h b/tcg/sparc64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc64/tcg-target.h +++ b/tcg/sparc64/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions; #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 1 -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t, uintptr_t); #define TCG_TARGET_NEED_POOL_LABELS diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_HAS_MEMORY_BSWAP 1 /* not defined -- call should be eliminated at compile time */ -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t, uintptr_t); #endif /* TCG_TARGET_H */ diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr) { tb->jmp_target_addr[n] = addr; if (TCG_TARGET_HAS_direct_jump) { + /* + * Get the rx view of the structure, from which we find the + * executable code address, and tb_target_set_jmp_target can + * produce a pc-relative displacement to jmp_target_addr[n]. + */ + const TranslationBlock *c_tb = tcg_splitwx_to_rx(tb); uintptr_t offset = tb->jmp_insn_offset[n]; - uintptr_t tc_ptr = (uintptr_t)tb->tc.ptr; - uintptr_t jmp_rx = tc_ptr + offset; + uintptr_t jmp_rx = (uintptr_t)tb->tc.ptr + offset; uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff; - tb_target_set_jmp_target(tc_ptr, jmp_rx, jmp_rw, addr); + tb_target_set_jmp_target(c_tb, n, jmp_rx, jmp_rw); } } diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, tcg_out_call_int(s, target); } -void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, - uintptr_t jmp_rw, uintptr_t addr) +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t jmp_rx, uintptr_t jmp_rw) { + uintptr_t addr = tb->jmp_target_addr[n]; tcg_insn_unit i1, i2; TCGType rt = TCG_TYPE_I64; TCGReg rd = TCG_REG_TMP; diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) set_jmp_reset_offset(s, which); } +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t jmp_rx, uintptr_t jmp_rw) +{ + /* patch the branch destination */ + uintptr_t addr = tb->jmp_target_addr[n]; + qatomic_set((int32_t *)jmp_rw, addr - (jmp_rx + 4)); + /* no need to flush icache explicitly */ +} + static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/loongarch64/tcg-target.c.inc +++ b/tcg/loongarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_nop(TCGContext *s) tcg_out32(s, NOP); } -void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, - uintptr_t jmp_rw, uintptr_t addr) +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t jmp_rx, uintptr_t jmp_rw) { tcg_insn_unit i1, i2; ptrdiff_t upper, lower; + uintptr_t addr = tb->jmp_target_addr[n]; ptrdiff_t offset = (ptrdiff_t)(addr - jmp_rx) >> 2; if (offset == sextreg(offset, 0, 26)) { diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void ppc64_replace4(uintptr_t rx, uintptr_t rw, flush_idcache_range(rx, rw, 16); } -void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, - uintptr_t jmp_rw, uintptr_t addr) +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t jmp_rx, uintptr_t jmp_rw) { tcg_insn_unit i0, i1, i2, i3; - intptr_t tb_diff = addr - tc_ptr; + uintptr_t addr = tb->jmp_target_addr[n]; + intptr_t tb_diff = addr - (uintptr_t)tb->tc.ptr; intptr_t br_diff = addr - (jmp_rx + 4); intptr_t lo, hi; diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) set_jmp_reset_offset(s, which); } +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t jmp_rx, uintptr_t jmp_rw) +{ + /* patch the branch destination */ + uintptr_t addr = tb->jmp_target_addr[n]; + intptr_t disp = addr - (jmp_rx - 2); + qatomic_set((int32_t *)jmp_rw, disp / 2); + /* no need to flush icache explicitly */ +} + # define OP_32_64(x) \ case glue(glue(INDEX_op_,x),_i32): \ case glue(glue(INDEX_op_,x),_i64) diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc64/tcg-target.c.inc +++ b/tcg/sparc64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ void tcg_register_jit(const void *buf, size_t buf_size) tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); } -void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, - uintptr_t jmp_rw, uintptr_t addr) +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t jmp_rx, uintptr_t jmp_rw) { - intptr_t tb_disp = addr - tc_ptr; + uintptr_t addr = tb->jmp_target_addr[n]; + intptr_t tb_disp = addr - (uintptr_t)tb->tc.ptr; intptr_t br_disp = addr - jmp_rx; tcg_insn_unit i1, i2; -- 2.34.1 Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg.h | 3 +++ tcg/aarch64/tcg-target.h | 4 ---- tcg/arm/tcg-target.h | 5 ----- tcg/i386/tcg-target.h | 3 --- tcg/loongarch64/tcg-target.h | 3 --- tcg/mips/tcg-target.h | 5 ----- tcg/ppc/tcg-target.h | 4 ---- tcg/riscv/tcg-target.h | 4 ---- tcg/s390x/tcg-target.h | 4 ---- tcg/sparc64/tcg-target.h | 4 ---- tcg/tci/tcg-target.h | 4 ---- 11 files changed, 3 insertions(+), 40 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ void tcg_func_start(TCGContext *s); int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start); +void tb_target_set_jmp_target(const TranslationBlock *, int, + uintptr_t, uintptr_t); + void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size); TCGTemp *tcg_global_mem_new_internal(TCGType, TCGv_ptr, diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 0 - -void tb_target_set_jmp_target(const TranslationBlock *, int, - uintptr_t, uintptr_t); - #define TCG_TARGET_NEED_LDST_LABELS #define TCG_TARGET_NEED_POOL_LABELS diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_neon_instructions; #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 0 - -/* not defined -- call should be eliminated at compile time */ -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, - uintptr_t, uintptr_t); - #define TCG_TARGET_NEED_LDST_LABELS #define TCG_TARGET_NEED_POOL_LABELS diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_movbe; #define TCG_TARGET_extract_i64_valid(ofs, len) \ (((ofs) == 8 && (len) == 8) || ((ofs) + (len)) == 32) -void tb_target_set_jmp_target(const TranslationBlock *, int, - uintptr_t, uintptr_t); - /* This defines the natural memory order supported by this * architecture before guarantees made by various barrier * instructions. diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/loongarch64/tcg-target.h +++ b/tcg/loongarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_HAS_muluh_i64 1 #define TCG_TARGET_HAS_mulsh_i64 1 -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, - uintptr_t, uintptr_t); - #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_NEED_LDST_LABELS diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions; #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 1 -/* not defined -- call should be eliminated at compile time */ -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, - uintptr_t, uintptr_t) - QEMU_ERROR("code path is reachable"); - #define TCG_TARGET_NEED_LDST_LABELS #endif diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_vsx; #define TCG_TARGET_HAS_bitsel_vec have_vsx #define TCG_TARGET_HAS_cmpsel_vec 0 -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, - uintptr_t, uintptr_t); - #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 1 - #define TCG_TARGET_NEED_LDST_LABELS #define TCG_TARGET_NEED_POOL_LABELS diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_HAS_mulsh_i64 1 #endif -/* not defined -- call should be eliminated at compile time */ -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, - uintptr_t, uintptr_t); - #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_NEED_LDST_LABELS diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern uint64_t s390_facilities[3]; #define TCG_TARGET_HAS_MEMORY_BSWAP 1 #define TCG_TARGET_DEFAULT_MO (TCG_MO_ALL & ~TCG_MO_ST_LD) - -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, - uintptr_t jmp_rx, uintptr_t jmp_rw); - #define TCG_TARGET_NEED_LDST_LABELS #define TCG_TARGET_NEED_POOL_LABELS diff --git a/tcg/sparc64/tcg-target.h b/tcg/sparc64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc64/tcg-target.h +++ b/tcg/sparc64/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions; #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 1 - -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, - uintptr_t, uintptr_t); - #define TCG_TARGET_NEED_POOL_LABELS #endif diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_HAS_MEMORY_BSWAP 1 -/* not defined -- call should be eliminated at compile time */ -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, - uintptr_t, uintptr_t); - #endif /* TCG_TARGET_H */ -- 2.34.1 Install empty versions for !TCG_TARGET_HAS_direct_jump hosts. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/arm/tcg-target.c.inc | 6 ++++++ tcg/mips/tcg-target.c.inc | 6 ++++++ tcg/riscv/tcg-target.c.inc | 6 ++++++ tcg/tci/tcg-target.c.inc | 6 ++++++ 4 files changed, 24 insertions(+) diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) set_jmp_reset_offset(s, which); } +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t jmp_rx, uintptr_t jmp_rw) +{ + /* Always indirect, nothing to do */ +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) set_jmp_reset_offset(s, which); } +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t jmp_rx, uintptr_t jmp_rw) +{ + /* Always indirect, nothing to do */ +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) set_jmp_reset_offset(s, which); } +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t jmp_rx, uintptr_t jmp_rw) +{ + /* Always indirect, nothing to do */ +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) set_jmp_reset_offset(s, which); } +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t jmp_rx, uintptr_t jmp_rw) +{ + /* Always indirect, nothing to do */ +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) -- 2.34.1 We now have the option to generate direct or indirect goto_tb depending on the dynamic displacement, thus the define is no longer necessary or completely accurate. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target.h | 1 - tcg/arm/tcg-target.h | 1 - tcg/i386/tcg-target.h | 1 - tcg/loongarch64/tcg-target.h | 1 - tcg/mips/tcg-target.h | 1 - tcg/ppc/tcg-target.h | 1 - tcg/riscv/tcg-target.h | 1 - tcg/s390x/tcg-target.h | 1 - tcg/sparc64/tcg-target.h | 1 - tcg/tci/tcg-target.h | 1 - accel/tcg/cpu-exec.c | 23 +++++++++++------------ tcg/tcg.c | 1 - tcg/arm/tcg-target.c.inc | 1 - tcg/mips/tcg-target.c.inc | 1 - tcg/riscv/tcg-target.c.inc | 1 - tcg/s390x/tcg-target.c.inc | 3 +++ tcg/tci/tcg-target.c.inc | 1 - 17 files changed, 14 insertions(+), 27 deletions(-) diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_HAS_muls2_i64 0 #define TCG_TARGET_HAS_muluh_i64 1 #define TCG_TARGET_HAS_mulsh_i64 1 -#define TCG_TARGET_HAS_direct_jump 1 #define TCG_TARGET_HAS_v64 1 #define TCG_TARGET_HAS_v128 1 diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_neon_instructions; #define TCG_TARGET_HAS_mulsh_i32 0 #define TCG_TARGET_HAS_div_i32 use_idiv_instructions #define TCG_TARGET_HAS_rem_i32 0 -#define TCG_TARGET_HAS_direct_jump 0 #define TCG_TARGET_HAS_qemu_st8_i32 0 #define TCG_TARGET_HAS_v64 use_neon_instructions diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_movbe; #define TCG_TARGET_HAS_muls2_i32 1 #define TCG_TARGET_HAS_muluh_i32 0 #define TCG_TARGET_HAS_mulsh_i32 0 -#define TCG_TARGET_HAS_direct_jump 1 #if TCG_TARGET_REG_BITS == 64 /* Keep target addresses zero-extended in a register. */ diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/loongarch64/tcg-target.h +++ b/tcg/loongarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_HAS_clz_i32 1 #define TCG_TARGET_HAS_ctz_i32 1 #define TCG_TARGET_HAS_ctpop_i32 0 -#define TCG_TARGET_HAS_direct_jump 1 #define TCG_TARGET_HAS_brcond2 0 #define TCG_TARGET_HAS_setcond2 0 #define TCG_TARGET_HAS_qemu_st8_i32 0 diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions; #define TCG_TARGET_HAS_muluh_i32 1 #define TCG_TARGET_HAS_mulsh_i32 1 #define TCG_TARGET_HAS_bswap32_i32 1 -#define TCG_TARGET_HAS_direct_jump 0 #if TCG_TARGET_REG_BITS == 64 #define TCG_TARGET_HAS_add2_i32 0 diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_vsx; #define TCG_TARGET_HAS_muls2_i32 0 #define TCG_TARGET_HAS_muluh_i32 1 #define TCG_TARGET_HAS_mulsh_i32 1 -#define TCG_TARGET_HAS_direct_jump 1 #define TCG_TARGET_HAS_qemu_st8_i32 0 #if TCG_TARGET_REG_BITS == 64 diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_HAS_clz_i32 0 #define TCG_TARGET_HAS_ctz_i32 0 #define TCG_TARGET_HAS_ctpop_i32 0 -#define TCG_TARGET_HAS_direct_jump 0 #define TCG_TARGET_HAS_brcond2 1 #define TCG_TARGET_HAS_setcond2 1 #define TCG_TARGET_HAS_qemu_st8_i32 0 diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern uint64_t s390_facilities[3]; #define TCG_TARGET_HAS_mulsh_i32 0 #define TCG_TARGET_HAS_extrl_i64_i32 0 #define TCG_TARGET_HAS_extrh_i64_i32 0 -#define TCG_TARGET_HAS_direct_jump 1 #define TCG_TARGET_HAS_qemu_st8_i32 0 #define TCG_TARGET_HAS_div2_i64 1 diff --git a/tcg/sparc64/tcg-target.h b/tcg/sparc64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc64/tcg-target.h +++ b/tcg/sparc64/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions; #define TCG_TARGET_HAS_muls2_i32 1 #define TCG_TARGET_HAS_muluh_i32 0 #define TCG_TARGET_HAS_mulsh_i32 0 -#define TCG_TARGET_HAS_direct_jump 1 #define TCG_TARGET_HAS_qemu_st8_i32 0 #define TCG_TARGET_HAS_extrl_i64_i32 1 diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ #define TCG_TARGET_HAS_muls2_i32 1 #define TCG_TARGET_HAS_muluh_i32 0 #define TCG_TARGET_HAS_mulsh_i32 0 -#define TCG_TARGET_HAS_direct_jump 0 #define TCG_TARGET_HAS_qemu_st8_i32 0 #if TCG_TARGET_REG_BITS == 64 diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ void cpu_exec_step_atomic(CPUState *cpu) void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr) { + /* + * Get the rx view of the structure, from which we find the + * executable code address, and tb_target_set_jmp_target can + * produce a pc-relative displacement to jmp_target_addr[n]. + */ + const TranslationBlock *c_tb = tcg_splitwx_to_rx(tb); + uintptr_t offset = tb->jmp_insn_offset[n]; + uintptr_t jmp_rx = (uintptr_t)tb->tc.ptr + offset; + uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff; + tb->jmp_target_addr[n] = addr; - if (TCG_TARGET_HAS_direct_jump) { - /* - * Get the rx view of the structure, from which we find the - * executable code address, and tb_target_set_jmp_target can - * produce a pc-relative displacement to jmp_target_addr[n]. - */ - const TranslationBlock *c_tb = tcg_splitwx_to_rx(tb); - uintptr_t offset = tb->jmp_insn_offset[n]; - uintptr_t jmp_rx = (uintptr_t)tb->tc.ptr + offset; - uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff; - tb_target_set_jmp_target(c_tb, n, jmp_rx, jmp_rw); - } + tb_target_set_jmp_target(c_tb, n, jmp_rx, jmp_rw); } static inline void tb_add_jump(TranslationBlock *tb, int n, diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) * We will check for overflow at the end of the opcode loop in * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. */ - tcg_debug_assert(TCG_TARGET_HAS_direct_jump); s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s); } diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) intptr_t ptr, dif, dil; TCGReg base = TCG_REG_PC; - qemu_build_assert(!TCG_TARGET_HAS_direct_jump); ptr = get_jmp_target_addr(s, which); dif = tcg_pcrel_diff(s, (void *)ptr) - 8; dil = sextract32(dif, 0, 12); diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) static void tcg_out_goto_tb(TCGContext *s, int which) { /* indirect jump method */ - qemu_build_assert(!TCG_TARGET_HAS_direct_jump); tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO, get_jmp_target_addr(s, which)); tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0); diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) static void tcg_out_goto_tb(TCGContext *s, int which) { - qemu_build_assert(!TCG_TARGET_HAS_direct_jump); /* indirect jump method */ tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO, get_jmp_target_addr(s, which)); diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) void tb_target_set_jmp_target(const TranslationBlock *tb, int n, uintptr_t jmp_rx, uintptr_t jmp_rw) { + if (!HAVE_FACILITY(GEN_INST_EXT)) { + return; + } /* patch the branch destination */ uintptr_t addr = tb->jmp_target_addr[n]; intptr_t disp = addr - (jmp_rx - 2); diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) static void tcg_out_goto_tb(TCGContext *s, int which) { - qemu_build_assert(!TCG_TARGET_HAS_direct_jump); /* indirect jump method. */ tcg_out_op_p(s, INDEX_op_goto_tb, (void *)get_jmp_target_addr(s, which)); set_jmp_reset_offset(s, which); -- 2.34.1 The old implementation replaces two insns, swapping between b <dest> nop br x30 and adrp x30, <dest> addi x30, x30, lo12:<dest> br x30 There is a race condition in which a thread could be stopped at the PC of the second insn, and when restarted does not see the complete address computation and branches to nowhere. The new implemetation replaces only one insn, swapping between b <dest> br tmp and ldr tmp, <jmp_addr> br tmp Reported-by: hev <r@hev.cc> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target.h | 2 +- tcg/aarch64/tcg-target.c.inc | 66 +++++++++++++++--------------------- 2 files changed, 29 insertions(+), 39 deletions(-) diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ #define TCG_TARGET_INSN_UNIT_SIZE 4 #define TCG_TARGET_TLB_DISPLACEMENT_BITS 24 -#define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB) +#define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) typedef enum { TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3, diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, tcg_out_call_int(s, target); } -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, - uintptr_t jmp_rx, uintptr_t jmp_rw) -{ - uintptr_t addr = tb->jmp_target_addr[n]; - tcg_insn_unit i1, i2; - TCGType rt = TCG_TYPE_I64; - TCGReg rd = TCG_REG_TMP; - uint64_t pair; - - ptrdiff_t offset = addr - jmp_rx; - - if (offset == sextract64(offset, 0, 26)) { - i1 = I3206_B | ((offset >> 2) & 0x3ffffff); - i2 = NOP; - } else { - offset = (addr >> 12) - (jmp_rx >> 12); - - /* patch ADRP */ - i1 = I3406_ADRP | (offset & 3) << 29 | (offset & 0x1ffffc) << (5 - 2) | rd; - /* patch ADDI */ - i2 = I3401_ADDI | rt << 31 | (addr & 0xfff) << 10 | rd << 5 | rd; - } - pair = (uint64_t)i2 << 32 | i1; - qatomic_set((uint64_t *)jmp_rw, pair); - flush_idcache_range(jmp_rx, jmp_rw, 8); -} - static inline void tcg_out_goto_label(TCGContext *s, TCGLabel *l) { if (!l->has_value) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) static void tcg_out_goto_tb(TCGContext *s, int which) { /* - * Ensure that ADRP+ADD are 8-byte aligned so that an atomic - * write can be used to patch the target address. + * Direct branch, or indirect address load, will be patched + * by tb_target_set_jmp_target. Assert indirect load offset + * in range early, regardless of direct branch distance. */ - if ((uintptr_t)s->code_ptr & 7) { - tcg_out32(s, NOP); - } + intptr_t i_off = tcg_pcrel_diff(s, (void *)get_jmp_target_addr(s, which)); + tcg_debug_assert(i_off == sextract64(i_off, 0, 21)); + set_jmp_insn_offset(s, which); - /* - * actual branch destination will be patched by - * tb_target_set_jmp_target later - */ - tcg_out_insn(s, 3406, ADRP, TCG_REG_TMP, 0); - tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_TMP, TCG_REG_TMP, 0); + tcg_out32(s, I3206_B); tcg_out_insn(s, 3207, BR, TCG_REG_TMP); set_jmp_reset_offset(s, which); } +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t jmp_rx, uintptr_t jmp_rw) +{ + uintptr_t d_addr = tb->jmp_target_addr[n]; + ptrdiff_t d_offset = d_addr - jmp_rx; + tcg_insn_unit insn; + + /* Either directly branch, or indirect branch load. */ + if (d_offset == sextract64(d_offset, 0, 28)) { + insn = deposit32(I3206_B, 0, 26, d_offset >> 2); + } else { + uintptr_t i_addr = (uintptr_t)&tb->jmp_target_addr[n]; + ptrdiff_t i_offset = i_addr - jmp_rx; + + /* Note that we asserted this in range in tcg_out_goto_tb. */ + insn = deposit32(I3305_LDR | TCG_REG_TMP, 0, 5, i_offset >> 2); + } + qatomic_set((uint32_t *)jmp_rw, insn); + flush_idcache_range(jmp_rx, jmp_rw, 4); +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) -- 2.34.1 The old ppc64 implementation replaces 2 or 4 insns, which leaves a race condition in which a thread could be stopped at a PC in the middle of the sequence, and when restarted does not see the complete address computation and branches to nowhere. The new implemetation replaces only one insn, swapping between b <dest> and mtctr r31 falling through to a general-case indirect branch. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/ppc/tcg-target.h | 3 +- tcg/ppc/tcg-target.c.inc | 158 +++++++++++---------------------------- 2 files changed, 44 insertions(+), 117 deletions(-) diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -XXX,XX +XXX,XX @@ #ifdef _ARCH_PPC64 # define TCG_TARGET_REG_BITS 64 -# define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB) #else # define TCG_TARGET_REG_BITS 32 -# define MAX_CODE_GEN_BUFFER_SIZE (32 * MiB) #endif +#define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) #define TCG_TARGET_NB_REGS 64 #define TCG_TARGET_INSN_UNIT_SIZE 4 diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_mb(TCGContext *s, TCGArg a0) tcg_out32(s, insn); } -static inline uint64_t make_pair(tcg_insn_unit i1, tcg_insn_unit i2) -{ - if (HOST_BIG_ENDIAN) { - return (uint64_t)i1 << 32 | i2; - } - return (uint64_t)i2 << 32 | i1; -} - -static inline void ppc64_replace2(uintptr_t rx, uintptr_t rw, - tcg_insn_unit i0, tcg_insn_unit i1) -{ -#if TCG_TARGET_REG_BITS == 64 - qatomic_set((uint64_t *)rw, make_pair(i0, i1)); - flush_idcache_range(rx, rw, 8); -#else - qemu_build_not_reached(); -#endif -} - -static inline void ppc64_replace4(uintptr_t rx, uintptr_t rw, - tcg_insn_unit i0, tcg_insn_unit i1, - tcg_insn_unit i2, tcg_insn_unit i3) -{ - uint64_t p[2]; - - p[!HOST_BIG_ENDIAN] = make_pair(i0, i1); - p[HOST_BIG_ENDIAN] = make_pair(i2, i3); - - /* - * There's no convenient way to get the compiler to allocate a pair - * of registers at an even index, so copy into r6/r7 and clobber. - */ - asm("mr %%r6, %1\n\t" - "mr %%r7, %2\n\t" - "stq %%r6, %0" - : "=Q"(*(__int128 *)rw) : "r"(p[0]), "r"(p[1]) : "r6", "r7"); - flush_idcache_range(rx, rw, 16); -} - -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, - uintptr_t jmp_rx, uintptr_t jmp_rw) -{ - tcg_insn_unit i0, i1, i2, i3; - uintptr_t addr = tb->jmp_target_addr[n]; - intptr_t tb_diff = addr - (uintptr_t)tb->tc.ptr; - intptr_t br_diff = addr - (jmp_rx + 4); - intptr_t lo, hi; - - if (TCG_TARGET_REG_BITS == 32) { - intptr_t diff = addr - jmp_rx; - tcg_debug_assert(in_range_b(diff)); - qatomic_set((uint32_t *)jmp_rw, B | (diff & 0x3fffffc)); - flush_idcache_range(jmp_rx, jmp_rw, 4); - return; - } - - /* - * For 16-bit displacements, we can use a single add + branch. - * This happens quite often. - */ - if (tb_diff == (int16_t)tb_diff) { - i0 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, tb_diff); - i1 = B | (br_diff & 0x3fffffc); - ppc64_replace2(jmp_rx, jmp_rw, i0, i1); - return; - } - - lo = (int16_t)tb_diff; - hi = (int32_t)(tb_diff - lo); - assert(tb_diff == hi + lo); - i0 = ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, hi >> 16); - i1 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, lo); - - /* - * Without stq from 2.07, we can only update two insns, - * and those must be the ones that load the target address. - */ - if (!have_isa_2_07) { - ppc64_replace2(jmp_rx, jmp_rw, i0, i1); - return; - } - - /* - * For 26-bit displacements, we can use a direct branch. - * Otherwise we still need the indirect branch, which we - * must restore after a potential direct branch write. - */ - br_diff -= 4; - if (in_range_b(br_diff)) { - i2 = B | (br_diff & 0x3fffffc); - i3 = NOP; - } else { - i2 = MTSPR | RS(TCG_REG_TB) | CTR; - i3 = BCCTR | BO_ALWAYS; - } - ppc64_replace4(jmp_rx, jmp_rw, i0, i1, i2, i3); -} - static void tcg_out_call_int(TCGContext *s, int lk, const tcg_insn_unit *target) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) static void tcg_out_goto_tb(TCGContext *s, int which) { - /* Direct jump. */ - if (TCG_TARGET_REG_BITS == 64) { - /* Ensure the next insns are 8 or 16-byte aligned. */ - while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) { - tcg_out32(s, NOP); - } + uintptr_t ptr = get_jmp_target_addr(s, which); + + if (USE_REG_TB) { + ptrdiff_t offset = tcg_tbrel_diff(s, (void *)ptr); + tcg_out_mem_long(s, LD, LDX, TCG_REG_TB, TCG_REG_TB, offset); + + /* Direct branch will be patched by tb_target_set_jmp_target. */ set_jmp_insn_offset(s, which); - tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0)); - tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0)); tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR); + + /* When branch is out of range, fall through to indirect. */ + tcg_out32(s, BCCTR | BO_ALWAYS); + + /* For the unlinked case, need to reset TCG_REG_TB. */ + set_jmp_reset_offset(s, which); + tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB, + -tcg_current_code_size(s)); + } else { + /* Direct branch will be patched by tb_target_set_jmp_target. */ + set_jmp_insn_offset(s, which); + tcg_out32(s, NOP); + + /* When branch is out of range, fall through to indirect. */ + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, ptr - (int16_t)ptr); + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_REG_TMP1, (int16_t)ptr); + tcg_out32(s, MTSPR | RS(TCG_REG_TMP1) | CTR); tcg_out32(s, BCCTR | BO_ALWAYS); set_jmp_reset_offset(s, which); - if (USE_REG_TB) { - /* For the unlinked case, need to reset TCG_REG_TB. */ - tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB, - -tcg_current_code_size(s)); - } - } else { - set_jmp_insn_offset(s, which); - tcg_out32(s, B); - set_jmp_reset_offset(s, which); } } +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t jmp_rx, uintptr_t jmp_rw) +{ + uintptr_t addr = tb->jmp_target_addr[n]; + intptr_t diff = addr - jmp_rx; + tcg_insn_unit insn; + + if (in_range_b(diff)) { + insn = B | (diff & 0x3fffffc); + } else if (USE_REG_TB) { + insn = MTSPR | RS(TCG_REG_TB) | CTR; + } else { + insn = NOP; + } + + qatomic_set((uint32_t *)jmp_rw, insn); + flush_idcache_range(jmp_rx, jmp_rw, 4); +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) -- 2.34.1 This is always true for sparc64, so this is dead since 3a5f6805c7ca. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc64/tcg-target.c.inc | 62 ++++++++++++------------------------ 1 file changed, 21 insertions(+), 41 deletions(-) diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc64/tcg-target.c.inc +++ b/tcg/sparc64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { #endif #define TCG_REG_TB TCG_REG_I1 -#define USE_REG_TB (sizeof(void *) > 4) static const int tcg_target_reg_alloc_order[] = { TCG_REG_L0, @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, } /* A 13-bit constant relative to the TB. */ - if (!in_prologue && USE_REG_TB) { + if (!in_prologue) { test = tcg_tbrel_diff(s, (void *)arg); if (check_fit_ptr(test, 13)) { tcg_out_arithi(s, ret, TCG_REG_TB, test, ARITH_ADD); @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, } /* Use the constant pool, if possible. */ - if (!in_prologue && USE_REG_TB) { + if (!in_prologue) { new_pool_label(s, arg, R_SPARC_13, s->code_ptr, tcg_tbrel_diff(s, NULL)); tcg_out32(s, LDX | INSN_RD(ret) | INSN_RS1(TCG_REG_TB)); @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) #endif /* We choose TCG_REG_TB such that no move is required. */ - if (USE_REG_TB) { - QEMU_BUILD_BUG_ON(TCG_REG_TB != TCG_REG_I1); - tcg_regset_set_reg(s->reserved_regs, TCG_REG_TB); - } + QEMU_BUILD_BUG_ON(TCG_REG_TB != TCG_REG_I1); + tcg_regset_set_reg(s->reserved_regs, TCG_REG_TB); tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I1, 0, JMPL); /* delay slot */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); tcg_out_movi_imm13(s, TCG_REG_O0, a0); return; - } else if (USE_REG_TB) { + } else { intptr_t tb_diff = tcg_tbrel_diff(s, (void *)a0); if (check_fit_ptr(tb_diff, 13)) { tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) static void tcg_out_goto_tb(TCGContext *s, int which) { + int c; + /* Direct jump. */ - if (USE_REG_TB) { - /* make sure the patch is 8-byte aligned. */ - if ((intptr_t)s->code_ptr & 4) { - tcg_out_nop(s); - } - set_jmp_insn_offset(s, which); - tcg_out_sethi(s, TCG_REG_T1, 0); - tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR); - tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL); - tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); - } else { - set_jmp_insn_offset(s, which); - tcg_out32(s, CALL); + /* make sure the patch is 8-byte aligned. */ + if ((intptr_t)s->code_ptr & 4) { tcg_out_nop(s); } + set_jmp_insn_offset(s, which); + tcg_out_sethi(s, TCG_REG_T1, 0); + tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR); + tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL); + tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); set_jmp_reset_offset(s, which); /* * For the unlinked path of goto_tb, we need to reset TCG_REG_TB * to the beginning of this TB. */ - if (USE_REG_TB) { - int c = -tcg_current_code_size(s); - if (check_fit_i32(c, 13)) { - tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD); - } else { - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c); - tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); - } + c = -tcg_current_code_size(s); + if (check_fit_i32(c, 13)) { + tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD); + } else { + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c); + tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); } } @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, switch (opc) { case INDEX_op_goto_ptr: tcg_out_arithi(s, TCG_REG_G0, a0, 0, JMPL); - if (USE_REG_TB) { - tcg_out_mov_delay(s, TCG_REG_TB, a0); - } else { - tcg_out_nop(s); - } + tcg_out_mov_delay(s, TCG_REG_TB, a0); break; case INDEX_op_br: tcg_out_bpcc(s, COND_A, BPCC_PT, arg_label(a0)); @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(const TranslationBlock *tb, int n, tcg_debug_assert(tb_disp == (int32_t)tb_disp); tcg_debug_assert(br_disp == (int32_t)br_disp); - if (!USE_REG_TB) { - qatomic_set((uint32_t *)jmp_rw, - deposit32(CALL, 0, 30, br_disp >> 2)); - flush_idcache_range(jmp_rx, jmp_rw, 4); - return; - } - /* This does not exercise the range of the branch, but we do still need to be able to load the new value of TCG_REG_TB. But this does still happen quite often. */ -- 2.34.1 The old sparc64 implementation may replace two insns, which leaves a race condition in which a thread could be stopped at a PC in the middle of the sequence, and when restarted does not see the complete address computation and branches to nowhere. The new implemetation replaces only one insn, swapping between a direct branch and a direct call. The TCG_REG_TB register is loaded from tb->jmp_target_addr[] in the delay slot. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc64/tcg-target.c.inc | 87 +++++++++++++++--------------------- 1 file changed, 37 insertions(+), 50 deletions(-) diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc64/tcg-target.c.inc +++ b/tcg/sparc64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) static void tcg_out_goto_tb(TCGContext *s, int which) { - int c; + ptrdiff_t off = tcg_tbrel_diff(s, (void *)get_jmp_target_addr(s, which)); - /* Direct jump. */ - /* make sure the patch is 8-byte aligned. */ - if ((intptr_t)s->code_ptr & 4) { - tcg_out_nop(s); - } + /* Direct branch will be patched by tb_target_set_jmp_target. */ set_jmp_insn_offset(s, which); - tcg_out_sethi(s, TCG_REG_T1, 0); - tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR); - tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL); - tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); + tcg_out32(s, CALL); + /* delay slot */ + tcg_debug_assert(check_fit_ptr(off, 13)); + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TB, TCG_REG_TB, off); set_jmp_reset_offset(s, which); /* * For the unlinked path of goto_tb, we need to reset TCG_REG_TB * to the beginning of this TB. */ - c = -tcg_current_code_size(s); - if (check_fit_i32(c, 13)) { - tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD); + off = -tcg_current_code_size(s); + if (check_fit_i32(off, 13)) { + tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, off, ARITH_ADD); } else { - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c); + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, off); tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); } } +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t jmp_rx, uintptr_t jmp_rw) +{ + uintptr_t addr = tb->jmp_target_addr[n]; + intptr_t br_disp = (intptr_t)(addr - jmp_rx) >> 2; + tcg_insn_unit insn; + + br_disp >>= 2; + if (check_fit_ptr(br_disp, 19)) { + /* ba,pt %icc, addr */ + insn = deposit32(INSN_OP(0) | INSN_OP2(1) | INSN_COND(COND_A) + | BPCC_ICC | BPCC_PT, 0, 19, br_disp); + } else if (check_fit_ptr(br_disp, 22)) { + /* ba addr */ + insn = deposit32(INSN_OP(0) | INSN_OP2(2) | INSN_COND(COND_A), + 0, 22, br_disp); + } else { + /* The code_gen_buffer can't be larger than 2GB. */ + tcg_debug_assert(check_fit_ptr(br_disp, 30)); + /* call addr */ + insn = deposit32(CALL, 0, 30, br_disp); + } + + qatomic_set((uint32_t *)jmp_rw, insn); + flush_idcache_range(jmp_rx, jmp_rw, 4); +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -XXX,XX +XXX,XX @@ void tcg_register_jit(const void *buf, size_t buf_size) { tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); } - -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, - uintptr_t jmp_rx, uintptr_t jmp_rw) -{ - uintptr_t addr = tb->jmp_target_addr[n]; - intptr_t tb_disp = addr - (uintptr_t)tb->tc.ptr; - intptr_t br_disp = addr - jmp_rx; - tcg_insn_unit i1, i2; - - /* We can reach the entire address space for ILP32. - For LP64, the code_gen_buffer can't be larger than 2GB. */ - tcg_debug_assert(tb_disp == (int32_t)tb_disp); - tcg_debug_assert(br_disp == (int32_t)br_disp); - - /* This does not exercise the range of the branch, but we do - still need to be able to load the new value of TCG_REG_TB. - But this does still happen quite often. */ - if (check_fit_ptr(tb_disp, 13)) { - /* ba,pt %icc, addr */ - i1 = (INSN_OP(0) | INSN_OP2(1) | INSN_COND(COND_A) - | BPCC_ICC | BPCC_PT | INSN_OFF19(br_disp)); - i2 = (ARITH_ADD | INSN_RD(TCG_REG_TB) | INSN_RS1(TCG_REG_TB) - | INSN_IMM13(tb_disp)); - } else if (tb_disp >= 0) { - i1 = SETHI | INSN_RD(TCG_REG_T1) | ((tb_disp & 0xfffffc00) >> 10); - i2 = (ARITH_OR | INSN_RD(TCG_REG_T1) | INSN_RS1(TCG_REG_T1) - | INSN_IMM13(tb_disp & 0x3ff)); - } else { - i1 = SETHI | INSN_RD(TCG_REG_T1) | ((~tb_disp & 0xfffffc00) >> 10); - i2 = (ARITH_XOR | INSN_RD(TCG_REG_T1) | INSN_RS1(TCG_REG_T1) - | INSN_IMM13((tb_disp & 0x3ff) | -0x400)); - } - - qatomic_set((uint64_t *)jmp_rw, deposit64(i2, 32, 32, i1)); - flush_idcache_range(jmp_rx, jmp_rw, 8); -} -- 2.34.1 Now that tcg can handle direct and indirect goto_tb simultaneously, we can optimistically leave space for a direct branch and fall back to loading the pointer from the TB for an indirect branch. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/arm/tcg-target.c.inc | 52 ++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ typedef enum { ARITH_BIC = 0xe << 21, ARITH_MVN = 0xf << 21, + INSN_B = 0x0a000000, + INSN_CLZ = 0x016f0f10, INSN_RBIT = 0x06ff0f30, @@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) static void tcg_out_b_imm(TCGContext *s, ARMCond cond, int32_t offset) { - tcg_out32(s, (cond << 28) | 0x0a000000 | + tcg_out32(s, (cond << 28) | INSN_B | (((offset - 8) >> 2) & 0x00ffffff)); } @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) static void tcg_out_goto_tb(TCGContext *s, int which) { - /* Indirect jump method */ - intptr_t ptr, dif, dil; - TCGReg base = TCG_REG_PC; + uintptr_t i_addr; + intptr_t i_disp; - ptr = get_jmp_target_addr(s, which); - dif = tcg_pcrel_diff(s, (void *)ptr) - 8; - dil = sextract32(dif, 0, 12); - if (dif != dil) { + /* Direct branch will be patched by tb_target_set_jmp_target. */ + set_jmp_insn_offset(s, which); + tcg_out32(s, INSN_NOP); + + /* When branch is out of range, fall through to indirect. */ + i_addr = get_jmp_target_addr(s, which); + i_disp = tcg_pcrel_diff(s, (void *)i_addr) - 8; + tcg_debug_assert(i_disp < 0); + if (i_disp >= -0xfff) { + tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_PC, i_disp); + } else { /* * The TB is close, but outside the 12 bits addressable by * the load. We can extend this to 20 bits with a sub of a - * shifted immediate from pc. In the vastly unlikely event - * the code requires more than 1MB, we'll use 2 insns and - * be no worse off. + * shifted immediate from pc. */ - base = TCG_REG_R0; - tcg_out_movi32(s, COND_AL, base, ptr - dil); + int h = -i_disp; + int l = h & 0xfff; + + h = encode_imm_nofail(h - l); + tcg_out_dat_imm(s, COND_AL, ARITH_SUB, TCG_REG_R0, TCG_REG_PC, h); + tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_R0, l); } - tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, base, dil); set_jmp_reset_offset(s, which); } void tb_target_set_jmp_target(const TranslationBlock *tb, int n, uintptr_t jmp_rx, uintptr_t jmp_rw) { - /* Always indirect, nothing to do */ + uintptr_t addr = tb->jmp_target_addr[n]; + ptrdiff_t offset = addr - (jmp_rx + 8); + tcg_insn_unit insn; + + /* Either directly branch, or fall through to indirect branch. */ + if (offset == sextract64(offset, 0, 26)) { + /* B <addr> */ + insn = deposit32((COND_AL << 28) | INSN_B, 0, 24, offset >> 2); + } else { + insn = INSN_NOP; + } + + qatomic_set((uint32_t *)jmp_rw, insn); + flush_idcache_range(jmp_rx, jmp_rw, 4); } static void tcg_out_op(TCGContext *s, TCGOpcode opc, -- 2.34.1 Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target.c.inc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ typedef enum { #endif OPC_FENCE = 0x0000000f, + OPC_NOP = OPC_ADDI, /* nop = addi r0,r0,0 */ } RISCVInsn; /* @@ -XXX,XX +XXX,XX @@ static void tcg_out_nop_fill(tcg_insn_unit *p, int count) { int i; for (i = 0; i < count; ++i) { - p[i] = encode_i(OPC_ADDI, TCG_REG_ZERO, TCG_REG_ZERO, 0); + p[i] = OPC_NOP; } } -- 2.34.1 Now that tcg can handle direct and indirect goto_tb simultaneously, we can optimistically leave space for a direct branch and fall back to loading the pointer from the TB for an indirect branch. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target.c.inc | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) static void tcg_out_goto_tb(TCGContext *s, int which) { - /* indirect jump method */ + /* Direct branch will be patched by tb_target_set_jmp_target. */ + set_jmp_insn_offset(s, which); + tcg_out32(s, OPC_JAL); + + /* When branch is out of range, fall through to indirect. */ tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO, get_jmp_target_addr(s, which)); tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0); @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) void tb_target_set_jmp_target(const TranslationBlock *tb, int n, uintptr_t jmp_rx, uintptr_t jmp_rw) { - /* Always indirect, nothing to do */ + uintptr_t addr = tb->jmp_target_addr[n]; + ptrdiff_t offset = addr - jmp_rx; + tcg_insn_unit insn; + + /* Either directly branch, or fall through to indirect branch. */ + if (offset == sextreg(offset, 0, 20)) { + insn = encode_uj(OPC_JAL, TCG_REG_ZERO, offset); + } else { + insn = OPC_NOP; + } + qatomic_set((uint32_t *)jmp_rw, insn); + flush_idcache_range(jmp_rx, jmp_rw, 4); } static void tcg_out_op(TCGContext *s, TCGOpcode opc, -- 2.34.1 The following changes since commit b300c134465465385045ab705b68a42699688332: Merge tag 'pull-vfio-20230524' of https://github.com/legoater/qemu into staging (2023-05-24 14:23:41 -0700) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20230525 for you to fetch changes up to a30498fcea5a8b9c544324ccfb0186090104b229: tcg/riscv: Support CTZ, CLZ from Zbb (2023-05-25 15:29:36 +0000) ---------------------------------------------------------------- tcg/mips: - Constant formation improvements - Replace MIPS_BE with HOST_BIG_ENDIAN - General cleanups tcg/riscv: - Improve setcond - Support movcond - Support Zbb, Zba ---------------------------------------------------------------- Richard Henderson (23): tcg/mips: Move TCG_AREG0 to S8 tcg/mips: Move TCG_GUEST_BASE_REG to S7 tcg/mips: Unify TCG_GUEST_BASE_REG tests tcg/mips: Create and use TCG_REG_TB tcg/mips: Split out tcg_out_movi_one tcg/mips: Split out tcg_out_movi_two tcg/mips: Use the constant pool for 64-bit constants tcg/mips: Aggressively use the constant pool for n64 calls tcg/mips: Try tb-relative addresses in tcg_out_movi tcg/mips: Try three insns with shift and add in tcg_out_movi tcg/mips: Use qemu_build_not_reached for LO/HI_OFF tcg/mips: Replace MIPS_BE with HOST_BIG_ENDIAN disas/riscv: Decode czero.{eqz,nez} tcg/riscv: Probe for Zba, Zbb, Zicond extensions tcg/riscv: Support ANDN, ORN, XNOR from Zbb tcg/riscv: Support ADD.UW, SEXT.B, SEXT.H, ZEXT.H from Zba+Zbb tcg/riscv: Use ADD.UW for guest address generation tcg/riscv: Support rotates from Zbb tcg/riscv: Support REV8 from Zbb tcg/riscv: Support CPOP from Zbb tcg/riscv: Improve setcond expansion tcg/riscv: Implement movcond tcg/riscv: Support CTZ, CLZ from Zbb tcg/mips/tcg-target.h | 3 +- tcg/riscv/tcg-target-con-set.h | 3 + tcg/riscv/tcg-target-con-str.h | 1 + tcg/riscv/tcg-target.h | 48 ++-- disas/riscv.c | 6 + tcg/mips/tcg-target.c.inc | 308 ++++++++++++++++----- tcg/riscv/tcg-target.c.inc | 612 ++++++++++++++++++++++++++++++++++++----- 7 files changed, 825 insertions(+), 156 deletions(-) No functional change; just moving the saved reserved regs to the end. Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target.h | 2 +- tcg/mips/tcg-target.c.inc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { TCG_REG_RA, TCG_REG_CALL_STACK = TCG_REG_SP, - TCG_AREG0 = TCG_REG_S0, + TCG_AREG0 = TCG_REG_S8, } TCGReg; /* used for function call generation */ diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) } static const int tcg_target_callee_save_regs[] = { - TCG_REG_S0, /* used for the global env (TCG_AREG0) */ + TCG_REG_S0, TCG_REG_S1, TCG_REG_S2, TCG_REG_S3, @@ -XXX,XX +XXX,XX @@ static const int tcg_target_callee_save_regs[] = { TCG_REG_S5, TCG_REG_S6, TCG_REG_S7, - TCG_REG_S8, + TCG_REG_S8, /* used for the global env (TCG_AREG0) */ TCG_REG_RA, /* should be last for ABI compliance */ }; -- 2.34.1 No functional change; just moving the saved reserved regs to the end. Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target.c.inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { #define TCG_TMP3 TCG_REG_T7 #ifndef CONFIG_SOFTMMU -#define TCG_GUEST_BASE_REG TCG_REG_S1 +#define TCG_GUEST_BASE_REG TCG_REG_S7 #endif /* check if we really need so many registers :P */ @@ -XXX,XX +XXX,XX @@ static const int tcg_target_callee_save_regs[] = { TCG_REG_S4, TCG_REG_S5, TCG_REG_S6, - TCG_REG_S7, + TCG_REG_S7, /* used for guest_base */ TCG_REG_S8, /* used for the global env (TCG_AREG0) */ TCG_REG_RA, /* should be last for ABI compliance */ }; -- 2.34.1 In tcg_out_qemu_ld/st, we already check for guest_base matching int16_t. Mirror that when setting up TCG_GUEST_BASE_REG in the prologue. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target.c.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) } #ifndef CONFIG_SOFTMMU - if (guest_base) { + if (guest_base != (int16_t)guest_base) { tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base); tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG); } -- 2.34.1 This vastly reduces the size of code generated for 64-bit addresses. The code for exit_tb, for instance, where we load a (tagged) pointer to the current TB, goes from 0x400aa9725c: li v0,64 0x400aa97260: dsll v0,v0,0x10 0x400aa97264: ori v0,v0,0xaa9 0x400aa97268: dsll v0,v0,0x10 0x400aa9726c: j 0x400aa9703c 0x400aa97270: ori v0,v0,0x7083 to 0x400aa97240: j 0x400aa97040 0x400aa97244: daddiu v0,s6,-189 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target.c.inc | 69 +++++++++++++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 10 deletions(-) diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { #ifndef CONFIG_SOFTMMU #define TCG_GUEST_BASE_REG TCG_REG_S7 #endif +#if TCG_TARGET_REG_BITS == 64 +#define TCG_REG_TB TCG_REG_S6 +#else +#define TCG_REG_TB (qemu_build_not_reached(), TCG_REG_ZERO) +#endif /* check if we really need so many registers :P */ static const int tcg_target_reg_alloc_order[] = { @@ -XXX,XX +XXX,XX @@ static void tcg_out_clz(TCGContext *s, MIPSInsn opcv2, MIPSInsn opcv6, static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) { - TCGReg b0 = TCG_REG_ZERO; + TCGReg base = TCG_REG_ZERO; + int16_t lo = 0; - if (a0 & ~0xffff) { - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_V0, a0 & ~0xffff); - b0 = TCG_REG_V0; + if (a0) { + intptr_t ofs; + if (TCG_TARGET_REG_BITS == 64) { + ofs = tcg_tbrel_diff(s, (void *)a0); + lo = ofs; + if (ofs == lo) { + base = TCG_REG_TB; + } else { + base = TCG_REG_V0; + tcg_out_movi(s, TCG_TYPE_PTR, base, ofs - lo); + tcg_out_opc_reg(s, ALIAS_PADD, base, base, TCG_REG_TB); + } + } else { + ofs = a0; + lo = ofs; + base = TCG_REG_V0; + tcg_out_movi(s, TCG_TYPE_PTR, base, ofs - lo); + } } if (!tcg_out_opc_jmp(s, OPC_J, tb_ret_addr)) { tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)tb_ret_addr); tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0); } - tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff); + /* delay slot */ + tcg_out_opc_imm(s, ALIAS_PADDI, TCG_REG_V0, base, lo); } static void tcg_out_goto_tb(TCGContext *s, int which) { + intptr_t ofs = get_jmp_target_addr(s, which); + TCGReg base, dest; + /* indirect jump method */ - tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO, - get_jmp_target_addr(s, which)); - tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0); + if (TCG_TARGET_REG_BITS == 64) { + dest = TCG_REG_TB; + base = TCG_REG_TB; + ofs = tcg_tbrel_diff(s, (void *)ofs); + } else { + dest = TCG_TMP0; + base = TCG_REG_ZERO; + } + tcg_out_ld(s, TCG_TYPE_PTR, dest, base, ofs); + tcg_out_opc_reg(s, OPC_JR, 0, dest, 0); + /* delay slot */ tcg_out_nop(s); + set_jmp_reset_offset(s, which); + if (TCG_TARGET_REG_BITS == 64) { + /* For the unlinked case, need to reset TCG_REG_TB. */ + tcg_out_ldst(s, ALIAS_PADDI, TCG_REG_TB, TCG_REG_TB, + -tcg_current_code_size(s)); + } } void tb_target_set_jmp_target(const TranslationBlock *tb, int n, @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_goto_ptr: /* jmp to the given host address (could be epilogue) */ tcg_out_opc_reg(s, OPC_JR, 0, a0, 0); - tcg_out_nop(s); + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_TB, a0); + } else { + tcg_out_nop(s); + } break; case INDEX_op_br: tcg_out_brcond(s, TCG_COND_EQ, TCG_REG_ZERO, TCG_REG_ZERO, @@ -XXX,XX +XXX,XX @@ static const int tcg_target_callee_save_regs[] = { TCG_REG_S3, TCG_REG_S4, TCG_REG_S5, - TCG_REG_S6, + TCG_REG_S6, /* used for the tb base (TCG_REG_TB) */ TCG_REG_S7, /* used for guest_base */ TCG_REG_S8, /* used for the global env (TCG_AREG0) */ TCG_REG_RA, /* should be last for ABI compliance */ @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG); } #endif + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_TB, tcg_target_call_iarg_regs[1]); + } /* Call generated code */ tcg_out_opc_reg(s, OPC_JR, 0, tcg_target_call_iarg_regs[1], 0); @@ -XXX,XX +XXX,XX @@ static void tcg_target_init(TCGContext *s) tcg_regset_set_reg(s->reserved_regs, TCG_REG_RA); /* return address */ tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP); /* stack pointer */ tcg_regset_set_reg(s->reserved_regs, TCG_REG_GP); /* global pointer */ + if (TCG_TARGET_REG_BITS == 64) { + tcg_regset_set_reg(s->reserved_regs, TCG_REG_TB); /* tc->tc_ptr */ + } } typedef struct { -- 2.34.1 Emit all constants that can be loaded in exactly one insn. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target.c.inc | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) return true; } +static bool tcg_out_movi_one(TCGContext *s, TCGReg ret, tcg_target_long arg) +{ + if (arg == (int16_t)arg) { + tcg_out_opc_imm(s, OPC_ADDIU, ret, TCG_REG_ZERO, arg); + return true; + } + if (arg == (uint16_t)arg) { + tcg_out_opc_imm(s, OPC_ORI, ret, TCG_REG_ZERO, arg); + return true; + } + if (arg == (int32_t)arg && (arg & 0xffff) == 0) { + tcg_out_opc_imm(s, OPC_LUI, ret, TCG_REG_ZERO, arg >> 16); + return true; + } + return false; +} + static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg ret, tcg_target_long arg) { if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) { arg = (int32_t)arg; } - if (arg == (int16_t)arg) { - tcg_out_opc_imm(s, OPC_ADDIU, ret, TCG_REG_ZERO, arg); - return; - } - if (arg == (uint16_t)arg) { - tcg_out_opc_imm(s, OPC_ORI, ret, TCG_REG_ZERO, arg); + + if (tcg_out_movi_one(s, ret, arg)) { return; } + if (TCG_TARGET_REG_BITS == 32 || arg == (int32_t)arg) { tcg_out_opc_imm(s, OPC_LUI, ret, TCG_REG_ZERO, arg >> 16); } else { -- 2.34.1 Emit all 32-bit signed constants, which can be loaded in two insns. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target.c.inc | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_out_movi_one(TCGContext *s, TCGReg ret, tcg_target_long arg) return false; } +static bool tcg_out_movi_two(TCGContext *s, TCGReg ret, tcg_target_long arg) +{ + /* + * All signed 32-bit constants are loadable with two immediates, + * and everything else requires more work. + */ + if (arg == (int32_t)arg) { + if (!tcg_out_movi_one(s, ret, arg)) { + tcg_out_opc_imm(s, OPC_LUI, ret, TCG_REG_ZERO, arg >> 16); + tcg_out_opc_imm(s, OPC_ORI, ret, ret, arg & 0xffff); + } + return true; + } + return false; +} + static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg ret, tcg_target_long arg) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, arg = (int32_t)arg; } - if (tcg_out_movi_one(s, ret, arg)) { + /* Load all 32-bit constants. */ + if (tcg_out_movi_two(s, ret, arg)) { return; } - if (TCG_TARGET_REG_BITS == 32 || arg == (int32_t)arg) { - tcg_out_opc_imm(s, OPC_LUI, ret, TCG_REG_ZERO, arg >> 16); + tcg_out_movi(s, TCG_TYPE_I32, ret, arg >> 31 >> 1); + if (arg & 0xffff0000ull) { + tcg_out_dsll(s, ret, ret, 16); + tcg_out_opc_imm(s, OPC_ORI, ret, ret, arg >> 16); + tcg_out_dsll(s, ret, ret, 16); } else { - tcg_out_movi(s, TCG_TYPE_I32, ret, arg >> 31 >> 1); - if (arg & 0xffff0000ull) { - tcg_out_dsll(s, ret, ret, 16); - tcg_out_opc_imm(s, OPC_ORI, ret, ret, arg >> 16); - tcg_out_dsll(s, ret, ret, 16); - } else { - tcg_out_dsll(s, ret, ret, 32); - } + tcg_out_dsll(s, ret, ret, 32); } if (arg & 0xffff) { tcg_out_opc_imm(s, OPC_ORI, ret, ret, arg & 0xffff); -- 2.34.1 During normal processing, the constant pool is accessible via TCG_REG_TB. During the prologue, it is accessible via TCG_REG_T9. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target.h | 1 + tcg/mips/tcg-target.c.inc | 65 +++++++++++++++++++++++++++++---------- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions; #define TCG_TARGET_DEFAULT_MO 0 #define TCG_TARGET_NEED_LDST_LABELS +#define TCG_TARGET_NEED_POOL_LABELS #endif diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ */ #include "../tcg-ldst.c.inc" +#include "../tcg-pool.c.inc" #if HOST_BIG_ENDIAN # define MIPS_BE 1 @@ -XXX,XX +XXX,XX @@ static bool reloc_pc16(tcg_insn_unit *src_rw, const tcg_insn_unit *target) static bool patch_reloc(tcg_insn_unit *code_ptr, int type, intptr_t value, intptr_t addend) { - tcg_debug_assert(type == R_MIPS_PC16); - tcg_debug_assert(addend == 0); - return reloc_pc16(code_ptr, (const tcg_insn_unit *)value); + value += addend; + switch (type) { + case R_MIPS_PC16: + return reloc_pc16(code_ptr, (const tcg_insn_unit *)value); + case R_MIPS_16: + if (value != (int16_t)value) { + return false; + } + *code_ptr = deposit32(*code_ptr, 0, 16, value); + return true; + } + g_assert_not_reached(); } #define TCG_CT_CONST_ZERO 0x100 @@ -XXX,XX +XXX,XX @@ static void tcg_out_nop(TCGContext *s) tcg_out32(s, 0); } +static void tcg_out_nop_fill(tcg_insn_unit *p, int count) +{ + memset(p, 0, count * sizeof(tcg_insn_unit)); +} + static void tcg_out_dsll(TCGContext *s, TCGReg rd, TCGReg rt, TCGArg sa) { tcg_out_opc_sa64(s, OPC_DSLL, OPC_DSLL32, rd, rt, sa); @@ -XXX,XX +XXX,XX @@ static bool tcg_out_movi_two(TCGContext *s, TCGReg ret, tcg_target_long arg) return false; } -static void tcg_out_movi(TCGContext *s, TCGType type, - TCGReg ret, tcg_target_long arg) +static void tcg_out_movi_pool(TCGContext *s, TCGReg ret, + tcg_target_long arg, TCGReg tbreg) +{ + new_pool_label(s, arg, R_MIPS_16, s->code_ptr, tcg_tbrel_diff(s, NULL)); + tcg_out_opc_imm(s, OPC_LD, ret, tbreg, 0); +} + +static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, + tcg_target_long arg, TCGReg tbreg) { if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) { arg = (int32_t)arg; @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, if (tcg_out_movi_two(s, ret, arg)) { return; } + assert(TCG_TARGET_REG_BITS == 64); - tcg_out_movi(s, TCG_TYPE_I32, ret, arg >> 31 >> 1); - if (arg & 0xffff0000ull) { - tcg_out_dsll(s, ret, ret, 16); - tcg_out_opc_imm(s, OPC_ORI, ret, ret, arg >> 16); - tcg_out_dsll(s, ret, ret, 16); - } else { - tcg_out_dsll(s, ret, ret, 32); - } - if (arg & 0xffff) { - tcg_out_opc_imm(s, OPC_ORI, ret, ret, arg & 0xffff); - } + /* Otherwise, put 64-bit constants into the constant pool. */ + tcg_out_movi_pool(s, ret, arg, tbreg); +} + +static void tcg_out_movi(TCGContext *s, TCGType type, + TCGReg ret, tcg_target_long arg) +{ + TCGReg tbreg = TCG_TARGET_REG_BITS == 64 ? TCG_REG_TB : 0; + tcg_out_movi_int(s, type, ret, arg, tbreg); } static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs) @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) #ifndef CONFIG_SOFTMMU if (guest_base != (int16_t)guest_base) { - tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base); + /* + * The function call abi for n32 and n64 will have loaded $25 (t9) + * with the address of the prologue, so we can use that instead + * of TCG_REG_TB. + */ +#if TCG_TARGET_REG_BITS == 64 && !defined(__mips_abicalls) +# error "Unknown mips abi" +#endif + tcg_out_movi_int(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base, + TCG_TARGET_REG_BITS == 64 ? TCG_REG_T9 : 0); tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG); } #endif + if (TCG_TARGET_REG_BITS == 64) { tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_TB, tcg_target_call_iarg_regs[1]); } -- 2.34.1 Repeated calls to a single helper are common -- especially the ones for softmmu memory access. Prefer the constant pool to longer sequences to increase sharing. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target.c.inc | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret, static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) { - /* Note that the ABI requires the called function's address to be - loaded into T9, even if a direct branch is in range. */ - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T9, (uintptr_t)arg); + /* + * Note that __mips_abicalls requires the called function's address + * to be loaded into $25 (t9), even if a direct branch is in range. + * + * For n64, always drop the pointer into the constant pool. + * We can re-use helper addresses often and do not want any + * of the longer sequences tcg_out_movi may try. + */ + if (sizeof(uintptr_t) == 8) { + tcg_out_movi_pool(s, TCG_REG_T9, (uintptr_t)arg, TCG_REG_TB); + } else { + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T9, (uintptr_t)arg); + } /* But do try a direct branch, allowing the cpu better insn prefetch. */ if (tail) { -- 2.34.1 These addresses are often loaded by the qemu_ld/st slow path, for loading the retaddr value. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target.c.inc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_pool(TCGContext *s, TCGReg ret, static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, tcg_target_long arg, TCGReg tbreg) { + tcg_target_long tmp; + if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) { arg = (int32_t)arg; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, } assert(TCG_TARGET_REG_BITS == 64); + /* Load addresses within 2GB of TB with 1 or 3 insns. */ + tmp = tcg_tbrel_diff(s, (void *)arg); + if (tmp == (int16_t)tmp) { + tcg_out_opc_imm(s, OPC_DADDIU, ret, tbreg, tmp); + return; + } + if (tcg_out_movi_two(s, ret, tmp)) { + tcg_out_opc_reg(s, OPC_DADDU, ret, ret, tbreg); + return; + } + /* Otherwise, put 64-bit constants into the constant pool. */ tcg_out_movi_pool(s, ret, arg, tbreg); } -- 2.34.1 These sequences are inexpensive to test. Maxing out at three insns results in the same space as a load plus the constant pool entry. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target.c.inc | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, tcg_target_long arg, TCGReg tbreg) { tcg_target_long tmp; + int sh, lo; if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) { arg = (int32_t)arg; @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, return; } + /* + * Load bitmasks with a right-shift. This is good for things + * like 0x0fff_ffff_ffff_fff0: ADDUI r,0,0xff00 + DSRL r,r,4. + * or similarly using LUI. For this to work, bit 31 must be set. + */ + if (arg > 0 && (int32_t)arg < 0) { + sh = clz64(arg); + if (tcg_out_movi_one(s, ret, arg << sh)) { + tcg_out_dsrl(s, ret, ret, sh); + return; + } + } + + /* + * Load slightly larger constants using left-shift. + * Limit this sequence to 3 insns to avoid too much expansion. + */ + sh = ctz64(arg); + if (sh && tcg_out_movi_two(s, ret, arg >> sh)) { + tcg_out_dsll(s, ret, ret, sh); + return; + } + + /* + * Load slightly larger constants using left-shift and add/or. + * Prefer addi with a negative immediate when that would produce + * a larger shift. For this to work, bits 15 and 16 must be set. + */ + lo = arg & 0xffff; + if (lo) { + if ((arg & 0x18000) == 0x18000) { + lo = (int16_t)arg; + } + tmp = arg - lo; + sh = ctz64(tmp); + tmp >>= sh; + if (tcg_out_movi_one(s, ret, tmp)) { + tcg_out_dsll(s, ret, ret, sh); + tcg_out_opc_imm(s, lo < 0 ? OPC_DADDIU : OPC_ORI, ret, ret, lo); + return; + } + } + /* Otherwise, put 64-bit constants into the constant pool. */ tcg_out_movi_pool(s, ret, arg, tbreg); } -- 2.34.1 The new(ish) macro produces a compile-time error instead of a link-time error. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target.c.inc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ # define LO_OFF (MIPS_BE * 4) # define HI_OFF (4 - LO_OFF) #else -/* To assert at compile-time that these values are never used - for TCG_TARGET_REG_BITS == 64. */ -int link_error(void); -# define LO_OFF link_error() -# define HI_OFF link_error() +/* Assert at compile-time that these values are never used for 64-bit. */ +# define LO_OFF ({ qemu_build_not_reached(); 0; }) +# define HI_OFF ({ qemu_build_not_reached(); 0; }) #endif #ifdef CONFIG_DEBUG_TCG -- 2.34.1 Since e03b56863d2b, which replaced HOST_WORDS_BIGENDIAN with HOST_BIG_ENDIAN, there is no need to define a second symbol which is [0,1]. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target.c.inc | 46 +++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ #include "../tcg-ldst.c.inc" #include "../tcg-pool.c.inc" -#if HOST_BIG_ENDIAN -# define MIPS_BE 1 -#else -# define MIPS_BE 0 -#endif - #if TCG_TARGET_REG_BITS == 32 -# define LO_OFF (MIPS_BE * 4) +# define LO_OFF (HOST_BIG_ENDIAN * 4) # define HI_OFF (4 - LO_OFF) #else /* Assert at compile-time that these values are never used for 64-bit. */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, /* Prefer to load from offset 0 first, but allow for overlap. */ if (TCG_TARGET_REG_BITS == 64) { tcg_out_opc_imm(s, OPC_LD, lo, base, 0); - } else if (MIPS_BE ? hi != base : lo == base) { + } else if (HOST_BIG_ENDIAN ? hi != base : lo == base) { tcg_out_opc_imm(s, OPC_LW, hi, base, HI_OFF); tcg_out_opc_imm(s, OPC_LW, lo, base, LO_OFF); } else { @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, static void tcg_out_qemu_ld_unalign(TCGContext *s, TCGReg lo, TCGReg hi, TCGReg base, MemOp opc, TCGType type) { - const MIPSInsn lw1 = MIPS_BE ? OPC_LWL : OPC_LWR; - const MIPSInsn lw2 = MIPS_BE ? OPC_LWR : OPC_LWL; - const MIPSInsn ld1 = MIPS_BE ? OPC_LDL : OPC_LDR; - const MIPSInsn ld2 = MIPS_BE ? OPC_LDR : OPC_LDL; + const MIPSInsn lw1 = HOST_BIG_ENDIAN ? OPC_LWL : OPC_LWR; + const MIPSInsn lw2 = HOST_BIG_ENDIAN ? OPC_LWR : OPC_LWL; + const MIPSInsn ld1 = HOST_BIG_ENDIAN ? OPC_LDL : OPC_LDR; + const MIPSInsn ld2 = HOST_BIG_ENDIAN ? OPC_LDR : OPC_LDL; bool sgn = opc & MO_SIGN; switch (opc & MO_SIZE) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_unalign(TCGContext *s, TCGReg lo, TCGReg hi, tcg_out_opc_imm(s, ld1, lo, base, 0); tcg_out_opc_imm(s, ld2, lo, base, 7); } else { - tcg_out_opc_imm(s, lw1, MIPS_BE ? hi : lo, base, 0 + 0); - tcg_out_opc_imm(s, lw2, MIPS_BE ? hi : lo, base, 0 + 3); - tcg_out_opc_imm(s, lw1, MIPS_BE ? lo : hi, base, 4 + 0); - tcg_out_opc_imm(s, lw2, MIPS_BE ? lo : hi, base, 4 + 3); + tcg_out_opc_imm(s, lw1, HOST_BIG_ENDIAN ? hi : lo, base, 0 + 0); + tcg_out_opc_imm(s, lw2, HOST_BIG_ENDIAN ? hi : lo, base, 0 + 3); + tcg_out_opc_imm(s, lw1, HOST_BIG_ENDIAN ? lo : hi, base, 4 + 0); + tcg_out_opc_imm(s, lw2, HOST_BIG_ENDIAN ? lo : hi, base, 4 + 3); } break; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi, if (TCG_TARGET_REG_BITS == 64) { tcg_out_opc_imm(s, OPC_SD, lo, base, 0); } else { - tcg_out_opc_imm(s, OPC_SW, MIPS_BE ? hi : lo, base, 0); - tcg_out_opc_imm(s, OPC_SW, MIPS_BE ? lo : hi, base, 4); + tcg_out_opc_imm(s, OPC_SW, HOST_BIG_ENDIAN ? hi : lo, base, 0); + tcg_out_opc_imm(s, OPC_SW, HOST_BIG_ENDIAN ? lo : hi, base, 4); } break; default: @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi, static void tcg_out_qemu_st_unalign(TCGContext *s, TCGReg lo, TCGReg hi, TCGReg base, MemOp opc) { - const MIPSInsn sw1 = MIPS_BE ? OPC_SWL : OPC_SWR; - const MIPSInsn sw2 = MIPS_BE ? OPC_SWR : OPC_SWL; - const MIPSInsn sd1 = MIPS_BE ? OPC_SDL : OPC_SDR; - const MIPSInsn sd2 = MIPS_BE ? OPC_SDR : OPC_SDL; + const MIPSInsn sw1 = HOST_BIG_ENDIAN ? OPC_SWL : OPC_SWR; + const MIPSInsn sw2 = HOST_BIG_ENDIAN ? OPC_SWR : OPC_SWL; + const MIPSInsn sd1 = HOST_BIG_ENDIAN ? OPC_SDL : OPC_SDR; + const MIPSInsn sd2 = HOST_BIG_ENDIAN ? OPC_SDR : OPC_SDL; switch (opc & MO_SIZE) { case MO_16: @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_unalign(TCGContext *s, TCGReg lo, TCGReg hi, tcg_out_opc_imm(s, sd1, lo, base, 0); tcg_out_opc_imm(s, sd2, lo, base, 7); } else { - tcg_out_opc_imm(s, sw1, MIPS_BE ? hi : lo, base, 0 + 0); - tcg_out_opc_imm(s, sw2, MIPS_BE ? hi : lo, base, 0 + 3); - tcg_out_opc_imm(s, sw1, MIPS_BE ? lo : hi, base, 4 + 0); - tcg_out_opc_imm(s, sw2, MIPS_BE ? lo : hi, base, 4 + 3); + tcg_out_opc_imm(s, sw1, HOST_BIG_ENDIAN ? hi : lo, base, 0 + 0); + tcg_out_opc_imm(s, sw2, HOST_BIG_ENDIAN ? hi : lo, base, 0 + 3); + tcg_out_opc_imm(s, sw1, HOST_BIG_ENDIAN ? lo : hi, base, 4 + 0); + tcg_out_opc_imm(s, sw2, HOST_BIG_ENDIAN ? lo : hi, base, 4 + 3); } break; -- 2.34.1 Acked-by: Alistair Francis <alistair.francis@wdc.com> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- disas/riscv.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/disas/riscv.c b/disas/riscv.c index XXXXXXX..XXXXXXX 100644 --- a/disas/riscv.c +++ b/disas/riscv.c @@ -XXX,XX +XXX,XX @@ typedef enum { rv_op_cm_mvsa01 = 786, rv_op_cm_jt = 787, rv_op_cm_jalt = 788, + rv_op_czero_eqz = 789, + rv_op_czero_nez = 790, } rv_op; /* structures */ @@ -XXX,XX +XXX,XX @@ const rv_opcode_data opcode_data[] = { { "cm.mvsa01", rv_codec_zcmp_cm_mv, rv_fmt_rd_rs2, NULL, 0, 0, 0 }, { "cm.jt", rv_codec_zcmt_jt, rv_fmt_zcmt_index, NULL, 0 }, { "cm.jalt", rv_codec_zcmt_jt, rv_fmt_zcmt_index, NULL, 0 }, + { "czero.eqz", rv_codec_r, rv_fmt_rd_rs1_rs2, NULL, 0, 0, 0 }, + { "czero.nez", rv_codec_r, rv_fmt_rd_rs1_rs2, NULL, 0, 0, 0 }, }; /* CSR names */ @@ -XXX,XX +XXX,XX @@ static void decode_inst_opcode(rv_decode *dec, rv_isa isa) case 45: op = rv_op_minu; break; case 46: op = rv_op_max; break; case 47: op = rv_op_maxu; break; + case 075: op = rv_op_czero_eqz; break; + case 077: op = rv_op_czero_nez; break; case 130: op = rv_op_sh1add; break; case 132: op = rv_op_sh2add; break; case 134: op = rv_op_sh3add; break; -- 2.34.1 Define a useful subset of the extensions. Probe for them via compiler pre-processor feature macros and SIGILL. Acked-by: Alistair Francis <alistair.francis@wdc.com> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target.h | 6 +++ tcg/riscv/tcg-target.c.inc | 96 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_NORMAL #define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_NORMAL +#if defined(__riscv_arch_test) && defined(__riscv_zbb) +# define have_zbb true +#else +extern bool have_zbb; +#endif + /* optional instructions */ #define TCG_TARGET_HAS_movcond_i32 0 #define TCG_TARGET_HAS_div_i32 1 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const int tcg_target_call_iarg_regs[] = { TCG_REG_A7, }; +#ifndef have_zbb +bool have_zbb; +#endif +#if defined(__riscv_arch_test) && defined(__riscv_zba) +# define have_zba true +#else +static bool have_zba; +#endif +#if defined(__riscv_arch_test) && defined(__riscv_zicond) +# define have_zicond true +#else +static bool have_zicond; +#endif + static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) { tcg_debug_assert(kind == TCG_CALL_RET_NORMAL); @@ -XXX,XX +XXX,XX @@ typedef enum { OPC_FENCE = 0x0000000f, OPC_NOP = OPC_ADDI, /* nop = addi r0,r0,0 */ + + /* Zba: Bit manipulation extension, address generation */ + OPC_ADD_UW = 0x0800003b, + + /* Zbb: Bit manipulation extension, basic bit manipulaton */ + OPC_ANDN = 0x40007033, + OPC_CLZ = 0x60001013, + OPC_CLZW = 0x6000101b, + OPC_CPOP = 0x60201013, + OPC_CPOPW = 0x6020101b, + OPC_CTZ = 0x60101013, + OPC_CTZW = 0x6010101b, + OPC_ORN = 0x40006033, + OPC_REV8 = 0x6b805013, + OPC_ROL = 0x60001033, + OPC_ROLW = 0x6000103b, + OPC_ROR = 0x60005033, + OPC_RORW = 0x6000503b, + OPC_RORI = 0x60005013, + OPC_RORIW = 0x6000501b, + OPC_SEXT_B = 0x60401013, + OPC_SEXT_H = 0x60501013, + OPC_XNOR = 0x40004033, + OPC_ZEXT_H = 0x0800403b, + + /* Zicond: integer conditional operations */ + OPC_CZERO_EQZ = 0x0e005033, + OPC_CZERO_NEZ = 0x0e007033, } RISCVInsn; /* @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_RA, 0); } +static volatile sig_atomic_t got_sigill; + +static void sigill_handler(int signo, siginfo_t *si, void *data) +{ + /* Skip the faulty instruction */ + ucontext_t *uc = (ucontext_t *)data; + uc->uc_mcontext.__gregs[REG_PC] += 4; + + got_sigill = 1; +} + +static void tcg_target_detect_isa(void) +{ +#if !defined(have_zba) || !defined(have_zbb) || !defined(have_zicond) + /* + * TODO: It is expected that this will be determinable via + * linux riscv_hwprobe syscall, not yet merged. + * In the meantime, test via sigill. + */ + + struct sigaction sa_old, sa_new; + + memset(&sa_new, 0, sizeof(sa_new)); + sa_new.sa_flags = SA_SIGINFO; + sa_new.sa_sigaction = sigill_handler; + sigaction(SIGILL, &sa_new, &sa_old); + +#ifndef have_zba + /* Probe for Zba: add.uw zero,zero,zero. */ + got_sigill = 0; + asm volatile(".insn r 0x3b, 0, 0x04, zero, zero, zero" : : : "memory"); + have_zba = !got_sigill; +#endif + +#ifndef have_zbb + /* Probe for Zba: andn zero,zero,zero. */ + got_sigill = 0; + asm volatile(".insn r 0x33, 7, 0x20, zero, zero, zero" : : : "memory"); + have_zbb = !got_sigill; +#endif + +#ifndef have_zicond + /* Probe for Zicond: czero.eqz zero,zero,zero. */ + got_sigill = 0; + asm volatile(".insn r 0x33, 5, 0x07, zero, zero, zero" : : : "memory"); + have_zicond = !got_sigill; +#endif + + sigaction(SIGILL, &sa_old, NULL); +#endif +} + static void tcg_target_init(TCGContext *s) { + tcg_target_detect_isa(); + tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffff; tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffff; -- 2.34.1 Acked-by: Alistair Francis <alistair.francis@wdc.com> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target-con-set.h | 1 + tcg/riscv/tcg-target-con-str.h | 1 + tcg/riscv/tcg-target.h | 12 +++++----- tcg/riscv/tcg-target.c.inc | 41 ++++++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 6 deletions(-) diff --git a/tcg/riscv/tcg-target-con-set.h b/tcg/riscv/tcg-target-con-set.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target-con-set.h +++ b/tcg/riscv/tcg-target-con-set.h @@ -XXX,XX +XXX,XX @@ C_O0_I2(rZ, rZ) C_O1_I1(r, r) C_O1_I2(r, r, ri) C_O1_I2(r, r, rI) +C_O1_I2(r, r, rJ) C_O1_I2(r, rZ, rN) C_O1_I2(r, rZ, rZ) C_O2_I4(r, r, rZ, rZ, rM, rM) diff --git a/tcg/riscv/tcg-target-con-str.h b/tcg/riscv/tcg-target-con-str.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target-con-str.h +++ b/tcg/riscv/tcg-target-con-str.h @@ -XXX,XX +XXX,XX @@ REGS('r', ALL_GENERAL_REGS) * CONST(letter, TCG_CT_CONST_* bit set) */ CONST('I', TCG_CT_CONST_S12) +CONST('J', TCG_CT_CONST_J12) CONST('N', TCG_CT_CONST_N12) CONST('M', TCG_CT_CONST_M12) CONST('Z', TCG_CT_CONST_ZERO) diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_zbb; #define TCG_TARGET_HAS_bswap32_i32 0 #define TCG_TARGET_HAS_not_i32 1 #define TCG_TARGET_HAS_neg_i32 1 -#define TCG_TARGET_HAS_andc_i32 0 -#define TCG_TARGET_HAS_orc_i32 0 -#define TCG_TARGET_HAS_eqv_i32 0 +#define TCG_TARGET_HAS_andc_i32 have_zbb +#define TCG_TARGET_HAS_orc_i32 have_zbb +#define TCG_TARGET_HAS_eqv_i32 have_zbb #define TCG_TARGET_HAS_nand_i32 0 #define TCG_TARGET_HAS_nor_i32 0 #define TCG_TARGET_HAS_clz_i32 0 @@ -XXX,XX +XXX,XX @@ extern bool have_zbb; #define TCG_TARGET_HAS_bswap64_i64 0 #define TCG_TARGET_HAS_not_i64 1 #define TCG_TARGET_HAS_neg_i64 1 -#define TCG_TARGET_HAS_andc_i64 0 -#define TCG_TARGET_HAS_orc_i64 0 -#define TCG_TARGET_HAS_eqv_i64 0 +#define TCG_TARGET_HAS_andc_i64 have_zbb +#define TCG_TARGET_HAS_orc_i64 have_zbb +#define TCG_TARGET_HAS_eqv_i64 have_zbb #define TCG_TARGET_HAS_nand_i64 0 #define TCG_TARGET_HAS_nor_i64 0 #define TCG_TARGET_HAS_clz_i64 0 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) #define TCG_CT_CONST_S12 0x200 #define TCG_CT_CONST_N12 0x400 #define TCG_CT_CONST_M12 0x800 +#define TCG_CT_CONST_J12 0x1000 #define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32) @@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) if ((ct & TCG_CT_CONST_M12) && val >= -0x7ff && val <= 0x7ff) { return 1; } + /* + * Inverse of sign extended from 12 bits: ~[-0x800, 0x7ff]. + * Used to map ANDN back to ANDI, etc. + */ + if ((ct & TCG_CT_CONST_J12) && ~val >= -0x800 && ~val <= 0x7ff) { + return 1; + } return 0; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, } break; + case INDEX_op_andc_i32: + case INDEX_op_andc_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_ANDI, a0, a1, ~a2); + } else { + tcg_out_opc_reg(s, OPC_ANDN, a0, a1, a2); + } + break; + case INDEX_op_orc_i32: + case INDEX_op_orc_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_ORI, a0, a1, ~a2); + } else { + tcg_out_opc_reg(s, OPC_ORN, a0, a1, a2); + } + break; + case INDEX_op_eqv_i32: + case INDEX_op_eqv_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_XORI, a0, a1, ~a2); + } else { + tcg_out_opc_reg(s, OPC_XNOR, a0, a1, a2); + } + break; + case INDEX_op_not_i32: case INDEX_op_not_i64: tcg_out_opc_imm(s, OPC_XORI, a0, a1, -1); @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_xor_i64: return C_O1_I2(r, r, rI); + case INDEX_op_andc_i32: + case INDEX_op_andc_i64: + case INDEX_op_orc_i32: + case INDEX_op_orc_i64: + case INDEX_op_eqv_i32: + case INDEX_op_eqv_i64: + return C_O1_I2(r, r, rJ); + case INDEX_op_sub_i32: case INDEX_op_sub_i64: return C_O1_I2(r, rZ, rN); -- 2.34.1 Acked-by: Alistair Francis <alistair.francis@wdc.com> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target.c.inc | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg) static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg) { - tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16); - tcg_out_opc_imm(s, OPC_SRLIW, ret, ret, 16); + if (have_zbb) { + tcg_out_opc_reg(s, OPC_ZEXT_H, ret, arg, TCG_REG_ZERO); + } else { + tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16); + tcg_out_opc_imm(s, OPC_SRLIW, ret, ret, 16); + } } static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg) { - tcg_out_opc_imm(s, OPC_SLLI, ret, arg, 32); - tcg_out_opc_imm(s, OPC_SRLI, ret, ret, 32); + if (have_zba) { + tcg_out_opc_reg(s, OPC_ADD_UW, ret, arg, TCG_REG_ZERO); + } else { + tcg_out_opc_imm(s, OPC_SLLI, ret, arg, 32); + tcg_out_opc_imm(s, OPC_SRLI, ret, ret, 32); + } } static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) { - tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 24); - tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 24); + if (have_zbb) { + tcg_out_opc_imm(s, OPC_SEXT_B, ret, arg, 0); + } else { + tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 24); + tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 24); + } } static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) { - tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16); - tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 16); + if (have_zbb) { + tcg_out_opc_imm(s, OPC_SEXT_H, ret, arg, 0); + } else { + tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16); + tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 16); + } } static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg) -- 2.34.1 The instruction is a combined zero-extend and add. Use it for exactly that. Acked-by: Alistair Francis <alistair.francis@wdc.com> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target.c.inc | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, TCGReg *pbase, tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP0, TCG_REG_TMP1, 0); /* TLB Hit - translate address using addend. */ - addr_adj = addr_reg; - if (TARGET_LONG_BITS == 32) { - addr_adj = TCG_REG_TMP0; - tcg_out_ext32u(s, addr_adj, addr_reg); + if (TARGET_LONG_BITS == 64) { + tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, addr_reg, TCG_REG_TMP2); + } else if (have_zba) { + tcg_out_opc_reg(s, OPC_ADD_UW, TCG_REG_TMP0, addr_reg, TCG_REG_TMP2); + } else { + tcg_out_ext32u(s, TCG_REG_TMP0, addr_reg); + tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_REG_TMP0, TCG_REG_TMP2); } - tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_REG_TMP2, addr_adj); *pbase = TCG_REG_TMP0; #else + TCGReg base; + if (a_mask) { ldst = new_ldst_label(s); ldst->is_ld = is_ld; @@ -XXX,XX +XXX,XX @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, TCGReg *pbase, tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP1, TCG_REG_ZERO, 0); } - TCGReg base = addr_reg; - if (TARGET_LONG_BITS == 32) { - tcg_out_ext32u(s, TCG_REG_TMP0, base); - base = TCG_REG_TMP0; - } if (guest_base != 0) { - tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_GUEST_BASE_REG, base); base = TCG_REG_TMP0; + if (TARGET_LONG_BITS == 64) { + tcg_out_opc_reg(s, OPC_ADD, base, addr_reg, TCG_GUEST_BASE_REG); + } else if (have_zba) { + tcg_out_opc_reg(s, OPC_ADD_UW, base, addr_reg, TCG_GUEST_BASE_REG); + } else { + tcg_out_ext32u(s, base, addr_reg); + tcg_out_opc_reg(s, OPC_ADD, base, base, TCG_GUEST_BASE_REG); + } + } else if (TARGET_LONG_BITS == 64) { + base = addr_reg; + } else { + base = TCG_REG_TMP0; + tcg_out_ext32u(s, base, addr_reg); } *pbase = base; #endif -- 2.34.1 Acked-by: Alistair Francis <alistair.francis@wdc.com> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target.h | 4 ++-- tcg/riscv/tcg-target.c.inc | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_zbb; #define TCG_TARGET_HAS_div_i32 1 #define TCG_TARGET_HAS_rem_i32 1 #define TCG_TARGET_HAS_div2_i32 0 -#define TCG_TARGET_HAS_rot_i32 0 +#define TCG_TARGET_HAS_rot_i32 have_zbb #define TCG_TARGET_HAS_deposit_i32 0 #define TCG_TARGET_HAS_extract_i32 0 #define TCG_TARGET_HAS_sextract_i32 0 @@ -XXX,XX +XXX,XX @@ extern bool have_zbb; #define TCG_TARGET_HAS_div_i64 1 #define TCG_TARGET_HAS_rem_i64 1 #define TCG_TARGET_HAS_div2_i64 0 -#define TCG_TARGET_HAS_rot_i64 0 +#define TCG_TARGET_HAS_rot_i64 have_zbb #define TCG_TARGET_HAS_deposit_i64 0 #define TCG_TARGET_HAS_extract_i64 0 #define TCG_TARGET_HAS_sextract_i64 0 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, } break; + case INDEX_op_rotl_i32: + if (c2) { + tcg_out_opc_imm(s, OPC_RORIW, a0, a1, -a2 & 0x1f); + } else { + tcg_out_opc_reg(s, OPC_ROLW, a0, a1, a2); + } + break; + case INDEX_op_rotl_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_RORI, a0, a1, -a2 & 0x3f); + } else { + tcg_out_opc_reg(s, OPC_ROL, a0, a1, a2); + } + break; + + case INDEX_op_rotr_i32: + if (c2) { + tcg_out_opc_imm(s, OPC_RORIW, a0, a1, a2 & 0x1f); + } else { + tcg_out_opc_reg(s, OPC_RORW, a0, a1, a2); + } + break; + case INDEX_op_rotr_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_RORI, a0, a1, a2 & 0x3f); + } else { + tcg_out_opc_reg(s, OPC_ROR, a0, a1, a2); + } + break; + case INDEX_op_add2_i32: tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], const_args[4], const_args[5], false, true); @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_shl_i32: case INDEX_op_shr_i32: case INDEX_op_sar_i32: + case INDEX_op_rotl_i32: + case INDEX_op_rotr_i32: case INDEX_op_shl_i64: case INDEX_op_shr_i64: case INDEX_op_sar_i64: + case INDEX_op_rotl_i64: + case INDEX_op_rotr_i64: return C_O1_I2(r, r, ri); case INDEX_op_brcond_i32: -- 2.34.1 Acked-by: Alistair Francis <alistair.francis@wdc.com> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target.h | 10 +++++----- tcg/riscv/tcg-target.c.inc | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_zbb; #define TCG_TARGET_HAS_ext16s_i32 1 #define TCG_TARGET_HAS_ext8u_i32 1 #define TCG_TARGET_HAS_ext16u_i32 1 -#define TCG_TARGET_HAS_bswap16_i32 0 -#define TCG_TARGET_HAS_bswap32_i32 0 +#define TCG_TARGET_HAS_bswap16_i32 have_zbb +#define TCG_TARGET_HAS_bswap32_i32 have_zbb #define TCG_TARGET_HAS_not_i32 1 #define TCG_TARGET_HAS_neg_i32 1 #define TCG_TARGET_HAS_andc_i32 have_zbb @@ -XXX,XX +XXX,XX @@ extern bool have_zbb; #define TCG_TARGET_HAS_ext8u_i64 1 #define TCG_TARGET_HAS_ext16u_i64 1 #define TCG_TARGET_HAS_ext32u_i64 1 -#define TCG_TARGET_HAS_bswap16_i64 0 -#define TCG_TARGET_HAS_bswap32_i64 0 -#define TCG_TARGET_HAS_bswap64_i64 0 +#define TCG_TARGET_HAS_bswap16_i64 have_zbb +#define TCG_TARGET_HAS_bswap32_i64 have_zbb +#define TCG_TARGET_HAS_bswap64_i64 have_zbb #define TCG_TARGET_HAS_not_i64 1 #define TCG_TARGET_HAS_neg_i64 1 #define TCG_TARGET_HAS_andc_i64 have_zbb diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, } break; + case INDEX_op_bswap64_i64: + tcg_out_opc_imm(s, OPC_REV8, a0, a1, 0); + break; + case INDEX_op_bswap32_i32: + a2 = 0; + /* fall through */ + case INDEX_op_bswap32_i64: + tcg_out_opc_imm(s, OPC_REV8, a0, a1, 0); + if (a2 & TCG_BSWAP_OZ) { + tcg_out_opc_imm(s, OPC_SRLI, a0, a0, 32); + } else { + tcg_out_opc_imm(s, OPC_SRAI, a0, a0, 32); + } + break; + case INDEX_op_bswap16_i64: + case INDEX_op_bswap16_i32: + tcg_out_opc_imm(s, OPC_REV8, a0, a1, 0); + if (a2 & TCG_BSWAP_OZ) { + tcg_out_opc_imm(s, OPC_SRLI, a0, a0, 48); + } else { + tcg_out_opc_imm(s, OPC_SRAI, a0, a0, 48); + } + break; + case INDEX_op_add2_i32: tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], const_args[4], const_args[5], false, true); @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_extrl_i64_i32: case INDEX_op_extrh_i64_i32: case INDEX_op_ext_i32_i64: + case INDEX_op_bswap16_i32: + case INDEX_op_bswap32_i32: + case INDEX_op_bswap16_i64: + case INDEX_op_bswap32_i64: + case INDEX_op_bswap64_i64: return C_O1_I1(r, r); case INDEX_op_st8_i32: -- 2.34.1 Acked-by: Alistair Francis <alistair.francis@wdc.com> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target.h | 4 ++-- tcg/riscv/tcg-target.c.inc | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_zbb; #define TCG_TARGET_HAS_nor_i32 0 #define TCG_TARGET_HAS_clz_i32 0 #define TCG_TARGET_HAS_ctz_i32 0 -#define TCG_TARGET_HAS_ctpop_i32 0 +#define TCG_TARGET_HAS_ctpop_i32 have_zbb #define TCG_TARGET_HAS_brcond2 1 #define TCG_TARGET_HAS_setcond2 1 #define TCG_TARGET_HAS_qemu_st8_i32 0 @@ -XXX,XX +XXX,XX @@ extern bool have_zbb; #define TCG_TARGET_HAS_nor_i64 0 #define TCG_TARGET_HAS_clz_i64 0 #define TCG_TARGET_HAS_ctz_i64 0 -#define TCG_TARGET_HAS_ctpop_i64 0 +#define TCG_TARGET_HAS_ctpop_i64 have_zbb #define TCG_TARGET_HAS_add2_i64 1 #define TCG_TARGET_HAS_sub2_i64 1 #define TCG_TARGET_HAS_mulu2_i64 0 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, } break; + case INDEX_op_ctpop_i32: + tcg_out_opc_imm(s, OPC_CPOPW, a0, a1, 0); + break; + case INDEX_op_ctpop_i64: + tcg_out_opc_imm(s, OPC_CPOP, a0, a1, 0); + break; + case INDEX_op_add2_i32: tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], const_args[4], const_args[5], false, true); @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_bswap16_i64: case INDEX_op_bswap32_i64: case INDEX_op_bswap64_i64: + case INDEX_op_ctpop_i32: + case INDEX_op_ctpop_i64: return C_O1_I1(r, r); case INDEX_op_st8_i32: -- 2.34.1 Split out a helper function, tcg_out_setcond_int, which does not always produce the complete boolean result, but returns a set of flags to do so. Based on 21af16198425, the same improvement for loongarch64. Acked-by: Alistair Francis <alistair.francis@wdc.com> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target.c.inc | 164 +++++++++++++++++++++++++++---------- 1 file changed, 121 insertions(+), 43 deletions(-) diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1, tcg_out_opc_branch(s, op, arg1, arg2, 0); } -static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret, - TCGReg arg1, TCGReg arg2) +#define SETCOND_INV TCG_TARGET_NB_REGS +#define SETCOND_NEZ (SETCOND_INV << 1) +#define SETCOND_FLAGS (SETCOND_INV | SETCOND_NEZ) + +static int tcg_out_setcond_int(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg arg1, tcg_target_long arg2, bool c2) { + int flags = 0; + switch (cond) { - case TCG_COND_EQ: - tcg_out_opc_reg(s, OPC_SUB, ret, arg1, arg2); - tcg_out_opc_imm(s, OPC_SLTIU, ret, ret, 1); - break; - case TCG_COND_NE: - tcg_out_opc_reg(s, OPC_SUB, ret, arg1, arg2); - tcg_out_opc_reg(s, OPC_SLTU, ret, TCG_REG_ZERO, ret); - break; - case TCG_COND_LT: - tcg_out_opc_reg(s, OPC_SLT, ret, arg1, arg2); - break; - case TCG_COND_GE: - tcg_out_opc_reg(s, OPC_SLT, ret, arg1, arg2); - tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); - break; - case TCG_COND_LE: - tcg_out_opc_reg(s, OPC_SLT, ret, arg2, arg1); - tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); - break; - case TCG_COND_GT: - tcg_out_opc_reg(s, OPC_SLT, ret, arg2, arg1); - break; - case TCG_COND_LTU: - tcg_out_opc_reg(s, OPC_SLTU, ret, arg1, arg2); - break; - case TCG_COND_GEU: - tcg_out_opc_reg(s, OPC_SLTU, ret, arg1, arg2); - tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); - break; - case TCG_COND_LEU: - tcg_out_opc_reg(s, OPC_SLTU, ret, arg2, arg1); - tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); - break; - case TCG_COND_GTU: - tcg_out_opc_reg(s, OPC_SLTU, ret, arg2, arg1); + case TCG_COND_EQ: /* -> NE */ + case TCG_COND_GE: /* -> LT */ + case TCG_COND_GEU: /* -> LTU */ + case TCG_COND_GT: /* -> LE */ + case TCG_COND_GTU: /* -> LEU */ + cond = tcg_invert_cond(cond); + flags ^= SETCOND_INV; break; default: - g_assert_not_reached(); - break; - } + break; + } + + switch (cond) { + case TCG_COND_LE: + case TCG_COND_LEU: + /* + * If we have a constant input, the most efficient way to implement + * LE is by adding 1 and using LT. Watch out for wrap around for LEU. + * We don't need to care for this for LE because the constant input + * is constrained to signed 12-bit, and 0x800 is representable in the + * temporary register. + */ + if (c2) { + if (cond == TCG_COND_LEU) { + /* unsigned <= -1 is true */ + if (arg2 == -1) { + tcg_out_movi(s, TCG_TYPE_REG, ret, !(flags & SETCOND_INV)); + return ret; + } + cond = TCG_COND_LTU; + } else { + cond = TCG_COND_LT; + } + tcg_debug_assert(arg2 <= 0x7ff); + if (++arg2 == 0x800) { + tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_TMP0, arg2); + arg2 = TCG_REG_TMP0; + c2 = false; + } + } else { + TCGReg tmp = arg2; + arg2 = arg1; + arg1 = tmp; + cond = tcg_swap_cond(cond); /* LE -> GE */ + cond = tcg_invert_cond(cond); /* GE -> LT */ + flags ^= SETCOND_INV; + } + break; + default: + break; + } + + switch (cond) { + case TCG_COND_NE: + flags |= SETCOND_NEZ; + if (!c2) { + tcg_out_opc_reg(s, OPC_XOR, ret, arg1, arg2); + } else if (arg2 == 0) { + ret = arg1; + } else { + tcg_out_opc_imm(s, OPC_XORI, ret, arg1, arg2); + } + break; + + case TCG_COND_LT: + if (c2) { + tcg_out_opc_imm(s, OPC_SLTI, ret, arg1, arg2); + } else { + tcg_out_opc_reg(s, OPC_SLT, ret, arg1, arg2); + } + break; + + case TCG_COND_LTU: + if (c2) { + tcg_out_opc_imm(s, OPC_SLTIU, ret, arg1, arg2); + } else { + tcg_out_opc_reg(s, OPC_SLTU, ret, arg1, arg2); + } + break; + + default: + g_assert_not_reached(); + } + + return ret | flags; +} + +static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg arg1, tcg_target_long arg2, bool c2) +{ + int tmpflags = tcg_out_setcond_int(s, cond, ret, arg1, arg2, c2); + + if (tmpflags != ret) { + TCGReg tmp = tmpflags & ~SETCOND_FLAGS; + + switch (tmpflags & SETCOND_FLAGS) { + case SETCOND_INV: + /* Intermediate result is boolean: simply invert. */ + tcg_out_opc_imm(s, OPC_XORI, ret, tmp, 1); + break; + case SETCOND_NEZ: + /* Intermediate result is zero/non-zero: test != 0. */ + tcg_out_opc_reg(s, OPC_SLTU, ret, TCG_REG_ZERO, tmp); + break; + case SETCOND_NEZ | SETCOND_INV: + /* Intermediate result is zero/non-zero: test == 0. */ + tcg_out_opc_imm(s, OPC_SLTIU, ret, tmp, 1); + break; + default: + g_assert_not_reached(); + } + } } static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_setcond_i32: case INDEX_op_setcond_i64: - tcg_out_setcond(s, args[3], a0, a1, a2); + tcg_out_setcond(s, args[3], a0, a1, a2, c2); break; case INDEX_op_qemu_ld_a32_i32: @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_and_i64: case INDEX_op_or_i64: case INDEX_op_xor_i64: + case INDEX_op_setcond_i32: + case INDEX_op_setcond_i64: return C_O1_I2(r, r, rI); case INDEX_op_andc_i32: @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_divu_i32: case INDEX_op_rem_i32: case INDEX_op_remu_i32: - case INDEX_op_setcond_i32: case INDEX_op_mul_i64: case INDEX_op_mulsh_i64: case INDEX_op_muluh_i64: @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_divu_i64: case INDEX_op_rem_i64: case INDEX_op_remu_i64: - case INDEX_op_setcond_i64: return C_O1_I2(r, rZ, rZ); case INDEX_op_shl_i32: -- 2.34.1 Implement with and without Zicond. Without Zicond, we were letting the middle-end expand to a 5 insn sequence; better to use a branch over a single insn. Acked-by: Alistair Francis <alistair.francis@wdc.com> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target-con-set.h | 1 + tcg/riscv/tcg-target.h | 4 +- tcg/riscv/tcg-target.c.inc | 139 ++++++++++++++++++++++++++++++++- 3 files changed, 141 insertions(+), 3 deletions(-) diff --git a/tcg/riscv/tcg-target-con-set.h b/tcg/riscv/tcg-target-con-set.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target-con-set.h +++ b/tcg/riscv/tcg-target-con-set.h @@ -XXX,XX +XXX,XX @@ C_O1_I2(r, r, rI) C_O1_I2(r, r, rJ) C_O1_I2(r, rZ, rN) C_O1_I2(r, rZ, rZ) +C_O1_I4(r, r, rI, rM, rM) C_O2_I4(r, r, rZ, rZ, rM, rM) diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_zbb; #endif /* optional instructions */ -#define TCG_TARGET_HAS_movcond_i32 0 +#define TCG_TARGET_HAS_movcond_i32 1 #define TCG_TARGET_HAS_div_i32 1 #define TCG_TARGET_HAS_rem_i32 1 #define TCG_TARGET_HAS_div2_i32 0 @@ -XXX,XX +XXX,XX @@ extern bool have_zbb; #define TCG_TARGET_HAS_setcond2 1 #define TCG_TARGET_HAS_qemu_st8_i32 0 -#define TCG_TARGET_HAS_movcond_i64 0 +#define TCG_TARGET_HAS_movcond_i64 1 #define TCG_TARGET_HAS_div_i64 1 #define TCG_TARGET_HAS_rem_i64 1 #define TCG_TARGET_HAS_div2_i64 0 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) } /* * Sign extended from 12 bits, +/- matching: [-0x7ff, 0x7ff]. - * Used by addsub2, which may need the negative operation, + * Used by addsub2 and movcond, which may need the negative value, * and requires the modified constant to be representable. */ if ((ct & TCG_CT_CONST_M12) && val >= -0x7ff && val <= 0x7ff) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret, } } +static void tcg_out_movcond_zicond(TCGContext *s, TCGReg ret, TCGReg test_ne, + int val1, bool c_val1, + int val2, bool c_val2) +{ + if (val1 == 0) { + if (c_val2) { + tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_TMP1, val2); + val2 = TCG_REG_TMP1; + } + tcg_out_opc_reg(s, OPC_CZERO_NEZ, ret, val2, test_ne); + return; + } + + if (val2 == 0) { + if (c_val1) { + tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_TMP1, val1); + val1 = TCG_REG_TMP1; + } + tcg_out_opc_reg(s, OPC_CZERO_EQZ, ret, val1, test_ne); + return; + } + + if (c_val2) { + if (c_val1) { + tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_TMP1, val1 - val2); + } else { + tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_TMP1, val1, -val2); + } + tcg_out_opc_reg(s, OPC_CZERO_EQZ, ret, TCG_REG_TMP1, test_ne); + tcg_out_opc_imm(s, OPC_ADDI, ret, ret, val2); + return; + } + + if (c_val1) { + tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_TMP1, val2, -val1); + tcg_out_opc_reg(s, OPC_CZERO_NEZ, ret, TCG_REG_TMP1, test_ne); + tcg_out_opc_imm(s, OPC_ADDI, ret, ret, val1); + return; + } + + tcg_out_opc_reg(s, OPC_CZERO_NEZ, TCG_REG_TMP1, val2, test_ne); + tcg_out_opc_reg(s, OPC_CZERO_EQZ, TCG_REG_TMP0, val1, test_ne); + tcg_out_opc_reg(s, OPC_OR, ret, TCG_REG_TMP0, TCG_REG_TMP1); +} + +static void tcg_out_movcond_br1(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg cmp1, TCGReg cmp2, + int val, bool c_val) +{ + RISCVInsn op; + int disp = 8; + + tcg_debug_assert((unsigned)cond < ARRAY_SIZE(tcg_brcond_to_riscv)); + op = tcg_brcond_to_riscv[cond].op; + tcg_debug_assert(op != 0); + + if (tcg_brcond_to_riscv[cond].swap) { + tcg_out_opc_branch(s, op, cmp2, cmp1, disp); + } else { + tcg_out_opc_branch(s, op, cmp1, cmp2, disp); + } + if (c_val) { + tcg_out_opc_imm(s, OPC_ADDI, ret, TCG_REG_ZERO, val); + } else { + tcg_out_opc_imm(s, OPC_ADDI, ret, val, 0); + } +} + +static void tcg_out_movcond_br2(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg cmp1, TCGReg cmp2, + int val1, bool c_val1, + int val2, bool c_val2) +{ + TCGReg tmp; + + /* TCG optimizer reorders to prefer ret matching val2. */ + if (!c_val2 && ret == val2) { + cond = tcg_invert_cond(cond); + tcg_out_movcond_br1(s, cond, ret, cmp1, cmp2, val1, c_val1); + return; + } + + if (!c_val1 && ret == val1) { + tcg_out_movcond_br1(s, cond, ret, cmp1, cmp2, val2, c_val2); + return; + } + + tmp = (ret == cmp1 || ret == cmp2 ? TCG_REG_TMP1 : ret); + if (c_val1) { + tcg_out_movi(s, TCG_TYPE_REG, tmp, val1); + } else { + tcg_out_mov(s, TCG_TYPE_REG, tmp, val1); + } + tcg_out_movcond_br1(s, cond, tmp, cmp1, cmp2, val2, c_val2); + tcg_out_mov(s, TCG_TYPE_REG, ret, tmp); +} + +static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg cmp1, int cmp2, bool c_cmp2, + TCGReg val1, bool c_val1, + TCGReg val2, bool c_val2) +{ + int tmpflags; + TCGReg t; + + if (!have_zicond && (!c_cmp2 || cmp2 == 0)) { + tcg_out_movcond_br2(s, cond, ret, cmp1, cmp2, + val1, c_val1, val2, c_val2); + return; + } + + tmpflags = tcg_out_setcond_int(s, cond, TCG_REG_TMP0, cmp1, cmp2, c_cmp2); + t = tmpflags & ~SETCOND_FLAGS; + + if (have_zicond) { + if (tmpflags & SETCOND_INV) { + tcg_out_movcond_zicond(s, ret, t, val2, c_val2, val1, c_val1); + } else { + tcg_out_movcond_zicond(s, ret, t, val1, c_val1, val2, c_val2); + } + } else { + cond = tmpflags & SETCOND_INV ? TCG_COND_EQ : TCG_COND_NE; + tcg_out_movcond_br2(s, cond, ret, t, TCG_REG_ZERO, + val1, c_val1, val2, c_val2); + } +} + static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) { TCGReg link = tail ? TCG_REG_ZERO : TCG_REG_RA; @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_setcond(s, args[3], a0, a1, a2, c2); break; + case INDEX_op_movcond_i32: + case INDEX_op_movcond_i64: + tcg_out_movcond(s, args[5], a0, a1, a2, c2, + args[3], const_args[3], args[4], const_args[4]); + break; + case INDEX_op_qemu_ld_a32_i32: case INDEX_op_qemu_ld_a64_i32: tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I32); @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_brcond_i64: return C_O0_I2(rZ, rZ); + case INDEX_op_movcond_i32: + case INDEX_op_movcond_i64: + return C_O1_I4(r, r, rI, rM, rM); + case INDEX_op_add2_i32: case INDEX_op_add2_i64: case INDEX_op_sub2_i32: -- 2.34.1 Acked-by: Alistair Francis <alistair.francis@wdc.com> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target-con-set.h | 1 + tcg/riscv/tcg-target.h | 8 ++++---- tcg/riscv/tcg-target.c.inc | 35 ++++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/tcg/riscv/tcg-target-con-set.h b/tcg/riscv/tcg-target-con-set.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target-con-set.h +++ b/tcg/riscv/tcg-target-con-set.h @@ -XXX,XX +XXX,XX @@ C_O1_I2(r, r, rI) C_O1_I2(r, r, rJ) C_O1_I2(r, rZ, rN) C_O1_I2(r, rZ, rZ) +C_N1_I2(r, r, rM) C_O1_I4(r, r, rI, rM, rM) C_O2_I4(r, r, rZ, rZ, rM, rM) diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_zbb; #define TCG_TARGET_HAS_eqv_i32 have_zbb #define TCG_TARGET_HAS_nand_i32 0 #define TCG_TARGET_HAS_nor_i32 0 -#define TCG_TARGET_HAS_clz_i32 0 -#define TCG_TARGET_HAS_ctz_i32 0 +#define TCG_TARGET_HAS_clz_i32 have_zbb +#define TCG_TARGET_HAS_ctz_i32 have_zbb #define TCG_TARGET_HAS_ctpop_i32 have_zbb #define TCG_TARGET_HAS_brcond2 1 #define TCG_TARGET_HAS_setcond2 1 @@ -XXX,XX +XXX,XX @@ extern bool have_zbb; #define TCG_TARGET_HAS_eqv_i64 have_zbb #define TCG_TARGET_HAS_nand_i64 0 #define TCG_TARGET_HAS_nor_i64 0 -#define TCG_TARGET_HAS_clz_i64 0 -#define TCG_TARGET_HAS_ctz_i64 0 +#define TCG_TARGET_HAS_clz_i64 have_zbb +#define TCG_TARGET_HAS_ctz_i64 have_zbb #define TCG_TARGET_HAS_ctpop_i64 have_zbb #define TCG_TARGET_HAS_add2_i64 1 #define TCG_TARGET_HAS_sub2_i64 1 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret, } } +static void tcg_out_cltz(TCGContext *s, TCGType type, RISCVInsn insn, + TCGReg ret, TCGReg src1, int src2, bool c_src2) +{ + tcg_out_opc_imm(s, insn, ret, src1, 0); + + if (!c_src2 || src2 != (type == TCG_TYPE_I32 ? 32 : 64)) { + /* + * The requested zero result does not match the insn, so adjust. + * Note that constraints put 'ret' in a new register, so the + * computation above did not clobber either 'src1' or 'src2'. + */ + tcg_out_movcond(s, TCG_COND_EQ, ret, src1, 0, true, + src2, c_src2, ret, false); + } +} + static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) { TCGReg link = tail ? TCG_REG_ZERO : TCG_REG_RA; @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_opc_imm(s, OPC_CPOP, a0, a1, 0); break; + case INDEX_op_clz_i32: + tcg_out_cltz(s, TCG_TYPE_I32, OPC_CLZW, a0, a1, a2, c2); + break; + case INDEX_op_clz_i64: + tcg_out_cltz(s, TCG_TYPE_I64, OPC_CLZ, a0, a1, a2, c2); + break; + case INDEX_op_ctz_i32: + tcg_out_cltz(s, TCG_TYPE_I32, OPC_CTZW, a0, a1, a2, c2); + break; + case INDEX_op_ctz_i64: + tcg_out_cltz(s, TCG_TYPE_I64, OPC_CTZ, a0, a1, a2, c2); + break; + case INDEX_op_add2_i32: tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], const_args[4], const_args[5], false, true); @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_rotr_i64: return C_O1_I2(r, r, ri); + case INDEX_op_clz_i32: + case INDEX_op_clz_i64: + case INDEX_op_ctz_i32: + case INDEX_op_ctz_i64: + return C_N1_I2(r, r, rM); + case INDEX_op_brcond_i32: case INDEX_op_brcond_i64: return C_O0_I2(rZ, rZ); -- 2.34.1