The following changes since commit 36eae3a732a1f2aa81391e871ac0e9bb3233e7d7: Merge remote-tracking branch 'remotes/dgilbert-gitlab/tags/pull-migration-20220302b' into staging (2022-03-02 20:55:48 +0000) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20220303 for you to fetch changes up to f23e6de25c31cadd9a3b7122f9384e6b259ce37f: tcg/loongarch64: Support TCG_TARGET_SIGNED_ADDR32 (2022-03-03 10:47:20 -1000) ---------------------------------------------------------------- Reorder do_constant_folding_cond test to satisfy valgrind. Fix value of MAX_OPC_PARAM_IARGS. Add opcodes for vector nand, nor, eqv. Support vector nand, nor, eqv on PPC and S390X hosts. Support AVX512VL, AVX512BW, AVX512DQ, and AVX512VBMI2. Support 32-bit guest addresses as signed values. ---------------------------------------------------------------- Alex Bennée (1): tcg/optimize: only read val after const check Richard Henderson (28): tcg: Add opcodes for vector nand, nor, eqv tcg/ppc: Implement vector NAND, NOR, EQV tcg/s390x: Implement vector NAND, NOR, EQV tcg/i386: Detect AVX512 tcg/i386: Add tcg_out_evex_opc tcg/i386: Use tcg_can_emit_vec_op in expand_vec_cmp_noinv tcg/i386: Implement avx512 variable shifts tcg/i386: Implement avx512 scalar shift tcg/i386: Implement avx512 immediate sari shift tcg/i386: Implement avx512 immediate rotate tcg/i386: Implement avx512 variable rotate tcg/i386: Support avx512vbmi2 vector shift-double instructions tcg/i386: Expand vector word rotate as avx512vbmi2 shift-double tcg/i386: Remove rotls_vec from tcg_target_op_def tcg/i386: Expand scalar rotate with avx512 insns tcg/i386: Implement avx512 min/max/abs tcg/i386: Implement avx512 multiply tcg/i386: Implement more logical operations for avx512 tcg/i386: Implement bitsel for avx512 tcg: Add TCG_TARGET_SIGNED_ADDR32 accel/tcg: Split out g2h_tlbe accel/tcg: Support TCG_TARGET_SIGNED_ADDR32 for softmmu accel/tcg: Add guest_base_signed_addr32 for user-only linux-user: Support TCG_TARGET_SIGNED_ADDR32 tcg/aarch64: Support TCG_TARGET_SIGNED_ADDR32 tcg/mips: Support TCG_TARGET_SIGNED_ADDR32 tcg/riscv: Support TCG_TARGET_SIGNED_ADDR32 tcg/loongarch64: Support TCG_TARGET_SIGNED_ADDR32 Ziqiao Kong (1): tcg: Set MAX_OPC_PARAM_IARGS to 7 include/exec/cpu-all.h | 20 +- include/exec/cpu_ldst.h | 3 +- include/qemu/cpuid.h | 20 +- include/tcg/tcg-opc.h | 3 + include/tcg/tcg.h | 5 +- tcg/aarch64/tcg-target-sa32.h | 7 + tcg/aarch64/tcg-target.h | 3 + tcg/arm/tcg-target-sa32.h | 1 + tcg/arm/tcg-target.h | 3 + tcg/i386/tcg-target-con-set.h | 1 + tcg/i386/tcg-target-sa32.h | 1 + tcg/i386/tcg-target.h | 17 +- tcg/i386/tcg-target.opc.h | 3 + tcg/loongarch64/tcg-target-sa32.h | 1 + tcg/mips/tcg-target-sa32.h | 9 + tcg/ppc/tcg-target-sa32.h | 1 + tcg/ppc/tcg-target.h | 3 + tcg/riscv/tcg-target-sa32.h | 5 + tcg/s390x/tcg-target-sa32.h | 1 + tcg/s390x/tcg-target.h | 3 + tcg/sparc/tcg-target-sa32.h | 1 + tcg/tci/tcg-target-sa32.h | 1 + accel/tcg/cputlb.c | 36 ++-- bsd-user/main.c | 4 + linux-user/elfload.c | 62 ++++-- linux-user/main.c | 3 + tcg/optimize.c | 20 +- tcg/tcg-op-vec.c | 27 ++- tcg/tcg.c | 10 + tcg/aarch64/tcg-target.c.inc | 81 +++++--- tcg/i386/tcg-target.c.inc | 387 +++++++++++++++++++++++++++++++------- tcg/loongarch64/tcg-target.c.inc | 15 +- tcg/mips/tcg-target.c.inc | 10 +- tcg/ppc/tcg-target.c.inc | 15 ++ tcg/riscv/tcg-target.c.inc | 8 +- tcg/s390x/tcg-target.c.inc | 17 ++ tcg/tci/tcg-target.c.inc | 2 +- 37 files changed, 640 insertions(+), 169 deletions(-) create mode 100644 tcg/aarch64/tcg-target-sa32.h create mode 100644 tcg/arm/tcg-target-sa32.h create mode 100644 tcg/i386/tcg-target-sa32.h create mode 100644 tcg/loongarch64/tcg-target-sa32.h create mode 100644 tcg/mips/tcg-target-sa32.h create mode 100644 tcg/ppc/tcg-target-sa32.h create mode 100644 tcg/riscv/tcg-target-sa32.h create mode 100644 tcg/s390x/tcg-target-sa32.h create mode 100644 tcg/sparc/tcg-target-sa32.h create mode 100644 tcg/tci/tcg-target-sa32.h From: Alex Bennée <alex.bennee@linaro.org> valgrind pointed out that arg_info()->val can be undefined which will be the case if the arguments are not constant. The ordering of the checks will have ensured we never relied on an undefined value but for the sake of completeness re-order the code to be clear. Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> Message-Id: <20220209112142.3367525-1-alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool do_constant_folding_cond_eq(TCGCond c) static int do_constant_folding_cond(TCGType type, TCGArg x, TCGArg y, TCGCond c) { - uint64_t xv = arg_info(x)->val; - uint64_t yv = arg_info(y)->val; - if (arg_is_const(x) && arg_is_const(y)) { + uint64_t xv = arg_info(x)->val; + uint64_t yv = arg_info(y)->val; + switch (type) { case TCG_TYPE_I32: return do_constant_folding_cond_32(xv, yv, c); @@ -XXX,XX +XXX,XX @@ static int do_constant_folding_cond(TCGType type, TCGArg x, } } else if (args_are_copies(x, y)) { return do_constant_folding_cond_eq(c); - } else if (arg_is_const(y) && yv == 0) { + } else if (arg_is_const(y) && arg_info(y)->val == 0) { switch (c) { case TCG_COND_LTU: return 0; -- 2.25.1 From: Ziqiao Kong <ziqiaokong@gmail.com> The last entry of DEF_HELPERS_FLAGS_n is DEF_HELPER_FLAGS_7 and thus the MAX_OPC_PARAM_IARGS should be 7. Reviewed-by: Taylor Simpson <tsimpson@quicinc.com> Signed-off-by: Ziqiao Kong <ziqiaokong@gmail.com> Message-Id: <20220227113127.414533-2-ziqiaokong@gmail.com> Fixes: e6cadf49c3d ("tcg: Add support for a helper with 7 arguments") Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg.h | 2 +- tcg/tci/tcg-target.c.inc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ #else #define MAX_OPC_PARAM_PER_ARG 1 #endif -#define MAX_OPC_PARAM_IARGS 6 +#define MAX_OPC_PARAM_IARGS 7 #define MAX_OPC_PARAM_OARGS 1 #define MAX_OPC_PARAM_ARGS (MAX_OPC_PARAM_IARGS + MAX_OPC_PARAM_OARGS) diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const int tcg_target_reg_alloc_order[] = { TCG_REG_R0, }; -#if MAX_OPC_PARAM_IARGS != 6 +#if MAX_OPC_PARAM_IARGS != 7 # error Fix needed, number of supported input arguments changed! #endif -- 2.25.1 We've had placeholders for these opcodes for a while, and should have support on ppc, s390x and avx512 hosts. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg-opc.h | 3 +++ include/tcg/tcg.h | 3 +++ tcg/aarch64/tcg-target.h | 3 +++ tcg/arm/tcg-target.h | 3 +++ tcg/i386/tcg-target.h | 3 +++ tcg/ppc/tcg-target.h | 3 +++ tcg/s390x/tcg-target.h | 3 +++ tcg/optimize.c | 12 ++++++------ tcg/tcg-op-vec.c | 27 ++++++++++++++++++--------- tcg/tcg.c | 6 ++++++ 10 files changed, 51 insertions(+), 15 deletions(-) diff --git a/include/tcg/tcg-opc.h b/include/tcg/tcg-opc.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg-opc.h +++ b/include/tcg/tcg-opc.h @@ -XXX,XX +XXX,XX @@ DEF(or_vec, 1, 2, 0, IMPLVEC) DEF(xor_vec, 1, 2, 0, IMPLVEC) DEF(andc_vec, 1, 2, 0, IMPLVEC | IMPL(TCG_TARGET_HAS_andc_vec)) DEF(orc_vec, 1, 2, 0, IMPLVEC | IMPL(TCG_TARGET_HAS_orc_vec)) +DEF(nand_vec, 1, 2, 0, IMPLVEC | IMPL(TCG_TARGET_HAS_nand_vec)) +DEF(nor_vec, 1, 2, 0, IMPLVEC | IMPL(TCG_TARGET_HAS_nor_vec)) +DEF(eqv_vec, 1, 2, 0, IMPLVEC | IMPL(TCG_TARGET_HAS_eqv_vec)) DEF(not_vec, 1, 1, 0, IMPLVEC | IMPL(TCG_TARGET_HAS_not_vec)) DEF(shli_vec, 1, 1, 1, IMPLVEC | IMPL(TCG_TARGET_HAS_shi_vec)) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ typedef uint64_t TCGRegSet; #define TCG_TARGET_HAS_not_vec 0 #define TCG_TARGET_HAS_andc_vec 0 #define TCG_TARGET_HAS_orc_vec 0 +#define TCG_TARGET_HAS_nand_vec 0 +#define TCG_TARGET_HAS_nor_vec 0 +#define TCG_TARGET_HAS_eqv_vec 0 #define TCG_TARGET_HAS_roti_vec 0 #define TCG_TARGET_HAS_rots_vec 0 #define TCG_TARGET_HAS_rotv_vec 0 diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_HAS_andc_vec 1 #define TCG_TARGET_HAS_orc_vec 1 +#define TCG_TARGET_HAS_nand_vec 0 +#define TCG_TARGET_HAS_nor_vec 0 +#define TCG_TARGET_HAS_eqv_vec 0 #define TCG_TARGET_HAS_not_vec 1 #define TCG_TARGET_HAS_neg_vec 1 #define TCG_TARGET_HAS_abs_vec 1 diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_neon_instructions; #define TCG_TARGET_HAS_andc_vec 1 #define TCG_TARGET_HAS_orc_vec 1 +#define TCG_TARGET_HAS_nand_vec 0 +#define TCG_TARGET_HAS_nor_vec 0 +#define TCG_TARGET_HAS_eqv_vec 0 #define TCG_TARGET_HAS_not_vec 1 #define TCG_TARGET_HAS_neg_vec 1 #define TCG_TARGET_HAS_abs_vec 1 diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_movbe; #define TCG_TARGET_HAS_andc_vec 1 #define TCG_TARGET_HAS_orc_vec 0 +#define TCG_TARGET_HAS_nand_vec 0 +#define TCG_TARGET_HAS_nor_vec 0 +#define TCG_TARGET_HAS_eqv_vec 0 #define TCG_TARGET_HAS_not_vec 0 #define TCG_TARGET_HAS_neg_vec 0 #define TCG_TARGET_HAS_abs_vec 1 diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_vsx; #define TCG_TARGET_HAS_andc_vec 1 #define TCG_TARGET_HAS_orc_vec have_isa_2_07 +#define TCG_TARGET_HAS_nand_vec 0 +#define TCG_TARGET_HAS_nor_vec 0 +#define TCG_TARGET_HAS_eqv_vec 0 #define TCG_TARGET_HAS_not_vec 1 #define TCG_TARGET_HAS_neg_vec have_isa_3_00 #define TCG_TARGET_HAS_abs_vec 0 diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern uint64_t s390_facilities[3]; #define TCG_TARGET_HAS_andc_vec 1 #define TCG_TARGET_HAS_orc_vec HAVE_FACILITY(VECTOR_ENH1) +#define TCG_TARGET_HAS_nand_vec 0 +#define TCG_TARGET_HAS_nor_vec 0 +#define TCG_TARGET_HAS_eqv_vec 0 #define TCG_TARGET_HAS_not_vec 1 #define TCG_TARGET_HAS_neg_vec 1 #define TCG_TARGET_HAS_abs_vec 1 diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y) CASE_OP_32_64_VEC(orc): return x | ~y; - CASE_OP_32_64(eqv): + CASE_OP_32_64_VEC(eqv): return ~(x ^ y); - CASE_OP_32_64(nand): + CASE_OP_32_64_VEC(nand): return ~(x & y); - CASE_OP_32_64(nor): + CASE_OP_32_64_VEC(nor): return ~(x | y); case INDEX_op_clz_i32: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_dup2_vec: done = fold_dup2(&ctx, op); break; - CASE_OP_32_64(eqv): + CASE_OP_32_64_VEC(eqv): done = fold_eqv(&ctx, op); break; CASE_OP_32_64(extract): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(mulu2): done = fold_multiply2(&ctx, op); break; - CASE_OP_32_64(nand): + CASE_OP_32_64_VEC(nand): done = fold_nand(&ctx, op); break; CASE_OP_32_64(neg): done = fold_neg(&ctx, op); break; - CASE_OP_32_64(nor): + CASE_OP_32_64_VEC(nor): done = fold_nor(&ctx, op); break; CASE_OP_32_64_VEC(not): diff --git a/tcg/tcg-op-vec.c b/tcg/tcg-op-vec.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-op-vec.c +++ b/tcg/tcg-op-vec.c @@ -XXX,XX +XXX,XX @@ void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) { - /* TODO: Add TCG_TARGET_HAS_nand_vec when adding a backend supports it. */ - tcg_gen_and_vec(0, r, a, b); - tcg_gen_not_vec(0, r, r); + if (TCG_TARGET_HAS_nand_vec) { + vec_gen_op3(INDEX_op_nand_vec, 0, r, a, b); + } else { + tcg_gen_and_vec(0, r, a, b); + tcg_gen_not_vec(0, r, r); + } } void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) { - /* TODO: Add TCG_TARGET_HAS_nor_vec when adding a backend supports it. */ - tcg_gen_or_vec(0, r, a, b); - tcg_gen_not_vec(0, r, r); + if (TCG_TARGET_HAS_nor_vec) { + vec_gen_op3(INDEX_op_nor_vec, 0, r, a, b); + } else { + tcg_gen_or_vec(0, r, a, b); + tcg_gen_not_vec(0, r, r); + } } void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) { - /* TODO: Add TCG_TARGET_HAS_eqv_vec when adding a backend supports it. */ - tcg_gen_xor_vec(0, r, a, b); - tcg_gen_not_vec(0, r, r); + if (TCG_TARGET_HAS_eqv_vec) { + vec_gen_op3(INDEX_op_eqv_vec, 0, r, a, b); + } else { + tcg_gen_xor_vec(0, r, a, b); + tcg_gen_not_vec(0, r, r); + } } static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ bool tcg_op_supported(TCGOpcode op) return have_vec && TCG_TARGET_HAS_andc_vec; case INDEX_op_orc_vec: return have_vec && TCG_TARGET_HAS_orc_vec; + case INDEX_op_nand_vec: + return have_vec && TCG_TARGET_HAS_nand_vec; + case INDEX_op_nor_vec: + return have_vec && TCG_TARGET_HAS_nor_vec; + case INDEX_op_eqv_vec: + return have_vec && TCG_TARGET_HAS_eqv_vec; case INDEX_op_mul_vec: return have_vec && TCG_TARGET_HAS_mul_vec; case INDEX_op_shli_vec: -- 2.25.1 Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/ppc/tcg-target.h | 6 +++--- tcg/ppc/tcg-target.c.inc | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_vsx; #define TCG_TARGET_HAS_andc_vec 1 #define TCG_TARGET_HAS_orc_vec have_isa_2_07 -#define TCG_TARGET_HAS_nand_vec 0 -#define TCG_TARGET_HAS_nor_vec 0 -#define TCG_TARGET_HAS_eqv_vec 0 +#define TCG_TARGET_HAS_nand_vec have_isa_2_07 +#define TCG_TARGET_HAS_nor_vec 1 +#define TCG_TARGET_HAS_eqv_vec have_isa_2_07 #define TCG_TARGET_HAS_not_vec 1 #define TCG_TARGET_HAS_neg_vec have_isa_3_00 #define TCG_TARGET_HAS_abs_vec 0 diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) case INDEX_op_xor_vec: case INDEX_op_andc_vec: case INDEX_op_not_vec: + case INDEX_op_nor_vec: + case INDEX_op_eqv_vec: + case INDEX_op_nand_vec: return 1; case INDEX_op_orc_vec: return have_isa_2_07; @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, case INDEX_op_orc_vec: insn = VORC; break; + case INDEX_op_nand_vec: + insn = VNAND; + break; + case INDEX_op_nor_vec: + insn = VNOR; + break; + case INDEX_op_eqv_vec: + insn = VEQV; + break; case INDEX_op_cmp_vec: switch (args[3]) { @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_xor_vec: case INDEX_op_andc_vec: case INDEX_op_orc_vec: + case INDEX_op_nor_vec: + case INDEX_op_eqv_vec: + case INDEX_op_nand_vec: case INDEX_op_cmp_vec: case INDEX_op_ssadd_vec: case INDEX_op_sssub_vec: -- 2.25.1 Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/s390x/tcg-target.h | 6 +++--- tcg/s390x/tcg-target.c.inc | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern uint64_t s390_facilities[3]; #define TCG_TARGET_HAS_andc_vec 1 #define TCG_TARGET_HAS_orc_vec HAVE_FACILITY(VECTOR_ENH1) -#define TCG_TARGET_HAS_nand_vec 0 -#define TCG_TARGET_HAS_nor_vec 0 -#define TCG_TARGET_HAS_eqv_vec 0 +#define TCG_TARGET_HAS_nand_vec HAVE_FACILITY(VECTOR_ENH1) +#define TCG_TARGET_HAS_nor_vec 1 +#define TCG_TARGET_HAS_eqv_vec HAVE_FACILITY(VECTOR_ENH1) #define TCG_TARGET_HAS_not_vec 1 #define TCG_TARGET_HAS_neg_vec 1 #define TCG_TARGET_HAS_abs_vec 1 diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ typedef enum S390Opcode { VRRc_VMXL = 0xe7fd, VRRc_VN = 0xe768, VRRc_VNC = 0xe769, + VRRc_VNN = 0xe76e, VRRc_VNO = 0xe76b, + VRRc_VNX = 0xe76c, VRRc_VO = 0xe76a, VRRc_VOC = 0xe76f, VRRc_VPKS = 0xe797, /* we leave the m5 cs field 0 */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, case INDEX_op_xor_vec: tcg_out_insn(s, VRRc, VX, a0, a1, a2, 0); break; + case INDEX_op_nand_vec: + tcg_out_insn(s, VRRc, VNN, a0, a1, a2, 0); + break; + case INDEX_op_nor_vec: + tcg_out_insn(s, VRRc, VNO, a0, a1, a2, 0); + break; + case INDEX_op_eqv_vec: + tcg_out_insn(s, VRRc, VNX, a0, a1, a2, 0); + break; case INDEX_op_shli_vec: tcg_out_insn(s, VRSa, VESL, a0, a2, TCG_REG_NONE, a1, vece); @@ -XXX,XX +XXX,XX @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) case INDEX_op_and_vec: case INDEX_op_andc_vec: case INDEX_op_bitsel_vec: + case INDEX_op_eqv_vec: + case INDEX_op_nand_vec: case INDEX_op_neg_vec: + case INDEX_op_nor_vec: case INDEX_op_not_vec: case INDEX_op_or_vec: case INDEX_op_orc_vec: @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_or_vec: case INDEX_op_orc_vec: case INDEX_op_xor_vec: + case INDEX_op_nand_vec: + case INDEX_op_nor_vec: + case INDEX_op_eqv_vec: case INDEX_op_cmp_vec: case INDEX_op_mul_vec: case INDEX_op_rotlv_vec: -- 2.25.1 There are some operation sizes in some subsets of AVX512 that are missing from previous iterations of AVX. Detect them. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/qemu/cpuid.h | 20 +++++++++++++++++--- tcg/i386/tcg-target.h | 4 ++++ tcg/i386/tcg-target.c.inc | 24 ++++++++++++++++++++++-- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/include/qemu/cpuid.h b/include/qemu/cpuid.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/cpuid.h +++ b/include/qemu/cpuid.h @@ -XXX,XX +XXX,XX @@ #ifndef bit_AVX2 #define bit_AVX2 (1 << 5) #endif -#ifndef bit_AVX512F -#define bit_AVX512F (1 << 16) -#endif #ifndef bit_BMI2 #define bit_BMI2 (1 << 8) #endif +#ifndef bit_AVX512F +#define bit_AVX512F (1 << 16) +#endif +#ifndef bit_AVX512DQ +#define bit_AVX512DQ (1 << 17) +#endif +#ifndef bit_AVX512BW +#define bit_AVX512BW (1 << 30) +#endif +#ifndef bit_AVX512VL +#define bit_AVX512VL (1u << 31) +#endif + +/* Leaf 7, %ecx */ +#ifndef bit_AVX512VBMI2 +#define bit_AVX512VBMI2 (1 << 6) +#endif /* Leaf 0x80000001, %ecx */ #ifndef bit_LZCNT diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_bmi1; extern bool have_popcnt; extern bool have_avx1; extern bool have_avx2; +extern bool have_avx512bw; +extern bool have_avx512dq; +extern bool have_avx512vbmi2; +extern bool have_avx512vl; extern bool have_movbe; /* optional instructions */ diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ bool have_bmi1; bool have_popcnt; bool have_avx1; bool have_avx2; +bool have_avx512bw; +bool have_avx512dq; +bool have_avx512vbmi2; +bool have_avx512vl; bool have_movbe; #ifdef CONFIG_CPUID_H @@ -XXX,XX +XXX,XX @@ static void tcg_out_nop_fill(tcg_insn_unit *p, int count) static void tcg_target_init(TCGContext *s) { #ifdef CONFIG_CPUID_H - unsigned a, b, c, d, b7 = 0; + unsigned a, b, c, d, b7 = 0, c7 = 0; unsigned max = __get_cpuid_max(0, 0); if (max >= 7) { /* BMI1 is available on AMD Piledriver and Intel Haswell CPUs. */ - __cpuid_count(7, 0, a, b7, c, d); + __cpuid_count(7, 0, a, b7, c7, d); have_bmi1 = (b7 & bit_BMI) != 0; have_bmi2 = (b7 & bit_BMI2) != 0; } @@ -XXX,XX +XXX,XX @@ static void tcg_target_init(TCGContext *s) if ((xcrl & 6) == 6) { have_avx1 = (c & bit_AVX) != 0; have_avx2 = (b7 & bit_AVX2) != 0; + + /* + * There are interesting instructions in AVX512, so long + * as we have AVX512VL, which indicates support for EVEX + * on sizes smaller than 512 bits. We are required to + * check that OPMASK and all extended ZMM state are enabled + * even if we're not using them -- the insns will fault. + */ + if ((xcrl & 0xe0) == 0xe0 + && (b7 & bit_AVX512F) + && (b7 & bit_AVX512VL)) { + have_avx512vl = true; + have_avx512bw = (b7 & bit_AVX512BW) != 0; + have_avx512dq = (b7 & bit_AVX512DQ) != 0; + have_avx512vbmi2 = (c7 & bit_AVX512VBMI2) != 0; + } } } } -- 2.25.1 The evex encoding is added here, for use in a subsequent patch. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 51 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) #define P_SIMDF3 0x20000 /* 0xf3 opcode prefix */ #define P_SIMDF2 0x40000 /* 0xf2 opcode prefix */ #define P_VEXL 0x80000 /* Set VEX.L = 1 */ +#define P_EVEX 0x100000 /* Requires EVEX encoding */ #define OPC_ARITH_EvIz (0x81) #define OPC_ARITH_EvIb (0x83) @@ -XXX,XX +XXX,XX @@ static void tcg_out_vex_opc(TCGContext *s, int opc, int r, int v, tcg_out8(s, opc); } +static void tcg_out_evex_opc(TCGContext *s, int opc, int r, int v, + int rm, int index) +{ + /* The entire 4-byte evex prefix; with R' and V' set. */ + uint32_t p = 0x08041062; + int mm, pp; + + tcg_debug_assert(have_avx512vl); + + /* EVEX.mm */ + if (opc & P_EXT3A) { + mm = 3; + } else if (opc & P_EXT38) { + mm = 2; + } else if (opc & P_EXT) { + mm = 1; + } else { + g_assert_not_reached(); + } + + /* EVEX.pp */ + if (opc & P_DATA16) { + pp = 1; /* 0x66 */ + } else if (opc & P_SIMDF3) { + pp = 2; /* 0xf3 */ + } else if (opc & P_SIMDF2) { + pp = 3; /* 0xf2 */ + } else { + pp = 0; + } + + p = deposit32(p, 8, 2, mm); + p = deposit32(p, 13, 1, (rm & 8) == 0); /* EVEX.RXB.B */ + p = deposit32(p, 14, 1, (index & 8) == 0); /* EVEX.RXB.X */ + p = deposit32(p, 15, 1, (r & 8) == 0); /* EVEX.RXB.R */ + p = deposit32(p, 16, 2, pp); + p = deposit32(p, 19, 4, ~v); + p = deposit32(p, 23, 1, (opc & P_VEXW) != 0); + p = deposit32(p, 29, 2, (opc & P_VEXL) != 0); + + tcg_out32(s, p); + tcg_out8(s, opc); +} + static void tcg_out_vex_modrm(TCGContext *s, int opc, int r, int v, int rm) { - tcg_out_vex_opc(s, opc, r, v, rm, 0); + if (opc & P_EVEX) { + tcg_out_evex_opc(s, opc, r, v, rm, 0); + } else { + tcg_out_vex_opc(s, opc, r, v, rm, 0); + } tcg_out8(s, 0xc0 | (LOWREGMASK(r) << 3) | LOWREGMASK(rm)); } -- 2.25.1 The condition for UMIN/UMAX availability is about to change; use the canonical version. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool expand_vec_cmp_noinv(TCGType type, unsigned vece, TCGv_vec v0, fixup = NEED_SWAP | NEED_INV; break; case TCG_COND_LEU: - if (vece <= MO_32) { + if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece)) { fixup = NEED_UMIN; } else { fixup = NEED_BIAS | NEED_INV; } break; case TCG_COND_GTU: - if (vece <= MO_32) { + if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece)) { fixup = NEED_UMIN | NEED_INV; } else { fixup = NEED_BIAS; } break; case TCG_COND_GEU: - if (vece <= MO_32) { + if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece)) { fixup = NEED_UMAX; } else { fixup = NEED_BIAS | NEED_SWAP | NEED_INV; } break; case TCG_COND_LTU: - if (vece <= MO_32) { + if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece)) { fixup = NEED_UMAX | NEED_INV; } else { fixup = NEED_BIAS | NEED_SWAP; -- 2.25.1 AVX512VL has VPSRAVQ, and AVX512BW has VPSLLVW, VPSRAVW, VPSRLVW. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) #define OPC_VPBROADCASTQ (0x59 | P_EXT38 | P_DATA16) #define OPC_VPERMQ (0x00 | P_EXT3A | P_DATA16 | P_VEXW) #define OPC_VPERM2I128 (0x46 | P_EXT3A | P_DATA16 | P_VEXL) +#define OPC_VPSLLVW (0x12 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) #define OPC_VPSLLVD (0x47 | P_EXT38 | P_DATA16) #define OPC_VPSLLVQ (0x47 | P_EXT38 | P_DATA16 | P_VEXW) +#define OPC_VPSRAVW (0x11 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) #define OPC_VPSRAVD (0x46 | P_EXT38 | P_DATA16) +#define OPC_VPSRAVQ (0x46 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) +#define OPC_VPSRLVW (0x10 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) #define OPC_VPSRLVD (0x45 | P_EXT38 | P_DATA16) #define OPC_VPSRLVQ (0x45 | P_EXT38 | P_DATA16 | P_VEXW) #define OPC_VZEROUPPER (0x77 | P_EXT) @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, OPC_PMAXUB, OPC_PMAXUW, OPC_PMAXUD, OPC_UD2 }; static int const shlv_insn[4] = { - /* TODO: AVX512 adds support for MO_16. */ - OPC_UD2, OPC_UD2, OPC_VPSLLVD, OPC_VPSLLVQ + OPC_UD2, OPC_VPSLLVW, OPC_VPSLLVD, OPC_VPSLLVQ }; static int const shrv_insn[4] = { - /* TODO: AVX512 adds support for MO_16. */ - OPC_UD2, OPC_UD2, OPC_VPSRLVD, OPC_VPSRLVQ + OPC_UD2, OPC_VPSRLVW, OPC_VPSRLVD, OPC_VPSRLVQ }; static int const sarv_insn[4] = { - /* TODO: AVX512 adds support for MO_16, MO_64. */ - OPC_UD2, OPC_UD2, OPC_VPSRAVD, OPC_UD2 + OPC_UD2, OPC_VPSRAVW, OPC_VPSRAVD, OPC_VPSRAVQ }; static int const shls_insn[4] = { OPC_UD2, OPC_PSLLW, OPC_PSLLD, OPC_PSLLQ @@ -XXX,XX +XXX,XX @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) case INDEX_op_shlv_vec: case INDEX_op_shrv_vec: - return have_avx2 && vece >= MO_32; + switch (vece) { + case MO_16: + return have_avx512bw; + case MO_32: + case MO_64: + return have_avx2; + } + return 0; case INDEX_op_sarv_vec: - return have_avx2 && vece == MO_32; + switch (vece) { + case MO_16: + return have_avx512bw; + case MO_32: + return have_avx2; + case MO_64: + return have_avx512vl; + } + return 0; case INDEX_op_rotlv_vec: case INDEX_op_rotrv_vec: return have_avx2 && vece >= MO_32 ? -1 : 0; -- 2.25.1 AVX512VL has VPSRAQ. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) #define OPC_PSLLQ (0xf3 | P_EXT | P_DATA16) #define OPC_PSRAW (0xe1 | P_EXT | P_DATA16) #define OPC_PSRAD (0xe2 | P_EXT | P_DATA16) +#define OPC_VPSRAQ (0x72 | P_EXT | P_DATA16 | P_VEXW | P_EVEX) #define OPC_PSRLW (0xd1 | P_EXT | P_DATA16) #define OPC_PSRLD (0xd2 | P_EXT | P_DATA16) #define OPC_PSRLQ (0xd3 | P_EXT | P_DATA16) @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, OPC_UD2, OPC_PSRLW, OPC_PSRLD, OPC_PSRLQ }; static int const sars_insn[4] = { - OPC_UD2, OPC_PSRAW, OPC_PSRAD, OPC_UD2 + OPC_UD2, OPC_PSRAW, OPC_PSRAD, OPC_VPSRAQ }; static int const abs_insn[4] = { /* TODO: AVX512 adds support for MO_64. */ @@ -XXX,XX +XXX,XX @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) case INDEX_op_shrs_vec: return vece >= MO_16; case INDEX_op_sars_vec: - return vece >= MO_16 && vece <= MO_32; + switch (vece) { + case MO_16: + case MO_32: + return 1; + case MO_64: + return have_avx512vl; + } + return 0; case INDEX_op_rotls_vec: return vece >= MO_16 ? -1 : 0; -- 2.25.1 AVX512 has VPSRAQ with immediate operand, in the same form as with AVX, but requires EVEX encoding and W1. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_shli_vec: + insn = shift_imm_insn[vece]; sub = 6; goto gen_shift; case INDEX_op_shri_vec: + insn = shift_imm_insn[vece]; sub = 2; goto gen_shift; case INDEX_op_sari_vec: - tcg_debug_assert(vece != MO_64); + if (vece == MO_64) { + insn = OPC_PSHIFTD_Ib | P_VEXW | P_EVEX; + } else { + insn = shift_imm_insn[vece]; + } sub = 4; gen_shift: tcg_debug_assert(vece != MO_8); - insn = shift_imm_insn[vece]; if (type == TCG_TYPE_V256) { insn |= P_VEXL; } @@ -XXX,XX +XXX,XX @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) return vece == MO_8 ? -1 : 1; case INDEX_op_sari_vec: - /* We must expand the operation for MO_8. */ - if (vece == MO_8) { + switch (vece) { + case MO_8: return -1; - } - /* We can emulate this for MO_64, but it does not pay off - unless we're producing at least 4 values. */ - if (vece == MO_64) { + case MO_16: + case MO_32: + return 1; + case MO_64: + if (have_avx512vl) { + return 1; + } + /* + * We can emulate this for MO_64, but it does not pay off + * unless we're producing at least 4 values. + */ return type >= TCG_TYPE_V256 ? -1 : 0; } - return 1; + return 0; case INDEX_op_shls_vec: case INDEX_op_shrs_vec: -- 2.25.1 AVX512VL has VPROLD and VPROLQ, layered onto the same opcode as PSHIFTD, but requires EVEX encoding and W1. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.h | 2 +- tcg/i386/tcg-target.c.inc | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_movbe; #define TCG_TARGET_HAS_not_vec 0 #define TCG_TARGET_HAS_neg_vec 0 #define TCG_TARGET_HAS_abs_vec 1 -#define TCG_TARGET_HAS_roti_vec 0 +#define TCG_TARGET_HAS_roti_vec have_avx512vl #define TCG_TARGET_HAS_rots_vec 0 #define TCG_TARGET_HAS_rotv_vec 0 #define TCG_TARGET_HAS_shi_vec 1 diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) #define OPC_PSHUFLW (0x70 | P_EXT | P_SIMDF2) #define OPC_PSHUFHW (0x70 | P_EXT | P_SIMDF3) #define OPC_PSHIFTW_Ib (0x71 | P_EXT | P_DATA16) /* /2 /6 /4 */ -#define OPC_PSHIFTD_Ib (0x72 | P_EXT | P_DATA16) /* /2 /6 /4 */ +#define OPC_PSHIFTD_Ib (0x72 | P_EXT | P_DATA16) /* /1 /2 /6 /4 */ #define OPC_PSHIFTQ_Ib (0x73 | P_EXT | P_DATA16) /* /2 /6 /4 */ #define OPC_PSLLW (0xf1 | P_EXT | P_DATA16) #define OPC_PSLLD (0xf2 | P_EXT | P_DATA16) @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, insn = shift_imm_insn[vece]; } sub = 4; + goto gen_shift; + case INDEX_op_rotli_vec: + insn = OPC_PSHIFTD_Ib | P_EVEX; /* VPROL[DQ] */ + if (vece == MO_64) { + insn |= P_VEXW; + } + sub = 1; + goto gen_shift; gen_shift: tcg_debug_assert(vece != MO_8); if (type == TCG_TYPE_V256) { @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_shli_vec: case INDEX_op_shri_vec: case INDEX_op_sari_vec: + case INDEX_op_rotli_vec: case INDEX_op_x86_psrldq_vec: return C_O1_I1(x, x); @@ -XXX,XX +XXX,XX @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) case INDEX_op_xor_vec: case INDEX_op_andc_vec: return 1; - case INDEX_op_rotli_vec: case INDEX_op_cmp_vec: case INDEX_op_cmpsel_vec: return -1; + case INDEX_op_rotli_vec: + return have_avx512vl && vece >= MO_32 ? 1 : -1; + case INDEX_op_shli_vec: case INDEX_op_shri_vec: /* We must expand the operation for MO_8. */ -- 2.25.1 AVX512VL has VPROLVD and VPRORVQ. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.h | 2 +- tcg/i386/tcg-target.c.inc | 25 ++++++++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_movbe; #define TCG_TARGET_HAS_abs_vec 1 #define TCG_TARGET_HAS_roti_vec have_avx512vl #define TCG_TARGET_HAS_rots_vec 0 -#define TCG_TARGET_HAS_rotv_vec 0 +#define TCG_TARGET_HAS_rotv_vec have_avx512vl #define TCG_TARGET_HAS_shi_vec 1 #define TCG_TARGET_HAS_shs_vec 1 #define TCG_TARGET_HAS_shv_vec have_avx2 diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) #define OPC_VPBROADCASTQ (0x59 | P_EXT38 | P_DATA16) #define OPC_VPERMQ (0x00 | P_EXT3A | P_DATA16 | P_VEXW) #define OPC_VPERM2I128 (0x46 | P_EXT3A | P_DATA16 | P_VEXL) +#define OPC_VPROLVD (0x15 | P_EXT38 | P_DATA16 | P_EVEX) +#define OPC_VPROLVQ (0x15 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) +#define OPC_VPRORVD (0x14 | P_EXT38 | P_DATA16 | P_EVEX) +#define OPC_VPRORVQ (0x14 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) #define OPC_VPSLLVW (0x12 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) #define OPC_VPSLLVD (0x47 | P_EXT38 | P_DATA16) #define OPC_VPSLLVQ (0x47 | P_EXT38 | P_DATA16 | P_VEXW) @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, static int const umax_insn[4] = { OPC_PMAXUB, OPC_PMAXUW, OPC_PMAXUD, OPC_UD2 }; + static int const rotlv_insn[4] = { + OPC_UD2, OPC_UD2, OPC_VPROLVD, OPC_VPROLVQ + }; + static int const rotrv_insn[4] = { + OPC_UD2, OPC_UD2, OPC_VPRORVD, OPC_VPRORVQ + }; static int const shlv_insn[4] = { OPC_UD2, OPC_VPSLLVW, OPC_VPSLLVD, OPC_VPSLLVQ }; @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, case INDEX_op_sarv_vec: insn = sarv_insn[vece]; goto gen_simd; + case INDEX_op_rotlv_vec: + insn = rotlv_insn[vece]; + goto gen_simd; + case INDEX_op_rotrv_vec: + insn = rotrv_insn[vece]; + goto gen_simd; case INDEX_op_shls_vec: insn = shls_insn[vece]; goto gen_simd; @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_shlv_vec: case INDEX_op_shrv_vec: case INDEX_op_sarv_vec: + case INDEX_op_rotlv_vec: + case INDEX_op_rotrv_vec: case INDEX_op_shls_vec: case INDEX_op_shrs_vec: case INDEX_op_sars_vec: @@ -XXX,XX +XXX,XX @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) return 0; case INDEX_op_rotlv_vec: case INDEX_op_rotrv_vec: - return have_avx2 && vece >= MO_32 ? -1 : 0; + switch (vece) { + case MO_32: + case MO_64: + return have_avx512vl ? 1 : have_avx2 ? -1 : 0; + } + return 0; case INDEX_op_mul_vec: if (vece == MO_8) { -- 2.25.1 We will use VPSHLD, VPSHLDV and VPSHRDV for 16-bit rotates. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target-con-set.h | 1 + tcg/i386/tcg-target.opc.h | 3 +++ tcg/i386/tcg-target.c.inc | 38 +++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/tcg/i386/tcg-target-con-set.h b/tcg/i386/tcg-target-con-set.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target-con-set.h +++ b/tcg/i386/tcg-target-con-set.h @@ -XXX,XX +XXX,XX @@ C_O1_I2(r, r, rI) C_O1_I2(x, x, x) C_N1_I2(r, r, r) C_N1_I2(r, r, rW) +C_O1_I3(x, 0, x, x) C_O1_I3(x, x, x, x) C_O1_I4(r, r, re, r, 0) C_O1_I4(r, r, r, ri, ri) diff --git a/tcg/i386/tcg-target.opc.h b/tcg/i386/tcg-target.opc.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.opc.h +++ b/tcg/i386/tcg-target.opc.h @@ -XXX,XX +XXX,XX @@ DEF(x86_psrldq_vec, 1, 1, 1, IMPLVEC) DEF(x86_vperm2i128_vec, 1, 2, 1, IMPLVEC) DEF(x86_punpckl_vec, 1, 2, 0, IMPLVEC) DEF(x86_punpckh_vec, 1, 2, 0, IMPLVEC) +DEF(x86_vpshldi_vec, 1, 2, 1, IMPLVEC) +DEF(x86_vpshldv_vec, 1, 3, 0, IMPLVEC) +DEF(x86_vpshrdv_vec, 1, 3, 0, IMPLVEC) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) #define OPC_VPROLVQ (0x15 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) #define OPC_VPRORVD (0x14 | P_EXT38 | P_DATA16 | P_EVEX) #define OPC_VPRORVQ (0x14 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) +#define OPC_VPSHLDW (0x70 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX) +#define OPC_VPSHLDD (0x71 | P_EXT3A | P_DATA16 | P_EVEX) +#define OPC_VPSHLDQ (0x71 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX) +#define OPC_VPSHLDVW (0x70 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) +#define OPC_VPSHLDVD (0x71 | P_EXT38 | P_DATA16 | P_EVEX) +#define OPC_VPSHLDVQ (0x71 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) +#define OPC_VPSHRDVW (0x72 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) +#define OPC_VPSHRDVD (0x73 | P_EXT38 | P_DATA16 | P_EVEX) +#define OPC_VPSHRDVQ (0x73 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) #define OPC_VPSLLVW (0x12 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) #define OPC_VPSLLVD (0x47 | P_EXT38 | P_DATA16) #define OPC_VPSLLVQ (0x47 | P_EXT38 | P_DATA16 | P_VEXW) @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, static int const sars_insn[4] = { OPC_UD2, OPC_PSRAW, OPC_PSRAD, OPC_VPSRAQ }; + static int const vpshldi_insn[4] = { + OPC_UD2, OPC_VPSHLDW, OPC_VPSHLDD, OPC_VPSHLDQ + }; + static int const vpshldv_insn[4] = { + OPC_UD2, OPC_VPSHLDVW, OPC_VPSHLDVD, OPC_VPSHLDVQ + }; + static int const vpshrdv_insn[4] = { + OPC_UD2, OPC_VPSHRDVW, OPC_VPSHRDVD, OPC_VPSHRDVQ + }; static int const abs_insn[4] = { /* TODO: AVX512 adds support for MO_64. */ OPC_PABSB, OPC_PABSW, OPC_PABSD, OPC_UD2 @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, case INDEX_op_x86_packus_vec: insn = packus_insn[vece]; goto gen_simd; + case INDEX_op_x86_vpshldv_vec: + insn = vpshldv_insn[vece]; + a1 = a2; + a2 = args[3]; + goto gen_simd; + case INDEX_op_x86_vpshrdv_vec: + insn = vpshrdv_insn[vece]; + a1 = a2; + a2 = args[3]; + goto gen_simd; #if TCG_TARGET_REG_BITS == 32 case INDEX_op_dup2_vec: /* First merge the two 32-bit inputs to a single 64-bit element. */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, insn = OPC_VPERM2I128; sub = args[3]; goto gen_simd_imm8; + case INDEX_op_x86_vpshldi_vec: + insn = vpshldi_insn[vece]; + sub = args[3]; + goto gen_simd_imm8; gen_simd_imm8: + tcg_debug_assert(insn != OPC_UD2); if (type == TCG_TYPE_V256) { insn |= P_VEXL; } @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_x86_vperm2i128_vec: case INDEX_op_x86_punpckl_vec: case INDEX_op_x86_punpckh_vec: + case INDEX_op_x86_vpshldi_vec: #if TCG_TARGET_REG_BITS == 32 case INDEX_op_dup2_vec: #endif @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_x86_psrldq_vec: return C_O1_I1(x, x); + case INDEX_op_x86_vpshldv_vec: + case INDEX_op_x86_vpshrdv_vec: + return C_O1_I3(x, 0, x, x); + case INDEX_op_x86_vpblendvb_vec: return C_O1_I3(x, x, x, x); -- 2.25.1 While there are no specific 16-bit rotate instructions, there are double-word shifts, which can perform the same operation. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) case INDEX_op_rotlv_vec: case INDEX_op_rotrv_vec: switch (vece) { + case MO_16: + return have_avx512vbmi2 ? -1 : 0; case MO_32: case MO_64: return have_avx512vl ? 1 : have_avx2 ? -1 : 0; @@ -XXX,XX +XXX,XX @@ static void expand_vec_rotli(TCGType type, unsigned vece, return; } + if (have_avx512vbmi2) { + vec_gen_4(INDEX_op_x86_vpshldi_vec, type, vece, + tcgv_vec_arg(v0), tcgv_vec_arg(v1), tcgv_vec_arg(v1), imm); + return; + } + t = tcg_temp_new_vec(type); tcg_gen_shli_vec(vece, t, v1, imm); tcg_gen_shri_vec(vece, v0, v1, (8 << vece) - imm); @@ -XXX,XX +XXX,XX @@ static void expand_vec_rotls(TCGType type, unsigned vece, static void expand_vec_rotv(TCGType type, unsigned vece, TCGv_vec v0, TCGv_vec v1, TCGv_vec sh, bool right) { - TCGv_vec t = tcg_temp_new_vec(type); + TCGv_vec t; + if (have_avx512vbmi2) { + vec_gen_4(right ? INDEX_op_x86_vpshrdv_vec : INDEX_op_x86_vpshldv_vec, + type, vece, tcgv_vec_arg(v0), tcgv_vec_arg(v1), + tcgv_vec_arg(v1), tcgv_vec_arg(sh)); + return; + } + + t = tcg_temp_new_vec(type); tcg_gen_dupi_vec(vece, t, 8 << vece); tcg_gen_sub_vec(vece, t, t, sh); if (right) { -- 2.25.1 There is no such instruction on x86, so we should not be pretending it has arguments. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 1 - 1 file changed, 1 deletion(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_shls_vec: case INDEX_op_shrs_vec: case INDEX_op_sars_vec: - case INDEX_op_rotls_vec: case INDEX_op_cmp_vec: case INDEX_op_x86_shufps_vec: case INDEX_op_x86_blend_vec: -- 2.25.1 Expand 32-bit and 64-bit scalar rotate with VPRO[LR]V; expand 16-bit scalar rotate with VPSHLDV. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 49 +++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void expand_vec_rotli(TCGType type, unsigned vece, tcg_temp_free_vec(t); } -static void expand_vec_rotls(TCGType type, unsigned vece, - TCGv_vec v0, TCGv_vec v1, TCGv_i32 lsh) -{ - TCGv_i32 rsh; - TCGv_vec t; - - tcg_debug_assert(vece != MO_8); - - t = tcg_temp_new_vec(type); - rsh = tcg_temp_new_i32(); - - tcg_gen_neg_i32(rsh, lsh); - tcg_gen_andi_i32(rsh, rsh, (8 << vece) - 1); - tcg_gen_shls_vec(vece, t, v1, lsh); - tcg_gen_shrs_vec(vece, v0, v1, rsh); - tcg_gen_or_vec(vece, v0, v0, t); - tcg_temp_free_vec(t); - tcg_temp_free_i32(rsh); -} - static void expand_vec_rotv(TCGType type, unsigned vece, TCGv_vec v0, TCGv_vec v1, TCGv_vec sh, bool right) { @@ -XXX,XX +XXX,XX @@ static void expand_vec_rotv(TCGType type, unsigned vece, TCGv_vec v0, tcg_temp_free_vec(t); } +static void expand_vec_rotls(TCGType type, unsigned vece, + TCGv_vec v0, TCGv_vec v1, TCGv_i32 lsh) +{ + TCGv_vec t = tcg_temp_new_vec(type); + + tcg_debug_assert(vece != MO_8); + + if (vece >= MO_32 ? have_avx512vl : have_avx512vbmi2) { + tcg_gen_dup_i32_vec(vece, t, lsh); + if (vece >= MO_32) { + tcg_gen_rotlv_vec(vece, v0, v1, t); + } else { + expand_vec_rotv(type, vece, v0, v1, t, false); + } + } else { + TCGv_i32 rsh = tcg_temp_new_i32(); + + tcg_gen_neg_i32(rsh, lsh); + tcg_gen_andi_i32(rsh, rsh, (8 << vece) - 1); + tcg_gen_shls_vec(vece, t, v1, lsh); + tcg_gen_shrs_vec(vece, v0, v1, rsh); + tcg_gen_or_vec(vece, v0, v0, t); + + tcg_temp_free_i32(rsh); + } + + tcg_temp_free_vec(t); +} + static void expand_vec_mul(TCGType type, unsigned vece, TCGv_vec v0, TCGv_vec v1, TCGv_vec v2) { -- 2.25.1 AVX512VL has VPABSQ, VPMAXSQ, VPMAXUQ, VPMINSQ, VPMINUQ. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) #define OPC_PABSB (0x1c | P_EXT38 | P_DATA16) #define OPC_PABSW (0x1d | P_EXT38 | P_DATA16) #define OPC_PABSD (0x1e | P_EXT38 | P_DATA16) +#define OPC_VPABSQ (0x1f | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) #define OPC_PACKSSDW (0x6b | P_EXT | P_DATA16) #define OPC_PACKSSWB (0x63 | P_EXT | P_DATA16) #define OPC_PACKUSDW (0x2b | P_EXT38 | P_DATA16) @@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) #define OPC_PMAXSB (0x3c | P_EXT38 | P_DATA16) #define OPC_PMAXSW (0xee | P_EXT | P_DATA16) #define OPC_PMAXSD (0x3d | P_EXT38 | P_DATA16) +#define OPC_VPMAXSQ (0x3d | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) #define OPC_PMAXUB (0xde | P_EXT | P_DATA16) #define OPC_PMAXUW (0x3e | P_EXT38 | P_DATA16) #define OPC_PMAXUD (0x3f | P_EXT38 | P_DATA16) +#define OPC_VPMAXUQ (0x3f | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) #define OPC_PMINSB (0x38 | P_EXT38 | P_DATA16) #define OPC_PMINSW (0xea | P_EXT | P_DATA16) #define OPC_PMINSD (0x39 | P_EXT38 | P_DATA16) +#define OPC_VPMINSQ (0x39 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) #define OPC_PMINUB (0xda | P_EXT | P_DATA16) #define OPC_PMINUW (0x3a | P_EXT38 | P_DATA16) #define OPC_PMINUD (0x3b | P_EXT38 | P_DATA16) +#define OPC_VPMINUQ (0x3b | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) #define OPC_PMOVSXBW (0x20 | P_EXT38 | P_DATA16) #define OPC_PMOVSXWD (0x23 | P_EXT38 | P_DATA16) #define OPC_PMOVSXDQ (0x25 | P_EXT38 | P_DATA16) @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, OPC_PACKUSWB, OPC_PACKUSDW, OPC_UD2, OPC_UD2 }; static int const smin_insn[4] = { - OPC_PMINSB, OPC_PMINSW, OPC_PMINSD, OPC_UD2 + OPC_PMINSB, OPC_PMINSW, OPC_PMINSD, OPC_VPMINSQ }; static int const smax_insn[4] = { - OPC_PMAXSB, OPC_PMAXSW, OPC_PMAXSD, OPC_UD2 + OPC_PMAXSB, OPC_PMAXSW, OPC_PMAXSD, OPC_VPMAXSQ }; static int const umin_insn[4] = { - OPC_PMINUB, OPC_PMINUW, OPC_PMINUD, OPC_UD2 + OPC_PMINUB, OPC_PMINUW, OPC_PMINUD, OPC_VPMINUQ }; static int const umax_insn[4] = { - OPC_PMAXUB, OPC_PMAXUW, OPC_PMAXUD, OPC_UD2 + OPC_PMAXUB, OPC_PMAXUW, OPC_PMAXUD, OPC_VPMAXUQ }; static int const rotlv_insn[4] = { OPC_UD2, OPC_UD2, OPC_VPROLVD, OPC_VPROLVQ @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, OPC_UD2, OPC_VPSHRDVW, OPC_VPSHRDVD, OPC_VPSHRDVQ }; static int const abs_insn[4] = { - /* TODO: AVX512 adds support for MO_64. */ - OPC_PABSB, OPC_PABSW, OPC_PABSD, OPC_UD2 + OPC_PABSB, OPC_PABSW, OPC_PABSD, OPC_VPABSQ }; TCGType type = vecl + TCG_TYPE_V64; @@ -XXX,XX +XXX,XX @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) case INDEX_op_umin_vec: case INDEX_op_umax_vec: case INDEX_op_abs_vec: - return vece <= MO_32; + return vece <= MO_32 || have_avx512vl; default: return 0; -- 2.25.1 AVX512DQ has VPMULLQ. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) #define OPC_PMOVZXDQ (0x35 | P_EXT38 | P_DATA16) #define OPC_PMULLW (0xd5 | P_EXT | P_DATA16) #define OPC_PMULLD (0x40 | P_EXT38 | P_DATA16) +#define OPC_VPMULLQ (0x40 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) #define OPC_POR (0xeb | P_EXT | P_DATA16) #define OPC_PSHUFB (0x00 | P_EXT38 | P_DATA16) #define OPC_PSHUFD (0x70 | P_EXT | P_DATA16) @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, OPC_PSUBUB, OPC_PSUBUW, OPC_UD2, OPC_UD2 }; static int const mul_insn[4] = { - OPC_UD2, OPC_PMULLW, OPC_PMULLD, OPC_UD2 + OPC_UD2, OPC_PMULLW, OPC_PMULLD, OPC_VPMULLQ }; static int const shift_imm_insn[4] = { OPC_UD2, OPC_PSHIFTW_Ib, OPC_PSHIFTD_Ib, OPC_PSHIFTQ_Ib @@ -XXX,XX +XXX,XX @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) return 0; case INDEX_op_mul_vec: - if (vece == MO_8) { - /* We can expand the operation for MO_8. */ + switch (vece) { + case MO_8: return -1; - } - if (vece == MO_64) { - return 0; + case MO_64: + return have_avx512dq; } return 1; -- 2.25.1 AVX512VL has a general ternary logic operation, VPTERNLOGQ, which can implement NOT, ORC, NAND, NOR, EQV. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.h | 10 +++++----- tcg/i386/tcg-target.c.inc | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_movbe; #define TCG_TARGET_HAS_v256 have_avx2 #define TCG_TARGET_HAS_andc_vec 1 -#define TCG_TARGET_HAS_orc_vec 0 -#define TCG_TARGET_HAS_nand_vec 0 -#define TCG_TARGET_HAS_nor_vec 0 -#define TCG_TARGET_HAS_eqv_vec 0 -#define TCG_TARGET_HAS_not_vec 0 +#define TCG_TARGET_HAS_orc_vec have_avx512vl +#define TCG_TARGET_HAS_nand_vec have_avx512vl +#define TCG_TARGET_HAS_nor_vec have_avx512vl +#define TCG_TARGET_HAS_eqv_vec have_avx512vl +#define TCG_TARGET_HAS_not_vec have_avx512vl #define TCG_TARGET_HAS_neg_vec 0 #define TCG_TARGET_HAS_abs_vec 1 #define TCG_TARGET_HAS_roti_vec have_avx512vl diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) #define OPC_VPSRLVW (0x10 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) #define OPC_VPSRLVD (0x45 | P_EXT38 | P_DATA16) #define OPC_VPSRLVQ (0x45 | P_EXT38 | P_DATA16 | P_VEXW) +#define OPC_VPTERNLOGQ (0x25 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX) #define OPC_VZEROUPPER (0x77 | P_EXT) #define OPC_XCHG_ax_r32 (0x90) @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, insn = vpshldi_insn[vece]; sub = args[3]; goto gen_simd_imm8; + + case INDEX_op_not_vec: + insn = OPC_VPTERNLOGQ; + a2 = a1; + sub = 0x33; /* !B */ + goto gen_simd_imm8; + case INDEX_op_nor_vec: + insn = OPC_VPTERNLOGQ; + sub = 0x11; /* norCB */ + goto gen_simd_imm8; + case INDEX_op_nand_vec: + insn = OPC_VPTERNLOGQ; + sub = 0x77; /* nandCB */ + goto gen_simd_imm8; + case INDEX_op_eqv_vec: + insn = OPC_VPTERNLOGQ; + sub = 0x99; /* xnorCB */ + goto gen_simd_imm8; + case INDEX_op_orc_vec: + insn = OPC_VPTERNLOGQ; + sub = 0xdd; /* orB!C */ + goto gen_simd_imm8; + gen_simd_imm8: tcg_debug_assert(insn != OPC_UD2); if (type == TCG_TYPE_V256) { @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_or_vec: case INDEX_op_xor_vec: case INDEX_op_andc_vec: + case INDEX_op_orc_vec: + case INDEX_op_nand_vec: + case INDEX_op_nor_vec: + case INDEX_op_eqv_vec: case INDEX_op_ssadd_vec: case INDEX_op_usadd_vec: case INDEX_op_sssub_vec: @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_abs_vec: case INDEX_op_dup_vec: + case INDEX_op_not_vec: case INDEX_op_shli_vec: case INDEX_op_shri_vec: case INDEX_op_sari_vec: @@ -XXX,XX +XXX,XX @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) case INDEX_op_or_vec: case INDEX_op_xor_vec: case INDEX_op_andc_vec: + case INDEX_op_orc_vec: + case INDEX_op_nand_vec: + case INDEX_op_nor_vec: + case INDEX_op_eqv_vec: + case INDEX_op_not_vec: return 1; case INDEX_op_cmp_vec: case INDEX_op_cmpsel_vec: -- 2.25.1 The general ternary logic operation can implement BITSEL. Funnel the 4-operand operation into three variants of the 3-operand instruction, depending on input operand overlap. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.h | 2 +- tcg/i386/tcg-target.c.inc | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool have_movbe; #define TCG_TARGET_HAS_mul_vec 1 #define TCG_TARGET_HAS_sat_vec 1 #define TCG_TARGET_HAS_minmax_vec 1 -#define TCG_TARGET_HAS_bitsel_vec 0 +#define TCG_TARGET_HAS_bitsel_vec have_avx512vl #define TCG_TARGET_HAS_cmpsel_vec -1 #define TCG_TARGET_deposit_i32_valid(ofs, len) \ diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, TCGType type = vecl + TCG_TYPE_V64; int insn, sub; - TCGArg a0, a1, a2; + TCGArg a0, a1, a2, a3; a0 = args[0]; a1 = args[1]; @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, sub = 0xdd; /* orB!C */ goto gen_simd_imm8; + case INDEX_op_bitsel_vec: + insn = OPC_VPTERNLOGQ; + a3 = args[3]; + if (a0 == a1) { + a1 = a2; + a2 = a3; + sub = 0xca; /* A?B:C */ + } else if (a0 == a2) { + a2 = a3; + sub = 0xe2; /* B?A:C */ + } else { + tcg_out_mov(s, type, a0, a3); + sub = 0xb8; /* B?C:A */ + } + goto gen_simd_imm8; + gen_simd_imm8: tcg_debug_assert(insn != OPC_UD2); if (type == TCG_TYPE_V256) { @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_x86_vpshrdv_vec: return C_O1_I3(x, 0, x, x); + case INDEX_op_bitsel_vec: case INDEX_op_x86_vpblendvb_vec: return C_O1_I3(x, x, x, x); @@ -XXX,XX +XXX,XX @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) case INDEX_op_nor_vec: case INDEX_op_eqv_vec: case INDEX_op_not_vec: + case INDEX_op_bitsel_vec: return 1; case INDEX_op_cmp_vec: case INDEX_op_cmpsel_vec: -- 2.25.1 Define as 0 for all tcg hosts. Put this in a separate header, because we'll want this in places that do not ordinarily have access to all of tcg/tcg.h. Reviewed-by: WANG Xuerui <git@xen0n.name> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target-sa32.h | 1 + tcg/arm/tcg-target-sa32.h | 1 + tcg/i386/tcg-target-sa32.h | 1 + tcg/loongarch64/tcg-target-sa32.h | 1 + tcg/mips/tcg-target-sa32.h | 1 + tcg/ppc/tcg-target-sa32.h | 1 + tcg/riscv/tcg-target-sa32.h | 1 + tcg/s390x/tcg-target-sa32.h | 1 + tcg/sparc/tcg-target-sa32.h | 1 + tcg/tci/tcg-target-sa32.h | 1 + tcg/tcg.c | 4 ++++ 11 files changed, 14 insertions(+) create mode 100644 tcg/aarch64/tcg-target-sa32.h create mode 100644 tcg/arm/tcg-target-sa32.h create mode 100644 tcg/i386/tcg-target-sa32.h create mode 100644 tcg/loongarch64/tcg-target-sa32.h create mode 100644 tcg/mips/tcg-target-sa32.h create mode 100644 tcg/ppc/tcg-target-sa32.h create mode 100644 tcg/riscv/tcg-target-sa32.h create mode 100644 tcg/s390x/tcg-target-sa32.h create mode 100644 tcg/sparc/tcg-target-sa32.h create mode 100644 tcg/tci/tcg-target-sa32.h diff --git a/tcg/aarch64/tcg-target-sa32.h b/tcg/aarch64/tcg-target-sa32.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/aarch64/tcg-target-sa32.h @@ -0,0 +1 @@ +#define TCG_TARGET_SIGNED_ADDR32 0 diff --git a/tcg/arm/tcg-target-sa32.h b/tcg/arm/tcg-target-sa32.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/arm/tcg-target-sa32.h @@ -0,0 +1 @@ +#define TCG_TARGET_SIGNED_ADDR32 0 diff --git a/tcg/i386/tcg-target-sa32.h b/tcg/i386/tcg-target-sa32.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/i386/tcg-target-sa32.h @@ -0,0 +1 @@ +#define TCG_TARGET_SIGNED_ADDR32 0 diff --git a/tcg/loongarch64/tcg-target-sa32.h b/tcg/loongarch64/tcg-target-sa32.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/loongarch64/tcg-target-sa32.h @@ -0,0 +1 @@ +#define TCG_TARGET_SIGNED_ADDR32 0 diff --git a/tcg/mips/tcg-target-sa32.h b/tcg/mips/tcg-target-sa32.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/mips/tcg-target-sa32.h @@ -0,0 +1 @@ +#define TCG_TARGET_SIGNED_ADDR32 0 diff --git a/tcg/ppc/tcg-target-sa32.h b/tcg/ppc/tcg-target-sa32.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/ppc/tcg-target-sa32.h @@ -0,0 +1 @@ +#define TCG_TARGET_SIGNED_ADDR32 0 diff --git a/tcg/riscv/tcg-target-sa32.h b/tcg/riscv/tcg-target-sa32.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/riscv/tcg-target-sa32.h @@ -0,0 +1 @@ +#define TCG_TARGET_SIGNED_ADDR32 0 diff --git a/tcg/s390x/tcg-target-sa32.h b/tcg/s390x/tcg-target-sa32.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/s390x/tcg-target-sa32.h @@ -0,0 +1 @@ +#define TCG_TARGET_SIGNED_ADDR32 0 diff --git a/tcg/sparc/tcg-target-sa32.h b/tcg/sparc/tcg-target-sa32.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/sparc/tcg-target-sa32.h @@ -0,0 +1 @@ +#define TCG_TARGET_SIGNED_ADDR32 0 diff --git a/tcg/tci/tcg-target-sa32.h b/tcg/tci/tcg-target-sa32.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/tci/tcg-target-sa32.h @@ -0,0 +1 @@ +#define TCG_TARGET_SIGNED_ADDR32 0 diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ #include "exec/log.h" #include "tcg/tcg-ldst.h" #include "tcg-internal.h" +#include "tcg-target-sa32.h" + +/* Sanity check for TCG_TARGET_SIGNED_ADDR32. */ +QEMU_BUILD_BUG_ON(TCG_TARGET_REG_BITS == 32 && TCG_TARGET_SIGNED_ADDR32); #ifdef CONFIG_TCG_INTERPRETER #include <ffi.h> -- 2.25.1 Create a new function to combine a CPUTLBEntry addend with the guest address to form a host address. Reviewed-by: WANG Xuerui <git@xen0n.name> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cputlb.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ static inline size_t sizeof_tlb(CPUTLBDescFast *fast) return fast->mask + (1 << CPU_TLB_ENTRY_BITS); } +static inline uintptr_t g2h_tlbe(const CPUTLBEntry *tlb, target_ulong gaddr) +{ + return tlb->addend + (uintptr_t)gaddr; +} + static void tlb_window_reset(CPUTLBDesc *desc, int64_t ns, size_t max_entries) { @@ -XXX,XX +XXX,XX @@ static void tlb_reset_dirty_range_locked(CPUTLBEntry *tlb_entry, if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_DISCARD_WRITE | TLB_NOTDIRTY)) == 0) { - addr &= TARGET_PAGE_MASK; - addr += tlb_entry->addend; + addr = g2h_tlbe(tlb_entry, addr & TARGET_PAGE_MASK); if ((addr - start) < length) { #if TCG_OVERSIZED_GUEST tlb_entry->addr_write |= TLB_NOTDIRTY; @@ -XXX,XX +XXX,XX @@ tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, return -1; } - p = (void *)((uintptr_t)addr + entry->addend); + p = (void *)g2h_tlbe(entry, addr); if (hostp) { *hostp = p; } @@ -XXX,XX +XXX,XX @@ static int probe_access_internal(CPUArchState *env, target_ulong addr, } /* Everything else is RAM. */ - *phost = (void *)((uintptr_t)addr + entry->addend); + *phost = (void *)g2h_tlbe(entry, addr); return flags; } @@ -XXX,XX +XXX,XX @@ bool tlb_plugin_lookup(CPUState *cpu, target_ulong addr, int mmu_idx, data->v.io.offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr; } else { data->is_io = false; - data->v.ram.hostaddr = (void *)((uintptr_t)addr + tlbe->addend); + data->v.ram.hostaddr = (void *)g2h_tlbe(tlbe, addr); } return true; } else { @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, goto stop_the_world; } - hostaddr = (void *)((uintptr_t)addr + tlbe->addend); + hostaddr = (void *)g2h_tlbe(tlbe, addr); if (unlikely(tlb_addr & TLB_NOTDIRTY)) { notdirty_write(env_cpu(env), addr, size, @@ -XXX,XX +XXX,XX @@ load_helper(CPUArchState *env, target_ulong addr, MemOpIdx oi, access_type, op ^ (need_swap * MO_BSWAP)); } - haddr = (void *)((uintptr_t)addr + entry->addend); + haddr = (void *)g2h_tlbe(entry, addr); /* * Keep these two load_memop separate to ensure that the compiler @@ -XXX,XX +XXX,XX @@ load_helper(CPUArchState *env, target_ulong addr, MemOpIdx oi, return res & MAKE_64BIT_MASK(0, size * 8); } - haddr = (void *)((uintptr_t)addr + entry->addend); + haddr = (void *)g2h_tlbe(entry, addr); return load_memop(haddr, op); } @@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val, notdirty_write(env_cpu(env), addr, size, iotlbentry, retaddr); } - haddr = (void *)((uintptr_t)addr + entry->addend); + haddr = (void *)g2h_tlbe(entry, addr); /* * Keep these two store_memop separate to ensure that the compiler @@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val, return; } - haddr = (void *)((uintptr_t)addr + entry->addend); + haddr = (void *)g2h_tlbe(entry, addr); store_memop(haddr, val, op); } -- 2.25.1 When TCG_TARGET_SIGNED_ADDR32 is set, adjust the tlb addend to allow the 32-bit guest address to be sign extended within the 64-bit host register instead of zero extended. This will simplify tcg hosts like MIPS, RISC-V, and LoongArch, which naturally sign-extend 32-bit values, in contrast to x86_64 and AArch64 which zero-extend them. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cputlb.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ #include "qemu/plugin-memory.h" #endif #include "tcg/tcg-ldst.h" +#include "tcg-target-sa32.h" /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */ /* #define DEBUG_TLB */ @@ -XXX,XX +XXX,XX @@ static inline size_t sizeof_tlb(CPUTLBDescFast *fast) static inline uintptr_t g2h_tlbe(const CPUTLBEntry *tlb, target_ulong gaddr) { + if (TCG_TARGET_SIGNED_ADDR32 && TARGET_LONG_BITS == 32) { + return tlb->addend + (int32_t)gaddr; + } return tlb->addend + (uintptr_t)gaddr; } @@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, desc->iotlb[index].attrs = attrs; /* Now calculate the new entry */ - tn.addend = addend - vaddr_page; + + if (TCG_TARGET_SIGNED_ADDR32 && TARGET_LONG_BITS == 32) { + tn.addend = addend - (int32_t)vaddr_page; + } else { + tn.addend = addend - vaddr_page; + } + if (prot & PAGE_READ) { tn.addr_read = address; if (wp_flags & BP_MEM_READ) { -- 2.25.1 While the host may prefer to treat 32-bit addresses as signed, there are edge cases of guests that cannot be implemented with addresses 0x7fff_ffff and 0x8000_0000 being non-consecutive. Therefore, default to guest_base_signed_addr32 false, and allow probe_guest_base to determine whether it is possible to set it to true. A tcg backend which sets TCG_TARGET_SIGNED_ADDR32 will have to cope with either setting for user-only. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/cpu-all.h | 16 ++++++++++++++++ include/exec/cpu_ldst.h | 3 ++- bsd-user/main.c | 4 ++++ linux-user/main.c | 3 +++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu-all.h +++ b/include/exec/cpu-all.h @@ -XXX,XX +XXX,XX @@ static inline void tswap64s(uint64_t *s) #if defined(CONFIG_USER_ONLY) #include "exec/user/abitypes.h" +#include "tcg-target-sa32.h" /* On some host systems the guest address space is reserved on the host. * This allows the guest address space to be offset to a convenient location. @@ -XXX,XX +XXX,XX @@ extern uintptr_t guest_base; extern bool have_guest_base; extern unsigned long reserved_va; +#if TCG_TARGET_SIGNED_ADDR32 && TARGET_LONG_BITS == 32 +extern bool guest_base_signed_addr32; +#else +#define guest_base_signed_addr32 false +#endif + +static inline void set_guest_base_signed_addr32(void) +{ +#ifdef guest_base_signed_addr32 + qemu_build_not_reached(); +#else + guest_base_signed_addr32 = true; +#endif +} + /* * Limit the guest addresses as best we can. * diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu_ldst.h +++ b/include/exec/cpu_ldst.h @@ -XXX,XX +XXX,XX @@ static inline abi_ptr cpu_untagged_addr(CPUState *cs, abi_ptr x) /* All direct uses of g2h and h2g need to go away for usermode softmmu. */ static inline void *g2h_untagged(abi_ptr x) { - return (void *)((uintptr_t)(x) + guest_base); + uintptr_t hx = guest_base_signed_addr32 ? (int32_t)x : (uintptr_t)x; + return (void *)(guest_base + hx); } static inline void *g2h(CPUState *cs, abi_ptr x) diff --git a/bsd-user/main.c b/bsd-user/main.c index XXXXXXX..XXXXXXX 100644 --- a/bsd-user/main.c +++ b/bsd-user/main.c @@ -XXX,XX +XXX,XX @@ int singlestep; uintptr_t guest_base; bool have_guest_base; +#ifndef guest_base_signed_addr32 +bool guest_base_signed_addr32; +#endif + /* * When running 32-on-64 we should make sure we can fit all of the possible * guest address space into a contiguous chunk of virtual host memory. diff --git a/linux-user/main.c b/linux-user/main.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/main.c +++ b/linux-user/main.c @@ -XXX,XX +XXX,XX @@ static const char *seed_optarg; unsigned long mmap_min_addr; uintptr_t guest_base; bool have_guest_base; +#ifndef guest_base_signed_addr32 +bool guest_base_signed_addr32; +#endif /* * Used to implement backwards-compatibility for the `-strace`, and -- 2.25.1 When using reserved_va, which is the default for a 64-bit host and a 32-bit guest, set guest_base_signed_addr32 if requested by TCG_TARGET_SIGNED_ADDR32, and the executable layout allows. Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/cpu-all.h | 4 --- linux-user/elfload.c | 62 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu-all.h +++ b/include/exec/cpu-all.h @@ -XXX,XX +XXX,XX @@ extern const TargetPageBits target_page; #define PAGE_RESET 0x0040 /* For linux-user, indicates that the page is MAP_ANON. */ #define PAGE_ANON 0x0080 - -#if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY) -/* FIXME: Code that sets/uses this is broken and needs to go away. */ #define PAGE_RESERVED 0x0100 -#endif /* Target-specific bits that will be used via page_get_flags(). */ #define PAGE_TARGET_1 0x0200 #define PAGE_TARGET_2 0x0400 diff --git a/linux-user/elfload.c b/linux-user/elfload.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/elfload.c +++ b/linux-user/elfload.c @@ -XXX,XX +XXX,XX @@ static void pgb_dynamic(const char *image_name, long align) static void pgb_reserved_va(const char *image_name, abi_ulong guest_loaddr, abi_ulong guest_hiaddr, long align) { - int flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE; + int flags = (MAP_ANONYMOUS | MAP_PRIVATE | + MAP_NORESERVE | MAP_FIXED_NOREPLACE); + unsigned long local_rva = reserved_va; + bool protect_wrap = false; void *addr, *test; - if (guest_hiaddr > reserved_va) { + if (guest_hiaddr > local_rva) { error_report("%s: requires more than reserved virtual " "address space (0x%" PRIx64 " > 0x%lx)", - image_name, (uint64_t)guest_hiaddr, reserved_va); + image_name, (uint64_t)guest_hiaddr, local_rva); exit(EXIT_FAILURE); } - /* Widen the "image" to the entire reserved address space. */ - pgb_static(image_name, 0, reserved_va, align); + if (TCG_TARGET_SIGNED_ADDR32 && TARGET_LONG_BITS == 32) { + if (guest_loaddr < 0x80000000u && guest_hiaddr > 0x80000000u) { + /* + * The executable itself wraps on signed addresses. + * Without per-page translation, we must keep the + * guest address 0x7fff_ffff adjacent to 0x8000_0000 + * consecutive in host memory: unsigned addresses. + */ + } else { + set_guest_base_signed_addr32(); + if (local_rva <= 0x80000000u) { + /* No guest addresses are "negative": win! */ + } else { + /* Begin by allocating the entire address space. */ + local_rva = 0xfffffffful + 1; + protect_wrap = true; + } + } + } - /* osdep.h defines this as 0 if it's missing */ - flags |= MAP_FIXED_NOREPLACE; + /* Widen the "image" to the entire reserved address space. */ + pgb_static(image_name, 0, local_rva, align); + assert(guest_base != 0); /* Reserve the memory on the host. */ - assert(guest_base != 0); test = g2h_untagged(0); - addr = mmap(test, reserved_va, PROT_NONE, flags, -1, 0); + addr = mmap(test, local_rva, PROT_NONE, flags, -1, 0); if (addr == MAP_FAILED || addr != test) { + /* + * If protect_wrap, we could try again with the original reserved_va + * setting, but the edge case of low ulimit vm setting on a 64-bit + * host is probably useless. + */ error_report("Unable to reserve 0x%lx bytes of virtual address " - "space at %p (%s) for use as guest address space (check your" - "virtual memory ulimit setting, min_mmap_addr or reserve less " - "using -R option)", reserved_va, test, strerror(errno)); + "space at %p (%s) for use as guest address space " + "(check your virtual memory ulimit setting, " + "min_mmap_addr or reserve less using -R option)", + local_rva, test, strerror(errno)); exit(EXIT_FAILURE); } + if (protect_wrap) { + /* + * Prevent the page just before 0x80000000 from being allocated. + * This prevents a single guest object/allocation from crossing + * the signed wrap, and thus being discontiguous in host memory. + */ + page_set_flags(0x7fffffff & TARGET_PAGE_MASK, 0x80000000u, + PAGE_RESERVED); + /* Adjust guest_base so that 0 is in the middle of the reservation. */ + guest_base += 0x80000000ul; + } + qemu_log_mask(CPU_LOG_PAGE, "%s: base @ %p for %lu bytes\n", __func__, addr, reserved_va); } -- 2.25.1 AArch64 has both sign and zero-extending addressing modes, which means that either treatment of guest addresses is equally efficient. Enabling this for AArch64 gives us testing of the feature in CI. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target-sa32.h | 8 +++- tcg/aarch64/tcg-target.c.inc | 81 ++++++++++++++++++++++++----------- 2 files changed, 64 insertions(+), 25 deletions(-) diff --git a/tcg/aarch64/tcg-target-sa32.h b/tcg/aarch64/tcg-target-sa32.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target-sa32.h +++ b/tcg/aarch64/tcg-target-sa32.h @@ -1 +1,7 @@ -#define TCG_TARGET_SIGNED_ADDR32 0 +/* + * AArch64 has both SXTW and UXTW addressing modes, which means that + * it is agnostic to how guest addresses should be represented. + * Because aarch64 is more common than the other hosts that will + * want to use this feature, enable it for continuous testing. + */ +#define TCG_TARGET_SIGNED_ADDR32 1 diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ typedef enum { LDST_LD_S_W = 3, /* load and sign-extend into Wt */ } AArch64LdstType; +/* + * See aarch64/instrs/extendreg/DecodeRegExtend + * But note that option<1> == 0 is UNDEFINED for LDR/STR. + */ +typedef enum { + LDST_EXT_UXTW = 2, /* zero-extend from uint32_t */ + LDST_EXT_UXTX = 3, /* zero-extend from uint64_t (i.e. no extension) */ + LDST_EXT_SXTW = 6, /* sign-extend from int32_t */ +} AArch64LdstExt; + /* We encode the format of the insn into the beginning of the name, so that we can have the preprocessor help "typecheck" the insn vs the output function. Arm didn't provide us with nice names for the formats, so we @@ -XXX,XX +XXX,XX @@ static void tcg_out_insn_3617(TCGContext *s, AArch64Insn insn, bool q, } static void tcg_out_insn_3310(TCGContext *s, AArch64Insn insn, - TCGReg rd, TCGReg base, TCGType ext, + TCGReg rd, TCGReg base, AArch64LdstExt option, TCGReg regoff) { /* Note the AArch64Insn constants above are for C3.3.12. Adjust. */ tcg_out32(s, insn | I3312_TO_I3310 | regoff << 16 | - 0x4000 | ext << 13 | base << 5 | (rd & 0x1f)); + option << 13 | base << 5 | (rd & 0x1f)); } static void tcg_out_insn_3312(TCGContext *s, AArch64Insn insn, @@ -XXX,XX +XXX,XX @@ static void tcg_out_ldst(TCGContext *s, AArch64Insn insn, TCGReg rd, /* Worst-case scenario, move offset to temp register, use reg offset. */ tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, offset); - tcg_out_ldst_r(s, insn, rd, rn, TCG_TYPE_I64, TCG_REG_TMP); + tcg_out_ldst_r(s, insn, rd, rn, LDST_EXT_UXTX, TCG_REG_TMP); } static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp memop, TCGType ext, TCGReg data_r, TCGReg addr_r, - TCGType otype, TCGReg off_r) + AArch64LdstExt option, TCGReg off_r) { switch (memop & MO_SSIZE) { case MO_UB: - tcg_out_ldst_r(s, I3312_LDRB, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_LDRB, data_r, addr_r, option, off_r); break; case MO_SB: tcg_out_ldst_r(s, ext ? I3312_LDRSBX : I3312_LDRSBW, - data_r, addr_r, otype, off_r); + data_r, addr_r, option, off_r); break; case MO_UW: - tcg_out_ldst_r(s, I3312_LDRH, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_LDRH, data_r, addr_r, option, off_r); break; case MO_SW: tcg_out_ldst_r(s, (ext ? I3312_LDRSHX : I3312_LDRSHW), - data_r, addr_r, otype, off_r); + data_r, addr_r, option, off_r); break; case MO_UL: - tcg_out_ldst_r(s, I3312_LDRW, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_LDRW, data_r, addr_r, option, off_r); break; case MO_SL: - tcg_out_ldst_r(s, I3312_LDRSWX, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_LDRSWX, data_r, addr_r, option, off_r); break; case MO_UQ: - tcg_out_ldst_r(s, I3312_LDRX, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_LDRX, data_r, addr_r, option, off_r); break; default: tcg_abort(); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp memop, TCGType ext, static void tcg_out_qemu_st_direct(TCGContext *s, MemOp memop, TCGReg data_r, TCGReg addr_r, - TCGType otype, TCGReg off_r) + AArch64LdstExt option, TCGReg off_r) { switch (memop & MO_SIZE) { case MO_8: - tcg_out_ldst_r(s, I3312_STRB, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_STRB, data_r, addr_r, option, off_r); break; case MO_16: - tcg_out_ldst_r(s, I3312_STRH, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_STRH, data_r, addr_r, option, off_r); break; case MO_32: - tcg_out_ldst_r(s, I3312_STRW, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_STRW, data_r, addr_r, option, off_r); break; case MO_64: - tcg_out_ldst_r(s, I3312_STRX, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_STRX, data_r, addr_r, option, off_r); break; default: tcg_abort(); } } +/* + * Bits for the option field of LDR/STR (register), + * for application to a guest address. + */ +static AArch64LdstExt ldst_ext_option(void) +{ +#ifdef CONFIG_USER_ONLY + bool signed_addr32 = guest_base_signed_addr32; +#else + bool signed_addr32 = TCG_TARGET_SIGNED_ADDR32; +#endif + + if (TARGET_LONG_BITS == 64) { + return LDST_EXT_UXTX; + } else if (signed_addr32) { + return LDST_EXT_SXTW; + } else { + return LDST_EXT_UXTW; + } +} + static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, MemOpIdx oi, TCGType ext) { MemOp memop = get_memop(oi); - const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32; + AArch64LdstExt option = ldst_ext_option(); /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((memop & MO_BSWAP) == 0); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 1); tcg_out_qemu_ld_direct(s, memop, ext, data_reg, - TCG_REG_X1, otype, addr_reg); + TCG_REG_X1, option, addr_reg); add_qemu_ldst_label(s, true, oi, ext, data_reg, addr_reg, s->code_ptr, label_ptr); #else /* !CONFIG_SOFTMMU */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, } if (USE_GUEST_BASE) { tcg_out_qemu_ld_direct(s, memop, ext, data_reg, - TCG_REG_GUEST_BASE, otype, addr_reg); + TCG_REG_GUEST_BASE, option, addr_reg); } else { + /* This case is always a 64-bit guest with no extension. */ tcg_out_qemu_ld_direct(s, memop, ext, data_reg, - addr_reg, TCG_TYPE_I64, TCG_REG_XZR); + addr_reg, LDST_EXT_UXTX, TCG_REG_XZR); } #endif /* CONFIG_SOFTMMU */ } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, MemOpIdx oi) { MemOp memop = get_memop(oi); - const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32; + AArch64LdstExt option = ldst_ext_option(); /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((memop & MO_BSWAP) == 0); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 0); tcg_out_qemu_st_direct(s, memop, data_reg, - TCG_REG_X1, otype, addr_reg); + TCG_REG_X1, option, addr_reg); add_qemu_ldst_label(s, false, oi, (memop & MO_SIZE)== MO_64, data_reg, addr_reg, s->code_ptr, label_ptr); #else /* !CONFIG_SOFTMMU */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, } if (USE_GUEST_BASE) { tcg_out_qemu_st_direct(s, memop, data_reg, - TCG_REG_GUEST_BASE, otype, addr_reg); + TCG_REG_GUEST_BASE, option, addr_reg); } else { + /* This case is always a 64-bit guest with no extension. */ tcg_out_qemu_st_direct(s, memop, data_reg, - addr_reg, TCG_TYPE_I64, TCG_REG_XZR); + addr_reg, LDST_EXT_UXTX, TCG_REG_XZR); } #endif /* CONFIG_SOFTMMU */ } -- 2.25.1 All 32-bit mips operations sign-extend the output, so we are easily able to keep TCG_TYPE_I32 values sign-extended in host registers. Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target-sa32.h | 8 ++++++++ tcg/mips/tcg-target.c.inc | 10 ++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tcg/mips/tcg-target-sa32.h b/tcg/mips/tcg-target-sa32.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target-sa32.h +++ b/tcg/mips/tcg-target-sa32.h @@ -1 +1,9 @@ +/* + * Do not set TCG_TARGET_SIGNED_ADDR32 for mips32; + * TCG expects this to only be set for 64-bit hosts. + */ +#ifdef __mips64 +#define TCG_TARGET_SIGNED_ADDR32 1 +#else #define TCG_TARGET_SIGNED_ADDR32 0 +#endif diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_tlb_load(TCGContext *s, TCGReg base, TCGReg addrl, TCG_TMP0, TCG_TMP3, cmp_off); } - /* Zero extend a 32-bit guest address for a 64-bit host. */ - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { - tcg_out_ext32u(s, base, addrl); - addrl = base; - } - /* * Mask the page bits, keeping the alignment bits to compare against. * For unaligned accesses, compare against the end of the access to @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) data_regl, data_regh, addr_regl, addr_regh, s->code_ptr, label_ptr); #else - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { + if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS && !guest_base_signed_addr32) { tcg_out_ext32u(s, base, addr_regl); addr_regl = base; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) data_regl, data_regh, addr_regl, addr_regh, s->code_ptr, label_ptr); #else - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { + if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS && !guest_base_signed_addr32) { tcg_out_ext32u(s, base, addr_regl); addr_regl = base; } -- 2.25.1 All RV64 32-bit operations sign-extend the output, so we are easily able to keep TCG_TYPE_I32 values sign-extended in host registers. Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target-sa32.h | 6 +++++- tcg/riscv/tcg-target.c.inc | 8 ++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tcg/riscv/tcg-target-sa32.h b/tcg/riscv/tcg-target-sa32.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target-sa32.h +++ b/tcg/riscv/tcg-target-sa32.h @@ -1 +1,5 @@ -#define TCG_TARGET_SIGNED_ADDR32 0 +/* + * Do not set TCG_TARGET_SIGNED_ADDR32 for RV32; + * TCG expects this to only be set for 64-bit hosts. + */ +#define TCG_TARGET_SIGNED_ADDR32 (__riscv_xlen == 64) diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_tlb_load(TCGContext *s, TCGReg addrl, tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP0, TCG_REG_TMP1, 0); /* TLB Hit - translate address using addend. */ - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { - tcg_out_ext32u(s, TCG_REG_TMP0, addrl); - addrl = TCG_REG_TMP0; - } tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_REG_TMP2, addrl); } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) data_regl, data_regh, addr_regl, addr_regh, s->code_ptr, label_ptr); #else - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { + if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS && !guest_base_signed_addr32) { tcg_out_ext32u(s, base, addr_regl); addr_regl = base; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) data_regl, data_regh, addr_regl, addr_regh, s->code_ptr, label_ptr); #else - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { + if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS && !guest_base_signed_addr32) { tcg_out_ext32u(s, base, addr_regl); addr_regl = base; } -- 2.25.1 All 32-bit LoongArch operations sign-extend the output, so we are easily able to keep TCG_TYPE_I32 values sign-extended in host registers. Cc: WANG Xuerui <git@xen0n.name> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/loongarch64/tcg-target-sa32.h | 2 +- tcg/loongarch64/tcg-target.c.inc | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/tcg/loongarch64/tcg-target-sa32.h b/tcg/loongarch64/tcg-target-sa32.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/loongarch64/tcg-target-sa32.h +++ b/tcg/loongarch64/tcg-target-sa32.h @@ -1 +1 @@ -#define TCG_TARGET_SIGNED_ADDR32 0 +#define TCG_TARGET_SIGNED_ADDR32 1 diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/loongarch64/tcg-target.c.inc +++ b/tcg/loongarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) return tcg_out_fail_alignment(s, l); } -#endif /* CONFIG_SOFTMMU */ - /* * `ext32u` the address register into the temp register given, * if target is 32-bit, no-op otherwise. @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) static TCGReg tcg_out_zext_addr_if_32_bit(TCGContext *s, TCGReg addr, TCGReg tmp) { - if (TARGET_LONG_BITS == 32) { + if (TARGET_LONG_BITS == 32 && !guest_base_signed_addr32) { tcg_out_ext32u(s, tmp, addr); return tmp; } return addr; } +#endif /* CONFIG_SOFTMMU */ static void tcg_out_qemu_ld_indexed(TCGContext *s, TCGReg rd, TCGReg rj, TCGReg rk, MemOp opc, TCGType type) @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, TCGType type) tcg_insn_unit *label_ptr[1]; #else unsigned a_bits; -#endif TCGReg base; +#endif data_regl = *args++; addr_regl = *args++; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, TCGType type) #if defined(CONFIG_SOFTMMU) tcg_out_tlb_load(s, addr_regl, oi, label_ptr, 1); - base = tcg_out_zext_addr_if_32_bit(s, addr_regl, TCG_REG_TMP0); - tcg_out_qemu_ld_indexed(s, data_regl, base, TCG_REG_TMP2, opc, type); + tcg_out_qemu_ld_indexed(s, data_regl, addr_regl, TCG_REG_TMP2, opc, type); add_qemu_ldst_label(s, 1, oi, type, data_regl, addr_regl, s->code_ptr, label_ptr); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args) tcg_insn_unit *label_ptr[1]; #else unsigned a_bits; -#endif TCGReg base; +#endif data_regl = *args++; addr_regl = *args++; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args) #if defined(CONFIG_SOFTMMU) tcg_out_tlb_load(s, addr_regl, oi, label_ptr, 0); - base = tcg_out_zext_addr_if_32_bit(s, addr_regl, TCG_REG_TMP0); - tcg_out_qemu_st_indexed(s, data_regl, base, TCG_REG_TMP2, opc); + tcg_out_qemu_st_indexed(s, data_regl, addr_regl, TCG_REG_TMP2, opc); add_qemu_ldst_label(s, 0, oi, 0, /* type param is unused for stores */ data_regl, addr_regl, -- 2.25.1 The following changes since commit e93ded1bf6c94ab95015b33e188bc8b0b0c32670: Merge tag 'testing-pull-request-2022-08-30' of https://gitlab.com/thuth/qemu into staging (2022-08-31 18:19:03 -0400) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20220901 for you to fetch changes up to 20011be2e30b8aa8ef1fc258485f00c688703deb: target/riscv: Make translator stop before the end of a page (2022-09-01 07:43:08 +0100) ---------------------------------------------------------------- Respect PROT_EXEC in user-only mode. Fix s390x, i386 and riscv for translations crossing a page. ---------------------------------------------------------------- Ilya Leoshkevich (4): linux-user: Clear translations on mprotect() accel/tcg: Introduce is_same_page() target/s390x: Make translator stop before the end of a page target/i386: Make translator stop before the end of a page Richard Henderson (16): linux-user/arm: Mark the commpage executable linux-user/hppa: Allocate page zero as a commpage linux-user/x86_64: Allocate vsyscall page as a commpage linux-user: Honor PT_GNU_STACK tests/tcg/i386: Move smc_code2 to an executable section accel/tcg: Properly implement get_page_addr_code for user-only accel/tcg: Unlock mmap_lock after longjmp accel/tcg: Make tb_htable_lookup static accel/tcg: Move qemu_ram_addr_from_host_nofail to physmem.c accel/tcg: Use probe_access_internal for softmmu get_page_addr_code_hostp accel/tcg: Document the faulting lookup in tb_lookup_cmp accel/tcg: Remove translator_ldsw accel/tcg: Add pc and host_pc params to gen_intermediate_code accel/tcg: Add fast path for translator_ld* target/riscv: Add MAX_INSN_LEN and insn_len target/riscv: Make translator stop before the end of a page include/elf.h | 1 + include/exec/cpu-common.h | 1 + include/exec/exec-all.h | 89 ++++++++---------------- include/exec/translator.h | 96 ++++++++++++++++--------- linux-user/arm/target_cpu.h | 4 +- linux-user/qemu.h | 1 + accel/tcg/cpu-exec.c | 143 ++++++++++++++++++++------------------ accel/tcg/cputlb.c | 93 +++++++------------------ accel/tcg/translate-all.c | 29 ++++---- accel/tcg/translator.c | 135 ++++++++++++++++++++++++++--------- accel/tcg/user-exec.c | 17 ++++- linux-user/elfload.c | 82 ++++++++++++++++++++-- linux-user/mmap.c | 6 +- softmmu/physmem.c | 12 ++++ target/alpha/translate.c | 5 +- target/arm/translate.c | 5 +- target/avr/translate.c | 5 +- target/cris/translate.c | 5 +- target/hexagon/translate.c | 6 +- target/hppa/translate.c | 5 +- target/i386/tcg/translate.c | 71 +++++++++++-------- target/loongarch/translate.c | 6 +- target/m68k/translate.c | 5 +- target/microblaze/translate.c | 5 +- target/mips/tcg/translate.c | 5 +- target/nios2/translate.c | 5 +- target/openrisc/translate.c | 6 +- target/ppc/translate.c | 5 +- target/riscv/translate.c | 32 +++++++-- target/rx/translate.c | 5 +- target/s390x/tcg/translate.c | 20 ++++-- target/sh4/translate.c | 5 +- target/sparc/translate.c | 5 +- target/tricore/translate.c | 6 +- target/xtensa/translate.c | 6 +- tests/tcg/i386/test-i386.c | 2 +- tests/tcg/riscv64/noexec.c | 79 +++++++++++++++++++++ tests/tcg/s390x/noexec.c | 106 ++++++++++++++++++++++++++++ tests/tcg/x86_64/noexec.c | 75 ++++++++++++++++++++ tests/tcg/multiarch/noexec.c.inc | 139 ++++++++++++++++++++++++++++++++++++ tests/tcg/riscv64/Makefile.target | 1 + tests/tcg/s390x/Makefile.target | 1 + tests/tcg/x86_64/Makefile.target | 3 +- 43 files changed, 966 insertions(+), 367 deletions(-) create mode 100644 tests/tcg/riscv64/noexec.c create mode 100644 tests/tcg/s390x/noexec.c create mode 100644 tests/tcg/x86_64/noexec.c create mode 100644 tests/tcg/multiarch/noexec.c.inc We're about to start validating PAGE_EXEC, which means that we've got to mark the commpage executable. We had been placing the commpage outside of reserved_va, which was incorrect and lead to an abort. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- linux-user/arm/target_cpu.h | 4 ++-- linux-user/elfload.c | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/linux-user/arm/target_cpu.h b/linux-user/arm/target_cpu.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/arm/target_cpu.h +++ b/linux-user/arm/target_cpu.h @@ -XXX,XX +XXX,XX @@ static inline unsigned long arm_max_reserved_va(CPUState *cs) } else { /* * We need to be able to map the commpage. - * See validate_guest_space in linux-user/elfload.c. + * See init_guest_commpage in linux-user/elfload.c. */ - return 0xffff0000ul; + return 0xfffffffful; } } #define MAX_RESERVED_VA arm_max_reserved_va diff --git a/linux-user/elfload.c b/linux-user/elfload.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/elfload.c +++ b/linux-user/elfload.c @@ -XXX,XX +XXX,XX @@ enum { static bool init_guest_commpage(void) { - void *want = g2h_untagged(HI_COMMPAGE & -qemu_host_page_size); + abi_ptr commpage = HI_COMMPAGE & -qemu_host_page_size; + void *want = g2h_untagged(commpage); void *addr = mmap(want, qemu_host_page_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0); @@ -XXX,XX +XXX,XX @@ static bool init_guest_commpage(void) perror("Protecting guest commpage"); exit(EXIT_FAILURE); } + + page_set_flags(commpage, commpage + qemu_host_page_size, + PAGE_READ | PAGE_EXEC | PAGE_VALID); return true; } -- 2.34.1 While there are no target-specific nonfaulting probes, generic code may grow some uses at some point. Note that the attrs argument was incorrect -- it should have been MEMTXATTRS_UNSPECIFIED. Just use the simpler interface. Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/avr/helper.c | 46 ++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/target/avr/helper.c b/target/avr/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/helper.c +++ b/target/avr/helper.c @@ -XXX,XX +XXX,XX @@ bool avr_cpu_tlb_fill(CPUState *cs, vaddr address, int size, MMUAccessType access_type, int mmu_idx, bool probe, uintptr_t retaddr) { - int prot = 0; - MemTxAttrs attrs = {}; + int prot, page_size = TARGET_PAGE_SIZE; uint32_t paddr; address &= TARGET_PAGE_MASK; if (mmu_idx == MMU_CODE_IDX) { - /* access to code in flash */ + /* Access to code in flash. */ paddr = OFFSET_CODE + address; prot = PAGE_READ | PAGE_EXEC; - if (paddr + TARGET_PAGE_SIZE > OFFSET_DATA) { + if (paddr >= OFFSET_DATA) { + /* + * This should not be possible via any architectural operations. + * There is certainly not an exception that we can deliver. + * Accept probing that might come from generic code. + */ + if (probe) { + return false; + } error_report("execution left flash memory"); abort(); } - } else if (address < NUMBER_OF_CPU_REGISTERS + NUMBER_OF_IO_REGISTERS) { - /* - * access to CPU registers, exit and rebuilt this TB to use full access - * incase it touches specially handled registers like SREG or SP - */ - AVRCPU *cpu = AVR_CPU(cs); - CPUAVRState *env = &cpu->env; - env->fullacc = 1; - cpu_loop_exit_restore(cs, retaddr); } else { - /* access to memory. nothing special */ + /* Access to memory. */ paddr = OFFSET_DATA + address; prot = PAGE_READ | PAGE_WRITE; + if (address < NUMBER_OF_CPU_REGISTERS + NUMBER_OF_IO_REGISTERS) { + /* + * Access to CPU registers, exit and rebuilt this TB to use + * full access in case it touches specially handled registers + * like SREG or SP. For probing, set page_size = 1, in order + * to force tlb_fill to be called for the next access. + */ + if (probe) { + page_size = 1; + } else { + AVRCPU *cpu = AVR_CPU(cs); + CPUAVRState *env = &cpu->env; + env->fullacc = 1; + cpu_loop_exit_restore(cs, retaddr); + } + } } - tlb_set_page_with_attrs(cs, address, paddr, attrs, prot, - mmu_idx, TARGET_PAGE_SIZE); - + tlb_set_page(cs, address, paddr, prot, mmu_idx, page_size); return true; } -- 2.34.1 There is no need to go through cc->tcg_ops when we know what value that must have. Reviewed-by: Michael Rolnik <mrolnik@gmail.com> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/avr/helper.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/target/avr/helper.c b/target/avr/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/helper.c +++ b/target/avr/helper.c @@ -XXX,XX +XXX,XX @@ bool avr_cpu_exec_interrupt(CPUState *cs, int interrupt_request) { bool ret = false; - CPUClass *cc = CPU_GET_CLASS(cs); AVRCPU *cpu = AVR_CPU(cs); CPUAVRState *env = &cpu->env; if (interrupt_request & CPU_INTERRUPT_RESET) { if (cpu_interrupts_enabled(env)) { cs->exception_index = EXCP_RESET; - cc->tcg_ops->do_interrupt(cs); + avr_cpu_do_interrupt(cs); cs->interrupt_request &= ~CPU_INTERRUPT_RESET; @@ -XXX,XX +XXX,XX @@ bool avr_cpu_exec_interrupt(CPUState *cs, int interrupt_request) if (cpu_interrupts_enabled(env) && env->intsrc != 0) { int index = ctz32(env->intsrc); cs->exception_index = EXCP_INT(index); - cc->tcg_ops->do_interrupt(cs); + avr_cpu_do_interrupt(cs); env->intsrc &= env->intsrc - 1; /* clear the interrupt */ if (!env->intsrc) { -- 2.34.1 We're about to start validating PAGE_EXEC, which means that we've got to mark page zero executable. We had been special casing this entirely within translate. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- linux-user/elfload.c | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/linux-user/elfload.c b/linux-user/elfload.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/elfload.c +++ b/linux-user/elfload.c @@ -XXX,XX +XXX,XX @@ static inline void init_thread(struct target_pt_regs *regs, regs->gr[31] = infop->entry; } +#define LO_COMMPAGE 0 + +static bool init_guest_commpage(void) +{ + void *want = g2h_untagged(LO_COMMPAGE); + void *addr = mmap(want, qemu_host_page_size, PROT_NONE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0); + + if (addr == MAP_FAILED) { + perror("Allocating guest commpage"); + exit(EXIT_FAILURE); + } + if (addr != want) { + return false; + } + + /* + * On Linux, page zero is normally marked execute only + gateway. + * Normal read or write is supposed to fail (thus PROT_NONE above), + * but specific offsets have kernel code mapped to raise permissions + * and implement syscalls. Here, simply mark the page executable. + * Special case the entry points during translation (see do_page_zero). + */ + page_set_flags(LO_COMMPAGE, LO_COMMPAGE + TARGET_PAGE_SIZE, + PAGE_EXEC | PAGE_VALID); + return true; +} + #endif /* TARGET_HPPA */ #ifdef TARGET_XTENSA @@ -XXX,XX +XXX,XX @@ static abi_ulong create_elf_tables(abi_ulong p, int argc, int envc, } #if defined(HI_COMMPAGE) -#define LO_COMMPAGE 0 +#define LO_COMMPAGE -1 #elif defined(LO_COMMPAGE) #define HI_COMMPAGE 0 #else #define HI_COMMPAGE 0 -#define LO_COMMPAGE 0 +#define LO_COMMPAGE -1 #define init_guest_commpage() true #endif @@ -XXX,XX +XXX,XX @@ static void pgb_static(const char *image_name, abi_ulong orig_loaddr, } else { offset = -(HI_COMMPAGE & -align); } - } else if (LO_COMMPAGE != 0) { + } else if (LO_COMMPAGE != -1) { loaddr = MIN(loaddr, LO_COMMPAGE & -align); } -- 2.34.1 We're about to start validating PAGE_EXEC, which means that we've got to mark the vsyscall page executable. We had been special casing this entirely within translate. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- linux-user/elfload.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/linux-user/elfload.c b/linux-user/elfload.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/elfload.c +++ b/linux-user/elfload.c @@ -XXX,XX +XXX,XX @@ static void elf_core_copy_regs(target_elf_gregset_t *regs, const CPUX86State *en (*regs)[26] = tswapreg(env->segs[R_GS].selector & 0xffff); } +#if ULONG_MAX >= TARGET_VSYSCALL_PAGE +#define INIT_GUEST_COMMPAGE +static bool init_guest_commpage(void) +{ + /* + * The vsyscall page is at a high negative address aka kernel space, + * which means that we cannot actually allocate it with target_mmap. + * We still should be able to use page_set_flags, unless the user + * has specified -R reserved_va, which would trigger an assert(). + */ + if (reserved_va != 0 && + TARGET_VSYSCALL_PAGE + TARGET_PAGE_SIZE >= reserved_va) { + error_report("Cannot allocate vsyscall page"); + exit(EXIT_FAILURE); + } + page_set_flags(TARGET_VSYSCALL_PAGE, + TARGET_VSYSCALL_PAGE + TARGET_PAGE_SIZE, + PAGE_EXEC | PAGE_VALID); + return true; +} +#endif #else #define ELF_START_MMAP 0x80000000 @@ -XXX,XX +XXX,XX @@ static abi_ulong create_elf_tables(abi_ulong p, int argc, int envc, #else #define HI_COMMPAGE 0 #define LO_COMMPAGE -1 +#ifndef INIT_GUEST_COMMPAGE #define init_guest_commpage() true #endif +#endif static void pgb_fail_in_use(const char *image_name) { -- 2.34.1 We cannot deliver two interrupts simultaneously; the first interrupt handler must execute first. Reviewed-by: Michael Rolnik <mrolnik@gmail.com> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/avr/helper.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/target/avr/helper.c b/target/avr/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/helper.c +++ b/target/avr/helper.c @@ -XXX,XX +XXX,XX @@ bool avr_cpu_exec_interrupt(CPUState *cs, int interrupt_request) { - bool ret = false; AVRCPU *cpu = AVR_CPU(cs); CPUAVRState *env = &cpu->env; @@ -XXX,XX +XXX,XX @@ bool avr_cpu_exec_interrupt(CPUState *cs, int interrupt_request) avr_cpu_do_interrupt(cs); cs->interrupt_request &= ~CPU_INTERRUPT_RESET; - - ret = true; + return true; } } if (interrupt_request & CPU_INTERRUPT_HARD) { @@ -XXX,XX +XXX,XX @@ bool avr_cpu_exec_interrupt(CPUState *cs, int interrupt_request) if (!env->intsrc) { cs->interrupt_request &= ~CPU_INTERRUPT_HARD; } - - ret = true; + return true; } } - return ret; + return false; } void avr_cpu_do_interrupt(CPUState *cs) -- 2.34.1 This bit is not saved across interrupts, so we must delay delivering the interrupt until the skip has been processed. Resolves: https://gitlab.com/qemu-project/qemu/-/issues/1118 Reviewed-by: Michael Rolnik <mrolnik@gmail.com> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/avr/helper.c | 9 +++++++++ target/avr/translate.c | 26 ++++++++++++++++++++++---- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/target/avr/helper.c b/target/avr/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/helper.c +++ b/target/avr/helper.c @@ -XXX,XX +XXX,XX @@ bool avr_cpu_exec_interrupt(CPUState *cs, int interrupt_request) AVRCPU *cpu = AVR_CPU(cs); CPUAVRState *env = &cpu->env; + /* + * We cannot separate a skip from the next instruction, + * as the skip would not be preserved across the interrupt. + * Separating the two insn normally only happens at page boundaries. + */ + if (env->skip) { + return false; + } + if (interrupt_request & CPU_INTERRUPT_RESET) { if (cpu_interrupts_enabled(env)) { cs->exception_index = EXCP_RESET; diff --git a/target/avr/translate.c b/target/avr/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/translate.c +++ b/target/avr/translate.c @@ -XXX,XX +XXX,XX @@ static void avr_tr_translate_insn(DisasContextBase *dcbase, CPUState *cs) if (skip_label) { canonicalize_skip(ctx); gen_set_label(skip_label); - if (ctx->base.is_jmp == DISAS_NORETURN) { + + switch (ctx->base.is_jmp) { + case DISAS_NORETURN: ctx->base.is_jmp = DISAS_CHAIN; + break; + case DISAS_NEXT: + if (ctx->base.tb->flags & TB_FLAGS_SKIP) { + ctx->base.is_jmp = DISAS_TOO_MANY; + } + break; + default: + break; } } @@ -XXX,XX +XXX,XX @@ static void avr_tr_tb_stop(DisasContextBase *dcbase, CPUState *cs) { DisasContext *ctx = container_of(dcbase, DisasContext, base); bool nonconst_skip = canonicalize_skip(ctx); + /* + * Because we disable interrupts while env->skip is set, + * we must return to the main loop to re-evaluate afterward. + */ + bool force_exit = ctx->base.tb->flags & TB_FLAGS_SKIP; switch (ctx->base.is_jmp) { case DISAS_NORETURN: @@ -XXX,XX +XXX,XX @@ static void avr_tr_tb_stop(DisasContextBase *dcbase, CPUState *cs) case DISAS_NEXT: case DISAS_TOO_MANY: case DISAS_CHAIN: - if (!nonconst_skip) { + if (!nonconst_skip && !force_exit) { /* Note gen_goto_tb checks singlestep. */ gen_goto_tb(ctx, 1, ctx->npc); break; @@ -XXX,XX +XXX,XX @@ static void avr_tr_tb_stop(DisasContextBase *dcbase, CPUState *cs) tcg_gen_movi_tl(cpu_pc, ctx->npc); /* fall through */ case DISAS_LOOKUP: - tcg_gen_lookup_and_goto_ptr(); - break; + if (!force_exit) { + tcg_gen_lookup_and_goto_ptr(); + break; + } + /* fall through */ case DISAS_EXIT: tcg_gen_exit_tb(NULL, 0); break; -- 2.34.1 Map the stack executable if required by default or on demand. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/elf.h | 1 + linux-user/qemu.h | 1 + linux-user/elfload.c | 19 ++++++++++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/include/elf.h b/include/elf.h index XXXXXXX..XXXXXXX 100644 --- a/include/elf.h +++ b/include/elf.h @@ -XXX,XX +XXX,XX @@ typedef int64_t Elf64_Sxword; #define PT_LOPROC 0x70000000 #define PT_HIPROC 0x7fffffff +#define PT_GNU_STACK (PT_LOOS + 0x474e551) #define PT_GNU_PROPERTY (PT_LOOS + 0x474e553) #define PT_MIPS_REGINFO 0x70000000 diff --git a/linux-user/qemu.h b/linux-user/qemu.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/qemu.h +++ b/linux-user/qemu.h @@ -XXX,XX +XXX,XX @@ struct image_info { uint32_t elf_flags; int personality; abi_ulong alignment; + bool exec_stack; /* Generic semihosting knows about these pointers. */ abi_ulong arg_strings; /* strings for argv */ diff --git a/linux-user/elfload.c b/linux-user/elfload.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/elfload.c +++ b/linux-user/elfload.c @@ -XXX,XX +XXX,XX @@ static bool init_guest_commpage(void) #define ELF_ARCH EM_386 #define ELF_PLATFORM get_elf_platform() +#define EXSTACK_DEFAULT true static const char *get_elf_platform(void) { @@ -XXX,XX +XXX,XX @@ static void elf_core_copy_regs(target_elf_gregset_t *regs, const CPUX86State *en #define ELF_ARCH EM_ARM #define ELF_CLASS ELFCLASS32 +#define EXSTACK_DEFAULT true static inline void init_thread(struct target_pt_regs *regs, struct image_info *infop) @@ -XXX,XX +XXX,XX @@ static inline void init_thread(struct target_pt_regs *regs, #else #define ELF_CLASS ELFCLASS32 +#define EXSTACK_DEFAULT true #endif @@ -XXX,XX +XXX,XX @@ static void elf_core_copy_regs(target_elf_gregset_t *regs, const CPUPPCState *en #define ELF_CLASS ELFCLASS64 #define ELF_ARCH EM_LOONGARCH +#define EXSTACK_DEFAULT true #define elf_check_arch(x) ((x) == EM_LOONGARCH) @@ -XXX,XX +XXX,XX @@ static uint32_t get_elf_hwcap(void) #define ELF_CLASS ELFCLASS32 #endif #define ELF_ARCH EM_MIPS +#define EXSTACK_DEFAULT true #ifdef TARGET_ABI_MIPSN32 #define elf_check_abi(x) ((x) & EF_MIPS_ABI2) @@ -XXX,XX +XXX,XX @@ static inline void init_thread(struct target_pt_regs *regs, #define bswaptls(ptr) bswap32s(ptr) #endif +#ifndef EXSTACK_DEFAULT +#define EXSTACK_DEFAULT false +#endif + #include "elf.h" /* We must delay the following stanzas until after "elf.h". */ @@ -XXX,XX +XXX,XX @@ static abi_ulong setup_arg_pages(struct linux_binprm *bprm, struct image_info *info) { abi_ulong size, error, guard; + int prot; size = guest_stack_size; if (size < STACK_LOWER_LIMIT) { @@ -XXX,XX +XXX,XX @@ static abi_ulong setup_arg_pages(struct linux_binprm *bprm, guard = qemu_real_host_page_size(); } - error = target_mmap(0, size + guard, PROT_READ | PROT_WRITE, + prot = PROT_READ | PROT_WRITE; + if (info->exec_stack) { + prot |= PROT_EXEC; + } + error = target_mmap(0, size + guard, prot, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (error == -1) { perror("mmap stack"); @@ -XXX,XX +XXX,XX @@ static void load_elf_image(const char *image_name, int image_fd, */ loaddr = -1, hiaddr = 0; info->alignment = 0; + info->exec_stack = EXSTACK_DEFAULT; for (i = 0; i < ehdr->e_phnum; ++i) { struct elf_phdr *eppnt = phdr + i; if (eppnt->p_type == PT_LOAD) { @@ -XXX,XX +XXX,XX @@ static void load_elf_image(const char *image_name, int image_fd, if (!parse_elf_properties(image_fd, info, eppnt, bprm_buf, &err)) { goto exit_errmsg; } + } else if (eppnt->p_type == PT_GNU_STACK) { + info->exec_stack = eppnt->p_flags & PF_X; } } -- 2.34.1 From: Ilya Leoshkevich <iii@linux.ibm.com> Currently it's possible to execute pages that do not have PAGE_EXEC if there is an existing translation block. Fix by invalidating TBs that touch the affected pages. Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com> Message-Id: <20220817150506.592862-2-iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- linux-user/mmap.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/linux-user/mmap.c b/linux-user/mmap.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/mmap.c +++ b/linux-user/mmap.c @@ -XXX,XX +XXX,XX @@ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot) goto error; } } + page_set_flags(start, start + len, page_flags); - mmap_unlock(); - return 0; + tb_invalidate_phys_range(start, start + len); + ret = 0; + error: mmap_unlock(); return ret; -- 2.34.1 We're about to start validating PAGE_EXEC, which means that we've got to put this code into a section that is both writable and executable. Note that this test did not run on hardware beforehand either. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tests/tcg/i386/test-i386.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tcg/i386/test-i386.c b/tests/tcg/i386/test-i386.c index XXXXXXX..XXXXXXX 100644 --- a/tests/tcg/i386/test-i386.c +++ b/tests/tcg/i386/test-i386.c @@ -XXX,XX +XXX,XX @@ uint8_t code[] = { 0xc3, /* ret */ }; -asm(".section \".data\"\n" +asm(".section \".data_x\",\"awx\"\n" "smc_code2:\n" "movl 4(%esp), %eax\n" "movl %eax, smc_patch_addr2 + 1\n" -- 2.34.1 From: Ilya Leoshkevich <iii@linux.ibm.com> Introduce a function that checks whether a given address is on the same page as where disassembly started. Having it improves readability of the following patches. Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com> Message-Id: <20220811095534.241224-3-iii@linux.ibm.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> [rth: Make the DisasContextBase parameter const.] Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/translator.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/exec/translator.h b/include/exec/translator.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/translator.h +++ b/include/exec/translator.h @@ -XXX,XX +XXX,XX @@ FOR_EACH_TRANSLATOR_LD(GEN_TRANSLATOR_LD) #undef GEN_TRANSLATOR_LD +/* + * Return whether addr is on the same page as where disassembly started. + * Translators can use this to enforce the rule that only single-insn + * translation blocks are allowed to cross page boundaries. + */ +static inline bool is_same_page(const DisasContextBase *db, target_ulong addr) +{ + return ((addr ^ db->pc_first) & TARGET_PAGE_MASK) == 0; +} + #endif /* EXEC__TRANSLATOR_H */ -- 2.34.1 The current implementation is a no-op, simply returning addr. This is incorrect, because we ought to be checking the page permissions for execution. Make get_page_addr_code inline for both implementations. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Acked-by: Alistair Francis <alistair.francis@wdc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/exec-all.h | 85 ++++++++++++++--------------------------- accel/tcg/cputlb.c | 5 --- accel/tcg/user-exec.c | 14 +++++++ 3 files changed, 42 insertions(+), 62 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ struct MemoryRegionSection *iotlb_to_section(CPUState *cpu, hwaddr index, MemTxAttrs attrs); #endif -#if defined(CONFIG_USER_ONLY) -void mmap_lock(void); -void mmap_unlock(void); -bool have_mmap_lock(void); - /** - * get_page_addr_code() - user-mode version + * get_page_addr_code_hostp() * @env: CPUArchState * @addr: guest virtual address of guest code * - * Returns @addr. + * See get_page_addr_code() (full-system version) for documentation on the + * return value. + * + * Sets *@hostp (when @hostp is non-NULL) as follows. + * If the return value is -1, sets *@hostp to NULL. Otherwise, sets *@hostp + * to the host address where @addr's content is kept. + * + * Note: this function can trigger an exception. + */ +tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, + void **hostp); + +/** + * get_page_addr_code() + * @env: CPUArchState + * @addr: guest virtual address of guest code + * + * If we cannot translate and execute from the entire RAM page, or if + * the region is not backed by RAM, returns -1. Otherwise, returns the + * ram_addr_t corresponding to the guest code at @addr. + * + * Note: this function can trigger an exception. */ static inline tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr) { - return addr; + return get_page_addr_code_hostp(env, addr, NULL); } -/** - * get_page_addr_code_hostp() - user-mode version - * @env: CPUArchState - * @addr: guest virtual address of guest code - * - * Returns @addr. - * - * If @hostp is non-NULL, sets *@hostp to the host address where @addr's content - * is kept. - */ -static inline tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, - target_ulong addr, - void **hostp) -{ - if (hostp) { - *hostp = g2h_untagged(addr); - } - return addr; -} +#if defined(CONFIG_USER_ONLY) +void mmap_lock(void); +void mmap_unlock(void); +bool have_mmap_lock(void); /** * adjust_signal_pc: @@ -XXX,XX +XXX,XX @@ G_NORETURN void cpu_loop_exit_sigbus(CPUState *cpu, target_ulong addr, static inline void mmap_lock(void) {} static inline void mmap_unlock(void) {} -/** - * get_page_addr_code() - full-system version - * @env: CPUArchState - * @addr: guest virtual address of guest code - * - * If we cannot translate and execute from the entire RAM page, or if - * the region is not backed by RAM, returns -1. Otherwise, returns the - * ram_addr_t corresponding to the guest code at @addr. - * - * Note: this function can trigger an exception. - */ -tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr); - -/** - * get_page_addr_code_hostp() - full-system version - * @env: CPUArchState - * @addr: guest virtual address of guest code - * - * See get_page_addr_code() (full-system version) for documentation on the - * return value. - * - * Sets *@hostp (when @hostp is non-NULL) as follows. - * If the return value is -1, sets *@hostp to NULL. Otherwise, sets *@hostp - * to the host address where @addr's content is kept. - * - * Note: this function can trigger an exception. - */ -tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, - void **hostp); - void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length); void tlb_set_dirty(CPUState *cpu, target_ulong vaddr); diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, return qemu_ram_addr_from_host_nofail(p); } -tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr) -{ - return get_page_addr_code_hostp(env, addr, NULL); -} - static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size, CPUIOTLBEntry *iotlbentry, uintptr_t retaddr) { diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -XXX,XX +XXX,XX @@ void *probe_access(CPUArchState *env, target_ulong addr, int size, return size ? g2h(env_cpu(env), addr) : NULL; } +tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, + void **hostp) +{ + int flags; + + flags = probe_access_internal(env, addr, 1, MMU_INST_FETCH, false, 0); + g_assert(flags == 0); + + if (hostp) { + *hostp = g2h_untagged(addr); + } + return addr; +} + /* The softmmu versions of these helpers are in cputlb.c. */ /* -- 2.34.1 The mmap_lock is held around tb_gen_code. While the comment is correct that the lock is dropped when tb_gen_code runs out of memory, the lock is *not* dropped when an exception is raised reading code for translation. Acked-by: Alistair Francis <alistair.francis@wdc.com> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cpu-exec.c | 12 ++++++------ accel/tcg/user-exec.c | 3 --- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ void cpu_exec_step_atomic(CPUState *cpu) cpu_tb_exec(cpu, tb, &tb_exit); cpu_exec_exit(cpu); } else { - /* - * The mmap_lock is dropped by tb_gen_code if it runs out of - * memory. - */ #ifndef CONFIG_SOFTMMU clear_helper_retaddr(); - tcg_debug_assert(!have_mmap_lock()); + if (have_mmap_lock()) { + mmap_unlock(); + } #endif if (qemu_mutex_iothread_locked()) { qemu_mutex_unlock_iothread(); @@ -XXX,XX +XXX,XX @@ int cpu_exec(CPUState *cpu) #ifndef CONFIG_SOFTMMU clear_helper_retaddr(); - tcg_debug_assert(!have_mmap_lock()); + if (have_mmap_lock()) { + mmap_unlock(); + } #endif if (qemu_mutex_iothread_locked()) { qemu_mutex_unlock_iothread(); diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -XXX,XX +XXX,XX @@ MMUAccessType adjust_signal_pc(uintptr_t *pc, bool is_write) * (and if the translator doesn't handle page boundaries correctly * there's little we can do about that here). Therefore, do not * trigger the unwinder. - * - * Like tb_gen_code, release the memory lock before cpu_loop_exit. */ - mmap_unlock(); *pc = 0; return MMU_INST_FETCH; } -- 2.34.1 The function is not used outside of cpu-exec.c. Move it and its subroutines up in the file, before the first use. Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/exec-all.h | 3 - accel/tcg/cpu-exec.c | 122 ++++++++++++++++++++-------------------- 2 files changed, 61 insertions(+), 64 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs); #endif void tb_flush(CPUState *cpu); void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr); -TranslationBlock *tb_htable_lookup(CPUState *cpu, target_ulong pc, - target_ulong cs_base, uint32_t flags, - uint32_t cflags); void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr); /* GETPC is the true target of the return instruction that we'll execute. */ diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ uint32_t curr_cflags(CPUState *cpu) return cflags; } +struct tb_desc { + target_ulong pc; + target_ulong cs_base; + CPUArchState *env; + tb_page_addr_t phys_page1; + uint32_t flags; + uint32_t cflags; + uint32_t trace_vcpu_dstate; +}; + +static bool tb_lookup_cmp(const void *p, const void *d) +{ + const TranslationBlock *tb = p; + const struct tb_desc *desc = d; + + if (tb->pc == desc->pc && + tb->page_addr[0] == desc->phys_page1 && + tb->cs_base == desc->cs_base && + tb->flags == desc->flags && + tb->trace_vcpu_dstate == desc->trace_vcpu_dstate && + tb_cflags(tb) == desc->cflags) { + /* check next page if needed */ + if (tb->page_addr[1] == -1) { + return true; + } else { + tb_page_addr_t phys_page2; + target_ulong virt_page2; + + virt_page2 = (desc->pc & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE; + phys_page2 = get_page_addr_code(desc->env, virt_page2); + if (tb->page_addr[1] == phys_page2) { + return true; + } + } + } + return false; +} + +static TranslationBlock *tb_htable_lookup(CPUState *cpu, target_ulong pc, + target_ulong cs_base, uint32_t flags, + uint32_t cflags) +{ + tb_page_addr_t phys_pc; + struct tb_desc desc; + uint32_t h; + + desc.env = cpu->env_ptr; + desc.cs_base = cs_base; + desc.flags = flags; + desc.cflags = cflags; + desc.trace_vcpu_dstate = *cpu->trace_dstate; + desc.pc = pc; + phys_pc = get_page_addr_code(desc.env, pc); + if (phys_pc == -1) { + return NULL; + } + desc.phys_page1 = phys_pc & TARGET_PAGE_MASK; + h = tb_hash_func(phys_pc, pc, flags, cflags, *cpu->trace_dstate); + return qht_lookup_custom(&tb_ctx.htable, &desc, h, tb_lookup_cmp); +} + /* Might cause an exception, so have a longjmp destination ready */ static inline TranslationBlock *tb_lookup(CPUState *cpu, target_ulong pc, target_ulong cs_base, @@ -XXX,XX +XXX,XX @@ void cpu_exec_step_atomic(CPUState *cpu) end_exclusive(); } -struct tb_desc { - target_ulong pc; - target_ulong cs_base; - CPUArchState *env; - tb_page_addr_t phys_page1; - uint32_t flags; - uint32_t cflags; - uint32_t trace_vcpu_dstate; -}; - -static bool tb_lookup_cmp(const void *p, const void *d) -{ - const TranslationBlock *tb = p; - const struct tb_desc *desc = d; - - if (tb->pc == desc->pc && - tb->page_addr[0] == desc->phys_page1 && - tb->cs_base == desc->cs_base && - tb->flags == desc->flags && - tb->trace_vcpu_dstate == desc->trace_vcpu_dstate && - tb_cflags(tb) == desc->cflags) { - /* check next page if needed */ - if (tb->page_addr[1] == -1) { - return true; - } else { - tb_page_addr_t phys_page2; - target_ulong virt_page2; - - virt_page2 = (desc->pc & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE; - phys_page2 = get_page_addr_code(desc->env, virt_page2); - if (tb->page_addr[1] == phys_page2) { - return true; - } - } - } - return false; -} - -TranslationBlock *tb_htable_lookup(CPUState *cpu, target_ulong pc, - target_ulong cs_base, uint32_t flags, - uint32_t cflags) -{ - tb_page_addr_t phys_pc; - struct tb_desc desc; - uint32_t h; - - desc.env = cpu->env_ptr; - desc.cs_base = cs_base; - desc.flags = flags; - desc.cflags = cflags; - desc.trace_vcpu_dstate = *cpu->trace_dstate; - desc.pc = pc; - phys_pc = get_page_addr_code(desc.env, pc); - if (phys_pc == -1) { - return NULL; - } - desc.phys_page1 = phys_pc & TARGET_PAGE_MASK; - h = tb_hash_func(phys_pc, pc, flags, cflags, *cpu->trace_dstate); - return qht_lookup_custom(&tb_ctx.htable, &desc, h, tb_lookup_cmp); -} - void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr) { if (TCG_TARGET_HAS_direct_jump) { -- 2.34.1 The base qemu_ram_addr_from_host function is already in softmmu/physmem.c; move the nofail version to be adjacent. Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/cpu-common.h | 1 + accel/tcg/cputlb.c | 12 ------------ softmmu/physmem.c | 12 ++++++++++++ 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu-common.h +++ b/include/exec/cpu-common.h @@ -XXX,XX +XXX,XX @@ typedef uintptr_t ram_addr_t; void qemu_ram_remap(ram_addr_t addr, ram_addr_t length); /* This should not be used by devices. */ ram_addr_t qemu_ram_addr_from_host(void *ptr); +ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr); RAMBlock *qemu_ram_block_by_name(const char *name); RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset, ram_addr_t *offset); diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ void tlb_set_page(CPUState *cpu, target_ulong vaddr, prot, mmu_idx, size); } -static inline ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr) -{ - ram_addr_t ram_addr; - - ram_addr = qemu_ram_addr_from_host(ptr); - if (ram_addr == RAM_ADDR_INVALID) { - error_report("Bad ram pointer %p", ptr); - abort(); - } - return ram_addr; -} - /* * Note: tlb_fill() can trigger a resize of the TLB. This means that all of the * caller's prior references to the TLB table (e.g. CPUTLBEntry pointers) must diff --git a/softmmu/physmem.c b/softmmu/physmem.c index XXXXXXX..XXXXXXX 100644 --- a/softmmu/physmem.c +++ b/softmmu/physmem.c @@ -XXX,XX +XXX,XX @@ ram_addr_t qemu_ram_addr_from_host(void *ptr) return block->offset + offset; } +ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr) +{ + ram_addr_t ram_addr; + + ram_addr = qemu_ram_addr_from_host(ptr); + if (ram_addr == RAM_ADDR_INVALID) { + error_report("Bad ram pointer %p", ptr); + abort(); + } + return ram_addr; +} + static MemTxResult flatview_read(FlatView *fv, hwaddr addr, MemTxAttrs attrs, void *buf, hwaddr len); static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs, -- 2.34.1 Simplify the implementation of get_page_addr_code_hostp by reusing the existing probe_access infrastructure. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cputlb.c | 76 ++++++++++++++++------------------------------ 1 file changed, 26 insertions(+), 50 deletions(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index, victim_tlb_hit(env, mmu_idx, index, offsetof(CPUTLBEntry, TY), \ (ADDR) & TARGET_PAGE_MASK) -/* - * Return a ram_addr_t for the virtual address for execution. - * - * Return -1 if we can't translate and execute from an entire page - * of RAM. This will force us to execute by loading and translating - * one insn at a time, without caching. - * - * NOTE: This function will trigger an exception if the page is - * not executable. - */ -tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, - void **hostp) -{ - uintptr_t mmu_idx = cpu_mmu_index(env, true); - uintptr_t index = tlb_index(env, mmu_idx, addr); - CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); - void *p; - - if (unlikely(!tlb_hit(entry->addr_code, addr))) { - if (!VICTIM_TLB_HIT(addr_code, addr)) { - tlb_fill(env_cpu(env), addr, 0, MMU_INST_FETCH, mmu_idx, 0); - index = tlb_index(env, mmu_idx, addr); - entry = tlb_entry(env, mmu_idx, addr); - - if (unlikely(entry->addr_code & TLB_INVALID_MASK)) { - /* - * The MMU protection covers a smaller range than a target - * page, so we must redo the MMU check for every insn. - */ - return -1; - } - } - assert(tlb_hit(entry->addr_code, addr)); - } - - if (unlikely(entry->addr_code & TLB_MMIO)) { - /* The region is not backed by RAM. */ - if (hostp) { - *hostp = NULL; - } - return -1; - } - - p = (void *)((uintptr_t)addr + entry->addend); - if (hostp) { - *hostp = p; - } - return qemu_ram_addr_from_host_nofail(p); -} - static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size, CPUIOTLBEntry *iotlbentry, uintptr_t retaddr) { @@ -XXX,XX +XXX,XX @@ void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr, return flags ? NULL : host; } +/* + * Return a ram_addr_t for the virtual address for execution. + * + * Return -1 if we can't translate and execute from an entire page + * of RAM. This will force us to execute by loading and translating + * one insn at a time, without caching. + * + * NOTE: This function will trigger an exception if the page is + * not executable. + */ +tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, + void **hostp) +{ + void *p; + + (void)probe_access_internal(env, addr, 1, MMU_INST_FETCH, + cpu_mmu_index(env, true), false, &p, 0); + if (p == NULL) { + return -1; + } + if (hostp) { + *hostp = p; + } + return qemu_ram_addr_from_host_nofail(p); +} + #ifdef CONFIG_PLUGIN /* * Perform a TLB lookup and populate the qemu_plugin_hwaddr structure. -- 2.34.1 It was non-obvious to me why we can raise an exception in the middle of a comparison function, but it works. While nearby, use TARGET_PAGE_ALIGN instead of open-coding. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cpu-exec.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ static bool tb_lookup_cmp(const void *p, const void *d) tb_page_addr_t phys_page2; target_ulong virt_page2; - virt_page2 = (desc->pc & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE; + /* + * We know that the first page matched, and an otherwise valid TB + * encountered an incomplete instruction at the end of that page, + * therefore we know that generating a new TB from the current PC + * must also require reading from the next page -- even if the + * second pages do not match, and therefore the resulting insn + * is different for the new TB. Therefore any exception raised + * here by the faulting lookup is not premature. + */ + virt_page2 = TARGET_PAGE_ALIGN(desc->pc); phys_page2 = get_page_addr_code(desc->env, virt_page2); if (tb->page_addr[1] == phys_page2) { return true; -- 2.34.1 The only user can easily use translator_lduw and adjust the type to signed during the return. Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/translator.h | 1 - target/i386/tcg/translate.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/include/exec/translator.h b/include/exec/translator.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/translator.h +++ b/include/exec/translator.h @@ -XXX,XX +XXX,XX @@ bool translator_use_goto_tb(DisasContextBase *db, target_ulong dest); #define FOR_EACH_TRANSLATOR_LD(F) \ F(translator_ldub, uint8_t, cpu_ldub_code, /* no swap */) \ - F(translator_ldsw, int16_t, cpu_ldsw_code, bswap16) \ F(translator_lduw, uint16_t, cpu_lduw_code, bswap16) \ F(translator_ldl, uint32_t, cpu_ldl_code, bswap32) \ F(translator_ldq, uint64_t, cpu_ldq_code, bswap64) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static inline uint8_t x86_ldub_code(CPUX86State *env, DisasContext *s) static inline int16_t x86_ldsw_code(CPUX86State *env, DisasContext *s) { - return translator_ldsw(env, &s->base, advance_pc(env, s, 2)); + return translator_lduw(env, &s->base, advance_pc(env, s, 2)); } static inline uint16_t x86_lduw_code(CPUX86State *env, DisasContext *s) -- 2.34.1 Pass these along to translator_loop -- pc may be used instead of tb->pc, and host_pc is currently unused. Adjust all targets at one time. Acked-by: Alistair Francis <alistair.francis@wdc.com> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/exec-all.h | 1 - include/exec/translator.h | 24 ++++++++++++++++++++---- accel/tcg/translate-all.c | 6 ++++-- accel/tcg/translator.c | 9 +++++---- target/alpha/translate.c | 5 +++-- target/arm/translate.c | 5 +++-- target/avr/translate.c | 5 +++-- target/cris/translate.c | 5 +++-- target/hexagon/translate.c | 6 ++++-- target/hppa/translate.c | 5 +++-- target/i386/tcg/translate.c | 5 +++-- target/loongarch/translate.c | 6 ++++-- target/m68k/translate.c | 5 +++-- target/microblaze/translate.c | 5 +++-- target/mips/tcg/translate.c | 5 +++-- target/nios2/translate.c | 5 +++-- target/openrisc/translate.c | 6 ++++-- target/ppc/translate.c | 5 +++-- target/riscv/translate.c | 5 +++-- target/rx/translate.c | 5 +++-- target/s390x/tcg/translate.c | 5 +++-- target/sh4/translate.c | 5 +++-- target/sparc/translate.c | 5 +++-- target/tricore/translate.c | 6 ++++-- target/xtensa/translate.c | 6 ++++-- 25 files changed, 97 insertions(+), 53 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ typedef ram_addr_t tb_page_addr_t; #define TB_PAGE_ADDR_FMT RAM_ADDR_FMT #endif -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns); void restore_state_to_opc(CPUArchState *env, TranslationBlock *tb, target_ulong *data); diff --git a/include/exec/translator.h b/include/exec/translator.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/translator.h +++ b/include/exec/translator.h @@ -XXX,XX +XXX,XX @@ #include "exec/translate-all.h" #include "tcg/tcg.h" +/** + * gen_intermediate_code + * @cpu: cpu context + * @tb: translation block + * @max_insns: max number of instructions to translate + * @pc: guest virtual program counter address + * @host_pc: host physical program counter address + * + * This function must be provided by the target, which should create + * the target-specific DisasContext, and then invoke translator_loop. + */ +void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc); /** * DisasJumpType: @@ -XXX,XX +XXX,XX @@ typedef struct TranslatorOps { /** * translator_loop: - * @ops: Target-specific operations. - * @db: Disassembly context. * @cpu: Target vCPU. * @tb: Translation block. * @max_insns: Maximum number of insns to translate. + * @pc: guest virtual program counter address + * @host_pc: host physical program counter address + * @ops: Target-specific operations. + * @db: Disassembly context. * * Generic translator loop. * @@ -XXX,XX +XXX,XX @@ typedef struct TranslatorOps { * - When single-stepping is enabled (system-wide or on the current vCPU). * - When too many instructions have been translated. */ -void translator_loop(const TranslatorOps *ops, DisasContextBase *db, - CPUState *cpu, TranslationBlock *tb, int max_insns); +void translator_loop(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc, + const TranslatorOps *ops, DisasContextBase *db); void translator_loop_temp_check(DisasContextBase *db); diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ #include "exec/cputlb.h" #include "exec/translate-all.h" +#include "exec/translator.h" #include "qemu/bitmap.h" #include "qemu/qemu-print.h" #include "qemu/timer.h" @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, TCGProfile *prof = &tcg_ctx->prof; int64_t ti; #endif + void *host_pc; assert_memory_lock(); qemu_thread_jit_write(); - phys_pc = get_page_addr_code(env, pc); + phys_pc = get_page_addr_code_hostp(env, pc, &host_pc); if (phys_pc == -1) { /* Generate a one-shot TB with 1 insn in it */ @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, tcg_func_start(tcg_ctx); tcg_ctx->cpu = env_cpu(env); - gen_intermediate_code(cpu, tb, max_insns); + gen_intermediate_code(cpu, tb, max_insns, pc, host_pc); assert(tb->size != 0); tcg_ctx->cpu = NULL; max_insns = tb->icount; diff --git a/accel/tcg/translator.c b/accel/tcg/translator.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translator.c +++ b/accel/tcg/translator.c @@ -XXX,XX +XXX,XX @@ static inline void translator_page_protect(DisasContextBase *dcbase, #endif } -void translator_loop(const TranslatorOps *ops, DisasContextBase *db, - CPUState *cpu, TranslationBlock *tb, int max_insns) +void translator_loop(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc, + const TranslatorOps *ops, DisasContextBase *db) { uint32_t cflags = tb_cflags(tb); bool plugin_enabled; /* Initialize DisasContext */ db->tb = tb; - db->pc_first = tb->pc; - db->pc_next = db->pc_first; + db->pc_first = pc; + db->pc_next = pc; db->is_jmp = DISAS_NEXT; db->num_insns = 0; db->max_insns = max_insns; diff --git a/target/alpha/translate.c b/target/alpha/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/alpha/translate.c +++ b/target/alpha/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps alpha_tr_ops = { .disas_log = alpha_tr_disas_log, }; -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc; - translator_loop(&alpha_tr_ops, &dc.base, cpu, tb, max_insns); + translator_loop(cpu, tb, max_insns, pc, host_pc, &alpha_tr_ops, &dc.base); } void restore_state_to_opc(CPUAlphaState *env, TranslationBlock *tb, diff --git a/target/arm/translate.c b/target/arm/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate.c +++ b/target/arm/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps thumb_translator_ops = { }; /* generate intermediate code for basic block 'tb'. */ -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc = { }; const TranslatorOps *ops = &arm_translator_ops; @@ -XXX,XX +XXX,XX @@ void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) } #endif - translator_loop(ops, &dc.base, cpu, tb, max_insns); + translator_loop(cpu, tb, max_insns, pc, host_pc, ops, &dc.base); } void restore_state_to_opc(CPUARMState *env, TranslationBlock *tb, diff --git a/target/avr/translate.c b/target/avr/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/translate.c +++ b/target/avr/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps avr_tr_ops = { .disas_log = avr_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc = { }; - translator_loop(&avr_tr_ops, &dc.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &avr_tr_ops, &dc.base); } void restore_state_to_opc(CPUAVRState *env, TranslationBlock *tb, diff --git a/target/cris/translate.c b/target/cris/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/cris/translate.c +++ b/target/cris/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps cris_tr_ops = { .disas_log = cris_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc; - translator_loop(&cris_tr_ops, &dc.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &cris_tr_ops, &dc.base); } void cris_cpu_dump_state(CPUState *cs, FILE *f, int flags) diff --git a/target/hexagon/translate.c b/target/hexagon/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/translate.c +++ b/target/hexagon/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps hexagon_tr_ops = { .disas_log = hexagon_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&hexagon_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, + &hexagon_tr_ops, &ctx.base); } #define NAME_LEN 64 diff --git a/target/hppa/translate.c b/target/hppa/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/hppa/translate.c +++ b/target/hppa/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps hppa_tr_ops = { .disas_log = hppa_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&hppa_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &hppa_tr_ops, &ctx.base); } void restore_state_to_opc(CPUHPPAState *env, TranslationBlock *tb, diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps i386_tr_ops = { }; /* generate intermediate code for basic block 'tb'. */ -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc; - translator_loop(&i386_tr_ops, &dc.base, cpu, tb, max_insns); + translator_loop(cpu, tb, max_insns, pc, host_pc, &i386_tr_ops, &dc.base); } void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb, diff --git a/target/loongarch/translate.c b/target/loongarch/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/loongarch/translate.c +++ b/target/loongarch/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps loongarch_tr_ops = { .disas_log = loongarch_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&loongarch_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, + &loongarch_tr_ops, &ctx.base); } void loongarch_translate_init(void) diff --git a/target/m68k/translate.c b/target/m68k/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/m68k/translate.c +++ b/target/m68k/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps m68k_tr_ops = { .disas_log = m68k_tr_disas_log, }; -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc; - translator_loop(&m68k_tr_ops, &dc.base, cpu, tb, max_insns); + translator_loop(cpu, tb, max_insns, pc, host_pc, &m68k_tr_ops, &dc.base); } static double floatx80_to_double(CPUM68KState *env, uint16_t high, uint64_t low) diff --git a/target/microblaze/translate.c b/target/microblaze/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/microblaze/translate.c +++ b/target/microblaze/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps mb_tr_ops = { .disas_log = mb_tr_disas_log, }; -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc; - translator_loop(&mb_tr_ops, &dc.base, cpu, tb, max_insns); + translator_loop(cpu, tb, max_insns, pc, host_pc, &mb_tr_ops, &dc.base); } void mb_cpu_dump_state(CPUState *cs, FILE *f, int flags) diff --git a/target/mips/tcg/translate.c b/target/mips/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/mips/tcg/translate.c +++ b/target/mips/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps mips_tr_ops = { .disas_log = mips_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&mips_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &mips_tr_ops, &ctx.base); } void mips_tcg_init(void) diff --git a/target/nios2/translate.c b/target/nios2/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/nios2/translate.c +++ b/target/nios2/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps nios2_tr_ops = { .disas_log = nios2_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc; - translator_loop(&nios2_tr_ops, &dc.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &nios2_tr_ops, &dc.base); } void nios2_cpu_dump_state(CPUState *cs, FILE *f, int flags) diff --git a/target/openrisc/translate.c b/target/openrisc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/openrisc/translate.c +++ b/target/openrisc/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps openrisc_tr_ops = { .disas_log = openrisc_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&openrisc_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, + &openrisc_tr_ops, &ctx.base); } void openrisc_cpu_dump_state(CPUState *cs, FILE *f, int flags) diff --git a/target/ppc/translate.c b/target/ppc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/translate.c +++ b/target/ppc/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps ppc_tr_ops = { .disas_log = ppc_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&ppc_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &ppc_tr_ops, &ctx.base); } void restore_state_to_opc(CPUPPCState *env, TranslationBlock *tb, diff --git a/target/riscv/translate.c b/target/riscv/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/translate.c +++ b/target/riscv/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps riscv_tr_ops = { .disas_log = riscv_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&riscv_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &riscv_tr_ops, &ctx.base); } void riscv_translate_init(void) diff --git a/target/rx/translate.c b/target/rx/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/rx/translate.c +++ b/target/rx/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps rx_tr_ops = { .disas_log = rx_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc; - translator_loop(&rx_tr_ops, &dc.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &rx_tr_ops, &dc.base); } void restore_state_to_opc(CPURXState *env, TranslationBlock *tb, diff --git a/target/s390x/tcg/translate.c b/target/s390x/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/tcg/translate.c +++ b/target/s390x/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps s390x_tr_ops = { .disas_log = s390x_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc; - translator_loop(&s390x_tr_ops, &dc.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &s390x_tr_ops, &dc.base); } void restore_state_to_opc(CPUS390XState *env, TranslationBlock *tb, diff --git a/target/sh4/translate.c b/target/sh4/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/sh4/translate.c +++ b/target/sh4/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps sh4_tr_ops = { .disas_log = sh4_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&sh4_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &sh4_tr_ops, &ctx.base); } void restore_state_to_opc(CPUSH4State *env, TranslationBlock *tb, diff --git a/target/sparc/translate.c b/target/sparc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/translate.c +++ b/target/sparc/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps sparc_tr_ops = { .disas_log = sparc_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc = {}; - translator_loop(&sparc_tr_ops, &dc.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, &sparc_tr_ops, &dc.base); } void sparc_tcg_init(void) diff --git a/target/tricore/translate.c b/target/tricore/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/tricore/translate.c +++ b/target/tricore/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps tricore_tr_ops = { }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext ctx; - translator_loop(&tricore_tr_ops, &ctx.base, cs, tb, max_insns); + translator_loop(cs, tb, max_insns, pc, host_pc, + &tricore_tr_ops, &ctx.base); } void diff --git a/target/xtensa/translate.c b/target/xtensa/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/xtensa/translate.c +++ b/target/xtensa/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps xtensa_translator_ops = { .disas_log = xtensa_tr_disas_log, }; -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) +void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns, + target_ulong pc, void *host_pc) { DisasContext dc = {}; - translator_loop(&xtensa_translator_ops, &dc.base, cpu, tb, max_insns); + translator_loop(cpu, tb, max_insns, pc, host_pc, + &xtensa_translator_ops, &dc.base); } void xtensa_cpu_dump_state(CPUState *cs, FILE *f, int flags) -- 2.34.1 Cache the translation from guest to host address, so we may use direct loads when we hit on the primary translation page. Look up the second translation page only once, during translation. This obviates another lookup of the second page within tb_gen_code after translation. Fixes a bug in that plugin_insn_append should be passed the bytes in the original memory order, not bswapped by pieces. Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/translator.h | 63 +++++++++++-------- accel/tcg/translate-all.c | 23 +++---- accel/tcg/translator.c | 126 +++++++++++++++++++++++++++++--------- 3 files changed, 141 insertions(+), 71 deletions(-) diff --git a/include/exec/translator.h b/include/exec/translator.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/translator.h +++ b/include/exec/translator.h @@ -XXX,XX +XXX,XX @@ typedef enum DisasJumpType { * Architecture-agnostic disassembly context. */ typedef struct DisasContextBase { - const TranslationBlock *tb; + TranslationBlock *tb; target_ulong pc_first; target_ulong pc_next; DisasJumpType is_jmp; int num_insns; int max_insns; bool singlestep_enabled; -#ifdef CONFIG_USER_ONLY - /* - * Guest address of the last byte of the last protected page. - * - * Pages containing the translated instructions are made non-writable in - * order to achieve consistency in case another thread is modifying the - * code while translate_insn() fetches the instruction bytes piecemeal. - * Such writer threads are blocked on mmap_lock() in page_unprotect(). - */ - target_ulong page_protect_end; -#endif + void *host_addr[2]; } DisasContextBase; /** @@ -XXX,XX +XXX,XX @@ bool translator_use_goto_tb(DisasContextBase *db, target_ulong dest); * the relevant information at translation time. */ -#define GEN_TRANSLATOR_LD(fullname, type, load_fn, swap_fn) \ - type fullname ## _swap(CPUArchState *env, DisasContextBase *dcbase, \ - abi_ptr pc, bool do_swap); \ - static inline type fullname(CPUArchState *env, \ - DisasContextBase *dcbase, abi_ptr pc) \ - { \ - return fullname ## _swap(env, dcbase, pc, false); \ +uint8_t translator_ldub(CPUArchState *env, DisasContextBase *db, abi_ptr pc); +uint16_t translator_lduw(CPUArchState *env, DisasContextBase *db, abi_ptr pc); +uint32_t translator_ldl(CPUArchState *env, DisasContextBase *db, abi_ptr pc); +uint64_t translator_ldq(CPUArchState *env, DisasContextBase *db, abi_ptr pc); + +static inline uint16_t +translator_lduw_swap(CPUArchState *env, DisasContextBase *db, + abi_ptr pc, bool do_swap) +{ + uint16_t ret = translator_lduw(env, db, pc); + if (do_swap) { + ret = bswap16(ret); } + return ret; +} -#define FOR_EACH_TRANSLATOR_LD(F) \ - F(translator_ldub, uint8_t, cpu_ldub_code, /* no swap */) \ - F(translator_lduw, uint16_t, cpu_lduw_code, bswap16) \ - F(translator_ldl, uint32_t, cpu_ldl_code, bswap32) \ - F(translator_ldq, uint64_t, cpu_ldq_code, bswap64) +static inline uint32_t +translator_ldl_swap(CPUArchState *env, DisasContextBase *db, + abi_ptr pc, bool do_swap) +{ + uint32_t ret = translator_ldl(env, db, pc); + if (do_swap) { + ret = bswap32(ret); + } + return ret; +} -FOR_EACH_TRANSLATOR_LD(GEN_TRANSLATOR_LD) - -#undef GEN_TRANSLATOR_LD +static inline uint64_t +translator_ldq_swap(CPUArchState *env, DisasContextBase *db, + abi_ptr pc, bool do_swap) +{ + uint64_t ret = translator_ldq_swap(env, db, pc, false); + if (do_swap) { + ret = bswap64(ret); + } + return ret; +} /* * Return whether addr is on the same page as where disassembly started. diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, { CPUArchState *env = cpu->env_ptr; TranslationBlock *tb, *existing_tb; - tb_page_addr_t phys_pc, phys_page2; - target_ulong virt_page2; + tb_page_addr_t phys_pc; tcg_insn_unit *gen_code_buf; int gen_code_size, search_size, max_insns; #ifdef CONFIG_PROFILER @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, tb->flags = flags; tb->cflags = cflags; tb->trace_vcpu_dstate = *cpu->trace_dstate; + tb->page_addr[0] = phys_pc; + tb->page_addr[1] = -1; tcg_ctx->tb_cflags = cflags; tb_overflow: @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, } /* - * If the TB is not associated with a physical RAM page then - * it must be a temporary one-insn TB, and we have nothing to do - * except fill in the page_addr[] fields. Return early before - * attempting to link to other TBs or add to the lookup table. + * If the TB is not associated with a physical RAM page then it must be + * a temporary one-insn TB, and we have nothing left to do. Return early + * before attempting to link to other TBs or add to the lookup table. */ - if (phys_pc == -1) { - tb->page_addr[0] = tb->page_addr[1] = -1; + if (tb->page_addr[0] == -1) { return tb; } @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, */ tcg_tb_insert(tb); - /* check next page if needed */ - virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK; - phys_page2 = -1; - if ((pc & TARGET_PAGE_MASK) != virt_page2) { - phys_page2 = get_page_addr_code(env, virt_page2); - } /* * No explicit memory barrier is required -- tb_link_page() makes the * TB visible in a consistent state. */ - existing_tb = tb_link_page(tb, phys_pc, phys_page2); + existing_tb = tb_link_page(tb, tb->page_addr[0], tb->page_addr[1]); /* if the TB already exists, discard what we just translated */ if (unlikely(existing_tb != tb)) { uintptr_t orig_aligned = (uintptr_t)gen_code_buf; diff --git a/accel/tcg/translator.c b/accel/tcg/translator.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translator.c +++ b/accel/tcg/translator.c @@ -XXX,XX +XXX,XX @@ bool translator_use_goto_tb(DisasContextBase *db, target_ulong dest) return ((db->pc_first ^ dest) & TARGET_PAGE_MASK) == 0; } -static inline void translator_page_protect(DisasContextBase *dcbase, - target_ulong pc) -{ -#ifdef CONFIG_USER_ONLY - dcbase->page_protect_end = pc | ~TARGET_PAGE_MASK; - page_protect(pc); -#endif -} - void translator_loop(CPUState *cpu, TranslationBlock *tb, int max_insns, target_ulong pc, void *host_pc, const TranslatorOps *ops, DisasContextBase *db) @@ -XXX,XX +XXX,XX @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int max_insns, db->num_insns = 0; db->max_insns = max_insns; db->singlestep_enabled = cflags & CF_SINGLE_STEP; - translator_page_protect(db, db->pc_next); + db->host_addr[0] = host_pc; + db->host_addr[1] = NULL; + +#ifdef CONFIG_USER_ONLY + page_protect(pc); +#endif ops->init_disas_context(db, cpu); tcg_debug_assert(db->is_jmp == DISAS_NEXT); /* no early exit */ @@ -XXX,XX +XXX,XX @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int max_insns, #endif } -static inline void translator_maybe_page_protect(DisasContextBase *dcbase, - target_ulong pc, size_t len) +static void *translator_access(CPUArchState *env, DisasContextBase *db, + target_ulong pc, size_t len) { -#ifdef CONFIG_USER_ONLY - target_ulong end = pc + len - 1; + void *host; + target_ulong base, end; + TranslationBlock *tb; - if (end > dcbase->page_protect_end) { - translator_page_protect(dcbase, end); + tb = db->tb; + + /* Use slow path if first page is MMIO. */ + if (unlikely(tb->page_addr[0] == -1)) { + return NULL; } + + end = pc + len - 1; + if (likely(is_same_page(db, end))) { + host = db->host_addr[0]; + base = db->pc_first; + } else { + host = db->host_addr[1]; + base = TARGET_PAGE_ALIGN(db->pc_first); + if (host == NULL) { + tb->page_addr[1] = + get_page_addr_code_hostp(env, base, &db->host_addr[1]); +#ifdef CONFIG_USER_ONLY + page_protect(end); #endif + /* We cannot handle MMIO as second page. */ + assert(tb->page_addr[1] != -1); + host = db->host_addr[1]; + } + + /* Use slow path when crossing pages. */ + if (is_same_page(db, pc)) { + return NULL; + } + } + + tcg_debug_assert(pc >= base); + return host + (pc - base); } -#define GEN_TRANSLATOR_LD(fullname, type, load_fn, swap_fn) \ - type fullname ## _swap(CPUArchState *env, DisasContextBase *dcbase, \ - abi_ptr pc, bool do_swap) \ - { \ - translator_maybe_page_protect(dcbase, pc, sizeof(type)); \ - type ret = load_fn(env, pc); \ - if (do_swap) { \ - ret = swap_fn(ret); \ - } \ - plugin_insn_append(pc, &ret, sizeof(ret)); \ - return ret; \ +uint8_t translator_ldub(CPUArchState *env, DisasContextBase *db, abi_ptr pc) +{ + uint8_t ret; + void *p = translator_access(env, db, pc, sizeof(ret)); + + if (p) { + plugin_insn_append(pc, p, sizeof(ret)); + return ldub_p(p); } + ret = cpu_ldub_code(env, pc); + plugin_insn_append(pc, &ret, sizeof(ret)); + return ret; +} -FOR_EACH_TRANSLATOR_LD(GEN_TRANSLATOR_LD) +uint16_t translator_lduw(CPUArchState *env, DisasContextBase *db, abi_ptr pc) +{ + uint16_t ret, plug; + void *p = translator_access(env, db, pc, sizeof(ret)); -#undef GEN_TRANSLATOR_LD + if (p) { + plugin_insn_append(pc, p, sizeof(ret)); + return lduw_p(p); + } + ret = cpu_lduw_code(env, pc); + plug = tswap16(ret); + plugin_insn_append(pc, &plug, sizeof(ret)); + return ret; +} + +uint32_t translator_ldl(CPUArchState *env, DisasContextBase *db, abi_ptr pc) +{ + uint32_t ret, plug; + void *p = translator_access(env, db, pc, sizeof(ret)); + + if (p) { + plugin_insn_append(pc, p, sizeof(ret)); + return ldl_p(p); + } + ret = cpu_ldl_code(env, pc); + plug = tswap32(ret); + plugin_insn_append(pc, &plug, sizeof(ret)); + return ret; +} + +uint64_t translator_ldq(CPUArchState *env, DisasContextBase *db, abi_ptr pc) +{ + uint64_t ret, plug; + void *p = translator_access(env, db, pc, sizeof(ret)); + + if (p) { + plugin_insn_append(pc, p, sizeof(ret)); + return ldq_p(p); + } + ret = cpu_ldq_code(env, pc); + plug = tswap64(ret); + plugin_insn_append(pc, &plug, sizeof(ret)); + return ret; +} -- 2.34.1 From: Ilya Leoshkevich <iii@linux.ibm.com> Right now translator stops right *after* the end of a page, which breaks reporting of fault locations when the last instruction of a multi-insn translation block crosses a page boundary. Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20220817150506.592862-3-iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/s390x/tcg/translate.c | 15 +++- tests/tcg/s390x/noexec.c | 106 +++++++++++++++++++++++ tests/tcg/multiarch/noexec.c.inc | 139 +++++++++++++++++++++++++++++++ tests/tcg/s390x/Makefile.target | 1 + 4 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 tests/tcg/s390x/noexec.c create mode 100644 tests/tcg/multiarch/noexec.c.inc diff --git a/target/s390x/tcg/translate.c b/target/s390x/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/tcg/translate.c +++ b/target/s390x/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static void s390x_tr_insn_start(DisasContextBase *dcbase, CPUState *cs) dc->insn_start = tcg_last_op(); } +static target_ulong get_next_pc(CPUS390XState *env, DisasContext *s, + uint64_t pc) +{ + uint64_t insn = ld_code2(env, s, pc); + + return pc + get_ilen((insn >> 8) & 0xff); +} + static void s390x_tr_translate_insn(DisasContextBase *dcbase, CPUState *cs) { CPUS390XState *env = cs->env_ptr; @@ -XXX,XX +XXX,XX @@ static void s390x_tr_translate_insn(DisasContextBase *dcbase, CPUState *cs) dc->base.is_jmp = translate_one(env, dc); if (dc->base.is_jmp == DISAS_NEXT) { - uint64_t page_start; - - page_start = dc->base.pc_first & TARGET_PAGE_MASK; - if (dc->base.pc_next - page_start >= TARGET_PAGE_SIZE || dc->ex_value) { + if (!is_same_page(dcbase, dc->base.pc_next) || + !is_same_page(dcbase, get_next_pc(env, dc, dc->base.pc_next)) || + dc->ex_value) { dc->base.is_jmp = DISAS_TOO_MANY; } } diff --git a/tests/tcg/s390x/noexec.c b/tests/tcg/s390x/noexec.c new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/tcg/s390x/noexec.c @@ -XXX,XX +XXX,XX @@ +#include "../multiarch/noexec.c.inc" + +static void *arch_mcontext_pc(const mcontext_t *ctx) +{ + return (void *)ctx->psw.addr; +} + +static int arch_mcontext_arg(const mcontext_t *ctx) +{ + return ctx->gregs[2]; +} + +static void arch_flush(void *p, int len) +{ +} + +extern char noexec_1[]; +extern char noexec_2[]; +extern char noexec_end[]; + +asm("noexec_1:\n" + " lgfi %r2,1\n" /* %r2 is 0 on entry, set 1. */ + "noexec_2:\n" + " lgfi %r2,2\n" /* %r2 is 0/1; set 2. */ + " br %r14\n" /* return */ + "noexec_end:"); + +extern char exrl_1[]; +extern char exrl_2[]; +extern char exrl_end[]; + +asm("exrl_1:\n" + " exrl %r0, exrl_2\n" + " br %r14\n" + "exrl_2:\n" + " lgfi %r2,2\n" + "exrl_end:"); + +int main(void) +{ + struct noexec_test noexec_tests[] = { + { + .name = "fallthrough", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2, + .entry_ofs = noexec_1 - noexec_2, + .expected_si_ofs = 0, + .expected_pc_ofs = 0, + .expected_arg = 1, + }, + { + .name = "jump", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2, + .entry_ofs = 0, + .expected_si_ofs = 0, + .expected_pc_ofs = 0, + .expected_arg = 0, + }, + { + .name = "exrl", + .test_code = exrl_1, + .test_len = exrl_end - exrl_1, + .page_ofs = exrl_1 - exrl_2, + .entry_ofs = exrl_1 - exrl_2, + .expected_si_ofs = 0, + .expected_pc_ofs = exrl_1 - exrl_2, + .expected_arg = 0, + }, + { + .name = "fallthrough [cross]", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2 - 2, + .entry_ofs = noexec_1 - noexec_2 - 2, + .expected_si_ofs = 0, + .expected_pc_ofs = -2, + .expected_arg = 1, + }, + { + .name = "jump [cross]", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2 - 2, + .entry_ofs = -2, + .expected_si_ofs = 0, + .expected_pc_ofs = -2, + .expected_arg = 0, + }, + { + .name = "exrl [cross]", + .test_code = exrl_1, + .test_len = exrl_end - exrl_1, + .page_ofs = exrl_1 - exrl_2 - 2, + .entry_ofs = exrl_1 - exrl_2 - 2, + .expected_si_ofs = 0, + .expected_pc_ofs = exrl_1 - exrl_2 - 2, + .expected_arg = 0, + }, + }; + + return test_noexec(noexec_tests, + sizeof(noexec_tests) / sizeof(noexec_tests[0])); +} diff --git a/tests/tcg/multiarch/noexec.c.inc b/tests/tcg/multiarch/noexec.c.inc new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/tcg/multiarch/noexec.c.inc @@ -XXX,XX +XXX,XX @@ +/* + * Common code for arch-specific MMU_INST_FETCH fault testing. + */ + +#define _GNU_SOURCE + +#include <assert.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <unistd.h> +#include <sys/mman.h> +#include <sys/ucontext.h> + +/* Forward declarations. */ + +static void *arch_mcontext_pc(const mcontext_t *ctx); +static int arch_mcontext_arg(const mcontext_t *ctx); +static void arch_flush(void *p, int len); + +/* Testing infrastructure. */ + +struct noexec_test { + const char *name; + const char *test_code; + int test_len; + int page_ofs; + int entry_ofs; + int expected_si_ofs; + int expected_pc_ofs; + int expected_arg; +}; + +static void *page_base; +static int page_size; +static const struct noexec_test *current_noexec_test; + +static void handle_err(const char *syscall) +{ + printf("[ FAILED ] %s: %s\n", syscall, strerror(errno)); + exit(EXIT_FAILURE); +} + +static void handle_segv(int sig, siginfo_t *info, void *ucontext) +{ + const struct noexec_test *test = current_noexec_test; + const mcontext_t *mc = &((ucontext_t *)ucontext)->uc_mcontext; + void *expected_si; + void *expected_pc; + void *pc; + int arg; + + if (test == NULL) { + printf("[ FAILED ] unexpected SEGV\n"); + exit(EXIT_FAILURE); + } + current_noexec_test = NULL; + + expected_si = page_base + test->expected_si_ofs; + if (info->si_addr != expected_si) { + printf("[ FAILED ] wrong si_addr (%p != %p)\n", + info->si_addr, expected_si); + exit(EXIT_FAILURE); + } + + pc = arch_mcontext_pc(mc); + expected_pc = page_base + test->expected_pc_ofs; + if (pc != expected_pc) { + printf("[ FAILED ] wrong pc (%p != %p)\n", pc, expected_pc); + exit(EXIT_FAILURE); + } + + arg = arch_mcontext_arg(mc); + if (arg != test->expected_arg) { + printf("[ FAILED ] wrong arg (%d != %d)\n", arg, test->expected_arg); + exit(EXIT_FAILURE); + } + + if (mprotect(page_base, page_size, + PROT_READ | PROT_WRITE | PROT_EXEC) < 0) { + handle_err("mprotect"); + } +} + +static void test_noexec_1(const struct noexec_test *test) +{ + void *start = page_base + test->page_ofs; + void (*fn)(int arg) = page_base + test->entry_ofs; + + memcpy(start, test->test_code, test->test_len); + arch_flush(start, test->test_len); + + /* Trigger TB creation in order to test invalidation. */ + fn(0); + + if (mprotect(page_base, page_size, PROT_NONE) < 0) { + handle_err("mprotect"); + } + + /* Trigger SEGV and check that handle_segv() ran. */ + current_noexec_test = test; + fn(0); + assert(current_noexec_test == NULL); +} + +static int test_noexec(struct noexec_test *tests, size_t n_tests) +{ + struct sigaction act; + size_t i; + + memset(&act, 0, sizeof(act)); + act.sa_sigaction = handle_segv; + act.sa_flags = SA_SIGINFO; + if (sigaction(SIGSEGV, &act, NULL) < 0) { + handle_err("sigaction"); + } + + page_size = getpagesize(); + page_base = mmap(NULL, 2 * page_size, + PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (page_base == MAP_FAILED) { + handle_err("mmap"); + } + page_base += page_size; + + for (i = 0; i < n_tests; i++) { + struct noexec_test *test = &tests[i]; + + printf("[ RUN ] %s\n", test->name); + test_noexec_1(test); + printf("[ OK ]\n"); + } + + printf("[ PASSED ]\n"); + return EXIT_SUCCESS; +} diff --git a/tests/tcg/s390x/Makefile.target b/tests/tcg/s390x/Makefile.target index XXXXXXX..XXXXXXX 100644 --- a/tests/tcg/s390x/Makefile.target +++ b/tests/tcg/s390x/Makefile.target @@ -XXX,XX +XXX,XX @@ TESTS+=shift TESTS+=trap TESTS+=signals-s390x TESTS+=branch-relative-long +TESTS+=noexec Z14_TESTS=vfminmax vfminmax: LDFLAGS+=-lm -- 2.34.1 From: Ilya Leoshkevich <iii@linux.ibm.com> Right now translator stops right *after* the end of a page, which breaks reporting of fault locations when the last instruction of a multi-insn translation block crosses a page boundary. An implementation, like the one arm and s390x have, would require an i386 length disassembler, which is burdensome to maintain. Another alternative would be to single-step at the end of a guest page, but this may come with a performance impact. Fix by snapshotting disassembly state and restoring it after we figure out we crossed a page boundary. This includes rolling back cc_op updates and emitted ops. Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Resolves: https://gitlab.com/qemu-project/qemu/-/issues/1143 Message-Id: <20220817150506.592862-4-iii@linux.ibm.com> [rth: Simplify end-of-insn cross-page checks.] Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/i386/tcg/translate.c | 64 ++++++++++++++++----------- tests/tcg/x86_64/noexec.c | 75 ++++++++++++++++++++++++++++++++ tests/tcg/x86_64/Makefile.target | 3 +- 3 files changed, 116 insertions(+), 26 deletions(-) create mode 100644 tests/tcg/x86_64/noexec.c diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -XXX,XX +XXX,XX @@ typedef struct DisasContext { TCGv_i64 tmp1_i64; sigjmp_buf jmpbuf; + TCGOp *prev_insn_end; } DisasContext; /* The environment in which user-only runs is constrained. */ @@ -XXX,XX +XXX,XX @@ static uint64_t advance_pc(CPUX86State *env, DisasContext *s, int num_bytes) { uint64_t pc = s->pc; + /* This is a subsequent insn that crosses a page boundary. */ + if (s->base.num_insns > 1 && + !is_same_page(&s->base, s->pc + num_bytes - 1)) { + siglongjmp(s->jmpbuf, 2); + } + s->pc += num_bytes; if (unlikely(s->pc - s->pc_start > X86_MAX_INSN_LENGTH)) { /* If the instruction's 16th byte is on a different page than the 1st, a @@ -XXX,XX +XXX,XX @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) int modrm, reg, rm, mod, op, opreg, val; target_ulong next_eip, tval; target_ulong pc_start = s->base.pc_next; + bool orig_cc_op_dirty = s->cc_op_dirty; + CCOp orig_cc_op = s->cc_op; s->pc_start = s->pc = pc_start; s->override = -1; @@ -XXX,XX +XXX,XX @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) s->rip_offset = 0; /* for relative ip address */ s->vex_l = 0; s->vex_v = 0; - if (sigsetjmp(s->jmpbuf, 0) != 0) { + switch (sigsetjmp(s->jmpbuf, 0)) { + case 0: + break; + case 1: gen_exception_gpf(s); return s->pc; + case 2: + /* Restore state that may affect the next instruction. */ + s->cc_op_dirty = orig_cc_op_dirty; + s->cc_op = orig_cc_op; + s->base.num_insns--; + tcg_remove_ops_after(s->prev_insn_end); + s->base.is_jmp = DISAS_TOO_MANY; + return pc_start; + default: + g_assert_not_reached(); } prefixes = 0; @@ -XXX,XX +XXX,XX @@ static void i386_tr_insn_start(DisasContextBase *dcbase, CPUState *cpu) { DisasContext *dc = container_of(dcbase, DisasContext, base); + dc->prev_insn_end = tcg_last_op(); tcg_gen_insn_start(dc->base.pc_next, dc->cc_op); } @@ -XXX,XX +XXX,XX @@ static void i386_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu) #endif pc_next = disas_insn(dc, cpu); - - if (dc->flags & (HF_TF_MASK | HF_INHIBIT_IRQ_MASK)) { - /* if single step mode, we generate only one instruction and - generate an exception */ - /* if irq were inhibited with HF_INHIBIT_IRQ_MASK, we clear - the flag and abort the translation to give the irqs a - chance to happen */ - dc->base.is_jmp = DISAS_TOO_MANY; - } else if ((tb_cflags(dc->base.tb) & CF_USE_ICOUNT) - && ((pc_next & TARGET_PAGE_MASK) - != ((pc_next + TARGET_MAX_INSN_SIZE - 1) - & TARGET_PAGE_MASK) - || (pc_next & ~TARGET_PAGE_MASK) == 0)) { - /* Do not cross the boundary of the pages in icount mode, - it can cause an exception. Do it only when boundary is - crossed by the first instruction in the block. - If current instruction already crossed the bound - it's ok, - because an exception hasn't stopped this code. - */ - dc->base.is_jmp = DISAS_TOO_MANY; - } else if ((pc_next - dc->base.pc_first) >= (TARGET_PAGE_SIZE - 32)) { - dc->base.is_jmp = DISAS_TOO_MANY; - } - dc->base.pc_next = pc_next; + + if (dc->base.is_jmp == DISAS_NEXT) { + if (dc->flags & (HF_TF_MASK | HF_INHIBIT_IRQ_MASK)) { + /* + * If single step mode, we generate only one instruction and + * generate an exception. + * If irq were inhibited with HF_INHIBIT_IRQ_MASK, we clear + * the flag and abort the translation to give the irqs a + * chance to happen. + */ + dc->base.is_jmp = DISAS_TOO_MANY; + } else if (!is_same_page(&dc->base, pc_next)) { + dc->base.is_jmp = DISAS_TOO_MANY; + } + } } static void i386_tr_tb_stop(DisasContextBase *dcbase, CPUState *cpu) diff --git a/tests/tcg/x86_64/noexec.c b/tests/tcg/x86_64/noexec.c new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/tcg/x86_64/noexec.c @@ -XXX,XX +XXX,XX @@ +#include "../multiarch/noexec.c.inc" + +static void *arch_mcontext_pc(const mcontext_t *ctx) +{ + return (void *)ctx->gregs[REG_RIP]; +} + +int arch_mcontext_arg(const mcontext_t *ctx) +{ + return ctx->gregs[REG_RDI]; +} + +static void arch_flush(void *p, int len) +{ +} + +extern char noexec_1[]; +extern char noexec_2[]; +extern char noexec_end[]; + +asm("noexec_1:\n" + " movq $1,%rdi\n" /* %rdi is 0 on entry, set 1. */ + "noexec_2:\n" + " movq $2,%rdi\n" /* %rdi is 0/1; set 2. */ + " ret\n" + "noexec_end:"); + +int main(void) +{ + struct noexec_test noexec_tests[] = { + { + .name = "fallthrough", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2, + .entry_ofs = noexec_1 - noexec_2, + .expected_si_ofs = 0, + .expected_pc_ofs = 0, + .expected_arg = 1, + }, + { + .name = "jump", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2, + .entry_ofs = 0, + .expected_si_ofs = 0, + .expected_pc_ofs = 0, + .expected_arg = 0, + }, + { + .name = "fallthrough [cross]", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2 - 2, + .entry_ofs = noexec_1 - noexec_2 - 2, + .expected_si_ofs = 0, + .expected_pc_ofs = -2, + .expected_arg = 1, + }, + { + .name = "jump [cross]", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2 - 2, + .entry_ofs = -2, + .expected_si_ofs = 0, + .expected_pc_ofs = -2, + .expected_arg = 0, + }, + }; + + return test_noexec(noexec_tests, + sizeof(noexec_tests) / sizeof(noexec_tests[0])); +} diff --git a/tests/tcg/x86_64/Makefile.target b/tests/tcg/x86_64/Makefile.target index XXXXXXX..XXXXXXX 100644 --- a/tests/tcg/x86_64/Makefile.target +++ b/tests/tcg/x86_64/Makefile.target @@ -XXX,XX +XXX,XX @@ include $(SRC_PATH)/tests/tcg/i386/Makefile.target ifeq ($(filter %-linux-user, $(TARGET)),$(TARGET)) X86_64_TESTS += vsyscall +X86_64_TESTS += noexec TESTS=$(MULTIARCH_TESTS) $(X86_64_TESTS) test-x86_64 else TESTS=$(MULTIARCH_TESTS) @@ -XXX,XX +XXX,XX @@ test-x86_64: LDFLAGS+=-lm -lc test-x86_64: test-i386.c test-i386.h test-i386-shift.h test-i386-muldiv.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -vsyscall: $(SRC_PATH)/tests/tcg/x86_64/vsyscall.c +%: $(SRC_PATH)/tests/tcg/x86_64/%.c $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -- 2.34.1 These will be useful in properly ending the TB. Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/riscv/translate.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/target/riscv/translate.c b/target/riscv/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/translate.c +++ b/target/riscv/translate.c @@ -XXX,XX +XXX,XX @@ static uint32_t opcode_at(DisasContextBase *dcbase, target_ulong pc) /* Include decoders for factored-out extensions */ #include "decode-XVentanaCondOps.c.inc" +/* The specification allows for longer insns, but not supported by qemu. */ +#define MAX_INSN_LEN 4 + +static inline int insn_len(uint16_t first_word) +{ + return (first_word & 3) == 3 ? 4 : 2; +} + static void decode_opc(CPURISCVState *env, DisasContext *ctx, uint16_t opcode) { /* @@ -XXX,XX +XXX,XX @@ static void decode_opc(CPURISCVState *env, DisasContext *ctx, uint16_t opcode) }; /* Check for compressed insn */ - if (extract16(opcode, 0, 2) != 3) { + if (insn_len(opcode) == 2) { if (!has_ext(ctx, RVC)) { gen_exception_illegal(ctx); } else { -- 2.34.1 Right now the translator stops right *after* the end of a page, which breaks reporting of fault locations when the last instruction of a multi-insn translation block crosses a page boundary. Resolves: https://gitlab.com/qemu-project/qemu/-/issues/1155 Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/riscv/translate.c | 17 +++++-- tests/tcg/riscv64/noexec.c | 79 +++++++++++++++++++++++++++++++ tests/tcg/riscv64/Makefile.target | 1 + 3 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 tests/tcg/riscv64/noexec.c diff --git a/target/riscv/translate.c b/target/riscv/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/translate.c +++ b/target/riscv/translate.c @@ -XXX,XX +XXX,XX @@ static void riscv_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu) } ctx->nftemp = 0; + /* Only the first insn within a TB is allowed to cross a page boundary. */ if (ctx->base.is_jmp == DISAS_NEXT) { - target_ulong page_start; - - page_start = ctx->base.pc_first & TARGET_PAGE_MASK; - if (ctx->base.pc_next - page_start >= TARGET_PAGE_SIZE) { + if (!is_same_page(&ctx->base, ctx->base.pc_next)) { ctx->base.is_jmp = DISAS_TOO_MANY; + } else { + unsigned page_ofs = ctx->base.pc_next & ~TARGET_PAGE_MASK; + + if (page_ofs > TARGET_PAGE_SIZE - MAX_INSN_LEN) { + uint16_t next_insn = cpu_lduw_code(env, ctx->base.pc_next); + int len = insn_len(next_insn); + + if (!is_same_page(&ctx->base, ctx->base.pc_next + len)) { + ctx->base.is_jmp = DISAS_TOO_MANY; + } + } } } } diff --git a/tests/tcg/riscv64/noexec.c b/tests/tcg/riscv64/noexec.c new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/tcg/riscv64/noexec.c @@ -XXX,XX +XXX,XX @@ +#include "../multiarch/noexec.c.inc" + +static void *arch_mcontext_pc(const mcontext_t *ctx) +{ + return (void *)ctx->__gregs[REG_PC]; +} + +static int arch_mcontext_arg(const mcontext_t *ctx) +{ + return ctx->__gregs[REG_A0]; +} + +static void arch_flush(void *p, int len) +{ + __builtin___clear_cache(p, p + len); +} + +extern char noexec_1[]; +extern char noexec_2[]; +extern char noexec_end[]; + +asm(".option push\n" + ".option norvc\n" + "noexec_1:\n" + " li a0,1\n" /* a0 is 0 on entry, set 1. */ + "noexec_2:\n" + " li a0,2\n" /* a0 is 0/1; set 2. */ + " ret\n" + "noexec_end:\n" + ".option pop"); + +int main(void) +{ + struct noexec_test noexec_tests[] = { + { + .name = "fallthrough", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2, + .entry_ofs = noexec_1 - noexec_2, + .expected_si_ofs = 0, + .expected_pc_ofs = 0, + .expected_arg = 1, + }, + { + .name = "jump", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2, + .entry_ofs = 0, + .expected_si_ofs = 0, + .expected_pc_ofs = 0, + .expected_arg = 0, + }, + { + .name = "fallthrough [cross]", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2 - 2, + .entry_ofs = noexec_1 - noexec_2 - 2, + .expected_si_ofs = 0, + .expected_pc_ofs = -2, + .expected_arg = 1, + }, + { + .name = "jump [cross]", + .test_code = noexec_1, + .test_len = noexec_end - noexec_1, + .page_ofs = noexec_1 - noexec_2 - 2, + .entry_ofs = -2, + .expected_si_ofs = 0, + .expected_pc_ofs = -2, + .expected_arg = 0, + }, + }; + + return test_noexec(noexec_tests, + sizeof(noexec_tests) / sizeof(noexec_tests[0])); +} diff --git a/tests/tcg/riscv64/Makefile.target b/tests/tcg/riscv64/Makefile.target index XXXXXXX..XXXXXXX 100644 --- a/tests/tcg/riscv64/Makefile.target +++ b/tests/tcg/riscv64/Makefile.target @@ -XXX,XX +XXX,XX @@ VPATH += $(SRC_PATH)/tests/tcg/riscv64 TESTS += test-div +TESTS += noexec -- 2.34.1