The following changes since commit 3ccf6cd0e3e1dfd663814640b3b18b55715d7a75: Merge remote-tracking branch 'remotes/kraxel/tags/audio-20210617-pull-request' into staging (2021-06-18 09:54:42 +0100) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20210619 for you to fetch changes up to 8169ec35eb766a12ad0ae898119060fde148ab61: util/oslib-win32: Fix fatal assertion in qemu_try_memalign (2021-06-19 11:09:11 -0700) ---------------------------------------------------------------- TCI cleanup and re-encoding Fixes for #367 and #390. Move TCGCond to tcg/tcg-cond.h. Fix for win32 qemu_try_memalign. ---------------------------------------------------------------- Alessandro Di Federico (1): tcg: expose TCGCond manipulation routines Richard Henderson (31): tcg: Combine dh_is_64bit and dh_is_signed to dh_typecode tcg: Add tcg_call_flags accel/tcg/plugin-gen: Drop inline markers plugins: Drop tcg_flags from struct qemu_plugin_dyn_cb accel/tcg: Add tcg call flags to plugins helpers tcg: Store the TCGHelperInfo in the TCGOp for call tcg: Add tcg_call_func tcg: Build ffi data structures for helpers tcg/tci: Improve tcg_target_call_clobber_regs tcg/tci: Move call-return regs to end of tcg_target_reg_alloc_order tcg/tci: Use ffi for calls tcg/tci: Reserve r13 for a temporary tcg/tci: Emit setcond before brcond tcg/tci: Remove tci_write_reg tcg/tci: Change encoding to uint32_t units tcg/tci: Implement goto_ptr tcg/tci: Implement movcond tcg/tci: Implement andc, orc, eqv, nand, nor tcg/tci: Implement extract, sextract tcg/tci: Implement clz, ctz, ctpop tcg/tci: Implement mulu2, muls2 tcg/tci: Implement add2, sub2 tcg/tci: Split out tci_qemu_ld, tci_qemu_st Revert "tcg/tci: Use exec/cpu_ldst.h interfaces" tcg/tci: Remove the qemu_ld/st_type macros tcg/tci: Use {set,clear}_helper_retaddr tests/tcg: Increase timeout for TCI accel/tcg: Probe the proper permissions for atomic ops tcg/sparc: Fix temp_allocate_frame vs sparc stack bias tcg: Allocate sufficient storage in temp_allocate_frame tcg: Restart when exhausting the stack frame Stefan Weil (1): util/oslib-win32: Fix fatal assertion in qemu_try_memalign configure | 3 + accel/tcg/atomic_template.h | 24 +- accel/tcg/plugin-helpers.h | 5 +- include/exec/helper-head.h | 37 +- include/exec/helper-tcg.h | 34 +- include/qemu/plugin.h | 1 - include/tcg/tcg-cond.h | 101 ++ include/tcg/tcg-opc.h | 4 +- include/tcg/tcg.h | 71 +- target/hppa/helper.h | 3 - target/i386/ops_sse_header.h | 3 - target/m68k/helper.h | 1 - target/ppc/helper.h | 3 - tcg/tcg-internal.h | 22 + tcg/tci/tcg-target-con-set.h | 1 + tcg/tci/tcg-target.h | 68 +- accel/tcg/cputlb.c | 95 +- accel/tcg/plugin-gen.c | 20 +- accel/tcg/user-exec.c | 8 +- plugins/core.c | 30 +- tcg/optimize.c | 3 +- tcg/tcg.c | 300 +++-- tcg/tci.c | 1203 ++++++++++---------- util/oslib-win32.c | 6 +- tcg/sparc/tcg-target.c.inc | 16 +- tcg/tci/tcg-target.c.inc | 550 ++++----- tcg/meson.build | 8 +- tcg/tci/README | 20 +- tests/docker/dockerfiles/alpine.docker | 1 + tests/docker/dockerfiles/centos8.docker | 1 + tests/docker/dockerfiles/debian10.docker | 1 + tests/docker/dockerfiles/fedora-i386-cross.docker | 1 + tests/docker/dockerfiles/fedora-win32-cross.docker | 1 + tests/docker/dockerfiles/fedora-win64-cross.docker | 1 + tests/docker/dockerfiles/fedora.docker | 1 + tests/docker/dockerfiles/ubuntu.docker | 1 + tests/docker/dockerfiles/ubuntu1804.docker | 1 + tests/docker/dockerfiles/ubuntu2004.docker | 1 + tests/tcg/Makefile.target | 6 +- 39 files changed, 1454 insertions(+), 1202 deletions(-) create mode 100644 include/tcg/tcg-cond.h We will shortly be interested in distinguishing pointers from integers in the helper's declaration, as well as a true void return. We currently have two parallel 1 bit fields; merge them and expand to a 3 bit field. Our current maximum is 7 helper arguments, plus the return makes 8 * 3 = 24 bits used within the uint32_t typemask. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/helper-head.h | 37 +++++-------------- include/exec/helper-tcg.h | 34 ++++++++--------- target/hppa/helper.h | 3 -- target/i386/ops_sse_header.h | 3 -- target/m68k/helper.h | 1 - target/ppc/helper.h | 3 -- tcg/tcg.c | 71 +++++++++++++++++++++--------------- 7 files changed, 67 insertions(+), 85 deletions(-) diff --git a/include/exec/helper-head.h b/include/exec/helper-head.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/helper-head.h +++ b/include/exec/helper-head.h @@ -XXX,XX +XXX,XX @@ #define dh_retvar_ptr tcgv_ptr_temp(retval) #define dh_retvar(t) glue(dh_retvar_, dh_alias(t)) -#define dh_is_64bit_void 0 -#define dh_is_64bit_noreturn 0 -#define dh_is_64bit_i32 0 -#define dh_is_64bit_i64 1 -#define dh_is_64bit_ptr (sizeof(void *) == 8) -#define dh_is_64bit_cptr dh_is_64bit_ptr -#define dh_is_64bit(t) glue(dh_is_64bit_, dh_alias(t)) - -#define dh_is_signed_void 0 -#define dh_is_signed_noreturn 0 -#define dh_is_signed_i32 0 -#define dh_is_signed_s32 1 -#define dh_is_signed_i64 0 -#define dh_is_signed_s64 1 -#define dh_is_signed_f16 0 -#define dh_is_signed_f32 0 -#define dh_is_signed_f64 0 -#define dh_is_signed_tl 0 -#define dh_is_signed_int 1 -/* ??? This is highly specific to the host cpu. There are even special - extension instructions that may be required, e.g. ia64's addp4. But - for now we don't support any 64-bit targets with 32-bit pointers. */ -#define dh_is_signed_ptr 0 -#define dh_is_signed_cptr dh_is_signed_ptr -#define dh_is_signed_env dh_is_signed_ptr -#define dh_is_signed(t) dh_is_signed_##t +#define dh_typecode_void 0 +#define dh_typecode_noreturn 0 +#define dh_typecode_i32 2 +#define dh_typecode_s32 3 +#define dh_typecode_i64 4 +#define dh_typecode_s64 5 +#define dh_typecode_ptr 6 +#define dh_typecode(t) glue(dh_typecode_, dh_alias(t)) #define dh_callflag_i32 0 #define dh_callflag_s32 0 @@ -XXX,XX +XXX,XX @@ #define dh_callflag_noreturn TCG_CALL_NO_RETURN #define dh_callflag(t) glue(dh_callflag_, dh_alias(t)) -#define dh_sizemask(t, n) \ - ((dh_is_64bit(t) << (n*2)) | (dh_is_signed(t) << (n*2+1))) +#define dh_typemask(t, n) (dh_typecode(t) << (n * 3)) #define dh_arg(t, n) \ glue(glue(tcgv_, dh_alias(t)), _temp)(glue(arg, n)) diff --git a/include/exec/helper-tcg.h b/include/exec/helper-tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/helper-tcg.h +++ b/include/exec/helper-tcg.h @@ -XXX,XX +XXX,XX @@ #define DEF_HELPER_FLAGS_0(NAME, FLAGS, ret) \ { .func = HELPER(NAME), .name = str(NAME), \ .flags = FLAGS | dh_callflag(ret), \ - .sizemask = dh_sizemask(ret, 0) }, + .typemask = dh_typemask(ret, 0) }, #define DEF_HELPER_FLAGS_1(NAME, FLAGS, ret, t1) \ { .func = HELPER(NAME), .name = str(NAME), \ .flags = FLAGS | dh_callflag(ret), \ - .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) }, + .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) }, #define DEF_HELPER_FLAGS_2(NAME, FLAGS, ret, t1, t2) \ { .func = HELPER(NAME), .name = str(NAME), \ .flags = FLAGS | dh_callflag(ret), \ - .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \ - | dh_sizemask(t2, 2) }, + .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ + | dh_typemask(t2, 2) }, #define DEF_HELPER_FLAGS_3(NAME, FLAGS, ret, t1, t2, t3) \ { .func = HELPER(NAME), .name = str(NAME), \ .flags = FLAGS | dh_callflag(ret), \ - .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \ - | dh_sizemask(t2, 2) | dh_sizemask(t3, 3) }, + .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ + | dh_typemask(t2, 2) | dh_typemask(t3, 3) }, #define DEF_HELPER_FLAGS_4(NAME, FLAGS, ret, t1, t2, t3, t4) \ { .func = HELPER(NAME), .name = str(NAME), \ .flags = FLAGS | dh_callflag(ret), \ - .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \ - | dh_sizemask(t2, 2) | dh_sizemask(t3, 3) | dh_sizemask(t4, 4) }, + .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ + | dh_typemask(t2, 2) | dh_typemask(t3, 3) | dh_typemask(t4, 4) }, #define DEF_HELPER_FLAGS_5(NAME, FLAGS, ret, t1, t2, t3, t4, t5) \ { .func = HELPER(NAME), .name = str(NAME), \ .flags = FLAGS | dh_callflag(ret), \ - .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \ - | dh_sizemask(t2, 2) | dh_sizemask(t3, 3) | dh_sizemask(t4, 4) \ - | dh_sizemask(t5, 5) }, + .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ + | dh_typemask(t2, 2) | dh_typemask(t3, 3) | dh_typemask(t4, 4) \ + | dh_typemask(t5, 5) }, #define DEF_HELPER_FLAGS_6(NAME, FLAGS, ret, t1, t2, t3, t4, t5, t6) \ { .func = HELPER(NAME), .name = str(NAME), \ .flags = FLAGS | dh_callflag(ret), \ - .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \ - | dh_sizemask(t2, 2) | dh_sizemask(t3, 3) | dh_sizemask(t4, 4) \ - | dh_sizemask(t5, 5) | dh_sizemask(t6, 6) }, + .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ + | dh_typemask(t2, 2) | dh_typemask(t3, 3) | dh_typemask(t4, 4) \ + | dh_typemask(t5, 5) | dh_typemask(t6, 6) }, #define DEF_HELPER_FLAGS_7(NAME, FLAGS, ret, t1, t2, t3, t4, t5, t6, t7) \ { .func = HELPER(NAME), .name = str(NAME), .flags = FLAGS, \ - .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \ - | dh_sizemask(t2, 2) | dh_sizemask(t3, 3) | dh_sizemask(t4, 4) \ - | dh_sizemask(t5, 5) | dh_sizemask(t6, 6) | dh_sizemask(t7, 7) }, + .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ + | dh_typemask(t2, 2) | dh_typemask(t3, 3) | dh_typemask(t4, 4) \ + | dh_typemask(t5, 5) | dh_typemask(t6, 6) | dh_typemask(t7, 7) }, #include "helper.h" #include "trace/generated-helpers.h" diff --git a/target/hppa/helper.h b/target/hppa/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/hppa/helper.h +++ b/target/hppa/helper.h @@ -XXX,XX +XXX,XX @@ #if TARGET_REGISTER_BITS == 64 # define dh_alias_tr i64 -# define dh_is_64bit_tr 1 #else # define dh_alias_tr i32 -# define dh_is_64bit_tr 0 #endif #define dh_ctype_tr target_ureg -#define dh_is_signed_tr 0 DEF_HELPER_2(excp, noreturn, env, int) DEF_HELPER_FLAGS_2(tsv, TCG_CALL_NO_WG, void, env, tr) diff --git a/target/i386/ops_sse_header.h b/target/i386/ops_sse_header.h index XXXXXXX..XXXXXXX 100644 --- a/target/i386/ops_sse_header.h +++ b/target/i386/ops_sse_header.h @@ -XXX,XX +XXX,XX @@ #define dh_ctype_Reg Reg * #define dh_ctype_ZMMReg ZMMReg * #define dh_ctype_MMXReg MMXReg * -#define dh_is_signed_Reg dh_is_signed_ptr -#define dh_is_signed_ZMMReg dh_is_signed_ptr -#define dh_is_signed_MMXReg dh_is_signed_ptr DEF_HELPER_3(glue(psrlw, SUFFIX), void, env, Reg, Reg) DEF_HELPER_3(glue(psraw, SUFFIX), void, env, Reg, Reg) diff --git a/target/m68k/helper.h b/target/m68k/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/m68k/helper.h +++ b/target/m68k/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_4(cas2l_parallel, void, env, i32, i32, i32) #define dh_alias_fp ptr #define dh_ctype_fp FPReg * -#define dh_is_signed_fp dh_is_signed_ptr DEF_HELPER_3(exts32, void, env, fp, s32) DEF_HELPER_3(extf32, void, env, fp, f32) diff --git a/target/ppc/helper.h b/target/ppc/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/helper.h +++ b/target/ppc/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_1(ftsqrt, TCG_CALL_NO_RWG_SE, i32, i64) #define dh_alias_avr ptr #define dh_ctype_avr ppc_avr_t * -#define dh_is_signed_avr dh_is_signed_ptr #define dh_alias_vsr ptr #define dh_ctype_vsr ppc_vsr_t * -#define dh_is_signed_vsr dh_is_signed_ptr DEF_HELPER_3(vavgub, void, avr, avr, avr) DEF_HELPER_3(vavguh, void, avr, avr, avr) @@ -XXX,XX +XXX,XX @@ DEF_HELPER_3(store_601_batu, void, env, i32, tl) #define dh_alias_fprp ptr #define dh_ctype_fprp ppc_fprp_t * -#define dh_is_signed_fprp dh_is_signed_ptr DEF_HELPER_4(dadd, void, env, fprp, fprp, fprp) DEF_HELPER_4(daddq, void, env, fprp, fprp, fprp) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ typedef struct TCGHelperInfo { void *func; const char *name; unsigned flags; - unsigned sizemask; + unsigned typemask; } TCGHelperInfo; #include "exec/helper-proto.h" @@ -XXX,XX +XXX,XX @@ bool tcg_op_supported(TCGOpcode op) void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) { int i, real_args, nb_rets, pi; - unsigned sizemask, flags; + unsigned typemask, flags; TCGHelperInfo *info; TCGOp *op; info = g_hash_table_lookup(helper_table, (gpointer)func); flags = info->flags; - sizemask = info->sizemask; + typemask = info->typemask; #ifdef CONFIG_PLUGIN /* detect non-plugin helpers */ @@ -XXX,XX +XXX,XX @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) && !defined(CONFIG_TCG_INTERPRETER) /* We have 64-bit values in one register, but need to pass as two separate parameters. Split them. */ - int orig_sizemask = sizemask; + int orig_typemask = typemask; int orig_nargs = nargs; TCGv_i64 retl, reth; TCGTemp *split_args[MAX_OPC_PARAM]; retl = NULL; reth = NULL; - if (sizemask != 0) { - for (i = real_args = 0; i < nargs; ++i) { - int is_64bit = sizemask & (1 << (i+1)*2); - if (is_64bit) { - TCGv_i64 orig = temp_tcgv_i64(args[i]); - TCGv_i32 h = tcg_temp_new_i32(); - TCGv_i32 l = tcg_temp_new_i32(); - tcg_gen_extr_i64_i32(l, h, orig); - split_args[real_args++] = tcgv_i32_temp(h); - split_args[real_args++] = tcgv_i32_temp(l); - } else { - split_args[real_args++] = args[i]; - } + typemask = 0; + for (i = real_args = 0; i < nargs; ++i) { + int argtype = extract32(orig_typemask, (i + 1) * 3, 3); + bool is_64bit = (argtype & ~1) == dh_typecode_i64; + + if (is_64bit) { + TCGv_i64 orig = temp_tcgv_i64(args[i]); + TCGv_i32 h = tcg_temp_new_i32(); + TCGv_i32 l = tcg_temp_new_i32(); + tcg_gen_extr_i64_i32(l, h, orig); + split_args[real_args++] = tcgv_i32_temp(h); + typemask |= dh_typecode_i32 << (real_args * 3); + split_args[real_args++] = tcgv_i32_temp(l); + typemask |= dh_typecode_i32 << (real_args * 3); + } else { + split_args[real_args++] = args[i]; + typemask |= argtype << (real_args * 3); } - nargs = real_args; - args = split_args; - sizemask = 0; } + nargs = real_args; + args = split_args; #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 for (i = 0; i < nargs; ++i) { - int is_64bit = sizemask & (1 << (i+1)*2); - int is_signed = sizemask & (2 << (i+1)*2); - if (!is_64bit) { + int argtype = extract32(typemask, (i + 1) * 3, 3); + bool is_32bit = (argtype & ~1) == dh_typecode_i32; + bool is_signed = argtype & 1; + + if (is_32bit) { TCGv_i64 temp = tcg_temp_new_i64(); TCGv_i64 orig = temp_tcgv_i64(args[i]); if (is_signed) { @@ -XXX,XX +XXX,XX @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) if (ret != NULL) { #if defined(__sparc__) && !defined(__arch64__) \ && !defined(CONFIG_TCG_INTERPRETER) - if (orig_sizemask & 1) { + if ((typemask & 6) == dh_typecode_i64) { /* The 32-bit ABI is going to return the 64-bit value in the %o0/%o1 register pair. Prepare for this by using two return temporaries, and reassemble below. */ @@ -XXX,XX +XXX,XX @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) nb_rets = 1; } #else - if (TCG_TARGET_REG_BITS < 64 && (sizemask & 1)) { + if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) { #ifdef HOST_WORDS_BIGENDIAN op->args[pi++] = temp_arg(ret + 1); op->args[pi++] = temp_arg(ret); @@ -XXX,XX +XXX,XX @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) real_args = 0; for (i = 0; i < nargs; i++) { - int is_64bit = sizemask & (1 << (i+1)*2); + int argtype = extract32(typemask, (i + 1) * 3, 3); + bool is_64bit = (argtype & ~1) == dh_typecode_i64; + if (TCG_TARGET_REG_BITS < 64 && is_64bit) { #ifdef TCG_TARGET_CALL_ALIGN_ARGS /* some targets want aligned 64 bit args */ @@ -XXX,XX +XXX,XX @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) && !defined(CONFIG_TCG_INTERPRETER) /* Free all of the parts we allocated above. */ for (i = real_args = 0; i < orig_nargs; ++i) { - int is_64bit = orig_sizemask & (1 << (i+1)*2); + int argtype = extract32(orig_typemask, (i + 1) * 3, 3); + bool is_64bit = (argtype & ~1) == dh_typecode_i64; + if (is_64bit) { tcg_temp_free_internal(args[real_args++]); tcg_temp_free_internal(args[real_args++]); @@ -XXX,XX +XXX,XX @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) real_args++; } } - if (orig_sizemask & 1) { + if ((orig_typemask & 6) == dh_typecode_i64) { /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them. Note that describing these as TCGv_i64 eliminates an unnecessary zero-extension that tcg_gen_concat_i32_i64 would create. */ @@ -XXX,XX +XXX,XX @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) } #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 for (i = 0; i < nargs; ++i) { - int is_64bit = sizemask & (1 << (i+1)*2); - if (!is_64bit) { + int argtype = extract32(typemask, (i + 1) * 3, 3); + bool is_32bit = (argtype & ~1) == dh_typecode_i32; + + if (is_32bit) { tcg_temp_free_internal(args[i]); } } -- 2.25.1 We're going to change how to look up the call flags from a TCGop, so extract it as a helper. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg-internal.h | 5 +++++ tcg/optimize.c | 3 ++- tcg/tcg.c | 14 ++++++-------- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tcg/tcg-internal.h b/tcg/tcg-internal.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-internal.h +++ b/tcg/tcg-internal.h @@ -XXX,XX +XXX,XX @@ bool tcg_region_alloc(TCGContext *s); void tcg_region_initial_alloc(TCGContext *s); void tcg_region_prologue_set(TCGContext *s); +static inline unsigned tcg_call_flags(TCGOp *op) +{ + return op->args[TCGOP_CALLO(op) + TCGOP_CALLI(op) + 1]; +} + #endif /* TCG_INTERNAL_H */ diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ #include "qemu/osdep.h" #include "tcg/tcg-op.h" +#include "tcg-internal.h" #define CASE_OP_32_64(x) \ glue(glue(case INDEX_op_, x), _i32): \ @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; case INDEX_op_call: - if (!(op->args[nb_oargs + nb_iargs + 1] + if (!(tcg_call_flags(op) & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) { for (i = 0; i < nb_globals; i++) { if (test_bit(i, temps_used.l)) { diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void tcg_dump_ops(TCGContext *s, bool have_prefs) nb_cargs = def->nb_cargs; /* function name, flags, out args */ - col += qemu_log(" %s %s,$0x%" TCG_PRIlx ",$%d", def->name, + col += qemu_log(" %s %s,$0x%x,$%d", def->name, tcg_find_helper(s, op->args[nb_oargs + nb_iargs]), - op->args[nb_oargs + nb_iargs + 1], nb_oargs); + tcg_call_flags(op), nb_oargs); for (i = 0; i < nb_oargs; i++) { col += qemu_log(",%s", tcg_get_arg_str(s, buf, sizeof(buf), op->args[i])); @@ -XXX,XX +XXX,XX @@ static void reachable_code_pass(TCGContext *s) QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { bool remove = dead; TCGLabel *label; - int call_flags; switch (op->opc) { case INDEX_op_set_label: @@ -XXX,XX +XXX,XX @@ static void reachable_code_pass(TCGContext *s) case INDEX_op_call: /* Notice noreturn helper calls, raising exceptions. */ - call_flags = op->args[TCGOP_CALLO(op) + TCGOP_CALLI(op) + 1]; - if (call_flags & TCG_CALL_NO_RETURN) { + if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) { dead = true; } break; @@ -XXX,XX +XXX,XX @@ static void liveness_pass_1(TCGContext *s) nb_oargs = TCGOP_CALLO(op); nb_iargs = TCGOP_CALLI(op); - call_flags = op->args[nb_oargs + nb_iargs + 1]; + call_flags = tcg_call_flags(op); /* pure functions can be removed if their result is unused */ if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { @@ -XXX,XX +XXX,XX @@ static bool liveness_pass_2(TCGContext *s) if (opc == INDEX_op_call) { nb_oargs = TCGOP_CALLO(op); nb_iargs = TCGOP_CALLI(op); - call_flags = op->args[nb_oargs + nb_iargs + 1]; + call_flags = tcg_call_flags(op); } else { nb_iargs = def->nb_iargs; nb_oargs = def->nb_oargs; @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) TCGRegSet allocated_regs; func_addr = (tcg_insn_unit *)(intptr_t)op->args[nb_oargs + nb_iargs]; - flags = op->args[nb_oargs + nb_iargs + 1]; + flags = tcg_call_flags(op); nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); if (nb_regs > nb_iargs) { -- 2.25.1 Let the compiler decide on inlining. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/plugin-gen.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -XXX,XX +XXX,XX @@ static void gen_empty_mem_helper(void) tcg_temp_free_ptr(ptr); } -static inline -void gen_plugin_cb_start(enum plugin_gen_from from, - enum plugin_gen_cb type, unsigned wr) +static void gen_plugin_cb_start(enum plugin_gen_from from, + enum plugin_gen_cb type, unsigned wr) { TCGOp *op; @@ -XXX,XX +XXX,XX @@ static void gen_wrapped(enum plugin_gen_from from, tcg_gen_plugin_cb_end(); } -static inline void plugin_gen_empty_callback(enum plugin_gen_from from) +static void plugin_gen_empty_callback(enum plugin_gen_from from) { switch (from) { case PLUGIN_GEN_AFTER_INSN: @@ -XXX,XX +XXX,XX @@ static bool op_rw(const TCGOp *op, const struct qemu_plugin_dyn_cb *cb) return !!(cb->rw & (w + 1)); } -static inline -void inject_cb_type(const GArray *cbs, TCGOp *begin_op, inject_fn inject, - op_ok_fn ok) +static void inject_cb_type(const GArray *cbs, TCGOp *begin_op, + inject_fn inject, op_ok_fn ok) { TCGOp *end_op; TCGOp *op; -- 2.25.1 As noted by qemu-plugins.h, enum qemu_plugin_cb_flags is currently unused -- plugins can neither read nor write guest registers. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/plugin-helpers.h | 1 - include/qemu/plugin.h | 1 - accel/tcg/plugin-gen.c | 8 ++++---- plugins/core.c | 30 ++++++------------------------ 4 files changed, 10 insertions(+), 30 deletions(-) diff --git a/accel/tcg/plugin-helpers.h b/accel/tcg/plugin-helpers.h index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/plugin-helpers.h +++ b/accel/tcg/plugin-helpers.h @@ -XXX,XX +XXX,XX @@ #ifdef CONFIG_PLUGIN -/* Note: no TCG flags because those are overwritten later */ DEF_HELPER_2(plugin_vcpu_udata_cb, void, i32, ptr) DEF_HELPER_4(plugin_vcpu_mem_cb, void, i32, i32, i64, ptr) #endif diff --git a/include/qemu/plugin.h b/include/qemu/plugin.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/plugin.h +++ b/include/qemu/plugin.h @@ -XXX,XX +XXX,XX @@ enum plugin_dyn_cb_subtype { struct qemu_plugin_dyn_cb { union qemu_plugin_cb_sig f; void *userp; - unsigned tcg_flags; enum plugin_dyn_cb_subtype type; /* @rw applies to mem callbacks only (both regular and inline) */ enum qemu_plugin_mem_rw rw; diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -XXX,XX +XXX,XX @@ static TCGOp *copy_st_ptr(TCGOp **begin_op, TCGOp *op) } static TCGOp *copy_call(TCGOp **begin_op, TCGOp *op, void *empty_func, - void *func, unsigned tcg_flags, int *cb_idx) + void *func, int *cb_idx) { /* copy all ops until the call */ do { @@ -XXX,XX +XXX,XX @@ static TCGOp *copy_call(TCGOp **begin_op, TCGOp *op, void *empty_func, tcg_debug_assert(i < MAX_OPC_PARAM_ARGS); } op->args[*cb_idx] = (uintptr_t)func; - op->args[*cb_idx + 1] = tcg_flags; + op->args[*cb_idx + 1] = (*begin_op)->args[*cb_idx + 1]; return op; } @@ -XXX,XX +XXX,XX @@ static TCGOp *append_udata_cb(const struct qemu_plugin_dyn_cb *cb, /* call */ op = copy_call(&begin_op, op, HELPER(plugin_vcpu_udata_cb), - cb->f.vcpu_udata, cb->tcg_flags, cb_idx); + cb->f.vcpu_udata, cb_idx); return op; } @@ -XXX,XX +XXX,XX @@ static TCGOp *append_mem_cb(const struct qemu_plugin_dyn_cb *cb, if (type == PLUGIN_GEN_CB_MEM) { /* call */ op = copy_call(&begin_op, op, HELPER(plugin_vcpu_mem_cb), - cb->f.vcpu_udata, cb->tcg_flags, cb_idx); + cb->f.vcpu_udata, cb_idx); } return op; diff --git a/plugins/core.c b/plugins/core.c index XXXXXXX..XXXXXXX 100644 --- a/plugins/core.c +++ b/plugins/core.c @@ -XXX,XX +XXX,XX @@ void plugin_register_inline_op(GArray **arr, dyn_cb->inline_insn.imm = imm; } -static inline uint32_t cb_to_tcg_flags(enum qemu_plugin_cb_flags flags) -{ - uint32_t ret; - - switch (flags) { - case QEMU_PLUGIN_CB_RW_REGS: - ret = 0; - break; - case QEMU_PLUGIN_CB_R_REGS: - ret = TCG_CALL_NO_WG; - break; - case QEMU_PLUGIN_CB_NO_REGS: - default: - ret = TCG_CALL_NO_RWG; - } - return ret; -} - -inline void -plugin_register_dyn_cb__udata(GArray **arr, - qemu_plugin_vcpu_udata_cb_t cb, - enum qemu_plugin_cb_flags flags, void *udata) +void plugin_register_dyn_cb__udata(GArray **arr, + qemu_plugin_vcpu_udata_cb_t cb, + enum qemu_plugin_cb_flags flags, + void *udata) { struct qemu_plugin_dyn_cb *dyn_cb = plugin_get_dyn_cb(arr); dyn_cb->userp = udata; - dyn_cb->tcg_flags = cb_to_tcg_flags(flags); + /* Note flags are discarded as unused. */ dyn_cb->f.vcpu_udata = cb; dyn_cb->type = PLUGIN_CB_REGULAR; } @@ -XXX,XX +XXX,XX @@ void plugin_register_vcpu_mem_cb(GArray **arr, dyn_cb = plugin_get_dyn_cb(arr); dyn_cb->userp = udata; - dyn_cb->tcg_flags = cb_to_tcg_flags(flags); + /* Note flags are discarded as unused. */ dyn_cb->type = PLUGIN_CB_REGULAR; dyn_cb->rw = rw; dyn_cb->f.generic = cb; -- 2.25.1 As noted by qemu-plugins.h, plugins can neither read nor write guest registers. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/plugin-helpers.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/accel/tcg/plugin-helpers.h b/accel/tcg/plugin-helpers.h index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/plugin-helpers.h +++ b/accel/tcg/plugin-helpers.h @@ -XXX,XX +XXX,XX @@ #ifdef CONFIG_PLUGIN -DEF_HELPER_2(plugin_vcpu_udata_cb, void, i32, ptr) -DEF_HELPER_4(plugin_vcpu_mem_cb, void, i32, i32, i64, ptr) +DEF_HELPER_FLAGS_2(plugin_vcpu_udata_cb, TCG_CALL_NO_RWG, void, i32, ptr) +DEF_HELPER_FLAGS_4(plugin_vcpu_mem_cb, TCG_CALL_NO_RWG, void, i32, i32, i64, ptr) #endif -- 2.25.1 This will give us both flags and typemask for use later. We also fix a dumping bug, wherein calls generated for plugins fail tcg_find_helper and print (null) instead of either a name or the raw function pointer. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg-internal.h | 14 ++++++++++++- tcg/tcg.c | 49 ++++++++++++++++++++-------------------------- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/tcg/tcg-internal.h b/tcg/tcg-internal.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-internal.h +++ b/tcg/tcg-internal.h @@ -XXX,XX +XXX,XX @@ #define TCG_HIGHWATER 1024 +typedef struct TCGHelperInfo { + void *func; + const char *name; + unsigned flags; + unsigned typemask; +} TCGHelperInfo; + extern TCGContext tcg_init_ctx; extern TCGContext **tcg_ctxs; extern unsigned int tcg_cur_ctxs; @@ -XXX,XX +XXX,XX @@ bool tcg_region_alloc(TCGContext *s); void tcg_region_initial_alloc(TCGContext *s); void tcg_region_prologue_set(TCGContext *s); +static inline const TCGHelperInfo *tcg_call_info(TCGOp *op) +{ + return (void *)(uintptr_t)op->args[TCGOP_CALLO(op) + TCGOP_CALLI(op) + 1]; +} + static inline unsigned tcg_call_flags(TCGOp *op) { - return op->args[TCGOP_CALLO(op) + TCGOP_CALLI(op) + 1]; + return tcg_call_info(op)->flags; } #endif /* TCG_INTERNAL_H */ diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ void tcg_pool_reset(TCGContext *s) s->pool_current = NULL; } -typedef struct TCGHelperInfo { - void *func; - const char *name; - unsigned flags; - unsigned typemask; -} TCGHelperInfo; - #include "exec/helper-proto.h" static const TCGHelperInfo all_helpers[] = { @@ -XXX,XX +XXX,XX @@ bool tcg_op_supported(TCGOpcode op) void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) { int i, real_args, nb_rets, pi; - unsigned typemask, flags; - TCGHelperInfo *info; + unsigned typemask; + const TCGHelperInfo *info; TCGOp *op; info = g_hash_table_lookup(helper_table, (gpointer)func); - flags = info->flags; typemask = info->typemask; #ifdef CONFIG_PLUGIN @@ -XXX,XX +XXX,XX @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) real_args++; } op->args[pi++] = (uintptr_t)func; - op->args[pi++] = flags; + op->args[pi++] = (uintptr_t)info; TCGOP_CALLI(op) = real_args; /* Make sure the fields didn't overflow. */ @@ -XXX,XX +XXX,XX @@ static char *tcg_get_arg_str(TCGContext *s, char *buf, return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg)); } -/* Find helper name. */ -static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val) -{ - const char *ret = NULL; - if (helper_table) { - TCGHelperInfo *info = g_hash_table_lookup(helper_table, (gpointer)val); - if (info) { - ret = info->name; - } - } - return ret; -} - static const char * const cond_name[] = { [TCG_COND_NEVER] = "never", @@ -XXX,XX +XXX,XX @@ static void tcg_dump_ops(TCGContext *s, bool have_prefs) col += qemu_log(" " TARGET_FMT_lx, a); } } else if (c == INDEX_op_call) { + const TCGHelperInfo *info = tcg_call_info(op); + void *func; + /* variable number of arguments */ nb_oargs = TCGOP_CALLO(op); nb_iargs = TCGOP_CALLI(op); nb_cargs = def->nb_cargs; - /* function name, flags, out args */ - col += qemu_log(" %s %s,$0x%x,$%d", def->name, - tcg_find_helper(s, op->args[nb_oargs + nb_iargs]), - tcg_call_flags(op), nb_oargs); + col += qemu_log(" %s ", def->name); + + /* + * Print the function name from TCGHelperInfo, if available. + * Note that plugins have a template function for the info, + * but the actual function pointer comes from the plugin. + */ + func = (void *)(uintptr_t)op->args[nb_oargs + nb_iargs]; + if (func == info->func) { + col += qemu_log("%s", info->name); + } else { + col += qemu_log("plugin(%p)", func); + } + + col += qemu_log("$0x%x,$%d", info->flags, nb_oargs); for (i = 0; i < nb_oargs; i++) { col += qemu_log(",%s", tcg_get_arg_str(s, buf, sizeof(buf), op->args[i])); -- 2.25.1 Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg-internal.h | 5 +++++ tcg/tcg.c | 5 ++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tcg/tcg-internal.h b/tcg/tcg-internal.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-internal.h +++ b/tcg/tcg-internal.h @@ -XXX,XX +XXX,XX @@ bool tcg_region_alloc(TCGContext *s); void tcg_region_initial_alloc(TCGContext *s); void tcg_region_prologue_set(TCGContext *s); +static inline void *tcg_call_func(TCGOp *op) +{ + return (void *)(uintptr_t)op->args[TCGOP_CALLO(op) + TCGOP_CALLI(op)]; +} + static inline const TCGHelperInfo *tcg_call_info(TCGOp *op) { return (void *)(uintptr_t)op->args[TCGOP_CALLO(op) + TCGOP_CALLI(op) + 1]; diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void tcg_dump_ops(TCGContext *s, bool have_prefs) } } else if (c == INDEX_op_call) { const TCGHelperInfo *info = tcg_call_info(op); - void *func; + void *func = tcg_call_func(op); /* variable number of arguments */ nb_oargs = TCGOP_CALLO(op); @@ -XXX,XX +XXX,XX @@ static void tcg_dump_ops(TCGContext *s, bool have_prefs) * Note that plugins have a template function for the info, * but the actual function pointer comes from the plugin. */ - func = (void *)(uintptr_t)op->args[nb_oargs + nb_iargs]; if (func == info->func) { col += qemu_log("%s", info->name); } else { @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) int allocate_args; TCGRegSet allocated_regs; - func_addr = (tcg_insn_unit *)(intptr_t)op->args[nb_oargs + nb_iargs]; + func_addr = tcg_call_func(op); flags = tcg_call_flags(op); nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); -- 2.25.1 Add libffi as a build requirement for TCI. Add libffi to the dockerfiles to satisfy that requirement. Construct an ffi_cif structure for each unique typemask. Record the result in a separate hash table for later lookup; this allows helper_table to stay const. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 58 +++++++++++++++++++ tcg/meson.build | 8 ++- tests/docker/dockerfiles/alpine.docker | 1 + tests/docker/dockerfiles/centos8.docker | 1 + tests/docker/dockerfiles/debian10.docker | 1 + .../dockerfiles/fedora-i386-cross.docker | 1 + .../dockerfiles/fedora-win32-cross.docker | 1 + .../dockerfiles/fedora-win64-cross.docker | 1 + tests/docker/dockerfiles/fedora.docker | 1 + tests/docker/dockerfiles/ubuntu.docker | 1 + tests/docker/dockerfiles/ubuntu1804.docker | 1 + tests/docker/dockerfiles/ubuntu2004.docker | 1 + 12 files changed, 75 insertions(+), 1 deletion(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ #include "exec/log.h" #include "tcg-internal.h" +#ifdef CONFIG_TCG_INTERPRETER +#include <ffi.h> +#endif + /* Forward declarations for functions declared in tcg-target.c.inc and used here. */ static void tcg_target_init(TCGContext *s); @@ -XXX,XX +XXX,XX @@ static const TCGHelperInfo all_helpers[] = { }; static GHashTable *helper_table; +#ifdef CONFIG_TCG_INTERPRETER +static GHashTable *ffi_table; + +static ffi_type * const typecode_to_ffi[8] = { + [dh_typecode_void] = &ffi_type_void, + [dh_typecode_i32] = &ffi_type_uint32, + [dh_typecode_s32] = &ffi_type_sint32, + [dh_typecode_i64] = &ffi_type_uint64, + [dh_typecode_s64] = &ffi_type_sint64, + [dh_typecode_ptr] = &ffi_type_pointer, +}; +#endif + static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; static void process_op_defs(TCGContext *s); static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, @@ -XXX,XX +XXX,XX @@ static void tcg_context_init(unsigned max_cpus) (gpointer)&all_helpers[i]); } +#ifdef CONFIG_TCG_INTERPRETER + /* g_direct_hash/equal for direct comparisons on uint32_t. */ + ffi_table = g_hash_table_new(NULL, NULL); + for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { + struct { + ffi_cif cif; + ffi_type *args[]; + } *ca; + uint32_t typemask = all_helpers[i].typemask; + gpointer hash = (gpointer)(uintptr_t)typemask; + ffi_status status; + int nargs; + + if (g_hash_table_lookup(ffi_table, hash)) { + continue; + } + + /* Ignoring the return type, find the last non-zero field. */ + nargs = 32 - clz32(typemask >> 3); + nargs = DIV_ROUND_UP(nargs, 3); + + ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); + ca->cif.rtype = typecode_to_ffi[typemask & 7]; + ca->cif.nargs = nargs; + + if (nargs != 0) { + ca->cif.arg_types = ca->args; + for (i = 0; i < nargs; ++i) { + int typecode = extract32(typemask, (i + 1) * 3, 3); + ca->args[i] = typecode_to_ffi[typecode]; + } + } + + status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, + ca->cif.rtype, ca->cif.arg_types); + assert(status == FFI_OK); + + g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif); + } +#endif + tcg_target_init(s); process_op_defs(s); diff --git a/tcg/meson.build b/tcg/meson.build index XXXXXXX..XXXXXXX 100644 --- a/tcg/meson.build +++ b/tcg/meson.build @@ -XXX,XX +XXX,XX @@ tcg_ss.add(files( 'tcg-op-gvec.c', 'tcg-op-vec.c', )) -tcg_ss.add(when: 'CONFIG_TCG_INTERPRETER', if_true: files('tci.c')) + +if get_option('tcg_interpreter') + libffi = dependency('libffi', version: '>=3.0', required: true, + method: 'pkg-config', kwargs: static_kwargs) + specific_ss.add(libffi) + specific_ss.add(files('tci.c')) +endif specific_ss.add_all(when: 'CONFIG_TCG', if_true: tcg_ss) diff --git a/tests/docker/dockerfiles/alpine.docker b/tests/docker/dockerfiles/alpine.docker index XXXXXXX..XXXXXXX 100644 --- a/tests/docker/dockerfiles/alpine.docker +++ b/tests/docker/dockerfiles/alpine.docker @@ -XXX,XX +XXX,XX @@ ENV PACKAGES \ libaio-dev \ libbpf-dev \ libcap-ng-dev \ + libffi-dev \ libjpeg-turbo-dev \ libnfs-dev \ libpng-dev \ diff --git a/tests/docker/dockerfiles/centos8.docker b/tests/docker/dockerfiles/centos8.docker index XXXXXXX..XXXXXXX 100644 --- a/tests/docker/dockerfiles/centos8.docker +++ b/tests/docker/dockerfiles/centos8.docker @@ -XXX,XX +XXX,XX @@ ENV PACKAGES \ libbpf-devel \ libepoxy-devel \ libfdt-devel \ + libffi-devel \ libgcrypt-devel \ lzo-devel \ make \ diff --git a/tests/docker/dockerfiles/debian10.docker b/tests/docker/dockerfiles/debian10.docker index XXXXXXX..XXXXXXX 100644 --- a/tests/docker/dockerfiles/debian10.docker +++ b/tests/docker/dockerfiles/debian10.docker @@ -XXX,XX +XXX,XX @@ RUN apt update && \ gdb-multiarch \ gettext \ git \ + libffi-dev \ libncurses5-dev \ ninja-build \ pkg-config \ diff --git a/tests/docker/dockerfiles/fedora-i386-cross.docker b/tests/docker/dockerfiles/fedora-i386-cross.docker index XXXXXXX..XXXXXXX 100644 --- a/tests/docker/dockerfiles/fedora-i386-cross.docker +++ b/tests/docker/dockerfiles/fedora-i386-cross.docker @@ -XXX,XX +XXX,XX @@ ENV PACKAGES \ findutils \ gcc \ git \ + libffi-devel.i686 \ libtasn1-devel.i686 \ libzstd-devel.i686 \ make \ diff --git a/tests/docker/dockerfiles/fedora-win32-cross.docker b/tests/docker/dockerfiles/fedora-win32-cross.docker index XXXXXXX..XXXXXXX 100644 --- a/tests/docker/dockerfiles/fedora-win32-cross.docker +++ b/tests/docker/dockerfiles/fedora-win32-cross.docker @@ -XXX,XX +XXX,XX @@ ENV PACKAGES \ mingw32-gmp \ mingw32-gnutls \ mingw32-gtk3 \ + mingw32-libffi \ mingw32-libjpeg-turbo \ mingw32-libpng \ mingw32-libtasn1 \ diff --git a/tests/docker/dockerfiles/fedora-win64-cross.docker b/tests/docker/dockerfiles/fedora-win64-cross.docker index XXXXXXX..XXXXXXX 100644 --- a/tests/docker/dockerfiles/fedora-win64-cross.docker +++ b/tests/docker/dockerfiles/fedora-win64-cross.docker @@ -XXX,XX +XXX,XX @@ ENV PACKAGES \ mingw64-glib2 \ mingw64-gmp \ mingw64-gtk3 \ + mingw64-libffi \ mingw64-libjpeg-turbo \ mingw64-libpng \ mingw64-libtasn1 \ diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker index XXXXXXX..XXXXXXX 100644 --- a/tests/docker/dockerfiles/fedora.docker +++ b/tests/docker/dockerfiles/fedora.docker @@ -XXX,XX +XXX,XX @@ ENV PACKAGES \ libepoxy-devel \ libfdt-devel \ libbpf-devel \ + libffi-devel \ libiscsi-devel \ libjpeg-devel \ libpmem-devel \ diff --git a/tests/docker/dockerfiles/ubuntu.docker b/tests/docker/dockerfiles/ubuntu.docker index XXXXXXX..XXXXXXX 100644 --- a/tests/docker/dockerfiles/ubuntu.docker +++ b/tests/docker/dockerfiles/ubuntu.docker @@ -XXX,XX +XXX,XX @@ ENV PACKAGES \ libdrm-dev \ libepoxy-dev \ libfdt-dev \ + libffi-dev \ libgbm-dev \ libgnutls28-dev \ libgtk-3-dev \ diff --git a/tests/docker/dockerfiles/ubuntu1804.docker b/tests/docker/dockerfiles/ubuntu1804.docker index XXXXXXX..XXXXXXX 100644 --- a/tests/docker/dockerfiles/ubuntu1804.docker +++ b/tests/docker/dockerfiles/ubuntu1804.docker @@ -XXX,XX +XXX,XX @@ ENV PACKAGES \ libdrm-dev \ libepoxy-dev \ libfdt-dev \ + libffi-dev \ libgbm-dev \ libgtk-3-dev \ libibverbs-dev \ diff --git a/tests/docker/dockerfiles/ubuntu2004.docker b/tests/docker/dockerfiles/ubuntu2004.docker index XXXXXXX..XXXXXXX 100644 --- a/tests/docker/dockerfiles/ubuntu2004.docker +++ b/tests/docker/dockerfiles/ubuntu2004.docker @@ -XXX,XX +XXX,XX @@ ENV PACKAGES flex bison \ libdrm-dev \ libepoxy-dev \ libfdt-dev \ + libffi-dev \ libgbm-dev \ libgtk-3-dev \ libibverbs-dev \ -- 2.25.1 The current setting is much too pessimistic. Indicating only the one or two registers that are actually assigned after a call should avoid unnecessary movement between the register array and the stack array. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci/tcg-target.c.inc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_init(TCGContext *s) tcg_target_available_regs[TCG_TYPE_I32] = BIT(TCG_TARGET_NB_REGS) - 1; /* Registers available for 64 bit operations. */ tcg_target_available_regs[TCG_TYPE_I64] = BIT(TCG_TARGET_NB_REGS) - 1; - /* TODO: Which registers should be set here? */ - tcg_target_call_clobber_regs = BIT(TCG_TARGET_NB_REGS) - 1; + /* + * The interpreter "registers" are in the local stack frame and + * cannot be clobbered by the called helper functions. However, + * the interpreter assumes a 64-bit return value and assigns to + * the return value registers. + */ + tcg_target_call_clobber_regs = + MAKE_64BIT_MASK(TCG_REG_R0, 64 / TCG_TARGET_REG_BITS); s->reserved_regs = 0; tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK); -- 2.25.1 As the only call-clobbered regs for TCI, these should receive the least priority. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci/tcg-target.c.inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) } static const int tcg_target_reg_alloc_order[] = { - TCG_REG_R0, - TCG_REG_R1, TCG_REG_R2, TCG_REG_R3, TCG_REG_R4, @@ -XXX,XX +XXX,XX @@ static const int tcg_target_reg_alloc_order[] = { TCG_REG_R13, TCG_REG_R14, TCG_REG_R15, + TCG_REG_R1, + TCG_REG_R0, }; #if MAX_OPC_PARAM_IARGS != 6 -- 2.25.1 This requires adjusting where arguments are stored. Place them on the stack at left-aligned positions. Adjust the stack frame to be at entirely positive offsets. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg.h | 1 + tcg/tci/tcg-target.h | 2 +- tcg/tcg.c | 64 +++++++++++++----- tcg/tci.c | 142 ++++++++++++++++++++++----------------- tcg/tci/tcg-target.c.inc | 50 +++++++------- 5 files changed, 153 insertions(+), 106 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ #define MAX_OPC_PARAM (4 + (MAX_OPC_PARAM_PER_ARG * MAX_OPC_PARAM_ARGS)) #define CPU_TEMP_BUF_NLONGS 128 +#define TCG_STATIC_FRAME_SIZE (CPU_TEMP_BUF_NLONGS * sizeof(long)) /* Default target word size to pointer size. */ #ifndef TCG_TARGET_REG_BITS diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { /* Used for function call generation. */ #define TCG_TARGET_CALL_STACK_OFFSET 0 -#define TCG_TARGET_STACK_ALIGN 16 +#define TCG_TARGET_STACK_ALIGN 8 #define HAVE_TCG_QEMU_TB_EXEC diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, intptr_t arg2); static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, TCGReg base, intptr_t ofs); +#ifdef CONFIG_TCG_INTERPRETER +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, + ffi_cif *cif); +#else static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target); +#endif static bool tcg_target_const_match(int64_t val, TCGType type, int ct); #ifdef TCG_TARGET_NEED_LDST_LABELS static int tcg_out_ldst_finalize(TCGContext *s); @@ -XXX,XX +XXX,XX @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) for (i = 0; i < nargs; i++) { int argtype = extract32(typemask, (i + 1) * 3, 3); bool is_64bit = (argtype & ~1) == dh_typecode_i64; + bool want_align = false; + +#if defined(CONFIG_TCG_INTERPRETER) + /* + * Align all arguments, so that they land in predictable places + * for passing off to ffi_call. + */ + want_align = true; +#elif defined(TCG_TARGET_CALL_ALIGN_ARGS) + /* Some targets want aligned 64 bit args */ + want_align = is_64bit; +#endif + + if (TCG_TARGET_REG_BITS < 64 && want_align && (real_args & 1)) { + op->args[pi++] = TCG_CALL_DUMMY_ARG; + real_args++; + } if (TCG_TARGET_REG_BITS < 64 && is_64bit) { -#ifdef TCG_TARGET_CALL_ALIGN_ARGS - /* some targets want aligned 64 bit args */ - if (real_args & 1) { - op->args[pi++] = TCG_CALL_DUMMY_ARG; - real_args++; - } -#endif - /* If stack grows up, then we will be placing successive - arguments at lower addresses, which means we need to - reverse the order compared to how we would normally - treat either big or little-endian. For those arguments - that will wind up in registers, this still works for - HPPA (the only current STACK_GROWSUP target) since the - argument registers are *also* allocated in decreasing - order. If another such target is added, this logic may - have to get more complicated to differentiate between - stack arguments and register arguments. */ + /* + * If stack grows up, then we will be placing successive + * arguments at lower addresses, which means we need to + * reverse the order compared to how we would normally + * treat either big or little-endian. For those arguments + * that will wind up in registers, this still works for + * HPPA (the only current STACK_GROWSUP target) since the + * argument registers are *also* allocated in decreasing + * order. If another such target is added, this logic may + * have to get more complicated to differentiate between + * stack arguments and register arguments. + */ #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP) op->args[pi++] = temp_arg(args[i] + 1); op->args[pi++] = temp_arg(args[i]); @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) const int nb_oargs = TCGOP_CALLO(op); const int nb_iargs = TCGOP_CALLI(op); const TCGLifeData arg_life = op->life; + const TCGHelperInfo *info; int flags, nb_regs, i; TCGReg reg; TCGArg arg; @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) TCGRegSet allocated_regs; func_addr = tcg_call_func(op); - flags = tcg_call_flags(op); + info = tcg_call_info(op); + flags = info->flags; nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); if (nb_regs > nb_iargs) { @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) save_globals(s, allocated_regs); } +#ifdef CONFIG_TCG_INTERPRETER + { + gpointer hash = (gpointer)(uintptr_t)info->typemask; + ffi_cif *cif = g_hash_table_lookup(ffi_table, hash); + assert(cif != NULL); + tcg_out_call(s, func_addr, cif); + } +#else tcg_out_call(s, func_addr); +#endif /* assign output registers and emit moves if needed */ for(i = 0; i < nb_oargs; i++) { diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ */ #include "qemu/osdep.h" +#include "qemu-common.h" +#include "tcg/tcg.h" /* MAX_OPC_PARAM_IARGS */ +#include "exec/cpu_ldst.h" +#include "tcg/tcg-op.h" +#include "qemu/compiler.h" +#include <ffi.h> -/* Enable TCI assertions only when debugging TCG (and without NDEBUG defined). - * Without assertions, the interpreter runs much faster. */ + +/* + * Enable TCI assertions only when debugging TCG (and without NDEBUG defined). + * Without assertions, the interpreter runs much faster. + */ #if defined(CONFIG_DEBUG_TCG) # define tci_assert(cond) assert(cond) #else # define tci_assert(cond) ((void)(cond)) #endif -#include "qemu-common.h" -#include "tcg/tcg.h" /* MAX_OPC_PARAM_IARGS */ -#include "exec/cpu_ldst.h" -#include "tcg/tcg-op.h" -#include "qemu/compiler.h" - -#if MAX_OPC_PARAM_IARGS != 6 -# error Fix needed, number of supported input arguments changed! -#endif -#if TCG_TARGET_REG_BITS == 32 -typedef uint64_t (*helper_function)(tcg_target_ulong, tcg_target_ulong, - tcg_target_ulong, tcg_target_ulong, - tcg_target_ulong, tcg_target_ulong, - tcg_target_ulong, tcg_target_ulong, - tcg_target_ulong, tcg_target_ulong, - tcg_target_ulong, tcg_target_ulong); -#else -typedef uint64_t (*helper_function)(tcg_target_ulong, tcg_target_ulong, - tcg_target_ulong, tcg_target_ulong, - tcg_target_ulong, tcg_target_ulong); -#endif - __thread uintptr_t tci_tb_ptr; -static tcg_target_ulong tci_read_reg(const tcg_target_ulong *regs, TCGReg index) -{ - tci_assert(index < TCG_TARGET_NB_REGS); - return regs[index]; -} - static void tci_write_reg(tcg_target_ulong *regs, TCGReg index, tcg_target_ulong value) { @@ -XXX,XX +XXX,XX @@ static tcg_target_ulong tci_read_label(const uint8_t **tb_ptr) * I = immediate (tcg_target_ulong) * l = label or pointer * m = immediate (TCGMemOpIdx) + * n = immediate (call return length) * r = register * s = signed ldst offset */ @@ -XXX,XX +XXX,XX @@ static void tci_args_l(const uint8_t **tb_ptr, void **l0) check_size(start, tb_ptr); } +static void tci_args_nll(const uint8_t **tb_ptr, uint8_t *n0, + void **l1, void **l2) +{ + const uint8_t *start = *tb_ptr; + + *n0 = tci_read_b(tb_ptr); + *l1 = (void *)tci_read_label(tb_ptr); + *l2 = (void *)tci_read_label(tb_ptr); + + check_size(start, tb_ptr); +} + static void tci_args_rr(const uint8_t **tb_ptr, TCGReg *r0, TCGReg *r1) { @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, { const uint8_t *tb_ptr = v_tb_ptr; tcg_target_ulong regs[TCG_TARGET_NB_REGS]; - long tcg_temps[CPU_TEMP_BUF_NLONGS]; - uintptr_t sp_value = (uintptr_t)(tcg_temps + CPU_TEMP_BUF_NLONGS); + uint64_t stack[(TCG_STATIC_CALL_ARGS_SIZE + TCG_STATIC_FRAME_SIZE) + / sizeof(uint64_t)]; + void *call_slots[TCG_STATIC_CALL_ARGS_SIZE / sizeof(uint64_t)]; regs[TCG_AREG0] = (tcg_target_ulong)env; - regs[TCG_REG_CALL_STACK] = sp_value; + regs[TCG_REG_CALL_STACK] = (uintptr_t)stack; + /* Other call_slots entries initialized at first use (see below). */ + call_slots[0] = NULL; tci_assert(tb_ptr); for (;;) { @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, #endif TCGMemOpIdx oi; int32_t ofs; - void *ptr; + void *ptr, *cif; /* Skip opcode and size entry. */ tb_ptr += 2; switch (opc) { case INDEX_op_call: - tci_args_l(&tb_ptr, &ptr); + /* + * Set up the ffi_avalue array once, delayed until now + * because many TB's do not make any calls. In tcg_gen_callN, + * we arranged for every real argument to be "left-aligned" + * in each 64-bit slot. + */ + if (unlikely(call_slots[0] == NULL)) { + for (int i = 0; i < ARRAY_SIZE(call_slots); ++i) { + call_slots[i] = &stack[i]; + } + } + + tci_args_nll(&tb_ptr, &len, &ptr, &cif); + + /* Helper functions may need to access the "return address" */ tci_tb_ptr = (uintptr_t)tb_ptr; -#if TCG_TARGET_REG_BITS == 32 - tmp64 = ((helper_function)ptr)(tci_read_reg(regs, TCG_REG_R0), - tci_read_reg(regs, TCG_REG_R1), - tci_read_reg(regs, TCG_REG_R2), - tci_read_reg(regs, TCG_REG_R3), - tci_read_reg(regs, TCG_REG_R4), - tci_read_reg(regs, TCG_REG_R5), - tci_read_reg(regs, TCG_REG_R6), - tci_read_reg(regs, TCG_REG_R7), - tci_read_reg(regs, TCG_REG_R8), - tci_read_reg(regs, TCG_REG_R9), - tci_read_reg(regs, TCG_REG_R10), - tci_read_reg(regs, TCG_REG_R11)); - tci_write_reg(regs, TCG_REG_R0, tmp64); - tci_write_reg(regs, TCG_REG_R1, tmp64 >> 32); -#else - tmp64 = ((helper_function)ptr)(tci_read_reg(regs, TCG_REG_R0), - tci_read_reg(regs, TCG_REG_R1), - tci_read_reg(regs, TCG_REG_R2), - tci_read_reg(regs, TCG_REG_R3), - tci_read_reg(regs, TCG_REG_R4), - tci_read_reg(regs, TCG_REG_R5)); - tci_write_reg(regs, TCG_REG_R0, tmp64); -#endif + + ffi_call(cif, ptr, stack, call_slots); + + /* Any result winds up "left-aligned" in the stack[0] slot. */ + switch (len) { + case 0: /* void */ + break; + case 1: /* uint32_t */ + /* + * Note that libffi has an odd special case in that it will + * always widen an integral result to ffi_arg. + */ + if (sizeof(ffi_arg) == 4) { + regs[TCG_REG_R0] = *(uint32_t *)stack; + break; + } + /* fall through */ + case 2: /* uint64_t */ + if (TCG_TARGET_REG_BITS == 32) { + tci_write_reg64(regs, TCG_REG_R1, TCG_REG_R0, stack[0]); + } else { + regs[TCG_REG_R0] = stack[0]; + } + break; + default: + g_assert_not_reached(); + } break; + case INDEX_op_br: tci_args_l(&tb_ptr, &ptr); tb_ptr = ptr; @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) TCGCond c; TCGMemOpIdx oi; uint8_t pos, len; - void *ptr; + void *ptr, *cif; const uint8_t *tb_ptr; status = info->read_memory_func(addr, buf, 2, info); @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) switch (op) { case INDEX_op_br: - case INDEX_op_call: case INDEX_op_exit_tb: case INDEX_op_goto_tb: tci_args_l(&tb_ptr, &ptr); info->fprintf_func(info->stream, "%-12s %p", op_name, ptr); break; + case INDEX_op_call: + tci_args_nll(&tb_ptr, &len, &ptr, &cif); + info->fprintf_func(info->stream, "%-12s %d, %p, %p", + op_name, len, ptr, cif); + break; + case INDEX_op_brcond_i32: case INDEX_op_brcond_i64: tci_args_rrcl(&tb_ptr, &r0, &r1, &c, &ptr); diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const int tcg_target_reg_alloc_order[] = { # error Fix needed, number of supported input arguments changed! #endif -static const int tcg_target_call_iarg_regs[] = { - TCG_REG_R0, - TCG_REG_R1, - TCG_REG_R2, - TCG_REG_R3, - TCG_REG_R4, - TCG_REG_R5, -#if TCG_TARGET_REG_BITS == 32 - /* 32 bit hosts need 2 * MAX_OPC_PARAM_IARGS registers. */ - TCG_REG_R6, - TCG_REG_R7, - TCG_REG_R8, - TCG_REG_R9, - TCG_REG_R10, - TCG_REG_R11, -#endif -}; +/* No call arguments via registers. All will be stored on the "stack". */ +static const int tcg_target_call_iarg_regs[] = { }; static const int tcg_target_call_oarg_regs[] = { TCG_REG_R0, @@ -XXX,XX +XXX,XX @@ static void tci_out_label(TCGContext *s, TCGLabel *label) static void stack_bounds_check(TCGReg base, target_long offset) { if (base == TCG_REG_CALL_STACK) { - tcg_debug_assert(offset < 0); - tcg_debug_assert(offset >= -(CPU_TEMP_BUF_NLONGS * sizeof(long))); + tcg_debug_assert(offset >= 0); + tcg_debug_assert(offset < (TCG_STATIC_CALL_ARGS_SIZE + + TCG_STATIC_FRAME_SIZE)); } } @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, } } -static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg) +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *func, + ffi_cif *cif) { uint8_t *old_code_ptr = s->code_ptr; + uint8_t which; + + if (cif->rtype == &ffi_type_void) { + which = 0; + } else if (cif->rtype->size == 4) { + which = 1; + } else { + tcg_debug_assert(cif->rtype->size == 8); + which = 2; + } tcg_out_op_t(s, INDEX_op_call); - tcg_out_i(s, (uintptr_t)arg); + tcg_out8(s, which); + tcg_out_i(s, (uintptr_t)func); + tcg_out_i(s, (uintptr_t)cif); + old_code_ptr[1] = s->code_ptr - old_code_ptr; } @@ -XXX,XX +XXX,XX @@ static void tcg_target_init(TCGContext *s) s->reserved_regs = 0; tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK); - /* We use negative offsets from "sp" so that we can distinguish - stores that might pretend to be call arguments. */ - tcg_set_frame(s, TCG_REG_CALL_STACK, - -CPU_TEMP_BUF_NLONGS * sizeof(long), - CPU_TEMP_BUF_NLONGS * sizeof(long)); + /* The call arguments come first, followed by the temp storage. */ + tcg_set_frame(s, TCG_REG_CALL_STACK, TCG_STATIC_CALL_ARGS_SIZE, + TCG_STATIC_FRAME_SIZE); } /* Generate global QEMU prologue and epilogue code. */ -- 2.25.1 We're about to adjust the offset range on host memory ops, and the format of branches. Both will require a temporary. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci/tcg-target.h | 1 + tcg/tci/tcg-target.c.inc | 1 + 2 files changed, 2 insertions(+) diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { TCG_REG_R14, TCG_REG_R15, + TCG_REG_TMP = TCG_REG_R13, TCG_AREG0 = TCG_REG_R14, TCG_REG_CALL_STACK = TCG_REG_R15, } TCGReg; diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_init(TCGContext *s) MAKE_64BIT_MASK(TCG_REG_R0, 64 / TCG_TARGET_REG_BITS); s->reserved_regs = 0; + tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP); tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK); /* The call arguments come first, followed by the temp storage. */ -- 2.25.1 The encoding planned for tci does not have enough room for brcond2, with 4 registers and a condition as input as well as the label. Resolve the condition into TCG_REG_TMP, and relax brcond to one register plus a label, considering the condition to always be reg != 0. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci.c | 68 ++++++++++------------------------------ tcg/tci/tcg-target.c.inc | 52 +++++++++++------------------- 2 files changed, 35 insertions(+), 85 deletions(-) diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ static void tci_args_nll(const uint8_t **tb_ptr, uint8_t *n0, check_size(start, tb_ptr); } +static void tci_args_rl(const uint8_t **tb_ptr, TCGReg *r0, void **l1) +{ + const uint8_t *start = *tb_ptr; + + *r0 = tci_read_r(tb_ptr); + *l1 = (void *)tci_read_label(tb_ptr); + + check_size(start, tb_ptr); +} + static void tci_args_rr(const uint8_t **tb_ptr, TCGReg *r0, TCGReg *r1) { @@ -XXX,XX +XXX,XX @@ static void tci_args_rrs(const uint8_t **tb_ptr, check_size(start, tb_ptr); } -static void tci_args_rrcl(const uint8_t **tb_ptr, - TCGReg *r0, TCGReg *r1, TCGCond *c2, void **l3) -{ - const uint8_t *start = *tb_ptr; - - *r0 = tci_read_r(tb_ptr); - *r1 = tci_read_r(tb_ptr); - *c2 = tci_read_b(tb_ptr); - *l3 = (void *)tci_read_label(tb_ptr); - - check_size(start, tb_ptr); -} - static void tci_args_rrrc(const uint8_t **tb_ptr, TCGReg *r0, TCGReg *r1, TCGReg *r2, TCGCond *c3) { @@ -XXX,XX +XXX,XX @@ static void tci_args_rrrr(const uint8_t **tb_ptr, check_size(start, tb_ptr); } -static void tci_args_rrrrcl(const uint8_t **tb_ptr, TCGReg *r0, TCGReg *r1, - TCGReg *r2, TCGReg *r3, TCGCond *c4, void **l5) -{ - const uint8_t *start = *tb_ptr; - - *r0 = tci_read_r(tb_ptr); - *r1 = tci_read_r(tb_ptr); - *r2 = tci_read_r(tb_ptr); - *r3 = tci_read_r(tb_ptr); - *c4 = tci_read_b(tb_ptr); - *l5 = (void *)tci_read_label(tb_ptr); - - check_size(start, tb_ptr); -} - static void tci_args_rrrrrc(const uint8_t **tb_ptr, TCGReg *r0, TCGReg *r1, TCGReg *r2, TCGReg *r3, TCGReg *r4, TCGCond *c5) { @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, break; #endif case INDEX_op_brcond_i32: - tci_args_rrcl(&tb_ptr, &r0, &r1, &condition, &ptr); - if (tci_compare32(regs[r0], regs[r1], condition)) { + tci_args_rl(&tb_ptr, &r0, &ptr); + if ((uint32_t)regs[r0]) { tb_ptr = ptr; } break; @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, T2 = tci_uint64(regs[r5], regs[r4]); tci_write_reg64(regs, r1, r0, T1 - T2); break; - case INDEX_op_brcond2_i32: - tci_args_rrrrcl(&tb_ptr, &r0, &r1, &r2, &r3, &condition, &ptr); - T1 = tci_uint64(regs[r1], regs[r0]); - T2 = tci_uint64(regs[r3], regs[r2]); - if (tci_compare64(T1, T2, condition)) { - tb_ptr = ptr; - continue; - } - break; case INDEX_op_mulu2_i32: tci_args_rrrr(&tb_ptr, &r0, &r1, &r2, &r3); tci_write_reg64(regs, r1, r0, (uint64_t)regs[r2] * regs[r3]); @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, break; #endif case INDEX_op_brcond_i64: - tci_args_rrcl(&tb_ptr, &r0, &r1, &condition, &ptr); - if (tci_compare64(regs[r0], regs[r1], condition)) { + tci_args_rl(&tb_ptr, &r0, &ptr); + if (regs[r0]) { tb_ptr = ptr; } break; @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) case INDEX_op_brcond_i32: case INDEX_op_brcond_i64: - tci_args_rrcl(&tb_ptr, &r0, &r1, &c, &ptr); - info->fprintf_func(info->stream, "%-12s %s, %s, %s, %p", - op_name, str_r(r0), str_r(r1), str_c(c), ptr); + tci_args_rl(&tb_ptr, &r0, &ptr); + info->fprintf_func(info->stream, "%-12s %s, 0, ne, %p", + op_name, str_r(r0), ptr); break; case INDEX_op_setcond_i32: @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) str_r(r3), str_r(r4), str_c(c)); break; - case INDEX_op_brcond2_i32: - tci_args_rrrrcl(&tb_ptr, &r0, &r1, &r2, &r3, &c, &ptr); - info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s, %s, %p", - op_name, str_r(r0), str_r(r1), - str_r(r2), str_r(r3), str_c(c), ptr); - break; - case INDEX_op_mulu2_i32: tci_args_rrrr(&tb_ptr, &r0, &r1, &r2, &r3); info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s", diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op_rI(TCGContext *s, TCGOpcode op, } #endif +static void tcg_out_op_rl(TCGContext *s, TCGOpcode op, TCGReg r0, TCGLabel *l1) +{ + uint8_t *old_code_ptr = s->code_ptr; + + tcg_out_op_t(s, op); + tcg_out_r(s, r0); + tci_out_label(s, l1); + + old_code_ptr[1] = s->code_ptr - old_code_ptr; +} + static void tcg_out_op_rr(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1) { uint8_t *old_code_ptr = s->code_ptr; @@ -XXX,XX +XXX,XX @@ static void tcg_out_op_rrs(TCGContext *s, TCGOpcode op, old_code_ptr[1] = s->code_ptr - old_code_ptr; } -static void tcg_out_op_rrcl(TCGContext *s, TCGOpcode op, - TCGReg r0, TCGReg r1, TCGCond c2, TCGLabel *l3) -{ - uint8_t *old_code_ptr = s->code_ptr; - - tcg_out_op_t(s, op); - tcg_out_r(s, r0); - tcg_out_r(s, r1); - tcg_out8(s, c2); - tci_out_label(s, l3); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; -} - static void tcg_out_op_rrrc(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1, TCGReg r2, TCGCond c3) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_op_rrrr(TCGContext *s, TCGOpcode op, old_code_ptr[1] = s->code_ptr - old_code_ptr; } -static void tcg_out_op_rrrrcl(TCGContext *s, TCGOpcode op, - TCGReg r0, TCGReg r1, TCGReg r2, TCGReg r3, - TCGCond c4, TCGLabel *l5) -{ - uint8_t *old_code_ptr = s->code_ptr; - - tcg_out_op_t(s, op); - tcg_out_r(s, r0); - tcg_out_r(s, r1); - tcg_out_r(s, r2); - tcg_out_r(s, r3); - tcg_out8(s, c4); - tci_out_label(s, l5); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; -} - static void tcg_out_op_rrrrrc(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1, TCGReg r2, TCGReg r3, TCGReg r4, TCGCond c5) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, break; CASE_32_64(brcond) - tcg_out_op_rrcl(s, opc, args[0], args[1], args[2], arg_label(args[3])); + tcg_out_op_rrrc(s, (opc == INDEX_op_brcond_i32 + ? INDEX_op_setcond_i32 : INDEX_op_setcond_i64), + TCG_REG_TMP, args[0], args[1], args[2]); + tcg_out_op_rl(s, opc, TCG_REG_TMP, arg_label(args[3])); break; CASE_32_64(neg) /* Optional (TCG_TARGET_HAS_neg_*). */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, args[3], args[4], args[5]); break; case INDEX_op_brcond2_i32: - tcg_out_op_rrrrcl(s, opc, args[0], args[1], args[2], - args[3], args[4], arg_label(args[5])); + tcg_out_op_rrrrrc(s, INDEX_op_setcond2_i32, TCG_REG_TMP, + args[0], args[1], args[2], args[3], args[4]); + tcg_out_op_rl(s, INDEX_op_brcond_i32, TCG_REG_TMP, arg_label(args[5])); break; case INDEX_op_mulu2_i32: tcg_out_op_rrrr(s, opc, args[0], args[1], args[2], args[3]); -- 2.25.1 Inline it into its one caller, tci_write_reg64. Drop the asserts that are redundant with tcg_read_r. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci.c | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ __thread uintptr_t tci_tb_ptr; -static void -tci_write_reg(tcg_target_ulong *regs, TCGReg index, tcg_target_ulong value) -{ - tci_assert(index < TCG_TARGET_NB_REGS); - tci_assert(index != TCG_AREG0); - tci_assert(index != TCG_REG_CALL_STACK); - regs[index] = value; -} - static void tci_write_reg64(tcg_target_ulong *regs, uint32_t high_index, uint32_t low_index, uint64_t value) { - tci_write_reg(regs, low_index, value); - tci_write_reg(regs, high_index, value >> 32); + regs[low_index] = value; + regs[high_index] = value >> 32; } /* Create a 64 bit value from two 32 bit values. */ -- 2.25.1 This removes all of the problems with unaligned accesses to the bytecode stream. With an 8-bit opcode at the bottom, we have 24 bits remaining, which are generally split into 6 4-bit slots. This fits well with the maximum length opcodes, e.g. INDEX_op_add2_i32, which have 6 register operands. We have, in previous patches, rearranged things such that there are no operations with a label which have more than one other operand. Which leaves us with a 20-bit field in which to encode a label, giving us a maximum TB size of 512k -- easily large. Change the INDEX_op_tci_movi_{i32,i64} opcodes to tci_mov[il]. The former puts the immediate in the upper 20 bits of the insn, like we do for the label displacement. The later uses a label to reference an entry in the constant pool. Thus, in the worst case we still have a single memory reference for any constant, but now the constants are out-of-line of the bytecode and can be shared between different moves saving space. Change INDEX_op_call to use a label to reference a pair of pointers in the constant pool. This removes the only slightly dodgy link with the layout of struct TCGHelperInfo. The re-encode cannot be done in pieces. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg-opc.h | 4 +- tcg/tci/tcg-target.h | 3 +- tcg/tci.c | 539 +++++++++++++++------------------------ tcg/tci/tcg-target.c.inc | 379 ++++++++++++--------------- tcg/tci/README | 20 +- 5 files changed, 383 insertions(+), 562 deletions(-) diff --git a/include/tcg/tcg-opc.h b/include/tcg/tcg-opc.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg-opc.h +++ b/include/tcg/tcg-opc.h @@ -XXX,XX +XXX,XX @@ DEF(last_generic, 0, 0, 0, TCG_OPF_NOT_PRESENT) #ifdef TCG_TARGET_INTERPRETER /* These opcodes are only for use between the tci generator and interpreter. */ -DEF(tci_movi_i32, 1, 0, 1, TCG_OPF_NOT_PRESENT) -DEF(tci_movi_i64, 1, 0, 1, TCG_OPF_64BIT | TCG_OPF_NOT_PRESENT) +DEF(tci_movi, 1, 0, 1, TCG_OPF_NOT_PRESENT) +DEF(tci_movl, 1, 0, 1, TCG_OPF_NOT_PRESENT) #endif #undef TLADDR_ARGS diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ #define TCG_TARGET_H #define TCG_TARGET_INTERPRETER 1 -#define TCG_TARGET_INSN_UNIT_SIZE 1 +#define TCG_TARGET_INSN_UNIT_SIZE 4 #define TCG_TARGET_TLB_DISPLACEMENT_BITS 32 #define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_STACK_ALIGN 8 #define HAVE_TCG_QEMU_TB_EXEC +#define TCG_TARGET_NEED_POOL_LABELS /* We could notice __i386__ or __s390x__ and reduce the barriers depending on the host. But if you want performance, you use the normal backend. diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ static uint64_t tci_uint64(uint32_t high, uint32_t low) return ((uint64_t)high << 32) + low; } -/* Read constant byte from bytecode. */ -static uint8_t tci_read_b(const uint8_t **tb_ptr) -{ - return *(tb_ptr[0]++); -} - -/* Read register number from bytecode. */ -static TCGReg tci_read_r(const uint8_t **tb_ptr) -{ - uint8_t regno = tci_read_b(tb_ptr); - tci_assert(regno < TCG_TARGET_NB_REGS); - return regno; -} - -/* Read constant (native size) from bytecode. */ -static tcg_target_ulong tci_read_i(const uint8_t **tb_ptr) -{ - tcg_target_ulong value = *(const tcg_target_ulong *)(*tb_ptr); - *tb_ptr += sizeof(value); - return value; -} - -/* Read unsigned constant (32 bit) from bytecode. */ -static uint32_t tci_read_i32(const uint8_t **tb_ptr) -{ - uint32_t value = *(const uint32_t *)(*tb_ptr); - *tb_ptr += sizeof(value); - return value; -} - -/* Read signed constant (32 bit) from bytecode. */ -static int32_t tci_read_s32(const uint8_t **tb_ptr) -{ - int32_t value = *(const int32_t *)(*tb_ptr); - *tb_ptr += sizeof(value); - return value; -} - -static tcg_target_ulong tci_read_label(const uint8_t **tb_ptr) -{ - return tci_read_i(tb_ptr); -} - /* * Load sets of arguments all at once. The naming convention is: * tci_args_<arguments> @@ -XXX,XX +XXX,XX @@ static tcg_target_ulong tci_read_label(const uint8_t **tb_ptr) * s = signed ldst offset */ -static void check_size(const uint8_t *start, const uint8_t **tb_ptr) +static void tci_args_l(uint32_t insn, const void *tb_ptr, void **l0) { - const uint8_t *old_code_ptr = start - 2; - uint8_t op_size = old_code_ptr[1]; - tci_assert(*tb_ptr == old_code_ptr + op_size); + int diff = sextract32(insn, 12, 20); + *l0 = diff ? (void *)tb_ptr + diff : NULL; } -static void tci_args_l(const uint8_t **tb_ptr, void **l0) +static void tci_args_nl(uint32_t insn, const void *tb_ptr, + uint8_t *n0, void **l1) { - const uint8_t *start = *tb_ptr; - - *l0 = (void *)tci_read_label(tb_ptr); - - check_size(start, tb_ptr); + *n0 = extract32(insn, 8, 4); + *l1 = sextract32(insn, 12, 20) + (void *)tb_ptr; } -static void tci_args_nll(const uint8_t **tb_ptr, uint8_t *n0, - void **l1, void **l2) +static void tci_args_rl(uint32_t insn, const void *tb_ptr, + TCGReg *r0, void **l1) { - const uint8_t *start = *tb_ptr; - - *n0 = tci_read_b(tb_ptr); - *l1 = (void *)tci_read_label(tb_ptr); - *l2 = (void *)tci_read_label(tb_ptr); - - check_size(start, tb_ptr); + *r0 = extract32(insn, 8, 4); + *l1 = sextract32(insn, 12, 20) + (void *)tb_ptr; } -static void tci_args_rl(const uint8_t **tb_ptr, TCGReg *r0, void **l1) +static void tci_args_rr(uint32_t insn, TCGReg *r0, TCGReg *r1) { - const uint8_t *start = *tb_ptr; - - *r0 = tci_read_r(tb_ptr); - *l1 = (void *)tci_read_label(tb_ptr); - - check_size(start, tb_ptr); + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); } -static void tci_args_rr(const uint8_t **tb_ptr, - TCGReg *r0, TCGReg *r1) +static void tci_args_ri(uint32_t insn, TCGReg *r0, tcg_target_ulong *i1) { - const uint8_t *start = *tb_ptr; - - *r0 = tci_read_r(tb_ptr); - *r1 = tci_read_r(tb_ptr); - - check_size(start, tb_ptr); + *r0 = extract32(insn, 8, 4); + *i1 = sextract32(insn, 12, 20); } -static void tci_args_ri(const uint8_t **tb_ptr, - TCGReg *r0, tcg_target_ulong *i1) +static void tci_args_rrm(uint32_t insn, TCGReg *r0, + TCGReg *r1, TCGMemOpIdx *m2) { - const uint8_t *start = *tb_ptr; - - *r0 = tci_read_r(tb_ptr); - *i1 = tci_read_i32(tb_ptr); - - check_size(start, tb_ptr); + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *m2 = extract32(insn, 20, 12); } -#if TCG_TARGET_REG_BITS == 64 -static void tci_args_rI(const uint8_t **tb_ptr, - TCGReg *r0, tcg_target_ulong *i1) +static void tci_args_rrr(uint32_t insn, TCGReg *r0, TCGReg *r1, TCGReg *r2) { - const uint8_t *start = *tb_ptr; - - *r0 = tci_read_r(tb_ptr); - *i1 = tci_read_i(tb_ptr); - - check_size(start, tb_ptr); -} -#endif - -static void tci_args_rrm(const uint8_t **tb_ptr, - TCGReg *r0, TCGReg *r1, TCGMemOpIdx *m2) -{ - const uint8_t *start = *tb_ptr; - - *r0 = tci_read_r(tb_ptr); - *r1 = tci_read_r(tb_ptr); - *m2 = tci_read_i32(tb_ptr); - - check_size(start, tb_ptr); + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *r2 = extract32(insn, 16, 4); } -static void tci_args_rrr(const uint8_t **tb_ptr, - TCGReg *r0, TCGReg *r1, TCGReg *r2) +static void tci_args_rrs(uint32_t insn, TCGReg *r0, TCGReg *r1, int32_t *i2) { - const uint8_t *start = *tb_ptr; - - *r0 = tci_read_r(tb_ptr); - *r1 = tci_read_r(tb_ptr); - *r2 = tci_read_r(tb_ptr); - - check_size(start, tb_ptr); + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *i2 = sextract32(insn, 16, 16); } -static void tci_args_rrs(const uint8_t **tb_ptr, - TCGReg *r0, TCGReg *r1, int32_t *i2) -{ - const uint8_t *start = *tb_ptr; - - *r0 = tci_read_r(tb_ptr); - *r1 = tci_read_r(tb_ptr); - *i2 = tci_read_s32(tb_ptr); - - check_size(start, tb_ptr); -} - -static void tci_args_rrrc(const uint8_t **tb_ptr, +static void tci_args_rrrc(uint32_t insn, TCGReg *r0, TCGReg *r1, TCGReg *r2, TCGCond *c3) { - const uint8_t *start = *tb_ptr; - - *r0 = tci_read_r(tb_ptr); - *r1 = tci_read_r(tb_ptr); - *r2 = tci_read_r(tb_ptr); - *c3 = tci_read_b(tb_ptr); - - check_size(start, tb_ptr); + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *r2 = extract32(insn, 16, 4); + *c3 = extract32(insn, 20, 4); } -static void tci_args_rrrm(const uint8_t **tb_ptr, +static void tci_args_rrrm(uint32_t insn, TCGReg *r0, TCGReg *r1, TCGReg *r2, TCGMemOpIdx *m3) { - const uint8_t *start = *tb_ptr; - - *r0 = tci_read_r(tb_ptr); - *r1 = tci_read_r(tb_ptr); - *r2 = tci_read_r(tb_ptr); - *m3 = tci_read_i32(tb_ptr); - - check_size(start, tb_ptr); + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *r2 = extract32(insn, 16, 4); + *m3 = extract32(insn, 20, 12); } -static void tci_args_rrrbb(const uint8_t **tb_ptr, TCGReg *r0, TCGReg *r1, +static void tci_args_rrrbb(uint32_t insn, TCGReg *r0, TCGReg *r1, TCGReg *r2, uint8_t *i3, uint8_t *i4) { - const uint8_t *start = *tb_ptr; - - *r0 = tci_read_r(tb_ptr); - *r1 = tci_read_r(tb_ptr); - *r2 = tci_read_r(tb_ptr); - *i3 = tci_read_b(tb_ptr); - *i4 = tci_read_b(tb_ptr); - - check_size(start, tb_ptr); + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *r2 = extract32(insn, 16, 4); + *i3 = extract32(insn, 20, 6); + *i4 = extract32(insn, 26, 6); } -static void tci_args_rrrrm(const uint8_t **tb_ptr, TCGReg *r0, TCGReg *r1, - TCGReg *r2, TCGReg *r3, TCGMemOpIdx *m4) +static void tci_args_rrrrr(uint32_t insn, TCGReg *r0, TCGReg *r1, + TCGReg *r2, TCGReg *r3, TCGReg *r4) { - const uint8_t *start = *tb_ptr; - - *r0 = tci_read_r(tb_ptr); - *r1 = tci_read_r(tb_ptr); - *r2 = tci_read_r(tb_ptr); - *r3 = tci_read_r(tb_ptr); - *m4 = tci_read_i32(tb_ptr); - - check_size(start, tb_ptr); + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *r2 = extract32(insn, 16, 4); + *r3 = extract32(insn, 20, 4); + *r4 = extract32(insn, 24, 4); } #if TCG_TARGET_REG_BITS == 32 -static void tci_args_rrrr(const uint8_t **tb_ptr, +static void tci_args_rrrr(uint32_t insn, TCGReg *r0, TCGReg *r1, TCGReg *r2, TCGReg *r3) { - const uint8_t *start = *tb_ptr; - - *r0 = tci_read_r(tb_ptr); - *r1 = tci_read_r(tb_ptr); - *r2 = tci_read_r(tb_ptr); - *r3 = tci_read_r(tb_ptr); - - check_size(start, tb_ptr); + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *r2 = extract32(insn, 16, 4); + *r3 = extract32(insn, 20, 4); } -static void tci_args_rrrrrc(const uint8_t **tb_ptr, TCGReg *r0, TCGReg *r1, +static void tci_args_rrrrrc(uint32_t insn, TCGReg *r0, TCGReg *r1, TCGReg *r2, TCGReg *r3, TCGReg *r4, TCGCond *c5) { - const uint8_t *start = *tb_ptr; - - *r0 = tci_read_r(tb_ptr); - *r1 = tci_read_r(tb_ptr); - *r2 = tci_read_r(tb_ptr); - *r3 = tci_read_r(tb_ptr); - *r4 = tci_read_r(tb_ptr); - *c5 = tci_read_b(tb_ptr); - - check_size(start, tb_ptr); + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *r2 = extract32(insn, 16, 4); + *r3 = extract32(insn, 20, 4); + *r4 = extract32(insn, 24, 4); + *c5 = extract32(insn, 28, 4); } -static void tci_args_rrrrrr(const uint8_t **tb_ptr, TCGReg *r0, TCGReg *r1, +static void tci_args_rrrrrr(uint32_t insn, TCGReg *r0, TCGReg *r1, TCGReg *r2, TCGReg *r3, TCGReg *r4, TCGReg *r5) { - const uint8_t *start = *tb_ptr; - - *r0 = tci_read_r(tb_ptr); - *r1 = tci_read_r(tb_ptr); - *r2 = tci_read_r(tb_ptr); - *r3 = tci_read_r(tb_ptr); - *r4 = tci_read_r(tb_ptr); - *r5 = tci_read_r(tb_ptr); - - check_size(start, tb_ptr); + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *r2 = extract32(insn, 16, 4); + *r3 = extract32(insn, 20, 4); + *r4 = extract32(insn, 24, 4); + *r5 = extract32(insn, 28, 4); } #endif @@ -XXX,XX +XXX,XX @@ static bool tci_compare64(uint64_t u0, uint64_t u1, TCGCond condition) uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, const void *v_tb_ptr) { - const uint8_t *tb_ptr = v_tb_ptr; + const uint32_t *tb_ptr = v_tb_ptr; tcg_target_ulong regs[TCG_TARGET_NB_REGS]; uint64_t stack[(TCG_STATIC_CALL_ARGS_SIZE + TCG_STATIC_FRAME_SIZE) / sizeof(uint64_t)]; @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, tci_assert(tb_ptr); for (;;) { - TCGOpcode opc = tb_ptr[0]; - TCGReg r0, r1, r2, r3; + uint32_t insn; + TCGOpcode opc; + TCGReg r0, r1, r2, r3, r4; tcg_target_ulong t1; TCGCond condition; target_ulong taddr; @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, uint32_t tmp32; uint64_t tmp64; #if TCG_TARGET_REG_BITS == 32 - TCGReg r4, r5; + TCGReg r5; uint64_t T1, T2; #endif TCGMemOpIdx oi; int32_t ofs; - void *ptr, *cif; + void *ptr; - /* Skip opcode and size entry. */ - tb_ptr += 2; + insn = *tb_ptr++; + opc = extract32(insn, 0, 8); switch (opc) { case INDEX_op_call: @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, } } - tci_args_nll(&tb_ptr, &len, &ptr, &cif); + tci_args_nl(insn, tb_ptr, &len, &ptr); /* Helper functions may need to access the "return address" */ tci_tb_ptr = (uintptr_t)tb_ptr; - ffi_call(cif, ptr, stack, call_slots); + { + void **pptr = ptr; + ffi_call(pptr[1], pptr[0], stack, call_slots); + } /* Any result winds up "left-aligned" in the stack[0] slot. */ switch (len) { @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, break; case INDEX_op_br: - tci_args_l(&tb_ptr, &ptr); + tci_args_l(insn, tb_ptr, &ptr); tb_ptr = ptr; continue; case INDEX_op_setcond_i32: - tci_args_rrrc(&tb_ptr, &r0, &r1, &r2, &condition); + tci_args_rrrc(insn, &r0, &r1, &r2, &condition); regs[r0] = tci_compare32(regs[r1], regs[r2], condition); break; #if TCG_TARGET_REG_BITS == 32 case INDEX_op_setcond2_i32: - tci_args_rrrrrc(&tb_ptr, &r0, &r1, &r2, &r3, &r4, &condition); + tci_args_rrrrrc(insn, &r0, &r1, &r2, &r3, &r4, &condition); T1 = tci_uint64(regs[r2], regs[r1]); T2 = tci_uint64(regs[r4], regs[r3]); regs[r0] = tci_compare64(T1, T2, condition); break; #elif TCG_TARGET_REG_BITS == 64 case INDEX_op_setcond_i64: - tci_args_rrrc(&tb_ptr, &r0, &r1, &r2, &condition); + tci_args_rrrc(insn, &r0, &r1, &r2, &condition); regs[r0] = tci_compare64(regs[r1], regs[r2], condition); break; #endif CASE_32_64(mov) - tci_args_rr(&tb_ptr, &r0, &r1); + tci_args_rr(insn, &r0, &r1); regs[r0] = regs[r1]; break; - case INDEX_op_tci_movi_i32: - tci_args_ri(&tb_ptr, &r0, &t1); + case INDEX_op_tci_movi: + tci_args_ri(insn, &r0, &t1); regs[r0] = t1; break; + case INDEX_op_tci_movl: + tci_args_rl(insn, tb_ptr, &r0, &ptr); + regs[r0] = *(tcg_target_ulong *)ptr; + break; /* Load/store operations (32 bit). */ CASE_32_64(ld8u) - tci_args_rrs(&tb_ptr, &r0, &r1, &ofs); + tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); regs[r0] = *(uint8_t *)ptr; break; CASE_32_64(ld8s) - tci_args_rrs(&tb_ptr, &r0, &r1, &ofs); + tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); regs[r0] = *(int8_t *)ptr; break; CASE_32_64(ld16u) - tci_args_rrs(&tb_ptr, &r0, &r1, &ofs); + tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); regs[r0] = *(uint16_t *)ptr; break; CASE_32_64(ld16s) - tci_args_rrs(&tb_ptr, &r0, &r1, &ofs); + tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); regs[r0] = *(int16_t *)ptr; break; case INDEX_op_ld_i32: CASE_64(ld32u) - tci_args_rrs(&tb_ptr, &r0, &r1, &ofs); + tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); regs[r0] = *(uint32_t *)ptr; break; CASE_32_64(st8) - tci_args_rrs(&tb_ptr, &r0, &r1, &ofs); + tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); *(uint8_t *)ptr = regs[r0]; break; CASE_32_64(st16) - tci_args_rrs(&tb_ptr, &r0, &r1, &ofs); + tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); *(uint16_t *)ptr = regs[r0]; break; case INDEX_op_st_i32: CASE_64(st32) - tci_args_rrs(&tb_ptr, &r0, &r1, &ofs); + tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); *(uint32_t *)ptr = regs[r0]; break; @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, /* Arithmetic operations (mixed 32/64 bit). */ CASE_32_64(add) - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] + regs[r2]; break; CASE_32_64(sub) - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] - regs[r2]; break; CASE_32_64(mul) - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] * regs[r2]; break; CASE_32_64(and) - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] & regs[r2]; break; CASE_32_64(or) - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] | regs[r2]; break; CASE_32_64(xor) - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] ^ regs[r2]; break; /* Arithmetic operations (32 bit). */ case INDEX_op_div_i32: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (int32_t)regs[r1] / (int32_t)regs[r2]; break; case INDEX_op_divu_i32: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (uint32_t)regs[r1] / (uint32_t)regs[r2]; break; case INDEX_op_rem_i32: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (int32_t)regs[r1] % (int32_t)regs[r2]; break; case INDEX_op_remu_i32: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (uint32_t)regs[r1] % (uint32_t)regs[r2]; break; /* Shift/rotate operations (32 bit). */ case INDEX_op_shl_i32: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (uint32_t)regs[r1] << (regs[r2] & 31); break; case INDEX_op_shr_i32: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (uint32_t)regs[r1] >> (regs[r2] & 31); break; case INDEX_op_sar_i32: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (int32_t)regs[r1] >> (regs[r2] & 31); break; #if TCG_TARGET_HAS_rot_i32 case INDEX_op_rotl_i32: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = rol32(regs[r1], regs[r2] & 31); break; case INDEX_op_rotr_i32: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = ror32(regs[r1], regs[r2] & 31); break; #endif #if TCG_TARGET_HAS_deposit_i32 case INDEX_op_deposit_i32: - tci_args_rrrbb(&tb_ptr, &r0, &r1, &r2, &pos, &len); + tci_args_rrrbb(insn, &r0, &r1, &r2, &pos, &len); regs[r0] = deposit32(regs[r1], pos, len, regs[r2]); break; #endif case INDEX_op_brcond_i32: - tci_args_rl(&tb_ptr, &r0, &ptr); + tci_args_rl(insn, tb_ptr, &r0, &ptr); if ((uint32_t)regs[r0]) { tb_ptr = ptr; } break; #if TCG_TARGET_REG_BITS == 32 case INDEX_op_add2_i32: - tci_args_rrrrrr(&tb_ptr, &r0, &r1, &r2, &r3, &r4, &r5); + tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); T1 = tci_uint64(regs[r3], regs[r2]); T2 = tci_uint64(regs[r5], regs[r4]); tci_write_reg64(regs, r1, r0, T1 + T2); break; case INDEX_op_sub2_i32: - tci_args_rrrrrr(&tb_ptr, &r0, &r1, &r2, &r3, &r4, &r5); + tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); T1 = tci_uint64(regs[r3], regs[r2]); T2 = tci_uint64(regs[r5], regs[r4]); tci_write_reg64(regs, r1, r0, T1 - T2); break; case INDEX_op_mulu2_i32: - tci_args_rrrr(&tb_ptr, &r0, &r1, &r2, &r3); + tci_args_rrrr(insn, &r0, &r1, &r2, &r3); tci_write_reg64(regs, r1, r0, (uint64_t)regs[r2] * regs[r3]); break; #endif /* TCG_TARGET_REG_BITS == 32 */ #if TCG_TARGET_HAS_ext8s_i32 || TCG_TARGET_HAS_ext8s_i64 CASE_32_64(ext8s) - tci_args_rr(&tb_ptr, &r0, &r1); + tci_args_rr(insn, &r0, &r1); regs[r0] = (int8_t)regs[r1]; break; #endif #if TCG_TARGET_HAS_ext16s_i32 || TCG_TARGET_HAS_ext16s_i64 CASE_32_64(ext16s) - tci_args_rr(&tb_ptr, &r0, &r1); + tci_args_rr(insn, &r0, &r1); regs[r0] = (int16_t)regs[r1]; break; #endif #if TCG_TARGET_HAS_ext8u_i32 || TCG_TARGET_HAS_ext8u_i64 CASE_32_64(ext8u) - tci_args_rr(&tb_ptr, &r0, &r1); + tci_args_rr(insn, &r0, &r1); regs[r0] = (uint8_t)regs[r1]; break; #endif #if TCG_TARGET_HAS_ext16u_i32 || TCG_TARGET_HAS_ext16u_i64 CASE_32_64(ext16u) - tci_args_rr(&tb_ptr, &r0, &r1); + tci_args_rr(insn, &r0, &r1); regs[r0] = (uint16_t)regs[r1]; break; #endif #if TCG_TARGET_HAS_bswap16_i32 || TCG_TARGET_HAS_bswap16_i64 CASE_32_64(bswap16) - tci_args_rr(&tb_ptr, &r0, &r1); + tci_args_rr(insn, &r0, &r1); regs[r0] = bswap16(regs[r1]); break; #endif #if TCG_TARGET_HAS_bswap32_i32 || TCG_TARGET_HAS_bswap32_i64 CASE_32_64(bswap32) - tci_args_rr(&tb_ptr, &r0, &r1); + tci_args_rr(insn, &r0, &r1); regs[r0] = bswap32(regs[r1]); break; #endif #if TCG_TARGET_HAS_not_i32 || TCG_TARGET_HAS_not_i64 CASE_32_64(not) - tci_args_rr(&tb_ptr, &r0, &r1); + tci_args_rr(insn, &r0, &r1); regs[r0] = ~regs[r1]; break; #endif #if TCG_TARGET_HAS_neg_i32 || TCG_TARGET_HAS_neg_i64 CASE_32_64(neg) - tci_args_rr(&tb_ptr, &r0, &r1); + tci_args_rr(insn, &r0, &r1); regs[r0] = -regs[r1]; break; #endif #if TCG_TARGET_REG_BITS == 64 - case INDEX_op_tci_movi_i64: - tci_args_rI(&tb_ptr, &r0, &t1); - regs[r0] = t1; - break; - /* Load/store operations (64 bit). */ case INDEX_op_ld32s_i64: - tci_args_rrs(&tb_ptr, &r0, &r1, &ofs); + tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); regs[r0] = *(int32_t *)ptr; break; case INDEX_op_ld_i64: - tci_args_rrs(&tb_ptr, &r0, &r1, &ofs); + tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); regs[r0] = *(uint64_t *)ptr; break; case INDEX_op_st_i64: - tci_args_rrs(&tb_ptr, &r0, &r1, &ofs); + tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); *(uint64_t *)ptr = regs[r0]; break; @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, /* Arithmetic operations (64 bit). */ case INDEX_op_div_i64: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (int64_t)regs[r1] / (int64_t)regs[r2]; break; case INDEX_op_divu_i64: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (uint64_t)regs[r1] / (uint64_t)regs[r2]; break; case INDEX_op_rem_i64: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (int64_t)regs[r1] % (int64_t)regs[r2]; break; case INDEX_op_remu_i64: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (uint64_t)regs[r1] % (uint64_t)regs[r2]; break; /* Shift/rotate operations (64 bit). */ case INDEX_op_shl_i64: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] << (regs[r2] & 63); break; case INDEX_op_shr_i64: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] >> (regs[r2] & 63); break; case INDEX_op_sar_i64: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (int64_t)regs[r1] >> (regs[r2] & 63); break; #if TCG_TARGET_HAS_rot_i64 case INDEX_op_rotl_i64: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = rol64(regs[r1], regs[r2] & 63); break; case INDEX_op_rotr_i64: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = ror64(regs[r1], regs[r2] & 63); break; #endif #if TCG_TARGET_HAS_deposit_i64 case INDEX_op_deposit_i64: - tci_args_rrrbb(&tb_ptr, &r0, &r1, &r2, &pos, &len); + tci_args_rrrbb(insn, &r0, &r1, &r2, &pos, &len); regs[r0] = deposit64(regs[r1], pos, len, regs[r2]); break; #endif case INDEX_op_brcond_i64: - tci_args_rl(&tb_ptr, &r0, &ptr); + tci_args_rl(insn, tb_ptr, &r0, &ptr); if (regs[r0]) { tb_ptr = ptr; } break; case INDEX_op_ext32s_i64: case INDEX_op_ext_i32_i64: - tci_args_rr(&tb_ptr, &r0, &r1); + tci_args_rr(insn, &r0, &r1); regs[r0] = (int32_t)regs[r1]; break; case INDEX_op_ext32u_i64: case INDEX_op_extu_i32_i64: - tci_args_rr(&tb_ptr, &r0, &r1); + tci_args_rr(insn, &r0, &r1); regs[r0] = (uint32_t)regs[r1]; break; #if TCG_TARGET_HAS_bswap64_i64 case INDEX_op_bswap64_i64: - tci_args_rr(&tb_ptr, &r0, &r1); + tci_args_rr(insn, &r0, &r1); regs[r0] = bswap64(regs[r1]); break; #endif @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, /* QEMU specific operations. */ case INDEX_op_exit_tb: - tci_args_l(&tb_ptr, &ptr); + tci_args_l(insn, tb_ptr, &ptr); return (uintptr_t)ptr; case INDEX_op_goto_tb: - tci_args_l(&tb_ptr, &ptr); + tci_args_l(insn, tb_ptr, &ptr); tb_ptr = *(void **)ptr; break; case INDEX_op_qemu_ld_i32: if (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS) { - tci_args_rrm(&tb_ptr, &r0, &r1, &oi); + tci_args_rrm(insn, &r0, &r1, &oi); taddr = regs[r1]; } else { - tci_args_rrrm(&tb_ptr, &r0, &r1, &r2, &oi); + tci_args_rrrm(insn, &r0, &r1, &r2, &oi); taddr = tci_uint64(regs[r2], regs[r1]); } switch (get_memop(oi) & (MO_BSWAP | MO_SSIZE)) { @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, case INDEX_op_qemu_ld_i64: if (TCG_TARGET_REG_BITS == 64) { - tci_args_rrm(&tb_ptr, &r0, &r1, &oi); + tci_args_rrm(insn, &r0, &r1, &oi); taddr = regs[r1]; } else if (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS) { - tci_args_rrrm(&tb_ptr, &r0, &r1, &r2, &oi); + tci_args_rrrm(insn, &r0, &r1, &r2, &oi); taddr = regs[r2]; } else { - tci_args_rrrrm(&tb_ptr, &r0, &r1, &r2, &r3, &oi); + tci_args_rrrrr(insn, &r0, &r1, &r2, &r3, &r4); taddr = tci_uint64(regs[r3], regs[r2]); + oi = regs[r4]; } switch (get_memop(oi) & (MO_BSWAP | MO_SSIZE)) { case MO_UB: @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, case INDEX_op_qemu_st_i32: if (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS) { - tci_args_rrm(&tb_ptr, &r0, &r1, &oi); + tci_args_rrm(insn, &r0, &r1, &oi); taddr = regs[r1]; } else { - tci_args_rrrm(&tb_ptr, &r0, &r1, &r2, &oi); + tci_args_rrrm(insn, &r0, &r1, &r2, &oi); taddr = tci_uint64(regs[r2], regs[r1]); } tmp32 = regs[r0]; @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, case INDEX_op_qemu_st_i64: if (TCG_TARGET_REG_BITS == 64) { - tci_args_rrm(&tb_ptr, &r0, &r1, &oi); + tci_args_rrm(insn, &r0, &r1, &oi); taddr = regs[r1]; tmp64 = regs[r0]; } else { if (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS) { - tci_args_rrrm(&tb_ptr, &r0, &r1, &r2, &oi); + tci_args_rrrm(insn, &r0, &r1, &r2, &oi); taddr = regs[r2]; } else { - tci_args_rrrrm(&tb_ptr, &r0, &r1, &r2, &r3, &oi); + tci_args_rrrrr(insn, &r0, &r1, &r2, &r3, &r4); taddr = tci_uint64(regs[r3], regs[r2]); + oi = regs[r4]; } tmp64 = tci_uint64(regs[r1], regs[r0]); } @@ -XXX,XX +XXX,XX @@ static const char *str_c(TCGCond c) /* Disassemble TCI bytecode. */ int print_insn_tci(bfd_vma addr, disassemble_info *info) { - uint8_t buf[256]; - int length, status; + const uint32_t *tb_ptr = (const void *)(uintptr_t)addr; const TCGOpDef *def; const char *op_name; + uint32_t insn; TCGOpcode op; - TCGReg r0, r1, r2, r3; + TCGReg r0, r1, r2, r3, r4; #if TCG_TARGET_REG_BITS == 32 - TCGReg r4, r5; + TCGReg r5; #endif tcg_target_ulong i1; int32_t s2; TCGCond c; TCGMemOpIdx oi; uint8_t pos, len; - void *ptr, *cif; - const uint8_t *tb_ptr; + void *ptr; - status = info->read_memory_func(addr, buf, 2, info); - if (status != 0) { - info->memory_error_func(status, addr, info); - return -1; - } - op = buf[0]; - length = buf[1]; + /* TCI is always the host, so we don't need to load indirect. */ + insn = *tb_ptr++; - if (length < 2) { - info->fprintf_func(info->stream, "invalid length %d", length); - return 1; - } - - status = info->read_memory_func(addr + 2, buf + 2, length - 2, info); - if (status != 0) { - info->memory_error_func(status, addr + 2, info); - return -1; - } + info->fprintf_func(info->stream, "%08x ", insn); + op = extract32(insn, 0, 8); def = &tcg_op_defs[op]; op_name = def->name; - tb_ptr = buf + 2; switch (op) { case INDEX_op_br: case INDEX_op_exit_tb: case INDEX_op_goto_tb: - tci_args_l(&tb_ptr, &ptr); + tci_args_l(insn, tb_ptr, &ptr); info->fprintf_func(info->stream, "%-12s %p", op_name, ptr); break; case INDEX_op_call: - tci_args_nll(&tb_ptr, &len, &ptr, &cif); - info->fprintf_func(info->stream, "%-12s %d, %p, %p", - op_name, len, ptr, cif); + tci_args_nl(insn, tb_ptr, &len, &ptr); + info->fprintf_func(info->stream, "%-12s %d, %p", op_name, len, ptr); break; case INDEX_op_brcond_i32: case INDEX_op_brcond_i64: - tci_args_rl(&tb_ptr, &r0, &ptr); + tci_args_rl(insn, tb_ptr, &r0, &ptr); info->fprintf_func(info->stream, "%-12s %s, 0, ne, %p", op_name, str_r(r0), ptr); break; case INDEX_op_setcond_i32: case INDEX_op_setcond_i64: - tci_args_rrrc(&tb_ptr, &r0, &r1, &r2, &c); + tci_args_rrrc(insn, &r0, &r1, &r2, &c); info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s", op_name, str_r(r0), str_r(r1), str_r(r2), str_c(c)); break; - case INDEX_op_tci_movi_i32: - tci_args_ri(&tb_ptr, &r0, &i1); + case INDEX_op_tci_movi: + tci_args_ri(insn, &r0, &i1); info->fprintf_func(info->stream, "%-12s %s, 0x%" TCG_PRIlx, op_name, str_r(r0), i1); break; -#if TCG_TARGET_REG_BITS == 64 - case INDEX_op_tci_movi_i64: - tci_args_rI(&tb_ptr, &r0, &i1); - info->fprintf_func(info->stream, "%-12s %s, 0x%" TCG_PRIlx, - op_name, str_r(r0), i1); + case INDEX_op_tci_movl: + tci_args_rl(insn, tb_ptr, &r0, &ptr); + info->fprintf_func(info->stream, "%-12s %s, %p", + op_name, str_r(r0), ptr); break; -#endif case INDEX_op_ld8u_i32: case INDEX_op_ld8u_i64: @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) case INDEX_op_st32_i64: case INDEX_op_st_i32: case INDEX_op_st_i64: - tci_args_rrs(&tb_ptr, &r0, &r1, &s2); + tci_args_rrs(insn, &r0, &r1, &s2); info->fprintf_func(info->stream, "%-12s %s, %s, %d", op_name, str_r(r0), str_r(r1), s2); break; @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) case INDEX_op_not_i64: case INDEX_op_neg_i32: case INDEX_op_neg_i64: - tci_args_rr(&tb_ptr, &r0, &r1); + tci_args_rr(insn, &r0, &r1); info->fprintf_func(info->stream, "%-12s %s, %s", op_name, str_r(r0), str_r(r1)); break; @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) case INDEX_op_rotl_i64: case INDEX_op_rotr_i32: case INDEX_op_rotr_i64: - tci_args_rrr(&tb_ptr, &r0, &r1, &r2); + tci_args_rrr(insn, &r0, &r1, &r2); info->fprintf_func(info->stream, "%-12s %s, %s, %s", op_name, str_r(r0), str_r(r1), str_r(r2)); break; case INDEX_op_deposit_i32: case INDEX_op_deposit_i64: - tci_args_rrrbb(&tb_ptr, &r0, &r1, &r2, &pos, &len); + tci_args_rrrbb(insn, &r0, &r1, &r2, &pos, &len); info->fprintf_func(info->stream, "%-12s %s, %s, %s, %d, %d", op_name, str_r(r0), str_r(r1), str_r(r2), pos, len); break; #if TCG_TARGET_REG_BITS == 32 case INDEX_op_setcond2_i32: - tci_args_rrrrrc(&tb_ptr, &r0, &r1, &r2, &r3, &r4, &c); + tci_args_rrrrrc(insn, &r0, &r1, &r2, &r3, &r4, &c); info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s, %s, %s", op_name, str_r(r0), str_r(r1), str_r(r2), str_r(r3), str_r(r4), str_c(c)); break; case INDEX_op_mulu2_i32: - tci_args_rrrr(&tb_ptr, &r0, &r1, &r2, &r3); + tci_args_rrrr(insn, &r0, &r1, &r2, &r3); info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s", op_name, str_r(r0), str_r(r1), str_r(r2), str_r(r3)); @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) case INDEX_op_add2_i32: case INDEX_op_sub2_i32: - tci_args_rrrrrr(&tb_ptr, &r0, &r1, &r2, &r3, &r4, &r5); + tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s, %s, %s", op_name, str_r(r0), str_r(r1), str_r(r2), str_r(r3), str_r(r4), str_r(r5)); @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) len += DIV_ROUND_UP(TARGET_LONG_BITS, TCG_TARGET_REG_BITS); switch (len) { case 2: - tci_args_rrm(&tb_ptr, &r0, &r1, &oi); + tci_args_rrm(insn, &r0, &r1, &oi); info->fprintf_func(info->stream, "%-12s %s, %s, %x", op_name, str_r(r0), str_r(r1), oi); break; case 3: - tci_args_rrrm(&tb_ptr, &r0, &r1, &r2, &oi); + tci_args_rrrm(insn, &r0, &r1, &r2, &oi); info->fprintf_func(info->stream, "%-12s %s, %s, %s, %x", op_name, str_r(r0), str_r(r1), str_r(r2), oi); break; case 4: - tci_args_rrrrm(&tb_ptr, &r0, &r1, &r2, &r3, &oi); - info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s, %x", + tci_args_rrrrr(insn, &r0, &r1, &r2, &r3, &r4); + info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s, %s", op_name, str_r(r0), str_r(r1), - str_r(r2), str_r(r3), oi); + str_r(r2), str_r(r3), str_r(r4)); break; default: g_assert_not_reached(); } break; + case 0: + /* tcg_out_nop_fill uses zeros */ + if (insn == 0) { + info->fprintf_func(info->stream, "align"); + break; + } + /* fall through */ + default: info->fprintf_func(info->stream, "illegal opcode %d", op); break; } - return length; + return sizeof(insn); } diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ * THE SOFTWARE. */ -/* TODO list: - * - See TODO comments in code. - */ - -/* Marker for missing code. */ -#define TODO() \ - do { \ - fprintf(stderr, "TODO %s:%u: %s()\n", \ - __FILE__, __LINE__, __func__); \ - tcg_abort(); \ - } while (0) - -/* Bitfield n...m (in 32 bit value). */ -#define BITS(n, m) (((0xffffffffU << (31 - n)) >> (31 - n + m)) << m) +#include "../tcg-pool.c.inc" static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) { @@ -XXX,XX +XXX,XX @@ static const char *const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { static bool patch_reloc(tcg_insn_unit *code_ptr, int type, intptr_t value, intptr_t addend) { - /* tcg_out_reloc always uses the same type, addend. */ - tcg_debug_assert(type == sizeof(tcg_target_long)); + intptr_t diff = value - (intptr_t)(code_ptr + 1); + tcg_debug_assert(addend == 0); - tcg_debug_assert(value != 0); - if (TCG_TARGET_REG_BITS == 32) { - tcg_patch32(code_ptr, value); - } else { - tcg_patch64(code_ptr, value); - } - return true; -} - -/* Write value (native size). */ -static void tcg_out_i(TCGContext *s, tcg_target_ulong v) -{ - if (TCG_TARGET_REG_BITS == 32) { - tcg_out32(s, v); - } else { - tcg_out64(s, v); - } -} - -/* Write opcode. */ -static void tcg_out_op_t(TCGContext *s, TCGOpcode op) -{ - tcg_out8(s, op); - tcg_out8(s, 0); -} - -/* Write register. */ -static void tcg_out_r(TCGContext *s, TCGArg t0) -{ - tcg_debug_assert(t0 < TCG_TARGET_NB_REGS); - tcg_out8(s, t0); -} - -/* Write label. */ -static void tci_out_label(TCGContext *s, TCGLabel *label) -{ - if (label->has_value) { - tcg_out_i(s, label->u.value); - tcg_debug_assert(label->u.value); - } else { - tcg_out_reloc(s, s->code_ptr, sizeof(tcg_target_ulong), label, 0); - s->code_ptr += sizeof(tcg_target_ulong); + tcg_debug_assert(type == 20); + + if (diff == sextract32(diff, 0, type)) { + tcg_patch32(code_ptr, deposit32(*code_ptr, 32 - type, type, diff)); + return true; } + return false; } static void stack_bounds_check(TCGReg base, target_long offset) @@ -XXX,XX +XXX,XX @@ static void stack_bounds_check(TCGReg base, target_long offset) static void tcg_out_op_l(TCGContext *s, TCGOpcode op, TCGLabel *l0) { - uint8_t *old_code_ptr = s->code_ptr; + tcg_insn_unit insn = 0; - tcg_out_op_t(s, op); - tci_out_label(s, l0); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; + tcg_out_reloc(s, s->code_ptr, 20, l0, 0); + insn = deposit32(insn, 0, 8, op); + tcg_out32(s, insn); } static void tcg_out_op_p(TCGContext *s, TCGOpcode op, void *p0) { - uint8_t *old_code_ptr = s->code_ptr; + tcg_insn_unit insn = 0; + intptr_t diff; - tcg_out_op_t(s, op); - tcg_out_i(s, (uintptr_t)p0); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; + /* Special case for exit_tb: map null -> 0. */ + if (p0 == NULL) { + diff = 0; + } else { + diff = p0 - (void *)(s->code_ptr + 1); + tcg_debug_assert(diff != 0); + if (diff != sextract32(diff, 0, 20)) { + tcg_raise_tb_overflow(s); + } + } + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 12, 20, diff); + tcg_out32(s, insn); } static void tcg_out_op_v(TCGContext *s, TCGOpcode op) { - uint8_t *old_code_ptr = s->code_ptr; - - tcg_out_op_t(s, op); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; + tcg_out32(s, (uint8_t)op); } static void tcg_out_op_ri(TCGContext *s, TCGOpcode op, TCGReg r0, int32_t i1) { - uint8_t *old_code_ptr = s->code_ptr; + tcg_insn_unit insn = 0; - tcg_out_op_t(s, op); - tcg_out_r(s, r0); - tcg_out32(s, i1); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; + tcg_debug_assert(i1 == sextract32(i1, 0, 20)); + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 20, i1); + tcg_out32(s, insn); } -#if TCG_TARGET_REG_BITS == 64 -static void tcg_out_op_rI(TCGContext *s, TCGOpcode op, - TCGReg r0, uint64_t i1) -{ - uint8_t *old_code_ptr = s->code_ptr; - - tcg_out_op_t(s, op); - tcg_out_r(s, r0); - tcg_out64(s, i1); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; -} -#endif - static void tcg_out_op_rl(TCGContext *s, TCGOpcode op, TCGReg r0, TCGLabel *l1) { - uint8_t *old_code_ptr = s->code_ptr; + tcg_insn_unit insn = 0; - tcg_out_op_t(s, op); - tcg_out_r(s, r0); - tci_out_label(s, l1); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; + tcg_out_reloc(s, s->code_ptr, 20, l1, 0); + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + tcg_out32(s, insn); } static void tcg_out_op_rr(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1) { - uint8_t *old_code_ptr = s->code_ptr; + tcg_insn_unit insn = 0; - tcg_out_op_t(s, op); - tcg_out_r(s, r0); - tcg_out_r(s, r1); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + tcg_out32(s, insn); } static void tcg_out_op_rrm(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1, TCGArg m2) { - uint8_t *old_code_ptr = s->code_ptr; + tcg_insn_unit insn = 0; - tcg_out_op_t(s, op); - tcg_out_r(s, r0); - tcg_out_r(s, r1); - tcg_out32(s, m2); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; + tcg_debug_assert(m2 == extract32(m2, 0, 12)); + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 20, 12, m2); + tcg_out32(s, insn); } static void tcg_out_op_rrr(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1, TCGReg r2) { - uint8_t *old_code_ptr = s->code_ptr; + tcg_insn_unit insn = 0; - tcg_out_op_t(s, op); - tcg_out_r(s, r0); - tcg_out_r(s, r1); - tcg_out_r(s, r2); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 4, r2); + tcg_out32(s, insn); } static void tcg_out_op_rrs(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1, intptr_t i2) { - uint8_t *old_code_ptr = s->code_ptr; + tcg_insn_unit insn = 0; - tcg_out_op_t(s, op); - tcg_out_r(s, r0); - tcg_out_r(s, r1); - tcg_debug_assert(i2 == (int32_t)i2); - tcg_out32(s, i2); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; + tcg_debug_assert(i2 == sextract32(i2, 0, 16)); + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 16, i2); + tcg_out32(s, insn); } static void tcg_out_op_rrrc(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1, TCGReg r2, TCGCond c3) { - uint8_t *old_code_ptr = s->code_ptr; + tcg_insn_unit insn = 0; - tcg_out_op_t(s, op); - tcg_out_r(s, r0); - tcg_out_r(s, r1); - tcg_out_r(s, r2); - tcg_out8(s, c3); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 4, r2); + insn = deposit32(insn, 20, 4, c3); + tcg_out32(s, insn); } static void tcg_out_op_rrrm(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1, TCGReg r2, TCGArg m3) { - uint8_t *old_code_ptr = s->code_ptr; + tcg_insn_unit insn = 0; - tcg_out_op_t(s, op); - tcg_out_r(s, r0); - tcg_out_r(s, r1); - tcg_out_r(s, r2); - tcg_out32(s, m3); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; + tcg_debug_assert(m3 == extract32(m3, 0, 12)); + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 4, r2); + insn = deposit32(insn, 20, 12, m3); + tcg_out32(s, insn); } static void tcg_out_op_rrrbb(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1, TCGReg r2, uint8_t b3, uint8_t b4) { - uint8_t *old_code_ptr = s->code_ptr; + tcg_insn_unit insn = 0; - tcg_out_op_t(s, op); - tcg_out_r(s, r0); - tcg_out_r(s, r1); - tcg_out_r(s, r2); - tcg_out8(s, b3); - tcg_out8(s, b4); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; + tcg_debug_assert(b3 == extract32(b3, 0, 6)); + tcg_debug_assert(b4 == extract32(b4, 0, 6)); + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 4, r2); + insn = deposit32(insn, 20, 6, b3); + insn = deposit32(insn, 26, 6, b4); + tcg_out32(s, insn); } -static void tcg_out_op_rrrrm(TCGContext *s, TCGOpcode op, TCGReg r0, - TCGReg r1, TCGReg r2, TCGReg r3, TCGArg m4) +static void tcg_out_op_rrrrr(TCGContext *s, TCGOpcode op, TCGReg r0, + TCGReg r1, TCGReg r2, TCGReg r3, TCGReg r4) { - uint8_t *old_code_ptr = s->code_ptr; + tcg_insn_unit insn = 0; - tcg_out_op_t(s, op); - tcg_out_r(s, r0); - tcg_out_r(s, r1); - tcg_out_r(s, r2); - tcg_out_r(s, r3); - tcg_out32(s, m4); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 4, r2); + insn = deposit32(insn, 20, 4, r3); + insn = deposit32(insn, 24, 4, r4); + tcg_out32(s, insn); } #if TCG_TARGET_REG_BITS == 32 static void tcg_out_op_rrrr(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1, TCGReg r2, TCGReg r3) { - uint8_t *old_code_ptr = s->code_ptr; + tcg_insn_unit insn = 0; - tcg_out_op_t(s, op); - tcg_out_r(s, r0); - tcg_out_r(s, r1); - tcg_out_r(s, r2); - tcg_out_r(s, r3); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 4, r2); + insn = deposit32(insn, 20, 4, r3); + tcg_out32(s, insn); } static void tcg_out_op_rrrrrc(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1, TCGReg r2, TCGReg r3, TCGReg r4, TCGCond c5) { - uint8_t *old_code_ptr = s->code_ptr; + tcg_insn_unit insn = 0; - tcg_out_op_t(s, op); - tcg_out_r(s, r0); - tcg_out_r(s, r1); - tcg_out_r(s, r2); - tcg_out_r(s, r3); - tcg_out_r(s, r4); - tcg_out8(s, c5); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 4, r2); + insn = deposit32(insn, 20, 4, r3); + insn = deposit32(insn, 24, 4, r4); + insn = deposit32(insn, 28, 4, c5); + tcg_out32(s, insn); } static void tcg_out_op_rrrrrr(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1, TCGReg r2, TCGReg r3, TCGReg r4, TCGReg r5) { - uint8_t *old_code_ptr = s->code_ptr; + tcg_insn_unit insn = 0; - tcg_out_op_t(s, op); - tcg_out_r(s, r0); - tcg_out_r(s, r1); - tcg_out_r(s, r2); - tcg_out_r(s, r3); - tcg_out_r(s, r4); - tcg_out_r(s, r5); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 4, r2); + insn = deposit32(insn, 20, 4, r3); + insn = deposit32(insn, 24, 4, r4); + insn = deposit32(insn, 28, 4, r5); + tcg_out32(s, insn); } #endif +static void tcg_out_ldst(TCGContext *s, TCGOpcode op, TCGReg val, + TCGReg base, intptr_t offset) +{ + stack_bounds_check(base, offset); + if (offset != sextract32(offset, 0, 16)) { + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, offset); + tcg_out_op_rrr(s, (TCG_TARGET_REG_BITS == 32 + ? INDEX_op_add_i32 : INDEX_op_add_i64), + TCG_REG_TMP, TCG_REG_TMP, base); + base = TCG_REG_TMP; + offset = 0; + } + tcg_out_op_rrs(s, op, val, base, offset); +} + static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg val, TCGReg base, intptr_t offset) { - stack_bounds_check(base, offset); switch (type) { case TCG_TYPE_I32: - tcg_out_op_rrs(s, INDEX_op_ld_i32, val, base, offset); + tcg_out_ldst(s, INDEX_op_ld_i32, val, base, offset); break; #if TCG_TARGET_REG_BITS == 64 case TCG_TYPE_I64: - tcg_out_op_rrs(s, INDEX_op_ld_i64, val, base, offset); + tcg_out_ldst(s, INDEX_op_ld_i64, val, base, offset); break; #endif default: @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, { switch (type) { case TCG_TYPE_I32: - tcg_out_op_ri(s, INDEX_op_tci_movi_i32, ret, arg); - break; #if TCG_TARGET_REG_BITS == 64 + arg = (int32_t)arg; + /* fall through */ case TCG_TYPE_I64: - tcg_out_op_rI(s, INDEX_op_tci_movi_i64, ret, arg); - break; #endif + break; default: g_assert_not_reached(); } + + if (arg == sextract32(arg, 0, 20)) { + tcg_out_op_ri(s, INDEX_op_tci_movi, ret, arg); + } else { + tcg_insn_unit insn = 0; + + new_pool_label(s, arg, 20, s->code_ptr, 0); + insn = deposit32(insn, 0, 8, INDEX_op_tci_movl); + insn = deposit32(insn, 8, 4, ret); + tcg_out32(s, insn); + } } static void tcg_out_call(TCGContext *s, const tcg_insn_unit *func, ffi_cif *cif) { - uint8_t *old_code_ptr = s->code_ptr; + tcg_insn_unit insn = 0; uint8_t which; if (cif->rtype == &ffi_type_void) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *func, tcg_debug_assert(cif->rtype->size == 8); which = 2; } - tcg_out_op_t(s, INDEX_op_call); - tcg_out8(s, which); - tcg_out_i(s, (uintptr_t)func); - tcg_out_i(s, (uintptr_t)cif); - - old_code_ptr[1] = s->code_ptr - old_code_ptr; + new_pool_l2(s, 20, s->code_ptr, 0, (uintptr_t)func, (uintptr_t)cif); + insn = deposit32(insn, 0, 8, INDEX_op_call); + insn = deposit32(insn, 8, 4, which); + tcg_out32(s, insn); } #if TCG_TARGET_REG_BITS == 64 @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_st_i32: CASE_64(st32) CASE_64(st) - stack_bounds_check(args[1], args[2]); - tcg_out_op_rrs(s, opc, args[0], args[1], args[2]); + tcg_out_ldst(s, opc, args[0], args[1], args[2]); break; CASE_32_64(add) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, } else if (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS) { tcg_out_op_rrrm(s, opc, args[0], args[1], args[2], args[3]); } else { - tcg_out_op_rrrrm(s, opc, args[0], args[1], - args[2], args[3], args[4]); + tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_TMP, args[4]); + tcg_out_op_rrrrr(s, opc, args[0], args[1], + args[2], args[3], TCG_REG_TMP); } break; @@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) return ct & TCG_CT_CONST; } +static void tcg_out_nop_fill(tcg_insn_unit *p, int count) +{ + memset(p, 0, sizeof(*p) * count); +} + static void tcg_target_init(TCGContext *s) { #if defined(CONFIG_DEBUG_TCG_INTERPRETER) diff --git a/tcg/tci/README b/tcg/tci/README index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/README +++ b/tcg/tci/README @@ -XXX,XX +XXX,XX @@ This is what TCI (Tiny Code Interpreter) does. Like each TCG host frontend, TCI implements the code generator in tcg-target.c.inc, tcg-target.h. Both files are in directory tcg/tci. -The additional file tcg/tci.c adds the interpreter. +The additional file tcg/tci.c adds the interpreter and disassembler. -The bytecode consists of opcodes (same numeric values as those used by -TCG), command length and arguments of variable size and number. +The bytecode consists of opcodes (with only a few exceptions, with +the same same numeric values and semantics as used by TCG), and up +to six arguments packed into a 32-bit integer. See comments in tci.c +for details on the encoding. 3) Usage @@ -XXX,XX +XXX,XX @@ suggest using this option. Setting it automatically would need additional code in configure which must be fixed when new native TCG implementations are added. -System emulation should work on any 32 or 64 bit host. -User mode emulation might work. Maybe a new linker script (*.ld) -is needed. Byte order might be wrong (on big endian hosts) -and need fixes in configure. - For hosts with native TCG, the interpreter TCI can be enabled by configure --enable-tcg-interpreter @@ -XXX,XX +XXX,XX @@ u1 = linux-user-test works in the interpreter. These opcodes raise a runtime exception, so it is possible to see where code must be added. -* The pseudo code is not optimized and still ugly. For hosts with special - alignment requirements, it needs some fixes (maybe aligned bytecode - would also improve speed for hosts which support byte alignment). - -* A better disassembler for the pseudo code would be nice (a very primitive - disassembler is included in tcg-target.c.inc). - * It might be useful to have a runtime option which selects the native TCG or TCI, so QEMU would have to include two TCGs. Today, selecting TCI is a configure option, so you need two compilations of QEMU. -- 2.25.1 This operation is critical to staying within the interpretation loop longer, which avoids the overhead of setup and teardown for many TBs. The check in tcg_prologue_init is disabled because TCI does want to use NULL to indicate exit, as opposed to branching to a real epilogue. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci/tcg-target-con-set.h | 1 + tcg/tci/tcg-target.h | 2 +- tcg/tcg.c | 8 +++++++- tcg/tci.c | 19 +++++++++++++++++++ tcg/tci/tcg-target.c.inc | 16 ++++++++++++++++ 5 files changed, 44 insertions(+), 2 deletions(-) diff --git a/tcg/tci/tcg-target-con-set.h b/tcg/tci/tcg-target-con-set.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target-con-set.h +++ b/tcg/tci/tcg-target-con-set.h @@ -XXX,XX +XXX,XX @@ * Each operand should be a sequence of constraint letters as defined by * tcg-target-con-str.h; the constraint combination is inclusive or. */ +C_O0_I1(r) C_O0_I2(r, r) C_O0_I3(r, r, r) C_O0_I4(r, r, r, r) diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ #define TCG_TARGET_HAS_muls2_i32 0 #define TCG_TARGET_HAS_muluh_i32 0 #define TCG_TARGET_HAS_mulsh_i32 0 -#define TCG_TARGET_HAS_goto_ptr 0 +#define TCG_TARGET_HAS_goto_ptr 1 #define TCG_TARGET_HAS_direct_jump 0 #define TCG_TARGET_HAS_qemu_st8_i32 0 diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ void tcg_prologue_init(TCGContext *s) } #endif - /* Assert that goto_ptr is implemented completely. */ +#ifndef CONFIG_TCG_INTERPRETER + /* + * Assert that goto_ptr is implemented completely, setting an epilogue. + * For tci, we use NULL as the signal to return from the interpreter, + * so skip this check. + */ if (TCG_TARGET_HAS_goto_ptr) { tcg_debug_assert(tcg_code_gen_epilogue != NULL); } +#endif } void tcg_func_start(TCGContext *s) diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ static void tci_args_l(uint32_t insn, const void *tb_ptr, void **l0) *l0 = diff ? (void *)tb_ptr + diff : NULL; } +static void tci_args_r(uint32_t insn, TCGReg *r0) +{ + *r0 = extract32(insn, 8, 4); +} + static void tci_args_nl(uint32_t insn, const void *tb_ptr, uint8_t *n0, void **l1) { @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, tb_ptr = *(void **)ptr; break; + case INDEX_op_goto_ptr: + tci_args_r(insn, &r0); + ptr = (void *)regs[r0]; + if (!ptr) { + return 0; + } + tb_ptr = ptr; + break; + case INDEX_op_qemu_ld_i32: if (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS) { tci_args_rrm(insn, &r0, &r1, &oi); @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) info->fprintf_func(info->stream, "%-12s %p", op_name, ptr); break; + case INDEX_op_goto_ptr: + tci_args_r(insn, &r0); + info->fprintf_func(info->stream, "%-12s %s", op_name, str_r(r0)); + break; + case INDEX_op_call: tci_args_nl(insn, tb_ptr, &len, &ptr); info->fprintf_func(info->stream, "%-12s %d, %p", op_name, len, ptr); diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) { switch (op) { + case INDEX_op_goto_ptr: + return C_O0_I1(r); + case INDEX_op_ld8u_i32: case INDEX_op_ld8s_i32: case INDEX_op_ld16u_i32: @@ -XXX,XX +XXX,XX @@ static void tcg_out_op_p(TCGContext *s, TCGOpcode op, void *p0) tcg_out32(s, insn); } +static void tcg_out_op_r(TCGContext *s, TCGOpcode op, TCGReg r0) +{ + tcg_insn_unit insn = 0; + + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + tcg_out32(s, insn); +} + static void tcg_out_op_v(TCGContext *s, TCGOpcode op) { tcg_out32(s, (uint8_t)op); @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, set_jmp_reset_offset(s, args[0]); break; + case INDEX_op_goto_ptr: + tcg_out_op_r(s, opc, args[0]); + break; + case INDEX_op_br: tcg_out_op_l(s, opc, arg_label(args[0])); break; -- 2.25.1 When this opcode is not available in the backend, tcg middle-end will expand this as a series of 5 opcodes. So implementing this saves bytecode space. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci/tcg-target.h | 4 ++-- tcg/tci.c | 16 +++++++++++++++- tcg/tci/tcg-target.c.inc | 10 +++++++--- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ #define TCG_TARGET_HAS_not_i32 1 #define TCG_TARGET_HAS_orc_i32 0 #define TCG_TARGET_HAS_rot_i32 1 -#define TCG_TARGET_HAS_movcond_i32 0 +#define TCG_TARGET_HAS_movcond_i32 1 #define TCG_TARGET_HAS_muls2_i32 0 #define TCG_TARGET_HAS_muluh_i32 0 #define TCG_TARGET_HAS_mulsh_i32 0 @@ -XXX,XX +XXX,XX @@ #define TCG_TARGET_HAS_not_i64 1 #define TCG_TARGET_HAS_orc_i64 0 #define TCG_TARGET_HAS_rot_i64 1 -#define TCG_TARGET_HAS_movcond_i64 0 +#define TCG_TARGET_HAS_movcond_i64 1 #define TCG_TARGET_HAS_muls2_i64 0 #define TCG_TARGET_HAS_add2_i32 0 #define TCG_TARGET_HAS_sub2_i32 0 diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ static void tci_args_rrrr(uint32_t insn, *r2 = extract32(insn, 16, 4); *r3 = extract32(insn, 20, 4); } +#endif static void tci_args_rrrrrc(uint32_t insn, TCGReg *r0, TCGReg *r1, TCGReg *r2, TCGReg *r3, TCGReg *r4, TCGCond *c5) @@ -XXX,XX +XXX,XX @@ static void tci_args_rrrrrc(uint32_t insn, TCGReg *r0, TCGReg *r1, *c5 = extract32(insn, 28, 4); } +#if TCG_TARGET_REG_BITS == 32 static void tci_args_rrrrrr(uint32_t insn, TCGReg *r0, TCGReg *r1, TCGReg *r2, TCGReg *r3, TCGReg *r4, TCGReg *r5) { @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, tci_args_rrrc(insn, &r0, &r1, &r2, &condition); regs[r0] = tci_compare32(regs[r1], regs[r2], condition); break; + case INDEX_op_movcond_i32: + tci_args_rrrrrc(insn, &r0, &r1, &r2, &r3, &r4, &condition); + tmp32 = tci_compare32(regs[r1], regs[r2], condition); + regs[r0] = regs[tmp32 ? r3 : r4]; + break; #if TCG_TARGET_REG_BITS == 32 case INDEX_op_setcond2_i32: tci_args_rrrrrc(insn, &r0, &r1, &r2, &r3, &r4, &condition); @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, tci_args_rrrc(insn, &r0, &r1, &r2, &condition); regs[r0] = tci_compare64(regs[r1], regs[r2], condition); break; + case INDEX_op_movcond_i64: + tci_args_rrrrrc(insn, &r0, &r1, &r2, &r3, &r4, &condition); + tmp32 = tci_compare64(regs[r1], regs[r2], condition); + regs[r0] = regs[tmp32 ? r3 : r4]; + break; #endif CASE_32_64(mov) tci_args_rr(insn, &r0, &r1); @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) op_name, str_r(r0), str_r(r1), str_r(r2), pos, len); break; -#if TCG_TARGET_REG_BITS == 32 + case INDEX_op_movcond_i32: + case INDEX_op_movcond_i64: case INDEX_op_setcond2_i32: tci_args_rrrrrc(insn, &r0, &r1, &r2, &r3, &r4, &c); info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s, %s, %s", @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) str_r(r3), str_r(r4), str_c(c)); break; +#if TCG_TARGET_REG_BITS == 32 case INDEX_op_mulu2_i32: tci_args_rrrr(insn, &r0, &r1, &r2, &r3); info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s", diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) return C_O0_I4(r, r, r, r); case INDEX_op_mulu2_i32: return C_O2_I2(r, r, r, r); +#endif + + case INDEX_op_movcond_i32: + case INDEX_op_movcond_i64: case INDEX_op_setcond2_i32: return C_O1_I4(r, r, r, r, r); -#endif case INDEX_op_qemu_ld_i32: return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS @@ -XXX,XX +XXX,XX @@ static void tcg_out_op_rrrr(TCGContext *s, TCGOpcode op, insn = deposit32(insn, 20, 4, r3); tcg_out32(s, insn); } +#endif static void tcg_out_op_rrrrrc(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1, TCGReg r2, @@ -XXX,XX +XXX,XX @@ static void tcg_out_op_rrrrrc(TCGContext *s, TCGOpcode op, tcg_out32(s, insn); } +#if TCG_TARGET_REG_BITS == 32 static void tcg_out_op_rrrrrr(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1, TCGReg r2, TCGReg r3, TCGReg r4, TCGReg r5) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_op_rrrc(s, opc, args[0], args[1], args[2], args[3]); break; -#if TCG_TARGET_REG_BITS == 32 + CASE_32_64(movcond) case INDEX_op_setcond2_i32: tcg_out_op_rrrrrc(s, opc, args[0], args[1], args[2], args[3], args[4], args[5]); break; -#endif CASE_32_64(ld8u) CASE_32_64(ld8s) -- 2.25.1 These were already present in tcg-target.c.inc, but not in the interpreter. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci/tcg-target.h | 20 ++++++++++---------- tcg/tci.c | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ #define TCG_TARGET_HAS_ext16s_i32 1 #define TCG_TARGET_HAS_ext8u_i32 1 #define TCG_TARGET_HAS_ext16u_i32 1 -#define TCG_TARGET_HAS_andc_i32 0 +#define TCG_TARGET_HAS_andc_i32 1 #define TCG_TARGET_HAS_deposit_i32 1 #define TCG_TARGET_HAS_extract_i32 0 #define TCG_TARGET_HAS_sextract_i32 0 #define TCG_TARGET_HAS_extract2_i32 0 -#define TCG_TARGET_HAS_eqv_i32 0 -#define TCG_TARGET_HAS_nand_i32 0 -#define TCG_TARGET_HAS_nor_i32 0 +#define TCG_TARGET_HAS_eqv_i32 1 +#define TCG_TARGET_HAS_nand_i32 1 +#define TCG_TARGET_HAS_nor_i32 1 #define TCG_TARGET_HAS_clz_i32 0 #define TCG_TARGET_HAS_ctz_i32 0 #define TCG_TARGET_HAS_ctpop_i32 0 #define TCG_TARGET_HAS_neg_i32 1 #define TCG_TARGET_HAS_not_i32 1 -#define TCG_TARGET_HAS_orc_i32 0 +#define TCG_TARGET_HAS_orc_i32 1 #define TCG_TARGET_HAS_rot_i32 1 #define TCG_TARGET_HAS_movcond_i32 1 #define TCG_TARGET_HAS_muls2_i32 0 @@ -XXX,XX +XXX,XX @@ #define TCG_TARGET_HAS_ext8u_i64 1 #define TCG_TARGET_HAS_ext16u_i64 1 #define TCG_TARGET_HAS_ext32u_i64 1 -#define TCG_TARGET_HAS_andc_i64 0 -#define TCG_TARGET_HAS_eqv_i64 0 -#define TCG_TARGET_HAS_nand_i64 0 -#define TCG_TARGET_HAS_nor_i64 0 +#define TCG_TARGET_HAS_andc_i64 1 +#define TCG_TARGET_HAS_eqv_i64 1 +#define TCG_TARGET_HAS_nand_i64 1 +#define TCG_TARGET_HAS_nor_i64 1 #define TCG_TARGET_HAS_clz_i64 0 #define TCG_TARGET_HAS_ctz_i64 0 #define TCG_TARGET_HAS_ctpop_i64 0 #define TCG_TARGET_HAS_neg_i64 1 #define TCG_TARGET_HAS_not_i64 1 -#define TCG_TARGET_HAS_orc_i64 0 +#define TCG_TARGET_HAS_orc_i64 1 #define TCG_TARGET_HAS_rot_i64 1 #define TCG_TARGET_HAS_movcond_i64 1 #define TCG_TARGET_HAS_muls2_i64 0 diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] ^ regs[r2]; break; +#if TCG_TARGET_HAS_andc_i32 || TCG_TARGET_HAS_andc_i64 + CASE_32_64(andc) + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] & ~regs[r2]; + break; +#endif +#if TCG_TARGET_HAS_orc_i32 || TCG_TARGET_HAS_orc_i64 + CASE_32_64(orc) + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] | ~regs[r2]; + break; +#endif +#if TCG_TARGET_HAS_eqv_i32 || TCG_TARGET_HAS_eqv_i64 + CASE_32_64(eqv) + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = ~(regs[r1] ^ regs[r2]); + break; +#endif +#if TCG_TARGET_HAS_nand_i32 || TCG_TARGET_HAS_nand_i64 + CASE_32_64(nand) + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = ~(regs[r1] & regs[r2]); + break; +#endif +#if TCG_TARGET_HAS_nor_i32 || TCG_TARGET_HAS_nor_i64 + CASE_32_64(nor) + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = ~(regs[r1] | regs[r2]); + break; +#endif /* Arithmetic operations (32 bit). */ @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) case INDEX_op_or_i64: case INDEX_op_xor_i32: case INDEX_op_xor_i64: + case INDEX_op_andc_i32: + case INDEX_op_andc_i64: + case INDEX_op_orc_i32: + case INDEX_op_orc_i64: + case INDEX_op_eqv_i32: + case INDEX_op_eqv_i64: + case INDEX_op_nand_i32: + case INDEX_op_nand_i64: + case INDEX_op_nor_i32: + case INDEX_op_nor_i64: case INDEX_op_div_i32: case INDEX_op_div_i64: case INDEX_op_rem_i32: -- 2.25.1 Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci/tcg-target.h | 8 ++++---- tcg/tci.c | 42 ++++++++++++++++++++++++++++++++++++++++ tcg/tci/tcg-target.c.inc | 32 ++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ #define TCG_TARGET_HAS_ext16u_i32 1 #define TCG_TARGET_HAS_andc_i32 1 #define TCG_TARGET_HAS_deposit_i32 1 -#define TCG_TARGET_HAS_extract_i32 0 -#define TCG_TARGET_HAS_sextract_i32 0 +#define TCG_TARGET_HAS_extract_i32 1 +#define TCG_TARGET_HAS_sextract_i32 1 #define TCG_TARGET_HAS_extract2_i32 0 #define TCG_TARGET_HAS_eqv_i32 1 #define TCG_TARGET_HAS_nand_i32 1 @@ -XXX,XX +XXX,XX @@ #define TCG_TARGET_HAS_bswap32_i64 1 #define TCG_TARGET_HAS_bswap64_i64 1 #define TCG_TARGET_HAS_deposit_i64 1 -#define TCG_TARGET_HAS_extract_i64 0 -#define TCG_TARGET_HAS_sextract_i64 0 +#define TCG_TARGET_HAS_extract_i64 1 +#define TCG_TARGET_HAS_sextract_i64 1 #define TCG_TARGET_HAS_extract2_i64 0 #define TCG_TARGET_HAS_div_i64 1 #define TCG_TARGET_HAS_rem_i64 1 diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ static void tci_args_rrs(uint32_t insn, TCGReg *r0, TCGReg *r1, int32_t *i2) *i2 = sextract32(insn, 16, 16); } +static void tci_args_rrbb(uint32_t insn, TCGReg *r0, TCGReg *r1, + uint8_t *i2, uint8_t *i3) +{ + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *i2 = extract32(insn, 16, 6); + *i3 = extract32(insn, 22, 6); +} + static void tci_args_rrrc(uint32_t insn, TCGReg *r0, TCGReg *r1, TCGReg *r2, TCGCond *c3) { @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, tci_args_rrrbb(insn, &r0, &r1, &r2, &pos, &len); regs[r0] = deposit32(regs[r1], pos, len, regs[r2]); break; +#endif +#if TCG_TARGET_HAS_extract_i32 + case INDEX_op_extract_i32: + tci_args_rrbb(insn, &r0, &r1, &pos, &len); + regs[r0] = extract32(regs[r1], pos, len); + break; +#endif +#if TCG_TARGET_HAS_sextract_i32 + case INDEX_op_sextract_i32: + tci_args_rrbb(insn, &r0, &r1, &pos, &len); + regs[r0] = sextract32(regs[r1], pos, len); + break; #endif case INDEX_op_brcond_i32: tci_args_rl(insn, tb_ptr, &r0, &ptr); @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, tci_args_rrrbb(insn, &r0, &r1, &r2, &pos, &len); regs[r0] = deposit64(regs[r1], pos, len, regs[r2]); break; +#endif +#if TCG_TARGET_HAS_extract_i64 + case INDEX_op_extract_i64: + tci_args_rrbb(insn, &r0, &r1, &pos, &len); + regs[r0] = extract64(regs[r1], pos, len); + break; +#endif +#if TCG_TARGET_HAS_sextract_i64 + case INDEX_op_sextract_i64: + tci_args_rrbb(insn, &r0, &r1, &pos, &len); + regs[r0] = sextract64(regs[r1], pos, len); + break; #endif case INDEX_op_brcond_i64: tci_args_rl(insn, tb_ptr, &r0, &ptr); @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) op_name, str_r(r0), str_r(r1), str_r(r2), pos, len); break; + case INDEX_op_extract_i32: + case INDEX_op_extract_i64: + case INDEX_op_sextract_i32: + case INDEX_op_sextract_i64: + tci_args_rrbb(insn, &r0, &r1, &pos, &len); + info->fprintf_func(info->stream, "%-12s %s,%s,%d,%d", + op_name, str_r(r0), str_r(r1), pos, len); + break; + case INDEX_op_movcond_i32: case INDEX_op_movcond_i64: case INDEX_op_setcond2_i32: diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_bswap32_i32: case INDEX_op_bswap32_i64: case INDEX_op_bswap64_i64: + case INDEX_op_extract_i32: + case INDEX_op_extract_i64: + case INDEX_op_sextract_i32: + case INDEX_op_sextract_i64: return C_O1_I1(r, r); case INDEX_op_st8_i32: @@ -XXX,XX +XXX,XX @@ static void tcg_out_op_rrs(TCGContext *s, TCGOpcode op, tcg_out32(s, insn); } +static void tcg_out_op_rrbb(TCGContext *s, TCGOpcode op, TCGReg r0, + TCGReg r1, uint8_t b2, uint8_t b3) +{ + tcg_insn_unit insn = 0; + + tcg_debug_assert(b2 == extract32(b2, 0, 6)); + tcg_debug_assert(b3 == extract32(b3, 0, 6)); + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 6, b2); + insn = deposit32(insn, 22, 6, b3); + tcg_out32(s, insn); +} + static void tcg_out_op_rrrc(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1, TCGReg r2, TCGCond c3) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, } break; + CASE_32_64(extract) /* Optional (TCG_TARGET_HAS_extract_*). */ + CASE_32_64(sextract) /* Optional (TCG_TARGET_HAS_sextract_*). */ + { + TCGArg pos = args[2], len = args[3]; + TCGArg max = tcg_op_defs[opc].flags & TCG_OPF_64BIT ? 64 : 32; + + tcg_debug_assert(pos < max); + tcg_debug_assert(pos + len <= max); + + tcg_out_op_rrbb(s, opc, args[0], args[1], pos, len); + } + break; + CASE_32_64(brcond) tcg_out_op_rrrc(s, (opc == INDEX_op_brcond_i32 ? INDEX_op_setcond_i32 : INDEX_op_setcond_i64), -- 2.25.1 Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci/tcg-target.h | 12 +++++------ tcg/tci.c | 44 ++++++++++++++++++++++++++++++++++++++++ tcg/tci/tcg-target.c.inc | 9 ++++++++ 3 files changed, 59 insertions(+), 6 deletions(-) diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ #define TCG_TARGET_HAS_eqv_i32 1 #define TCG_TARGET_HAS_nand_i32 1 #define TCG_TARGET_HAS_nor_i32 1 -#define TCG_TARGET_HAS_clz_i32 0 -#define TCG_TARGET_HAS_ctz_i32 0 -#define TCG_TARGET_HAS_ctpop_i32 0 +#define TCG_TARGET_HAS_clz_i32 1 +#define TCG_TARGET_HAS_ctz_i32 1 +#define TCG_TARGET_HAS_ctpop_i32 1 #define TCG_TARGET_HAS_neg_i32 1 #define TCG_TARGET_HAS_not_i32 1 #define TCG_TARGET_HAS_orc_i32 1 @@ -XXX,XX +XXX,XX @@ #define TCG_TARGET_HAS_eqv_i64 1 #define TCG_TARGET_HAS_nand_i64 1 #define TCG_TARGET_HAS_nor_i64 1 -#define TCG_TARGET_HAS_clz_i64 0 -#define TCG_TARGET_HAS_ctz_i64 0 -#define TCG_TARGET_HAS_ctpop_i64 0 +#define TCG_TARGET_HAS_clz_i64 1 +#define TCG_TARGET_HAS_ctz_i64 1 +#define TCG_TARGET_HAS_ctpop_i64 1 #define TCG_TARGET_HAS_neg_i64 1 #define TCG_TARGET_HAS_not_i64 1 #define TCG_TARGET_HAS_orc_i64 1 diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (uint32_t)regs[r1] % (uint32_t)regs[r2]; break; +#if TCG_TARGET_HAS_clz_i32 + case INDEX_op_clz_i32: + tci_args_rrr(insn, &r0, &r1, &r2); + tmp32 = regs[r1]; + regs[r0] = tmp32 ? clz32(tmp32) : regs[r2]; + break; +#endif +#if TCG_TARGET_HAS_ctz_i32 + case INDEX_op_ctz_i32: + tci_args_rrr(insn, &r0, &r1, &r2); + tmp32 = regs[r1]; + regs[r0] = tmp32 ? ctz32(tmp32) : regs[r2]; + break; +#endif +#if TCG_TARGET_HAS_ctpop_i32 + case INDEX_op_ctpop_i32: + tci_args_rr(insn, &r0, &r1); + regs[r0] = ctpop32(regs[r1]); + break; +#endif /* Shift/rotate operations (32 bit). */ @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (uint64_t)regs[r1] % (uint64_t)regs[r2]; break; +#if TCG_TARGET_HAS_clz_i64 + case INDEX_op_clz_i64: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] ? clz64(regs[r1]) : regs[r2]; + break; +#endif +#if TCG_TARGET_HAS_ctz_i64 + case INDEX_op_ctz_i64: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] ? ctz64(regs[r1]) : regs[r2]; + break; +#endif +#if TCG_TARGET_HAS_ctpop_i64 + case INDEX_op_ctpop_i64: + tci_args_rr(insn, &r0, &r1); + regs[r0] = ctpop64(regs[r1]); + break; +#endif /* Shift/rotate operations (64 bit). */ @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) case INDEX_op_not_i64: case INDEX_op_neg_i32: case INDEX_op_neg_i64: + case INDEX_op_ctpop_i32: + case INDEX_op_ctpop_i64: tci_args_rr(insn, &r0, &r1); info->fprintf_func(info->stream, "%-12s %s, %s", op_name, str_r(r0), str_r(r1)); @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) case INDEX_op_rotl_i64: case INDEX_op_rotr_i32: case INDEX_op_rotr_i64: + case INDEX_op_clz_i32: + case INDEX_op_clz_i64: + case INDEX_op_ctz_i32: + case INDEX_op_ctz_i64: tci_args_rrr(insn, &r0, &r1, &r2); info->fprintf_func(info->stream, "%-12s %s, %s, %s", op_name, str_r(r0), str_r(r1), str_r(r2)); diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_extract_i64: case INDEX_op_sextract_i32: case INDEX_op_sextract_i64: + case INDEX_op_ctpop_i32: + case INDEX_op_ctpop_i64: return C_O1_I1(r, r); case INDEX_op_st8_i32: @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_setcond_i64: case INDEX_op_deposit_i32: case INDEX_op_deposit_i64: + case INDEX_op_clz_i32: + case INDEX_op_clz_i64: + case INDEX_op_ctz_i32: + case INDEX_op_ctz_i64: return C_O1_I2(r, r, r); case INDEX_op_brcond_i32: @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, CASE_32_64(divu) /* Optional (TCG_TARGET_HAS_div_*). */ CASE_32_64(rem) /* Optional (TCG_TARGET_HAS_div_*). */ CASE_32_64(remu) /* Optional (TCG_TARGET_HAS_div_*). */ + CASE_32_64(clz) /* Optional (TCG_TARGET_HAS_clz_*). */ + CASE_32_64(ctz) /* Optional (TCG_TARGET_HAS_ctz_*). */ tcg_out_op_rrr(s, opc, args[0], args[1], args[2]); break; @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, CASE_32_64(bswap16) /* Optional (TCG_TARGET_HAS_bswap16_*). */ CASE_32_64(bswap32) /* Optional (TCG_TARGET_HAS_bswap32_*). */ CASE_64(bswap64) /* Optional (TCG_TARGET_HAS_bswap64_i64). */ + CASE_32_64(ctpop) /* Optional (TCG_TARGET_HAS_ctpop_*). */ tcg_out_op_rr(s, opc, args[0], args[1]); break; -- 2.25.1 We already had mulu2_i32 for a 32-bit host; expand this to 64-bit hosts as well. The muls2_i32 and the 64-bit opcodes are new. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci/tcg-target.h | 8 ++++---- tcg/tci.c | 35 +++++++++++++++++++++++++++++------ tcg/tci/tcg-target.c.inc | 16 ++++++++++------ 3 files changed, 43 insertions(+), 16 deletions(-) diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ #define TCG_TARGET_HAS_orc_i32 1 #define TCG_TARGET_HAS_rot_i32 1 #define TCG_TARGET_HAS_movcond_i32 1 -#define TCG_TARGET_HAS_muls2_i32 0 +#define TCG_TARGET_HAS_muls2_i32 1 #define TCG_TARGET_HAS_muluh_i32 0 #define TCG_TARGET_HAS_mulsh_i32 0 #define TCG_TARGET_HAS_goto_ptr 1 @@ -XXX,XX +XXX,XX @@ #define TCG_TARGET_HAS_orc_i64 1 #define TCG_TARGET_HAS_rot_i64 1 #define TCG_TARGET_HAS_movcond_i64 1 -#define TCG_TARGET_HAS_muls2_i64 0 +#define TCG_TARGET_HAS_muls2_i64 1 #define TCG_TARGET_HAS_add2_i32 0 #define TCG_TARGET_HAS_sub2_i32 0 -#define TCG_TARGET_HAS_mulu2_i32 0 +#define TCG_TARGET_HAS_mulu2_i32 1 #define TCG_TARGET_HAS_add2_i64 0 #define TCG_TARGET_HAS_sub2_i64 0 -#define TCG_TARGET_HAS_mulu2_i64 0 +#define TCG_TARGET_HAS_mulu2_i64 1 #define TCG_TARGET_HAS_muluh_i64 0 #define TCG_TARGET_HAS_mulsh_i64 0 #else diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ __thread uintptr_t tci_tb_ptr; static void tci_write_reg64(tcg_target_ulong *regs, uint32_t high_index, uint32_t low_index, uint64_t value) { - regs[low_index] = value; + regs[low_index] = (uint32_t)value; regs[high_index] = value >> 32; } @@ -XXX,XX +XXX,XX @@ static void tci_args_rrrrr(uint32_t insn, TCGReg *r0, TCGReg *r1, *r4 = extract32(insn, 24, 4); } -#if TCG_TARGET_REG_BITS == 32 static void tci_args_rrrr(uint32_t insn, TCGReg *r0, TCGReg *r1, TCGReg *r2, TCGReg *r3) { @@ -XXX,XX +XXX,XX @@ static void tci_args_rrrr(uint32_t insn, *r2 = extract32(insn, 16, 4); *r3 = extract32(insn, 20, 4); } -#endif static void tci_args_rrrrrc(uint32_t insn, TCGReg *r0, TCGReg *r1, TCGReg *r2, TCGReg *r3, TCGReg *r4, TCGCond *c5) @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, T2 = tci_uint64(regs[r5], regs[r4]); tci_write_reg64(regs, r1, r0, T1 - T2); break; +#endif /* TCG_TARGET_REG_BITS == 32 */ +#if TCG_TARGET_HAS_mulu2_i32 case INDEX_op_mulu2_i32: tci_args_rrrr(insn, &r0, &r1, &r2, &r3); - tci_write_reg64(regs, r1, r0, (uint64_t)regs[r2] * regs[r3]); + tmp64 = (uint64_t)(uint32_t)regs[r2] * (uint32_t)regs[r3]; + tci_write_reg64(regs, r1, r0, tmp64); break; -#endif /* TCG_TARGET_REG_BITS == 32 */ +#endif +#if TCG_TARGET_HAS_muls2_i32 + case INDEX_op_muls2_i32: + tci_args_rrrr(insn, &r0, &r1, &r2, &r3); + tmp64 = (int64_t)(int32_t)regs[r2] * (int32_t)regs[r3]; + tci_write_reg64(regs, r1, r0, tmp64); + break; +#endif #if TCG_TARGET_HAS_ext8s_i32 || TCG_TARGET_HAS_ext8s_i64 CASE_32_64(ext8s) tci_args_rr(insn, &r0, &r1); @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, regs[r0] = ctpop64(regs[r1]); break; #endif +#if TCG_TARGET_HAS_mulu2_i64 + case INDEX_op_mulu2_i64: + tci_args_rrrr(insn, &r0, &r1, &r2, &r3); + mulu64(&regs[r0], &regs[r1], regs[r2], regs[r3]); + break; +#endif +#if TCG_TARGET_HAS_muls2_i64 + case INDEX_op_muls2_i64: + tci_args_rrrr(insn, &r0, &r1, &r2, &r3); + muls64(&regs[r0], &regs[r1], regs[r2], regs[r3]); + break; +#endif /* Shift/rotate operations (64 bit). */ @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) str_r(r3), str_r(r4), str_c(c)); break; -#if TCG_TARGET_REG_BITS == 32 case INDEX_op_mulu2_i32: + case INDEX_op_mulu2_i64: + case INDEX_op_muls2_i32: + case INDEX_op_muls2_i64: tci_args_rrrr(insn, &r0, &r1, &r2, &r3); info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s", op_name, str_r(r0), str_r(r1), str_r(r2), str_r(r3)); break; +#if TCG_TARGET_REG_BITS == 32 case INDEX_op_add2_i32: case INDEX_op_sub2_i32: tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) return C_O2_I4(r, r, r, r, r, r); case INDEX_op_brcond2_i32: return C_O0_I4(r, r, r, r); - case INDEX_op_mulu2_i32: - return C_O2_I2(r, r, r, r); #endif + case INDEX_op_mulu2_i32: + case INDEX_op_mulu2_i64: + case INDEX_op_muls2_i32: + case INDEX_op_muls2_i64: + return C_O2_I2(r, r, r, r); + case INDEX_op_movcond_i32: case INDEX_op_movcond_i64: case INDEX_op_setcond2_i32: @@ -XXX,XX +XXX,XX @@ static void tcg_out_op_rrrrr(TCGContext *s, TCGOpcode op, TCGReg r0, tcg_out32(s, insn); } -#if TCG_TARGET_REG_BITS == 32 static void tcg_out_op_rrrr(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1, TCGReg r2, TCGReg r3) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_op_rrrr(TCGContext *s, TCGOpcode op, insn = deposit32(insn, 20, 4, r3); tcg_out32(s, insn); } -#endif static void tcg_out_op_rrrrrc(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1, TCGReg r2, @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, args[0], args[1], args[2], args[3], args[4]); tcg_out_op_rl(s, INDEX_op_brcond_i32, TCG_REG_TMP, arg_label(args[5])); break; - case INDEX_op_mulu2_i32: +#endif + + CASE_32_64(mulu2) + CASE_32_64(muls2) tcg_out_op_rrrr(s, opc, args[0], args[1], args[2], args[3]); break; -#endif case INDEX_op_qemu_ld_i32: case INDEX_op_qemu_st_i32: -- 2.25.1 We already had the 32-bit versions for a 32-bit host; expand this to 64-bit hosts as well. The 64-bit opcodes are new. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci/tcg-target.h | 8 ++++---- tcg/tci.c | 40 ++++++++++++++++++++++++++-------------- tcg/tci/tcg-target.c.inc | 15 ++++++++------- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ #define TCG_TARGET_HAS_rot_i64 1 #define TCG_TARGET_HAS_movcond_i64 1 #define TCG_TARGET_HAS_muls2_i64 1 -#define TCG_TARGET_HAS_add2_i32 0 -#define TCG_TARGET_HAS_sub2_i32 0 +#define TCG_TARGET_HAS_add2_i32 1 +#define TCG_TARGET_HAS_sub2_i32 1 #define TCG_TARGET_HAS_mulu2_i32 1 -#define TCG_TARGET_HAS_add2_i64 0 -#define TCG_TARGET_HAS_sub2_i64 0 +#define TCG_TARGET_HAS_add2_i64 1 +#define TCG_TARGET_HAS_sub2_i64 1 #define TCG_TARGET_HAS_mulu2_i64 1 #define TCG_TARGET_HAS_muluh_i64 0 #define TCG_TARGET_HAS_mulsh_i64 0 diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ static void tci_args_rrrrrc(uint32_t insn, TCGReg *r0, TCGReg *r1, *c5 = extract32(insn, 28, 4); } -#if TCG_TARGET_REG_BITS == 32 static void tci_args_rrrrrr(uint32_t insn, TCGReg *r0, TCGReg *r1, TCGReg *r2, TCGReg *r3, TCGReg *r4, TCGReg *r5) { @@ -XXX,XX +XXX,XX @@ static void tci_args_rrrrrr(uint32_t insn, TCGReg *r0, TCGReg *r1, *r4 = extract32(insn, 24, 4); *r5 = extract32(insn, 28, 4); } -#endif static bool tci_compare32(uint32_t u0, uint32_t u1, TCGCond condition) { @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, for (;;) { uint32_t insn; TCGOpcode opc; - TCGReg r0, r1, r2, r3, r4; + TCGReg r0, r1, r2, r3, r4, r5; tcg_target_ulong t1; TCGCond condition; target_ulong taddr; uint8_t pos, len; uint32_t tmp32; uint64_t tmp64; -#if TCG_TARGET_REG_BITS == 32 - TCGReg r5; uint64_t T1, T2; -#endif TCGMemOpIdx oi; int32_t ofs; void *ptr; @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, tb_ptr = ptr; } break; -#if TCG_TARGET_REG_BITS == 32 +#if TCG_TARGET_REG_BITS == 32 || TCG_TARGET_HAS_add2_i32 case INDEX_op_add2_i32: tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); T1 = tci_uint64(regs[r3], regs[r2]); T2 = tci_uint64(regs[r5], regs[r4]); tci_write_reg64(regs, r1, r0, T1 + T2); break; +#endif +#if TCG_TARGET_REG_BITS == 32 || TCG_TARGET_HAS_sub2_i32 case INDEX_op_sub2_i32: tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); T1 = tci_uint64(regs[r3], regs[r2]); T2 = tci_uint64(regs[r5], regs[r4]); tci_write_reg64(regs, r1, r0, T1 - T2); break; -#endif /* TCG_TARGET_REG_BITS == 32 */ +#endif #if TCG_TARGET_HAS_mulu2_i32 case INDEX_op_mulu2_i32: tci_args_rrrr(insn, &r0, &r1, &r2, &r3); @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, muls64(&regs[r0], &regs[r1], regs[r2], regs[r3]); break; #endif +#if TCG_TARGET_HAS_add2_i64 + case INDEX_op_add2_i64: + tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); + T1 = regs[r2] + regs[r4]; + T2 = regs[r3] + regs[r5] + (T1 < regs[r2]); + regs[r0] = T1; + regs[r1] = T2; + break; +#endif +#if TCG_TARGET_HAS_add2_i64 + case INDEX_op_sub2_i64: + tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); + T1 = regs[r2] - regs[r4]; + T2 = regs[r3] - regs[r5] - (regs[r2] < regs[r4]); + regs[r0] = T1; + regs[r1] = T2; + break; +#endif /* Shift/rotate operations (64 bit). */ @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) const char *op_name; uint32_t insn; TCGOpcode op; - TCGReg r0, r1, r2, r3, r4; -#if TCG_TARGET_REG_BITS == 32 - TCGReg r5; -#endif + TCGReg r0, r1, r2, r3, r4, r5; tcg_target_ulong i1; int32_t s2; TCGCond c; @@ -XXX,XX +XXX,XX @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) str_r(r2), str_r(r3)); break; -#if TCG_TARGET_REG_BITS == 32 case INDEX_op_add2_i32: + case INDEX_op_add2_i64: case INDEX_op_sub2_i32: + case INDEX_op_sub2_i64: tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s, %s, %s", op_name, str_r(r0), str_r(r1), str_r(r2), str_r(r3), str_r(r4), str_r(r5)); break; -#endif case INDEX_op_qemu_ld_i64: case INDEX_op_qemu_st_i64: diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_brcond_i64: return C_O0_I2(r, r); -#if TCG_TARGET_REG_BITS == 32 - /* TODO: Support R, R, R, R, RI, RI? Will it be faster? */ case INDEX_op_add2_i32: + case INDEX_op_add2_i64: case INDEX_op_sub2_i32: + case INDEX_op_sub2_i64: return C_O2_I4(r, r, r, r, r, r); + +#if TCG_TARGET_REG_BITS == 32 case INDEX_op_brcond2_i32: return C_O0_I4(r, r, r, r); #endif @@ -XXX,XX +XXX,XX @@ static void tcg_out_op_rrrrrc(TCGContext *s, TCGOpcode op, tcg_out32(s, insn); } -#if TCG_TARGET_REG_BITS == 32 static void tcg_out_op_rrrrrr(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1, TCGReg r2, TCGReg r3, TCGReg r4, TCGReg r5) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op_rrrrrr(TCGContext *s, TCGOpcode op, insn = deposit32(insn, 28, 4, r5); tcg_out32(s, insn); } -#endif static void tcg_out_ldst(TCGContext *s, TCGOpcode op, TCGReg val, TCGReg base, intptr_t offset) @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_op_rr(s, opc, args[0], args[1]); break; -#if TCG_TARGET_REG_BITS == 32 - case INDEX_op_add2_i32: - case INDEX_op_sub2_i32: + CASE_32_64(add2) + CASE_32_64(sub2) tcg_out_op_rrrrrr(s, opc, args[0], args[1], args[2], args[3], args[4], args[5]); break; + +#if TCG_TARGET_REG_BITS == 32 case INDEX_op_brcond2_i32: tcg_out_op_rrrrrc(s, INDEX_op_setcond2_i32, TCG_REG_TMP, args[0], args[1], args[2], args[3], args[4]); -- 2.25.1 We can share this code between 32-bit and 64-bit loads and stores. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci.c | 183 +++++++++++++++++++++--------------------------------- 1 file changed, 71 insertions(+), 112 deletions(-) diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ static bool tci_compare64(uint64_t u0, uint64_t u1, TCGCond condition) #define qemu_st_beq(X) \ cpu_stq_be_mmuidx_ra(env, taddr, X, get_mmuidx(oi), (uintptr_t)tb_ptr) +static uint64_t tci_qemu_ld(CPUArchState *env, target_ulong taddr, + TCGMemOpIdx oi, const void *tb_ptr) +{ + MemOp mop = get_memop(oi) & (MO_BSWAP | MO_SSIZE); + + switch (mop) { + case MO_UB: + return qemu_ld_ub; + case MO_SB: + return (int8_t)qemu_ld_ub; + case MO_LEUW: + return qemu_ld_leuw; + case MO_LESW: + return (int16_t)qemu_ld_leuw; + case MO_LEUL: + return qemu_ld_leul; + case MO_LESL: + return (int32_t)qemu_ld_leul; + case MO_LEQ: + return qemu_ld_leq; + case MO_BEUW: + return qemu_ld_beuw; + case MO_BESW: + return (int16_t)qemu_ld_beuw; + case MO_BEUL: + return qemu_ld_beul; + case MO_BESL: + return (int32_t)qemu_ld_beul; + case MO_BEQ: + return qemu_ld_beq; + default: + g_assert_not_reached(); + } +} + +static void tci_qemu_st(CPUArchState *env, target_ulong taddr, uint64_t val, + TCGMemOpIdx oi, const void *tb_ptr) +{ + MemOp mop = get_memop(oi) & (MO_BSWAP | MO_SSIZE); + + switch (mop) { + case MO_UB: + qemu_st_b(val); + break; + case MO_LEUW: + qemu_st_lew(val); + break; + case MO_LEUL: + qemu_st_lel(val); + break; + case MO_LEQ: + qemu_st_leq(val); + break; + case MO_BEUW: + qemu_st_bew(val); + break; + case MO_BEUL: + qemu_st_bel(val); + break; + case MO_BEQ: + qemu_st_beq(val); + break; + default: + g_assert_not_reached(); + } +} + #if TCG_TARGET_REG_BITS == 64 # define CASE_32_64(x) \ case glue(glue(INDEX_op_, x), _i64): \ @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, tci_args_rrrm(insn, &r0, &r1, &r2, &oi); taddr = tci_uint64(regs[r2], regs[r1]); } - switch (get_memop(oi) & (MO_BSWAP | MO_SSIZE)) { - case MO_UB: - tmp32 = qemu_ld_ub; - break; - case MO_SB: - tmp32 = (int8_t)qemu_ld_ub; - break; - case MO_LEUW: - tmp32 = qemu_ld_leuw; - break; - case MO_LESW: - tmp32 = (int16_t)qemu_ld_leuw; - break; - case MO_LEUL: - tmp32 = qemu_ld_leul; - break; - case MO_BEUW: - tmp32 = qemu_ld_beuw; - break; - case MO_BESW: - tmp32 = (int16_t)qemu_ld_beuw; - break; - case MO_BEUL: - tmp32 = qemu_ld_beul; - break; - default: - g_assert_not_reached(); - } + tmp32 = tci_qemu_ld(env, taddr, oi, tb_ptr); regs[r0] = tmp32; break; @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, taddr = tci_uint64(regs[r3], regs[r2]); oi = regs[r4]; } - switch (get_memop(oi) & (MO_BSWAP | MO_SSIZE)) { - case MO_UB: - tmp64 = qemu_ld_ub; - break; - case MO_SB: - tmp64 = (int8_t)qemu_ld_ub; - break; - case MO_LEUW: - tmp64 = qemu_ld_leuw; - break; - case MO_LESW: - tmp64 = (int16_t)qemu_ld_leuw; - break; - case MO_LEUL: - tmp64 = qemu_ld_leul; - break; - case MO_LESL: - tmp64 = (int32_t)qemu_ld_leul; - break; - case MO_LEQ: - tmp64 = qemu_ld_leq; - break; - case MO_BEUW: - tmp64 = qemu_ld_beuw; - break; - case MO_BESW: - tmp64 = (int16_t)qemu_ld_beuw; - break; - case MO_BEUL: - tmp64 = qemu_ld_beul; - break; - case MO_BESL: - tmp64 = (int32_t)qemu_ld_beul; - break; - case MO_BEQ: - tmp64 = qemu_ld_beq; - break; - default: - g_assert_not_reached(); - } + tmp64 = tci_qemu_ld(env, taddr, oi, tb_ptr); if (TCG_TARGET_REG_BITS == 32) { tci_write_reg64(regs, r1, r0, tmp64); } else { @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, taddr = tci_uint64(regs[r2], regs[r1]); } tmp32 = regs[r0]; - switch (get_memop(oi) & (MO_BSWAP | MO_SIZE)) { - case MO_UB: - qemu_st_b(tmp32); - break; - case MO_LEUW: - qemu_st_lew(tmp32); - break; - case MO_LEUL: - qemu_st_lel(tmp32); - break; - case MO_BEUW: - qemu_st_bew(tmp32); - break; - case MO_BEUL: - qemu_st_bel(tmp32); - break; - default: - g_assert_not_reached(); - } + tci_qemu_st(env, taddr, tmp32, oi, tb_ptr); break; case INDEX_op_qemu_st_i64: @@ -XXX,XX +XXX,XX @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, } tmp64 = tci_uint64(regs[r1], regs[r0]); } - switch (get_memop(oi) & (MO_BSWAP | MO_SIZE)) { - case MO_UB: - qemu_st_b(tmp64); - break; - case MO_LEUW: - qemu_st_lew(tmp64); - break; - case MO_LEUL: - qemu_st_lel(tmp64); - break; - case MO_LEQ: - qemu_st_leq(tmp64); - break; - case MO_BEUW: - qemu_st_bew(tmp64); - break; - case MO_BEUL: - qemu_st_bel(tmp64); - break; - case MO_BEQ: - qemu_st_beq(tmp64); - break; - default: - g_assert_not_reached(); - } + tci_qemu_st(env, taddr, tmp64, oi, tb_ptr); break; case INDEX_op_mb: -- 2.25.1 This reverts commit dc09f047eddec8f4a1991c4f5f4a428d7aa3f2c0. For tcg, tracepoints are expanded inline in tcg opcodes. Using a helper which generates a second tracepoint is incorrect. For system mode, the extraction and re-packing of MemOp and mmu_idx lost the alignment information from MemOp. So we were no longer raising alignment exceptions for !TARGET_ALIGNED_ONLY guests. This can be seen in tests/tcg/xtensa/test_load_store.S. For user mode, we must update to the new signature of g2h() so that the revert compiles. We can leave set_helper_retaddr for later. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci.c | 73 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ static bool tci_compare64(uint64_t u0, uint64_t u1, TCGCond condition) return result; } -#define qemu_ld_ub \ - cpu_ldub_mmuidx_ra(env, taddr, get_mmuidx(oi), (uintptr_t)tb_ptr) -#define qemu_ld_leuw \ - cpu_lduw_le_mmuidx_ra(env, taddr, get_mmuidx(oi), (uintptr_t)tb_ptr) -#define qemu_ld_leul \ - cpu_ldl_le_mmuidx_ra(env, taddr, get_mmuidx(oi), (uintptr_t)tb_ptr) -#define qemu_ld_leq \ - cpu_ldq_le_mmuidx_ra(env, taddr, get_mmuidx(oi), (uintptr_t)tb_ptr) -#define qemu_ld_beuw \ - cpu_lduw_be_mmuidx_ra(env, taddr, get_mmuidx(oi), (uintptr_t)tb_ptr) -#define qemu_ld_beul \ - cpu_ldl_be_mmuidx_ra(env, taddr, get_mmuidx(oi), (uintptr_t)tb_ptr) -#define qemu_ld_beq \ - cpu_ldq_be_mmuidx_ra(env, taddr, get_mmuidx(oi), (uintptr_t)tb_ptr) -#define qemu_st_b(X) \ - cpu_stb_mmuidx_ra(env, taddr, X, get_mmuidx(oi), (uintptr_t)tb_ptr) -#define qemu_st_lew(X) \ - cpu_stw_le_mmuidx_ra(env, taddr, X, get_mmuidx(oi), (uintptr_t)tb_ptr) -#define qemu_st_lel(X) \ - cpu_stl_le_mmuidx_ra(env, taddr, X, get_mmuidx(oi), (uintptr_t)tb_ptr) -#define qemu_st_leq(X) \ - cpu_stq_le_mmuidx_ra(env, taddr, X, get_mmuidx(oi), (uintptr_t)tb_ptr) -#define qemu_st_bew(X) \ - cpu_stw_be_mmuidx_ra(env, taddr, X, get_mmuidx(oi), (uintptr_t)tb_ptr) -#define qemu_st_bel(X) \ - cpu_stl_be_mmuidx_ra(env, taddr, X, get_mmuidx(oi), (uintptr_t)tb_ptr) -#define qemu_st_beq(X) \ - cpu_stq_be_mmuidx_ra(env, taddr, X, get_mmuidx(oi), (uintptr_t)tb_ptr) +#ifdef CONFIG_SOFTMMU +# define qemu_ld_ub \ + helper_ret_ldub_mmu(env, taddr, oi, (uintptr_t)tb_ptr) +# define qemu_ld_leuw \ + helper_le_lduw_mmu(env, taddr, oi, (uintptr_t)tb_ptr) +# define qemu_ld_leul \ + helper_le_ldul_mmu(env, taddr, oi, (uintptr_t)tb_ptr) +# define qemu_ld_leq \ + helper_le_ldq_mmu(env, taddr, oi, (uintptr_t)tb_ptr) +# define qemu_ld_beuw \ + helper_be_lduw_mmu(env, taddr, oi, (uintptr_t)tb_ptr) +# define qemu_ld_beul \ + helper_be_ldul_mmu(env, taddr, oi, (uintptr_t)tb_ptr) +# define qemu_ld_beq \ + helper_be_ldq_mmu(env, taddr, oi, (uintptr_t)tb_ptr) +# define qemu_st_b(X) \ + helper_ret_stb_mmu(env, taddr, X, oi, (uintptr_t)tb_ptr) +# define qemu_st_lew(X) \ + helper_le_stw_mmu(env, taddr, X, oi, (uintptr_t)tb_ptr) +# define qemu_st_lel(X) \ + helper_le_stl_mmu(env, taddr, X, oi, (uintptr_t)tb_ptr) +# define qemu_st_leq(X) \ + helper_le_stq_mmu(env, taddr, X, oi, (uintptr_t)tb_ptr) +# define qemu_st_bew(X) \ + helper_be_stw_mmu(env, taddr, X, oi, (uintptr_t)tb_ptr) +# define qemu_st_bel(X) \ + helper_be_stl_mmu(env, taddr, X, oi, (uintptr_t)tb_ptr) +# define qemu_st_beq(X) \ + helper_be_stq_mmu(env, taddr, X, oi, (uintptr_t)tb_ptr) +#else +# define qemu_ld_ub ldub_p(g2h(env_cpu(env), taddr)) +# define qemu_ld_leuw lduw_le_p(g2h(env_cpu(env), taddr)) +# define qemu_ld_leul (uint32_t)ldl_le_p(g2h(env_cpu(env), taddr)) +# define qemu_ld_leq ldq_le_p(g2h(env_cpu(env), taddr)) +# define qemu_ld_beuw lduw_be_p(g2h(env_cpu(env), taddr)) +# define qemu_ld_beul (uint32_t)ldl_be_p(g2h(env_cpu(env), taddr)) +# define qemu_ld_beq ldq_be_p(g2h(env_cpu(env), taddr)) +# define qemu_st_b(X) stb_p(g2h(env_cpu(env), taddr), X) +# define qemu_st_lew(X) stw_le_p(g2h(env_cpu(env), taddr), X) +# define qemu_st_lel(X) stl_le_p(g2h(env_cpu(env), taddr), X) +# define qemu_st_leq(X) stq_le_p(g2h(env_cpu(env), taddr), X) +# define qemu_st_bew(X) stw_be_p(g2h(env_cpu(env), taddr), X) +# define qemu_st_bel(X) stl_be_p(g2h(env_cpu(env), taddr), X) +# define qemu_st_beq(X) stq_be_p(g2h(env_cpu(env), taddr), X) +#endif static uint64_t tci_qemu_ld(CPUArchState *env, target_ulong taddr, TCGMemOpIdx oi, const void *tb_ptr) -- 2.25.1 These macros are only used in one place. By expanding, we get to apply some common-subexpression elimination and create some local variables. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci.c | 165 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 100 insertions(+), 65 deletions(-) diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ static bool tci_compare64(uint64_t u0, uint64_t u1, TCGCond condition) return result; } -#ifdef CONFIG_SOFTMMU -# define qemu_ld_ub \ - helper_ret_ldub_mmu(env, taddr, oi, (uintptr_t)tb_ptr) -# define qemu_ld_leuw \ - helper_le_lduw_mmu(env, taddr, oi, (uintptr_t)tb_ptr) -# define qemu_ld_leul \ - helper_le_ldul_mmu(env, taddr, oi, (uintptr_t)tb_ptr) -# define qemu_ld_leq \ - helper_le_ldq_mmu(env, taddr, oi, (uintptr_t)tb_ptr) -# define qemu_ld_beuw \ - helper_be_lduw_mmu(env, taddr, oi, (uintptr_t)tb_ptr) -# define qemu_ld_beul \ - helper_be_ldul_mmu(env, taddr, oi, (uintptr_t)tb_ptr) -# define qemu_ld_beq \ - helper_be_ldq_mmu(env, taddr, oi, (uintptr_t)tb_ptr) -# define qemu_st_b(X) \ - helper_ret_stb_mmu(env, taddr, X, oi, (uintptr_t)tb_ptr) -# define qemu_st_lew(X) \ - helper_le_stw_mmu(env, taddr, X, oi, (uintptr_t)tb_ptr) -# define qemu_st_lel(X) \ - helper_le_stl_mmu(env, taddr, X, oi, (uintptr_t)tb_ptr) -# define qemu_st_leq(X) \ - helper_le_stq_mmu(env, taddr, X, oi, (uintptr_t)tb_ptr) -# define qemu_st_bew(X) \ - helper_be_stw_mmu(env, taddr, X, oi, (uintptr_t)tb_ptr) -# define qemu_st_bel(X) \ - helper_be_stl_mmu(env, taddr, X, oi, (uintptr_t)tb_ptr) -# define qemu_st_beq(X) \ - helper_be_stq_mmu(env, taddr, X, oi, (uintptr_t)tb_ptr) -#else -# define qemu_ld_ub ldub_p(g2h(env_cpu(env), taddr)) -# define qemu_ld_leuw lduw_le_p(g2h(env_cpu(env), taddr)) -# define qemu_ld_leul (uint32_t)ldl_le_p(g2h(env_cpu(env), taddr)) -# define qemu_ld_leq ldq_le_p(g2h(env_cpu(env), taddr)) -# define qemu_ld_beuw lduw_be_p(g2h(env_cpu(env), taddr)) -# define qemu_ld_beul (uint32_t)ldl_be_p(g2h(env_cpu(env), taddr)) -# define qemu_ld_beq ldq_be_p(g2h(env_cpu(env), taddr)) -# define qemu_st_b(X) stb_p(g2h(env_cpu(env), taddr), X) -# define qemu_st_lew(X) stw_le_p(g2h(env_cpu(env), taddr), X) -# define qemu_st_lel(X) stl_le_p(g2h(env_cpu(env), taddr), X) -# define qemu_st_leq(X) stq_le_p(g2h(env_cpu(env), taddr), X) -# define qemu_st_bew(X) stw_be_p(g2h(env_cpu(env), taddr), X) -# define qemu_st_bel(X) stl_be_p(g2h(env_cpu(env), taddr), X) -# define qemu_st_beq(X) stq_be_p(g2h(env_cpu(env), taddr), X) -#endif - static uint64_t tci_qemu_ld(CPUArchState *env, target_ulong taddr, TCGMemOpIdx oi, const void *tb_ptr) { MemOp mop = get_memop(oi) & (MO_BSWAP | MO_SSIZE); +#ifdef CONFIG_SOFTMMU + uintptr_t ra = (uintptr_t)tb_ptr; + switch (mop) { case MO_UB: - return qemu_ld_ub; + return helper_ret_ldub_mmu(env, taddr, oi, ra); case MO_SB: - return (int8_t)qemu_ld_ub; + return helper_ret_ldsb_mmu(env, taddr, oi, ra); case MO_LEUW: - return qemu_ld_leuw; + return helper_le_lduw_mmu(env, taddr, oi, ra); case MO_LESW: - return (int16_t)qemu_ld_leuw; + return helper_le_ldsw_mmu(env, taddr, oi, ra); case MO_LEUL: - return qemu_ld_leul; + return helper_le_ldul_mmu(env, taddr, oi, ra); case MO_LESL: - return (int32_t)qemu_ld_leul; + return helper_le_ldsl_mmu(env, taddr, oi, ra); case MO_LEQ: - return qemu_ld_leq; + return helper_le_ldq_mmu(env, taddr, oi, ra); case MO_BEUW: - return qemu_ld_beuw; + return helper_be_lduw_mmu(env, taddr, oi, ra); case MO_BESW: - return (int16_t)qemu_ld_beuw; + return helper_be_ldsw_mmu(env, taddr, oi, ra); case MO_BEUL: - return qemu_ld_beul; + return helper_be_ldul_mmu(env, taddr, oi, ra); case MO_BESL: - return (int32_t)qemu_ld_beul; + return helper_be_ldsl_mmu(env, taddr, oi, ra); case MO_BEQ: - return qemu_ld_beq; + return helper_be_ldq_mmu(env, taddr, oi, ra); default: g_assert_not_reached(); } +#else + void *haddr = g2h(env_cpu(env), taddr); + uint64_t ret; + + switch (mop) { + case MO_UB: + ret = ldub_p(haddr); + break; + case MO_SB: + ret = ldsb_p(haddr); + break; + case MO_LEUW: + ret = lduw_le_p(haddr); + break; + case MO_LESW: + ret = ldsw_le_p(haddr); + break; + case MO_LEUL: + ret = (uint32_t)ldl_le_p(haddr); + break; + case MO_LESL: + ret = (int32_t)ldl_le_p(haddr); + break; + case MO_LEQ: + ret = ldq_le_p(haddr); + break; + case MO_BEUW: + ret = lduw_be_p(haddr); + break; + case MO_BESW: + ret = ldsw_be_p(haddr); + break; + case MO_BEUL: + ret = (uint32_t)ldl_be_p(haddr); + break; + case MO_BESL: + ret = (int32_t)ldl_be_p(haddr); + break; + case MO_BEQ: + ret = ldq_be_p(haddr); + break; + default: + g_assert_not_reached(); + } + return ret; +#endif } static void tci_qemu_st(CPUArchState *env, target_ulong taddr, uint64_t val, @@ -XXX,XX +XXX,XX @@ static void tci_qemu_st(CPUArchState *env, target_ulong taddr, uint64_t val, { MemOp mop = get_memop(oi) & (MO_BSWAP | MO_SSIZE); +#ifdef CONFIG_SOFTMMU + uintptr_t ra = (uintptr_t)tb_ptr; + switch (mop) { case MO_UB: - qemu_st_b(val); + helper_ret_stb_mmu(env, taddr, val, oi, ra); break; case MO_LEUW: - qemu_st_lew(val); + helper_le_stw_mmu(env, taddr, val, oi, ra); break; case MO_LEUL: - qemu_st_lel(val); + helper_le_stl_mmu(env, taddr, val, oi, ra); break; case MO_LEQ: - qemu_st_leq(val); + helper_le_stq_mmu(env, taddr, val, oi, ra); break; case MO_BEUW: - qemu_st_bew(val); + helper_be_stw_mmu(env, taddr, val, oi, ra); break; case MO_BEUL: - qemu_st_bel(val); + helper_be_stl_mmu(env, taddr, val, oi, ra); break; case MO_BEQ: - qemu_st_beq(val); + helper_be_stq_mmu(env, taddr, val, oi, ra); break; default: g_assert_not_reached(); } +#else + void *haddr = g2h(env_cpu(env), taddr); + + switch (mop) { + case MO_UB: + stb_p(haddr, val); + break; + case MO_LEUW: + stw_le_p(haddr, val); + break; + case MO_LEUL: + stl_le_p(haddr, val); + break; + case MO_LEQ: + stq_le_p(haddr, val); + break; + case MO_BEUW: + stw_be_p(haddr, val); + break; + case MO_BEUL: + stl_be_p(haddr, val); + break; + case MO_BEQ: + stq_be_p(haddr, val); + break; + default: + g_assert_not_reached(); + } +#endif } #if TCG_TARGET_REG_BITS == 64 -- 2.25.1 Wrap guest memory operations for tci like we do for cpu_ld*_data. We cannot actually use the cpu_ldst.h interface without duplicating the memory trace operations performed within, which will already have been expanded into the tcg opcode stream. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ static uint64_t tci_qemu_ld(CPUArchState *env, target_ulong taddr, TCGMemOpIdx oi, const void *tb_ptr) { MemOp mop = get_memop(oi) & (MO_BSWAP | MO_SSIZE); - -#ifdef CONFIG_SOFTMMU uintptr_t ra = (uintptr_t)tb_ptr; +#ifdef CONFIG_SOFTMMU switch (mop) { case MO_UB: return helper_ret_ldub_mmu(env, taddr, oi, ra); @@ -XXX,XX +XXX,XX @@ static uint64_t tci_qemu_ld(CPUArchState *env, target_ulong taddr, void *haddr = g2h(env_cpu(env), taddr); uint64_t ret; + set_helper_retaddr(ra); switch (mop) { case MO_UB: ret = ldub_p(haddr); @@ -XXX,XX +XXX,XX @@ static uint64_t tci_qemu_ld(CPUArchState *env, target_ulong taddr, default: g_assert_not_reached(); } + clear_helper_retaddr(); return ret; #endif } @@ -XXX,XX +XXX,XX @@ static void tci_qemu_st(CPUArchState *env, target_ulong taddr, uint64_t val, TCGMemOpIdx oi, const void *tb_ptr) { MemOp mop = get_memop(oi) & (MO_BSWAP | MO_SSIZE); - -#ifdef CONFIG_SOFTMMU uintptr_t ra = (uintptr_t)tb_ptr; +#ifdef CONFIG_SOFTMMU switch (mop) { case MO_UB: helper_ret_stb_mmu(env, taddr, val, oi, ra); @@ -XXX,XX +XXX,XX @@ static void tci_qemu_st(CPUArchState *env, target_ulong taddr, uint64_t val, #else void *haddr = g2h(env_cpu(env), taddr); + set_helper_retaddr(ra); switch (mop) { case MO_UB: stb_p(haddr, val); @@ -XXX,XX +XXX,XX @@ static void tci_qemu_st(CPUArchState *env, target_ulong taddr, uint64_t val, default: g_assert_not_reached(); } + clear_helper_retaddr(); #endif } -- 2.25.1 The longest test at the moment seems to be a (slower) aarch64 host, for which test-mmap takes 64 seconds. Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Acked-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Thomas Huth <thuth@redhat.com> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- configure | 3 +++ tests/tcg/Makefile.target | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/configure b/configure index XXXXXXX..XXXXXXX 100755 --- a/configure +++ b/configure @@ -XXX,XX +XXX,XX @@ fi if test "$optreset" = "yes" ; then echo "HAVE_OPTRESET=y" >> $config_host_mak fi +if test "$tcg" = "enabled" -a "$tcg_interpreter" = "true" ; then + echo "CONFIG_TCG_INTERPRETER=y" >> $config_host_mak +fi if test "$fdatasync" = "yes" ; then echo "CONFIG_FDATASYNC=y" >> $config_host_mak fi diff --git a/tests/tcg/Makefile.target b/tests/tcg/Makefile.target index XXXXXXX..XXXXXXX 100644 --- a/tests/tcg/Makefile.target +++ b/tests/tcg/Makefile.target @@ -XXX,XX +XXX,XX @@ LDFLAGS= QEMU_OPTS= -# If TCG debugging is enabled things are a lot slower -ifeq ($(CONFIG_DEBUG_TCG),y) +# If TCG debugging, or TCI is enabled things are a lot slower +ifneq ($(CONFIG_TCG_INTERPRETER),) +TIMEOUT=90 +else ifneq ($(CONFIG_DEBUG_TCG),) TIMEOUT=60 else TIMEOUT=15 -- 2.25.1 We had a single ATOMIC_MMU_LOOKUP macro that probed for read+write on all atomic ops. This is incorrect for plain atomic load and atomic store. For user-only, we rely on the host page permissions. Resolves: https://gitlab.com/qemu-project/qemu/-/issues/390 Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/atomic_template.h | 24 +++++----- accel/tcg/cputlb.c | 95 ++++++++++++++++++++++++++----------- accel/tcg/user-exec.c | 8 ++-- 3 files changed, 83 insertions(+), 44 deletions(-) diff --git a/accel/tcg/atomic_template.h b/accel/tcg/atomic_template.h index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/atomic_template.h +++ b/accel/tcg/atomic_template.h @@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) { ATOMIC_MMU_DECLS; - DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; + DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; DATA_TYPE ret; uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, ATOMIC_MMU_IDX); @@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) { ATOMIC_MMU_DECLS; - DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; + DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP_R; uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, ATOMIC_MMU_IDX); @@ -XXX,XX +XXX,XX @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val EXTRA_ARGS) { ATOMIC_MMU_DECLS; - DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; + DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_W; uint16_t info = trace_mem_build_info(SHIFT, false, 0, true, ATOMIC_MMU_IDX); @@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val EXTRA_ARGS) { ATOMIC_MMU_DECLS; - DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; + DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; DATA_TYPE ret; uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, ATOMIC_MMU_IDX); @@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ ABI_TYPE val EXTRA_ARGS) \ { \ ATOMIC_MMU_DECLS; \ - DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ + DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; \ DATA_TYPE ret; \ uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \ ATOMIC_MMU_IDX); \ @@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ ABI_TYPE xval EXTRA_ARGS) \ { \ ATOMIC_MMU_DECLS; \ - XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ + XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; \ XDATA_TYPE cmp, old, new, val = xval; \ uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \ ATOMIC_MMU_IDX); \ @@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) { ATOMIC_MMU_DECLS; - DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; + DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; DATA_TYPE ret; uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false, ATOMIC_MMU_IDX); @@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) { ATOMIC_MMU_DECLS; - DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; + DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP_R; uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false, ATOMIC_MMU_IDX); @@ -XXX,XX +XXX,XX @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val EXTRA_ARGS) { ATOMIC_MMU_DECLS; - DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; + DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_W; uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, true, ATOMIC_MMU_IDX); @@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val EXTRA_ARGS) { ATOMIC_MMU_DECLS; - DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; + DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; ABI_TYPE ret; uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false, ATOMIC_MMU_IDX); @@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ ABI_TYPE val EXTRA_ARGS) \ { \ ATOMIC_MMU_DECLS; \ - DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ + DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; \ DATA_TYPE ret; \ uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \ false, ATOMIC_MMU_IDX); \ @@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ ABI_TYPE xval EXTRA_ARGS) \ { \ ATOMIC_MMU_DECLS; \ - XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ + XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; \ XDATA_TYPE ldo, ldn, old, new, val = xval; \ uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \ false, ATOMIC_MMU_IDX); \ diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ bool tlb_plugin_lookup(CPUState *cpu, target_ulong addr, int mmu_idx, #endif -/* Probe for a read-modify-write atomic operation. Do not allow unaligned - * operations, or io operations to proceed. Return the host address. */ +/* + * Probe for an atomic operation. Do not allow unaligned operations, + * or io operations to proceed. Return the host address. + * + * @prot may be PAGE_READ, PAGE_WRITE, or PAGE_READ|PAGE_WRITE. + */ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + TCGMemOpIdx oi, int size, int prot, + uintptr_t retaddr) { size_t mmu_idx = get_mmuidx(oi); - uintptr_t index = tlb_index(env, mmu_idx, addr); - CPUTLBEntry *tlbe = tlb_entry(env, mmu_idx, addr); - target_ulong tlb_addr = tlb_addr_write(tlbe); MemOp mop = get_memop(oi); int a_bits = get_alignment_bits(mop); - int s_bits = mop & MO_SIZE; + uintptr_t index; + CPUTLBEntry *tlbe; + target_ulong tlb_addr; void *hostaddr; /* Adjust the given return address. */ @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, } /* Enforce qemu required alignment. */ - if (unlikely(addr & ((1 << s_bits) - 1))) { + if (unlikely(addr & (size - 1))) { /* We get here if guest alignment was not requested, or was not enforced by cpu_unaligned_access above. We might widen the access and emulate, but for now @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, goto stop_the_world; } + index = tlb_index(env, mmu_idx, addr); + tlbe = tlb_entry(env, mmu_idx, addr); + /* Check TLB entry and enforce page permissions. */ - if (!tlb_hit(tlb_addr, addr)) { - if (!VICTIM_TLB_HIT(addr_write, addr)) { - tlb_fill(env_cpu(env), addr, 1 << s_bits, MMU_DATA_STORE, - mmu_idx, retaddr); - index = tlb_index(env, mmu_idx, addr); - tlbe = tlb_entry(env, mmu_idx, addr); + if (prot & PAGE_WRITE) { + tlb_addr = tlb_addr_write(tlbe); + if (!tlb_hit(tlb_addr, addr)) { + if (!VICTIM_TLB_HIT(addr_write, addr)) { + tlb_fill(env_cpu(env), addr, size, + MMU_DATA_STORE, mmu_idx, retaddr); + index = tlb_index(env, mmu_idx, addr); + tlbe = tlb_entry(env, mmu_idx, addr); + } + tlb_addr = tlb_addr_write(tlbe) & ~TLB_INVALID_MASK; + } + + /* Let the guest notice RMW on a write-only page. */ + if ((prot & PAGE_READ) && + unlikely(tlbe->addr_read != (tlb_addr & ~TLB_NOTDIRTY))) { + tlb_fill(env_cpu(env), addr, size, + MMU_DATA_LOAD, mmu_idx, retaddr); + /* + * Since we don't support reads and writes to different addresses, + * and we do have the proper page loaded for write, this shouldn't + * ever return. But just in case, handle via stop-the-world. + */ + goto stop_the_world; + } + } else /* if (prot & PAGE_READ) */ { + tlb_addr = tlbe->addr_read; + if (!tlb_hit(tlb_addr, addr)) { + if (!VICTIM_TLB_HIT(addr_write, addr)) { + tlb_fill(env_cpu(env), addr, size, + MMU_DATA_LOAD, mmu_idx, retaddr); + index = tlb_index(env, mmu_idx, addr); + tlbe = tlb_entry(env, mmu_idx, addr); + } + tlb_addr = tlbe->addr_read & ~TLB_INVALID_MASK; } - tlb_addr = tlb_addr_write(tlbe) & ~TLB_INVALID_MASK; } /* Notice an IO access or a needs-MMU-lookup access */ @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, goto stop_the_world; } - /* Let the guest notice RMW on a write-only page. */ - if (unlikely(tlbe->addr_read != (tlb_addr & ~TLB_NOTDIRTY))) { - tlb_fill(env_cpu(env), addr, 1 << s_bits, MMU_DATA_LOAD, - mmu_idx, retaddr); - /* Since we don't support reads and writes to different addresses, - and we do have the proper page loaded for write, this shouldn't - ever return. But just in case, handle via stop-the-world. */ - goto stop_the_world; - } - hostaddr = (void *)((uintptr_t)addr + tlbe->addend); if (unlikely(tlb_addr & TLB_NOTDIRTY)) { - notdirty_write(env_cpu(env), addr, 1 << s_bits, + notdirty_write(env_cpu(env), addr, size, &env_tlb(env)->d[mmu_idx].iotlb[index], retaddr); } @@ -XXX,XX +XXX,XX @@ void cpu_stq_le_data(CPUArchState *env, target_ulong ptr, uint64_t val) #define ATOMIC_NAME(X) \ HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu)) #define ATOMIC_MMU_DECLS -#define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, retaddr) +#define ATOMIC_MMU_LOOKUP_RW \ + atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_READ | PAGE_WRITE, retaddr) +#define ATOMIC_MMU_LOOKUP_R \ + atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_READ, retaddr) +#define ATOMIC_MMU_LOOKUP_W \ + atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_WRITE, retaddr) #define ATOMIC_MMU_CLEANUP #define ATOMIC_MMU_IDX get_mmuidx(oi) @@ -XXX,XX +XXX,XX @@ void cpu_stq_le_data(CPUArchState *env, target_ulong ptr, uint64_t val) #undef EXTRA_ARGS #undef ATOMIC_NAME -#undef ATOMIC_MMU_LOOKUP +#undef ATOMIC_MMU_LOOKUP_RW +#undef ATOMIC_MMU_LOOKUP_R +#undef ATOMIC_MMU_LOOKUP_W + #define EXTRA_ARGS , TCGMemOpIdx oi #define ATOMIC_NAME(X) HELPER(glue(glue(atomic_ ## X, SUFFIX), END)) -#define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, GETPC()) +#define ATOMIC_MMU_LOOKUP_RW \ + atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_READ | PAGE_WRITE, GETPC()) +#define ATOMIC_MMU_LOOKUP_R \ + atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_READ, GETPC()) +#define ATOMIC_MMU_LOOKUP_W \ + atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_WRITE, GETPC()) #define DATA_SIZE 1 #include "atomic_template.h" diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, /* Macro to call the above, with local variables from the use context. */ #define ATOMIC_MMU_DECLS do {} while (0) -#define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, DATA_SIZE, GETPC()) +#define ATOMIC_MMU_LOOKUP_RW atomic_mmu_lookup(env, addr, DATA_SIZE, GETPC()) +#define ATOMIC_MMU_LOOKUP_R ATOMIC_MMU_LOOKUP_RW +#define ATOMIC_MMU_LOOKUP_W ATOMIC_MMU_LOOKUP_RW #define ATOMIC_MMU_CLEANUP do { clear_helper_retaddr(); } while (0) #define ATOMIC_MMU_IDX MMU_USER_IDX @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, #undef EXTRA_ARGS #undef ATOMIC_NAME -#undef ATOMIC_MMU_LOOKUP +#undef ATOMIC_MMU_LOOKUP_RW #define EXTRA_ARGS , TCGMemOpIdx oi, uintptr_t retaddr #define ATOMIC_NAME(X) \ HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu)) -#define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, DATA_SIZE, retaddr) +#define ATOMIC_MMU_LOOKUP_RW atomic_mmu_lookup(env, addr, DATA_SIZE, retaddr) #define DATA_SIZE 16 #include "atomic_template.h" -- 2.25.1 We should not be aligning the offset in temp_allocate_frame, because the odd offset produces an aligned address in the end. Instead, pass the logical offset into tcg_set_frame and add the stack bias last. Cc: qemu-stable@nongnu.org Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 9 +++------ tcg/sparc/tcg-target.c.inc | 16 ++++++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void check_regs(TCGContext *s) static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) { -#if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64) - /* Sparc64 stack is accessed with offset of 2047 */ - s->current_frame_offset = (s->current_frame_offset + - (tcg_target_long)sizeof(tcg_target_long) - 1) & - ~(sizeof(tcg_target_long) - 1); -#endif if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) > s->frame_end) { tcg_abort(); } ts->mem_offset = s->current_frame_offset; +#if defined(__sparc__) + ts->mem_offset += TCG_TARGET_STACK_BIAS; +#endif ts->mem_base = s->frame_temp; ts->mem_allocated = 1; s->current_frame_offset += sizeof(tcg_target_long); diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) { int tmp_buf_size, frame_size; - /* The TCG temp buffer is at the top of the frame, immediately - below the frame pointer. */ + /* + * The TCG temp buffer is at the top of the frame, immediately + * below the frame pointer. Use the logical (aligned) offset here; + * the stack bias is applied in temp_allocate_frame(). + */ tmp_buf_size = CPU_TEMP_BUF_NLONGS * (int)sizeof(long); - tcg_set_frame(s, TCG_REG_I6, TCG_TARGET_STACK_BIAS - tmp_buf_size, - tmp_buf_size); + tcg_set_frame(s, TCG_REG_I6, -tmp_buf_size, tmp_buf_size); - /* TCG_TARGET_CALL_STACK_OFFSET includes the stack bias, but is - otherwise the minimal frame usable by callees. */ + /* + * TCG_TARGET_CALL_STACK_OFFSET includes the stack bias, but is + * otherwise the minimal frame usable by callees. + */ frame_size = TCG_TARGET_CALL_STACK_OFFSET - TCG_TARGET_STACK_BIAS; frame_size += TCG_STATIC_CALL_ARGS_SIZE + tmp_buf_size; frame_size += TCG_TARGET_STACK_ALIGN - 1; -- 2.25.1 This function should have been updated for vector types when they were introduced. Fixes: d2fd745fe8b Resolves: https://gitlab.com/qemu-project/qemu/-/issues/367 Cc: qemu-stable@nongnu.org Tested-by: Stefan Weil <sw@weilnetz.de> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void check_regs(TCGContext *s) static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) { - if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) > - s->frame_end) { - tcg_abort(); + size_t size, align; + intptr_t off; + + switch (ts->type) { + case TCG_TYPE_I32: + size = align = 4; + break; + case TCG_TYPE_I64: + case TCG_TYPE_V64: + size = align = 8; + break; + case TCG_TYPE_V128: + size = align = 16; + break; + case TCG_TYPE_V256: + /* Note that we do not require aligned storage for V256. */ + size = 32, align = 16; + break; + default: + g_assert_not_reached(); } - ts->mem_offset = s->current_frame_offset; + + assert(align <= TCG_TARGET_STACK_ALIGN); + off = ROUND_UP(s->current_frame_offset, align); + assert(off + size <= s->frame_end); + s->current_frame_offset = off + size; + + ts->mem_offset = off; #if defined(__sparc__) ts->mem_offset += TCG_TARGET_STACK_BIAS; #endif ts->mem_base = s->frame_temp; ts->mem_allocated = 1; - s->current_frame_offset += sizeof(tcg_target_long); } static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); -- 2.25.1 Assume that we'll have fewer temps allocated after restarting with a fewer number of instructions. Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) assert(align <= TCG_TARGET_STACK_ALIGN); off = ROUND_UP(s->current_frame_offset, align); - assert(off + size <= s->frame_end); + + /* If we've exhausted the stack frame, restart with a smaller TB. */ + if (off + size > s->frame_end) { + tcg_raise_tb_overflow(s); + } s->current_frame_offset = off + size; ts->mem_offset = off; -- 2.25.1 From: Alessandro Di Federico <ale@rev.ng> This commit moves into a separate file routines used to manipulate TCGCond. These will be employed by the idef-parser. Signed-off-by: Alessandro Di Federico <ale@rev.ng> Signed-off-by: Paolo Montesel <babush@rev.ng> Message-Id: <20210619093713.1845446-2-ale.qemu@rev.ng> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg-cond.h | 101 +++++++++++++++++++++++++++++++++++++++++ include/tcg/tcg.h | 70 +--------------------------- 2 files changed, 102 insertions(+), 69 deletions(-) create mode 100644 include/tcg/tcg-cond.h diff --git a/include/tcg/tcg-cond.h b/include/tcg/tcg-cond.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/include/tcg/tcg-cond.h @@ -XXX,XX +XXX,XX @@ +/* + * Tiny Code Generator for QEMU + * + * Copyright (c) 2008 Fabrice Bellard + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef TCG_COND_H +#define TCG_COND_H + +/* + * Conditions. Note that these are laid out for easy manipulation by + * the functions below: + * bit 0 is used for inverting; + * bit 1 is signed, + * bit 2 is unsigned, + * bit 3 is used with bit 0 for swapping signed/unsigned. + */ +typedef enum { + /* non-signed */ + TCG_COND_NEVER = 0 | 0 | 0 | 0, + TCG_COND_ALWAYS = 0 | 0 | 0 | 1, + TCG_COND_EQ = 8 | 0 | 0 | 0, + TCG_COND_NE = 8 | 0 | 0 | 1, + /* signed */ + TCG_COND_LT = 0 | 0 | 2 | 0, + TCG_COND_GE = 0 | 0 | 2 | 1, + TCG_COND_LE = 8 | 0 | 2 | 0, + TCG_COND_GT = 8 | 0 | 2 | 1, + /* unsigned */ + TCG_COND_LTU = 0 | 4 | 0 | 0, + TCG_COND_GEU = 0 | 4 | 0 | 1, + TCG_COND_LEU = 8 | 4 | 0 | 0, + TCG_COND_GTU = 8 | 4 | 0 | 1, +} TCGCond; + +/* Invert the sense of the comparison. */ +static inline TCGCond tcg_invert_cond(TCGCond c) +{ + return (TCGCond)(c ^ 1); +} + +/* Swap the operands in a comparison. */ +static inline TCGCond tcg_swap_cond(TCGCond c) +{ + return c & 6 ? (TCGCond)(c ^ 9) : c; +} + +/* Create an "unsigned" version of a "signed" comparison. */ +static inline TCGCond tcg_unsigned_cond(TCGCond c) +{ + return c & 2 ? (TCGCond)(c ^ 6) : c; +} + +/* Create a "signed" version of an "unsigned" comparison. */ +static inline TCGCond tcg_signed_cond(TCGCond c) +{ + return c & 4 ? (TCGCond)(c ^ 6) : c; +} + +/* Must a comparison be considered unsigned? */ +static inline bool is_unsigned_cond(TCGCond c) +{ + return (c & 4) != 0; +} + +/* + * Create a "high" version of a double-word comparison. + * This removes equality from a LTE or GTE comparison. + */ +static inline TCGCond tcg_high_cond(TCGCond c) +{ + switch (c) { + case TCG_COND_GE: + case TCG_COND_LE: + case TCG_COND_GEU: + case TCG_COND_LEU: + return (TCGCond)(c ^ 8); + default: + return c; + } +} + +#endif /* TCG_COND_H */ diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ #include "tcg/tcg-mo.h" #include "tcg-target.h" #include "qemu/int128.h" +#include "tcg/tcg-cond.h" /* XXX: make safe guess about sizes */ #define MAX_OP_PER_INSTR 266 @@ -XXX,XX +XXX,XX @@ typedef TCGv_ptr TCGv_env; /* Used to align parameters. See the comment before tcgv_i32_temp. */ #define TCG_CALL_DUMMY_ARG ((TCGArg)0) -/* Conditions. Note that these are laid out for easy manipulation by - the functions below: - bit 0 is used for inverting; - bit 1 is signed, - bit 2 is unsigned, - bit 3 is used with bit 0 for swapping signed/unsigned. */ -typedef enum { - /* non-signed */ - TCG_COND_NEVER = 0 | 0 | 0 | 0, - TCG_COND_ALWAYS = 0 | 0 | 0 | 1, - TCG_COND_EQ = 8 | 0 | 0 | 0, - TCG_COND_NE = 8 | 0 | 0 | 1, - /* signed */ - TCG_COND_LT = 0 | 0 | 2 | 0, - TCG_COND_GE = 0 | 0 | 2 | 1, - TCG_COND_LE = 8 | 0 | 2 | 0, - TCG_COND_GT = 8 | 0 | 2 | 1, - /* unsigned */ - TCG_COND_LTU = 0 | 4 | 0 | 0, - TCG_COND_GEU = 0 | 4 | 0 | 1, - TCG_COND_LEU = 8 | 4 | 0 | 0, - TCG_COND_GTU = 8 | 4 | 0 | 1, -} TCGCond; - -/* Invert the sense of the comparison. */ -static inline TCGCond tcg_invert_cond(TCGCond c) -{ - return (TCGCond)(c ^ 1); -} - -/* Swap the operands in a comparison. */ -static inline TCGCond tcg_swap_cond(TCGCond c) -{ - return c & 6 ? (TCGCond)(c ^ 9) : c; -} - -/* Create an "unsigned" version of a "signed" comparison. */ -static inline TCGCond tcg_unsigned_cond(TCGCond c) -{ - return c & 2 ? (TCGCond)(c ^ 6) : c; -} - -/* Create a "signed" version of an "unsigned" comparison. */ -static inline TCGCond tcg_signed_cond(TCGCond c) -{ - return c & 4 ? (TCGCond)(c ^ 6) : c; -} - -/* Must a comparison be considered unsigned? */ -static inline bool is_unsigned_cond(TCGCond c) -{ - return (c & 4) != 0; -} - -/* Create a "high" version of a double-word comparison. - This removes equality from a LTE or GTE comparison. */ -static inline TCGCond tcg_high_cond(TCGCond c) -{ - switch (c) { - case TCG_COND_GE: - case TCG_COND_LE: - case TCG_COND_GEU: - case TCG_COND_LEU: - return (TCGCond)(c ^ 8); - default: - return c; - } -} - typedef enum TCGTempVal { TEMP_VAL_DEAD, TEMP_VAL_REG, -- 2.25.1 From: Stefan Weil <sw@weilnetz.de> The function is called with alignment == 0 which caused an assertion. Use the code from oslib-posix.c to fix that regression. Fixes: ed6f53f9ca9 Signed-off-by: Stefan Weil <sw@weilnetz.de> Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> Message-Id: <20210611105846.347954-1-sw@weilnetz.de> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- util/oslib-win32.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/util/oslib-win32.c b/util/oslib-win32.c index XXXXXXX..XXXXXXX 100644 --- a/util/oslib-win32.c +++ b/util/oslib-win32.c @@ -XXX,XX +XXX,XX @@ void *qemu_try_memalign(size_t alignment, size_t size) void *ptr; g_assert(size != 0); - g_assert(is_power_of_2(alignment)); + if (alignment < sizeof(void *)) { + alignment = sizeof(void *); + } else { + g_assert(is_power_of_2(alignment)); + } ptr = _aligned_malloc(size, alignment); trace_qemu_memalign(alignment, size, ptr); return ptr; -- 2.25.1 Changes since v1: * Added QEMU_ERROR to wrap __attribute__((error)) -- patch 12. r~ The following changes since commit 77f7c747193662edfadeeb3118d63eed0eac51a6: Merge remote-tracking branch 'remotes/huth-gitlab/tags/pull-request-2018-10-17' into staging (2018-10-18 13:40:19 +0100) are available in the Git repository at: https://github.com/rth7680/qemu.git tags/pull-tcg-20181018 for you to fetch changes up to 403f290c0603f35f2d09c982bf5549b6d0803ec1: cputlb: read CPUTLBEntry.addr_write atomically (2018-10-18 19:46:53 -0700) ---------------------------------------------------------------- Queued tcg patches. ---------------------------------------------------------------- Emilio G. Cota (10): tcg: access cpu->icount_decr.u16.high with atomics tcg: fix use of uninitialized variable under CONFIG_PROFILER tcg: plug holes in struct TCGProfile tcg: distribute tcg_time into TCG contexts target/alpha: remove tlb_flush from alpha_cpu_initfn target/unicore32: remove tlb_flush from uc32_init_fn exec: introduce tlb_init cputlb: fix assert_cpu_is_self macro cputlb: serialize tlb updates with env->tlb_lock cputlb: read CPUTLBEntry.addr_write atomically Richard Henderson (11): tcg: Implement CPU_LOG_TB_NOCHAIN during expansion tcg: Add tlb_index and tlb_entry helpers tcg: Split CONFIG_ATOMIC128 target/i386: Convert to HAVE_CMPXCHG128 target/arm: Convert to HAVE_CMPXCHG128 target/arm: Check HAVE_CMPXCHG128 at translate time target/ppc: Convert to HAVE_CMPXCHG128 and HAVE_ATOMIC128 target/s390x: Convert to HAVE_CMPXCHG128 and HAVE_ATOMIC128 target/s390x: Split do_cdsg, do_lpq, do_stpq target/s390x: Skip wout, cout helpers if op helper does not return target/s390x: Check HAVE_ATOMIC128 and HAVE_CMPXCHG128 at translate accel/tcg/atomic_template.h | 20 +++- accel/tcg/softmmu_template.h | 64 +++++----- include/exec/cpu-defs.h | 3 + include/exec/cpu_ldst.h | 30 ++++- include/exec/cpu_ldst_template.h | 25 ++-- include/exec/exec-all.h | 8 ++ include/qemu/atomic128.h | 153 ++++++++++++++++++++++++ include/qemu/compiler.h | 11 ++ include/qemu/timer.h | 1 - target/ppc/helper.h | 2 +- tcg/tcg.h | 20 ++-- accel/tcg/cpu-exec.c | 2 +- accel/tcg/cputlb.c | 235 +++++++++++++++++++----------------- accel/tcg/tcg-all.c | 2 +- accel/tcg/translate-all.c | 2 +- accel/tcg/user-exec.c | 5 +- cpus.c | 3 +- exec.c | 1 + monitor.c | 13 +- qom/cpu.c | 2 +- target/alpha/cpu.c | 1 - target/arm/helper-a64.c | 251 +++++++++++++++++++-------------------- target/arm/translate-a64.c | 38 +++--- target/i386/mem_helper.c | 9 +- target/ppc/mem_helper.c | 33 ++++- target/ppc/translate.c | 115 +++++++++--------- target/s390x/mem_helper.c | 202 +++++++++++++++---------------- target/s390x/translate.c | 45 +++++-- target/unicore32/cpu.c | 2 - tcg/tcg-op.c | 9 +- tcg/tcg.c | 25 +++- configure | 19 +++ 32 files changed, 839 insertions(+), 512 deletions(-) create mode 100644 include/qemu/atomic128.h Rather than test NOCHAIN before linking, do not emit the goto_tb opcode at all. We already do this for goto_ptr. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cpu-exec.c | 2 +- tcg/tcg-op.c | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ static inline TranslationBlock *tb_find(CPUState *cpu, } #endif /* See if we can patch the calling TB. */ - if (last_tb && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { + if (last_tb) { tb_add_jump(last_tb, tb_exit, tb); } return tb; diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -XXX,XX +XXX,XX @@ void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx) seen this numbered exit before, via tcg_gen_goto_tb. */ tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx)); #endif + /* When not chaining, exit without indicating a link. */ + if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { + val = 0; + } } else { /* This is an exit via the exitreq label. */ tcg_debug_assert(idx == TB_EXIT_REQUESTED); @@ -XXX,XX +XXX,XX @@ void tcg_gen_goto_tb(unsigned idx) tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0); tcg_ctx->goto_tb_issue_mask |= 1 << idx; #endif - tcg_gen_op1i(INDEX_op_goto_tb, idx); + /* When not chaining, we simply fall through to the "fallback" exit. */ + if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { + tcg_gen_op1i(INDEX_op_goto_tb, idx); + } } void tcg_gen_lookup_and_goto_ptr(void) -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> Consistently access u16.high with atomics to avoid undefined behaviour in MTTCG. Note that icount_decr.u16.low is only used in icount mode, so regular accesses to it are OK. Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181010144853.13005-2-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/tcg-all.c | 2 +- accel/tcg/translate-all.c | 2 +- qom/cpu.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/accel/tcg/tcg-all.c b/accel/tcg/tcg-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/tcg-all.c +++ b/accel/tcg/tcg-all.c @@ -XXX,XX +XXX,XX @@ static void tcg_handle_interrupt(CPUState *cpu, int mask) if (!qemu_cpu_is_self(cpu)) { qemu_cpu_kick(cpu); } else { - cpu->icount_decr.u16.high = -1; + atomic_set(&cpu->icount_decr.u16.high, -1); if (use_icount && !cpu->can_do_io && (mask & ~old_mask) != 0) { diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ void cpu_interrupt(CPUState *cpu, int mask) { g_assert(qemu_mutex_iothread_locked()); cpu->interrupt_request |= mask; - cpu->icount_decr.u16.high = -1; + atomic_set(&cpu->icount_decr.u16.high, -1); } /* diff --git a/qom/cpu.c b/qom/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/qom/cpu.c +++ b/qom/cpu.c @@ -XXX,XX +XXX,XX @@ static void cpu_common_reset(CPUState *cpu) cpu->mem_io_pc = 0; cpu->mem_io_vaddr = 0; cpu->icount_extra = 0; - cpu->icount_decr.u32 = 0; + atomic_set(&cpu->icount_decr.u32, 0); cpu->can_do_io = 1; cpu->exception_index = -1; cpu->crash_occurred = false; -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> We forgot to initialize n in commit 15fa08f845 ("tcg: Dynamically allocate TCGOps", 2017-12-29). Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181010144853.13005-3-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb) #ifdef CONFIG_PROFILER { - int n; + int n = 0; QTAILQ_FOREACH(op, &s->ops, link) { n++; -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> This plugs two 4-byte holes in 64-bit. Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181010144853.13005-4-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/tcg.h b/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.h +++ b/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ typedef struct TCGProfile { int64_t tb_count; int64_t op_count; /* total insn count */ int op_count_max; /* max insn per TB */ - int64_t temp_count; int temp_count_max; + int64_t temp_count; int64_t del_op_count; int64_t code_in_len; int64_t code_out_len; -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> When we implemented per-vCPU TCG contexts, we forgot to also distribute the tcg_time counter, which has remained as a global accessed without any serialization, leading to potentially missed counts. Fix it by distributing the field over the TCG contexts, embedding it into TCGProfile with a field called "cpu_exec_time", which is more descriptive than "tcg_time". Add a function to query this value directly, and for completeness, fill in the field in tcg_profile_snapshot, even though its callers do not use it. Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181010144853.13005-5-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/qemu/timer.h | 1 - tcg/tcg.h | 2 ++ cpus.c | 3 ++- monitor.c | 13 ++++++++++--- tcg/tcg.c | 23 +++++++++++++++++++++++ 5 files changed, 37 insertions(+), 5 deletions(-) diff --git a/include/qemu/timer.h b/include/qemu/timer.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/timer.h +++ b/include/qemu/timer.h @@ -XXX,XX +XXX,XX @@ static inline int64_t profile_getclock(void) return get_clock(); } -extern int64_t tcg_time; extern int64_t dev_time; #endif diff --git a/tcg/tcg.h b/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.h +++ b/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ typedef struct TCGOp { QEMU_BUILD_BUG_ON(NB_OPS > (1 << 8)); typedef struct TCGProfile { + int64_t cpu_exec_time; int64_t tb_count1; int64_t tb_count; int64_t op_count; /* total insn count */ @@ -XXX,XX +XXX,XX @@ int tcg_check_temp_count(void); #define tcg_check_temp_count() 0 #endif +int64_t tcg_cpu_exec_time(void); void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf); void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf); diff --git a/cpus.c b/cpus.c index XXXXXXX..XXXXXXX 100644 --- a/cpus.c +++ b/cpus.c @@ -XXX,XX +XXX,XX @@ static int tcg_cpu_exec(CPUState *cpu) ret = cpu_exec(cpu); cpu_exec_end(cpu); #ifdef CONFIG_PROFILER - tcg_time += profile_getclock() - ti; + atomic_set(&tcg_ctx->prof.cpu_exec_time, + tcg_ctx->prof.cpu_exec_time + profile_getclock() - ti); #endif return ret; } diff --git a/monitor.c b/monitor.c index XXXXXXX..XXXXXXX 100644 --- a/monitor.c +++ b/monitor.c @@ -XXX,XX +XXX,XX @@ #include "sysemu/cpus.h" #include "sysemu/iothread.h" #include "qemu/cutils.h" +#include "tcg/tcg.h" #if defined(TARGET_S390X) #include "hw/s390x/storage-keys.h" @@ -XXX,XX +XXX,XX @@ static void hmp_info_numa(Monitor *mon, const QDict *qdict) #ifdef CONFIG_PROFILER -int64_t tcg_time; int64_t dev_time; static void hmp_info_profile(Monitor *mon, const QDict *qdict) { + static int64_t last_cpu_exec_time; + int64_t cpu_exec_time; + int64_t delta; + + cpu_exec_time = tcg_cpu_exec_time(); + delta = cpu_exec_time - last_cpu_exec_time; + monitor_printf(mon, "async time %" PRId64 " (%0.3f)\n", dev_time, dev_time / (double)NANOSECONDS_PER_SECOND); monitor_printf(mon, "qemu time %" PRId64 " (%0.3f)\n", - tcg_time, tcg_time / (double)NANOSECONDS_PER_SECOND); - tcg_time = 0; + delta, delta / (double)NANOSECONDS_PER_SECOND); + last_cpu_exec_time = cpu_exec_time; dev_time = 0; } #else diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ /* Define to jump the ELF file used to communicate with GDB. */ #undef DEBUG_JIT +#include "qemu/error-report.h" #include "qemu/cutils.h" #include "qemu/host-utils.h" #include "qemu/timer.h" @@ -XXX,XX +XXX,XX @@ void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table) const TCGProfile *orig = &s->prof; if (counters) { + PROF_ADD(prof, orig, cpu_exec_time); PROF_ADD(prof, orig, tb_count1); PROF_ADD(prof, orig, tb_count); PROF_ADD(prof, orig, op_count); @@ -XXX,XX +XXX,XX @@ void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf) prof.table_op_count[i]); } } + +int64_t tcg_cpu_exec_time(void) +{ + unsigned int n_ctxs = atomic_read(&n_tcg_ctxs); + unsigned int i; + int64_t ret = 0; + + for (i = 0; i < n_ctxs; i++) { + const TCGContext *s = atomic_read(&tcg_ctxs[i]); + const TCGProfile *prof = &s->prof; + + ret += atomic_read(&prof->cpu_exec_time); + } + return ret; +} #else void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf) { cpu_fprintf(f, "[TCG profiler not compiled]\n"); } + +int64_t tcg_cpu_exec_time(void) +{ + error_report("%s: TCG profiler not compiled", __func__); + exit(EXIT_FAILURE); +} #endif -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> As far as I can tell tlb_flush does not need to be called this early. tlb_flush is eventually called after the CPU has been realized. This change paves the way to the introduction of tlb_init, which will be called from cpu_exec_realizefn. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181009174557.16125-2-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/alpha/cpu.c | 1 - 1 file changed, 1 deletion(-) diff --git a/target/alpha/cpu.c b/target/alpha/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/alpha/cpu.c +++ b/target/alpha/cpu.c @@ -XXX,XX +XXX,XX @@ static void alpha_cpu_initfn(Object *obj) CPUAlphaState *env = &cpu->env; cs->env_ptr = env; - tlb_flush(cs); env->lock_addr = -1; #if defined(CONFIG_USER_ONLY) -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> As far as I can tell tlb_flush does not need to be called this early. tlb_flush is eventually called after the CPU has been realized. This change paves the way to the introduction of tlb_init, which will be called from cpu_exec_realizefn. Cc: Guan Xuetao <gxt@mprc.pku.edu.cn> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181009174557.16125-3-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/unicore32/cpu.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/target/unicore32/cpu.c b/target/unicore32/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/unicore32/cpu.c +++ b/target/unicore32/cpu.c @@ -XXX,XX +XXX,XX @@ static void uc32_cpu_initfn(Object *obj) env->uncached_asr = ASR_MODE_PRIV; env->regs[31] = 0x03000000; #endif - - tlb_flush(cs); } static const VMStateDescription vmstate_uc32_cpu = { -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> Paves the way for the addition of a per-TLB lock. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181009174557.16125-4-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/exec-all.h | 8 ++++++++ accel/tcg/cputlb.c | 4 ++++ exec.c | 1 + 3 files changed, 13 insertions(+) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ void cpu_address_space_init(CPUState *cpu, int asidx, #if !defined(CONFIG_USER_ONLY) && defined(CONFIG_TCG) /* cputlb.c */ +/** + * tlb_init - initialize a CPU's TLB + * @cpu: CPU whose TLB should be initialized + */ +void tlb_init(CPUState *cpu); /** * tlb_flush_page: * @cpu: CPU whose TLB should be flushed @@ -XXX,XX +XXX,XX @@ void tlb_set_page(CPUState *cpu, target_ulong vaddr, void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, uintptr_t retaddr); #else +static inline void tlb_init(CPUState *cpu) +{ +} static inline void tlb_flush_page(CPUState *cpu, target_ulong addr) { } diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ QEMU_BUILD_BUG_ON(sizeof(target_ulong) > sizeof(run_on_cpu_data)); QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16); #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1) +void tlb_init(CPUState *cpu) +{ +} + /* flush_all_helper: run fn across all cpus * * If the wait flag is set then the src cpu's helper will be queued as diff --git a/exec.c b/exec.c index XXXXXXX..XXXXXXX 100644 --- a/exec.c +++ b/exec.c @@ -XXX,XX +XXX,XX @@ void cpu_exec_realizefn(CPUState *cpu, Error **errp) tcg_target_initialized = true; cc->tcg_initialize(); } + tlb_init(cpu); #ifndef CONFIG_USER_ONLY if (qdev_get_vmsd(DEVICE(cpu)) == NULL) { -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181009174557.16125-5-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cputlb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ } \ } while (0) -#define assert_cpu_is_self(this_cpu) do { \ +#define assert_cpu_is_self(cpu) do { \ if (DEBUG_TLB_GATE) { \ - g_assert(!cpu->created || qemu_cpu_is_self(cpu)); \ + g_assert(!(cpu)->created || qemu_cpu_is_self(cpu)); \ } \ } while (0) -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> Currently we rely on atomic operations for cross-CPU invalidations. There are two cases that these atomics miss: cross-CPU invalidations can race with either (1) vCPU threads flushing their TLB, which happens via memset, or (2) vCPUs calling tlb_reset_dirty on their TLB, which updates .addr_write with a regular store. This results in undefined behaviour, since we're mixing regular and atomic ops on concurrent accesses. Fix it by using tlb_lock, a per-vCPU lock. All updaters of tlb_table and the corresponding victim cache now hold the lock. The readers that do not hold tlb_lock must use atomic reads when reading .addr_write, since this field can be updated by other threads; the conversion to atomic reads is done in the next patch. Note that an alternative fix would be to expand the use of atomic ops. However, in the case of TLB flushes this would have a huge performance impact, since (1) TLB flushes can happen very frequently and (2) we currently use a full memory barrier to flush each TLB entry, and a TLB has many entries. Instead, acquiring the lock is barely slower than a full memory barrier since it is uncontended, and with a single lock acquisition we can flush the entire TLB. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181009174557.16125-6-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/cpu-defs.h | 3 + accel/tcg/cputlb.c | 155 ++++++++++++++++++++++------------------ 2 files changed, 87 insertions(+), 71 deletions(-) diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu-defs.h +++ b/include/exec/cpu-defs.h @@ -XXX,XX +XXX,XX @@ #endif #include "qemu/host-utils.h" +#include "qemu/thread.h" #include "qemu/queue.h" #ifdef CONFIG_TCG #include "tcg-target.h" @@ -XXX,XX +XXX,XX @@ typedef struct CPUIOTLBEntry { #define CPU_COMMON_TLB \ /* The meaning of the MMU modes is defined in the target code. */ \ + /* tlb_lock serializes updates to tlb_table and tlb_v_table */ \ + QemuSpin tlb_lock; \ CPUTLBEntry tlb_table[NB_MMU_MODES][CPU_TLB_SIZE]; \ CPUTLBEntry tlb_v_table[NB_MMU_MODES][CPU_VTLB_SIZE]; \ CPUIOTLBEntry iotlb[NB_MMU_MODES][CPU_TLB_SIZE]; \ diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16); void tlb_init(CPUState *cpu) { + CPUArchState *env = cpu->env_ptr; + + qemu_spin_init(&env->tlb_lock); } /* flush_all_helper: run fn across all cpus @@ -XXX,XX +XXX,XX @@ static void tlb_flush_nocheck(CPUState *cpu) atomic_set(&env->tlb_flush_count, env->tlb_flush_count + 1); tlb_debug("(count: %zu)\n", tlb_flush_count()); + /* + * tlb_table/tlb_v_table updates from any thread must hold tlb_lock. + * However, updates from the owner thread (as is the case here; see the + * above assert_cpu_is_self) do not need atomic_set because all reads + * that do not hold the lock are performed by the same owner thread. + */ + qemu_spin_lock(&env->tlb_lock); memset(env->tlb_table, -1, sizeof(env->tlb_table)); memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table)); + qemu_spin_unlock(&env->tlb_lock); + cpu_tb_jmp_cache_clear(cpu); env->vtlb_index = 0; @@ -XXX,XX +XXX,XX @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data) tlb_debug("start: mmu_idx:0x%04lx\n", mmu_idx_bitmask); + qemu_spin_lock(&env->tlb_lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { if (test_bit(mmu_idx, &mmu_idx_bitmask)) { @@ -XXX,XX +XXX,XX @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data) memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0])); } } + qemu_spin_unlock(&env->tlb_lock); cpu_tb_jmp_cache_clear(cpu); @@ -XXX,XX +XXX,XX @@ static inline bool tlb_hit_page_anyprot(CPUTLBEntry *tlb_entry, tlb_hit_page(tlb_entry->addr_code, page); } -static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong page) +/* Called with tlb_lock held */ +static inline void tlb_flush_entry_locked(CPUTLBEntry *tlb_entry, + target_ulong page) { if (tlb_hit_page_anyprot(tlb_entry, page)) { memset(tlb_entry, -1, sizeof(*tlb_entry)); } } -static inline void tlb_flush_vtlb_page(CPUArchState *env, int mmu_idx, - target_ulong page) +/* Called with tlb_lock held */ +static inline void tlb_flush_vtlb_page_locked(CPUArchState *env, int mmu_idx, + target_ulong page) { int k; + + assert_cpu_is_self(ENV_GET_CPU(env)); for (k = 0; k < CPU_VTLB_SIZE; k++) { - tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], page); + tlb_flush_entry_locked(&env->tlb_v_table[mmu_idx][k], page); } } @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data) addr &= TARGET_PAGE_MASK; i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); + qemu_spin_lock(&env->tlb_lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { - tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr); - tlb_flush_vtlb_page(env, mmu_idx, addr); + tlb_flush_entry_locked(&env->tlb_table[mmu_idx][i], addr); + tlb_flush_vtlb_page_locked(env, mmu_idx, addr); } + qemu_spin_unlock(&env->tlb_lock); tb_flush_jmp_cache(cpu, addr); } @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_by_mmuidx_async_work(CPUState *cpu, tlb_debug("page:%d addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n", page, addr, mmu_idx_bitmap); + qemu_spin_lock(&env->tlb_lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { if (test_bit(mmu_idx, &mmu_idx_bitmap)) { - tlb_flush_entry(&env->tlb_table[mmu_idx][page], addr); - tlb_flush_vtlb_page(env, mmu_idx, addr); + tlb_flush_entry_locked(&env->tlb_table[mmu_idx][page], addr); + tlb_flush_vtlb_page_locked(env, mmu_idx, addr); } } + qemu_spin_unlock(&env->tlb_lock); tb_flush_jmp_cache(cpu, addr); } @@ -XXX,XX +XXX,XX @@ void tlb_unprotect_code(ram_addr_t ram_addr) * most usual is detecting writes to code regions which may invalidate * generated code. * - * Because we want other vCPUs to respond to changes straight away we - * update the te->addr_write field atomically. If the TLB entry has - * been changed by the vCPU in the mean time we skip the update. + * Other vCPUs might be reading their TLBs during guest execution, so we update + * te->addr_write with atomic_set. We don't need to worry about this for + * oversized guests as MTTCG is disabled for them. * - * As this function uses atomic accesses we also need to ensure - * updates to tlb_entries follow the same access rules. We don't need - * to worry about this for oversized guests as MTTCG is disabled for - * them. + * Called with tlb_lock held. */ - -static void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start, - uintptr_t length) +static void tlb_reset_dirty_range_locked(CPUTLBEntry *tlb_entry, + uintptr_t start, uintptr_t length) { -#if TCG_OVERSIZED_GUEST uintptr_t addr = tlb_entry->addr_write; if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) { addr &= TARGET_PAGE_MASK; addr += tlb_entry->addend; if ((addr - start) < length) { +#if TCG_OVERSIZED_GUEST tlb_entry->addr_write |= TLB_NOTDIRTY; - } - } #else - /* paired with atomic_mb_set in tlb_set_page_with_attrs */ - uintptr_t orig_addr = atomic_mb_read(&tlb_entry->addr_write); - uintptr_t addr = orig_addr; - - if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) { - addr &= TARGET_PAGE_MASK; - addr += atomic_read(&tlb_entry->addend); - if ((addr - start) < length) { - uintptr_t notdirty_addr = orig_addr | TLB_NOTDIRTY; - atomic_cmpxchg(&tlb_entry->addr_write, orig_addr, notdirty_addr); + atomic_set(&tlb_entry->addr_write, + tlb_entry->addr_write | TLB_NOTDIRTY); +#endif } } -#endif } -/* For atomic correctness when running MTTCG we need to use the right - * primitives when copying entries */ -static inline void copy_tlb_helper(CPUTLBEntry *d, CPUTLBEntry *s, - bool atomic_set) +/* + * Called with tlb_lock held. + * Called only from the vCPU context, i.e. the TLB's owner thread. + */ +static inline void copy_tlb_helper_locked(CPUTLBEntry *d, const CPUTLBEntry *s) { -#if TCG_OVERSIZED_GUEST *d = *s; -#else - if (atomic_set) { - d->addr_read = s->addr_read; - d->addr_code = s->addr_code; - atomic_set(&d->addend, atomic_read(&s->addend)); - /* Pairs with flag setting in tlb_reset_dirty_range */ - atomic_mb_set(&d->addr_write, atomic_read(&s->addr_write)); - } else { - d->addr_read = s->addr_read; - d->addr_write = atomic_read(&s->addr_write); - d->addr_code = s->addr_code; - d->addend = atomic_read(&s->addend); - } -#endif } /* This is a cross vCPU call (i.e. another vCPU resetting the flags of - * the target vCPU). As such care needs to be taken that we don't - * dangerously race with another vCPU update. The only thing actually - * updated is the target TLB entry ->addr_write flags. + * the target vCPU). + * We must take tlb_lock to avoid racing with another vCPU update. The only + * thing actually updated is the target TLB entry ->addr_write flags. */ void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length) { @@ -XXX,XX +XXX,XX @@ void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length) int mmu_idx; env = cpu->env_ptr; + qemu_spin_lock(&env->tlb_lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { unsigned int i; for (i = 0; i < CPU_TLB_SIZE; i++) { - tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i], - start1, length); + tlb_reset_dirty_range_locked(&env->tlb_table[mmu_idx][i], start1, + length); } for (i = 0; i < CPU_VTLB_SIZE; i++) { - tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i], - start1, length); + tlb_reset_dirty_range_locked(&env->tlb_v_table[mmu_idx][i], start1, + length); } } + qemu_spin_unlock(&env->tlb_lock); } -static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr) +/* Called with tlb_lock held */ +static inline void tlb_set_dirty1_locked(CPUTLBEntry *tlb_entry, + target_ulong vaddr) { if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY)) { tlb_entry->addr_write = vaddr; @@ -XXX,XX +XXX,XX @@ void tlb_set_dirty(CPUState *cpu, target_ulong vaddr) vaddr &= TARGET_PAGE_MASK; i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); + qemu_spin_lock(&env->tlb_lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { - tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr); + tlb_set_dirty1_locked(&env->tlb_table[mmu_idx][i], vaddr); } for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { int k; for (k = 0; k < CPU_VTLB_SIZE; k++) { - tlb_set_dirty1(&env->tlb_v_table[mmu_idx][k], vaddr); + tlb_set_dirty1_locked(&env->tlb_v_table[mmu_idx][k], vaddr); } } + qemu_spin_unlock(&env->tlb_lock); } /* Our TLB does not support large pages, so remember the area covered by @@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat; } - /* Make sure there's no cached translation for the new page. */ - tlb_flush_vtlb_page(env, mmu_idx, vaddr_page); - code_address = address; iotlb = memory_region_section_get_iotlb(cpu, section, vaddr_page, paddr_page, xlat, prot, &address); @@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, index = (vaddr_page >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); te = &env->tlb_table[mmu_idx][index]; + /* + * Hold the TLB lock for the rest of the function. We could acquire/release + * the lock several times in the function, but it is faster to amortize the + * acquisition cost by acquiring it just once. Note that this leads to + * a longer critical section, but this is not a concern since the TLB lock + * is unlikely to be contended. + */ + qemu_spin_lock(&env->tlb_lock); + + /* Make sure there's no cached translation for the new page. */ + tlb_flush_vtlb_page_locked(env, mmu_idx, vaddr_page); + /* * Only evict the old entry to the victim tlb if it's for a * different page; otherwise just overwrite the stale data. @@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, CPUTLBEntry *tv = &env->tlb_v_table[mmu_idx][vidx]; /* Evict the old entry into the victim tlb. */ - copy_tlb_helper(tv, te, true); + copy_tlb_helper_locked(tv, te); env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index]; } @@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, } } - /* Pairs with flag setting in tlb_reset_dirty_range */ - copy_tlb_helper(te, &tn, true); - /* atomic_mb_set(&te->addr_write, write_address); */ + copy_tlb_helper_locked(te, &tn); + qemu_spin_unlock(&env->tlb_lock); } /* Add a new TLB entry, but without specifying the memory @@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index, size_t elt_ofs, target_ulong page) { size_t vidx; + + assert_cpu_is_self(ENV_GET_CPU(env)); for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) { CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx]; target_ulong cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs); @@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index, /* Found entry in victim tlb, swap tlb and iotlb. */ CPUTLBEntry tmptlb, *tlb = &env->tlb_table[mmu_idx][index]; - copy_tlb_helper(&tmptlb, tlb, false); - copy_tlb_helper(tlb, vtlb, true); - copy_tlb_helper(vtlb, &tmptlb, true); + qemu_spin_lock(&env->tlb_lock); + copy_tlb_helper_locked(&tmptlb, tlb); + copy_tlb_helper_locked(tlb, vtlb); + copy_tlb_helper_locked(vtlb, &tmptlb); + qemu_spin_unlock(&env->tlb_lock); CPUIOTLBEntry tmpio, *io = &env->iotlb[mmu_idx][index]; CPUIOTLBEntry *vio = &env->iotlb_v[mmu_idx][vidx]; -- 2.17.2 Isolate the computation of an index from an address into a helper before we change that function. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> [ cota: convert tlb_vaddr_to_host; use atomic_read on addr_write ] Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181009175129.17888-2-cota@braap.org> --- accel/tcg/softmmu_template.h | 64 +++++++++++++++++--------------- include/exec/cpu_ldst.h | 19 ++++++++-- include/exec/cpu_ldst_template.h | 25 +++++++------ accel/tcg/cputlb.c | 60 ++++++++++++++---------------- 4 files changed, 90 insertions(+), 78 deletions(-) diff --git a/accel/tcg/softmmu_template.h b/accel/tcg/softmmu_template.h index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/softmmu_template.h +++ b/accel/tcg/softmmu_template.h @@ -XXX,XX +XXX,XX @@ static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env, WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - unsigned mmu_idx = get_mmuidx(oi); - int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ; + uintptr_t mmu_idx = get_mmuidx(oi); + uintptr_t index = tlb_index(env, mmu_idx, addr); + CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); + target_ulong tlb_addr = entry->ADDR_READ; unsigned a_bits = get_alignment_bits(get_memop(oi)); uintptr_t haddr; DATA_TYPE res; @@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, READ_ACCESS_TYPE, mmu_idx, retaddr); } - tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ; + tlb_addr = entry->ADDR_READ; } /* Handle an IO access. */ @@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, return res; } - haddr = addr + env->tlb_table[mmu_idx][index].addend; + haddr = addr + entry->addend; #if DATA_SIZE == 1 res = glue(glue(ld, LSUFFIX), _p)((uint8_t *)haddr); #else @@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - unsigned mmu_idx = get_mmuidx(oi); - int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ; + uintptr_t mmu_idx = get_mmuidx(oi); + uintptr_t index = tlb_index(env, mmu_idx, addr); + CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); + target_ulong tlb_addr = entry->ADDR_READ; unsigned a_bits = get_alignment_bits(get_memop(oi)); uintptr_t haddr; DATA_TYPE res; @@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr, tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, READ_ACCESS_TYPE, mmu_idx, retaddr); } - tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ; + tlb_addr = entry->ADDR_READ; } /* Handle an IO access. */ @@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr, return res; } - haddr = addr + env->tlb_table[mmu_idx][index].addend; + haddr = addr + entry->addend; res = glue(glue(ld, LSUFFIX), _be_p)((uint8_t *)haddr); return res; } @@ -XXX,XX +XXX,XX @@ static inline void glue(io_write, SUFFIX)(CPUArchState *env, void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, TCGMemOpIdx oi, uintptr_t retaddr) { - unsigned mmu_idx = get_mmuidx(oi); - int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write; + uintptr_t mmu_idx = get_mmuidx(oi); + uintptr_t index = tlb_index(env, mmu_idx, addr); + CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); + target_ulong tlb_addr = entry->addr_write; unsigned a_bits = get_alignment_bits(get_memop(oi)); uintptr_t haddr; @@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); } - tlb_addr = env->tlb_table[mmu_idx][index].addr_write & ~TLB_INVALID_MASK; + tlb_addr = entry->addr_write & ~TLB_INVALID_MASK; } /* Handle an IO access. */ @@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, if (DATA_SIZE > 1 && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1 >= TARGET_PAGE_SIZE)) { - int i, index2; - target_ulong page2, tlb_addr2; + int i; + target_ulong page2; + CPUTLBEntry *entry2; do_unaligned_access: /* Ensure the second page is in the TLB. Note that the first page is already guaranteed to be filled, and that the second page cannot evict the first. */ page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK; - index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write; - if (!tlb_hit_page(tlb_addr2, page2) + entry2 = tlb_entry(env, mmu_idx, page2); + if (!tlb_hit_page(entry2->addr_write, page2) && !VICTIM_TLB_HIT(addr_write, page2)) { tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); @@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, return; } - haddr = addr + env->tlb_table[mmu_idx][index].addend; + haddr = addr + entry->addend; #if DATA_SIZE == 1 glue(glue(st, SUFFIX), _p)((uint8_t *)haddr, val); #else @@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, TCGMemOpIdx oi, uintptr_t retaddr) { - unsigned mmu_idx = get_mmuidx(oi); - int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write; + uintptr_t mmu_idx = get_mmuidx(oi); + uintptr_t index = tlb_index(env, mmu_idx, addr); + CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); + target_ulong tlb_addr = entry->addr_write; unsigned a_bits = get_alignment_bits(get_memop(oi)); uintptr_t haddr; @@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); } - tlb_addr = env->tlb_table[mmu_idx][index].addr_write & ~TLB_INVALID_MASK; + tlb_addr = entry->addr_write & ~TLB_INVALID_MASK; } /* Handle an IO access. */ @@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, if (DATA_SIZE > 1 && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1 >= TARGET_PAGE_SIZE)) { - int i, index2; - target_ulong page2, tlb_addr2; + int i; + target_ulong page2; + CPUTLBEntry *entry2; do_unaligned_access: /* Ensure the second page is in the TLB. Note that the first page is already guaranteed to be filled, and that the second page cannot evict the first. */ page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK; - index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write; - if (!tlb_hit_page(tlb_addr2, page2) + entry2 = tlb_entry(env, mmu_idx, page2); + if (!tlb_hit_page(entry2->addr_write, page2) && !VICTIM_TLB_HIT(addr_write, page2)) { tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); @@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, return; } - haddr = addr + env->tlb_table[mmu_idx][index].addend; + haddr = addr + entry->addend; glue(glue(st, SUFFIX), _be_p)((uint8_t *)haddr, val); } #endif /* DATA_SIZE > 1 */ diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu_ldst.h +++ b/include/exec/cpu_ldst.h @@ -XXX,XX +XXX,XX @@ extern __thread uintptr_t helper_retaddr; /* The memory helpers for tcg-generated code need tcg_target_long etc. */ #include "tcg.h" +/* Find the TLB index corresponding to the mmu_idx + address pair. */ +static inline uintptr_t tlb_index(CPUArchState *env, uintptr_t mmu_idx, + target_ulong addr) +{ + return (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); +} + +/* Find the TLB entry corresponding to the mmu_idx + address pair. */ +static inline CPUTLBEntry *tlb_entry(CPUArchState *env, uintptr_t mmu_idx, + target_ulong addr) +{ + return &env->tlb_table[mmu_idx][tlb_index(env, mmu_idx, addr)]; +} + #ifdef MMU_MODE0_SUFFIX #define CPU_MMU_INDEX 0 #define MEMSUFFIX MMU_MODE0_SUFFIX @@ -XXX,XX +XXX,XX @@ static inline void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr, #if defined(CONFIG_USER_ONLY) return g2h(addr); #else - int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - CPUTLBEntry *tlbentry = &env->tlb_table[mmu_idx][index]; + CPUTLBEntry *tlbentry = tlb_entry(env, mmu_idx, addr); abi_ptr tlb_addr; uintptr_t haddr; @@ -XXX,XX +XXX,XX @@ static inline void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr, return NULL; } - haddr = addr + env->tlb_table[mmu_idx][index].addend; + haddr = addr + tlbentry->addend; return (void *)haddr; #endif /* defined(CONFIG_USER_ONLY) */ } diff --git a/include/exec/cpu_ldst_template.h b/include/exec/cpu_ldst_template.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu_ldst_template.h +++ b/include/exec/cpu_ldst_template.h @@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_ld, USUFFIX), MEMSUFFIX), _ra)(CPUArchState *env, target_ulong ptr, uintptr_t retaddr) { - int page_index; + CPUTLBEntry *entry; RES_TYPE res; target_ulong addr; int mmu_idx; @@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_ld, USUFFIX), MEMSUFFIX), _ra)(CPUArchState *env, #endif addr = ptr; - page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); mmu_idx = CPU_MMU_INDEX; - if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ != + entry = tlb_entry(env, mmu_idx, addr); + if (unlikely(entry->ADDR_READ != (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) { oi = make_memop_idx(SHIFT, mmu_idx); res = glue(glue(helper_ret_ld, URETSUFFIX), MMUSUFFIX)(env, addr, oi, retaddr); } else { - uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend; + uintptr_t hostaddr = addr + entry->addend; res = glue(glue(ld, USUFFIX), _p)((uint8_t *)hostaddr); } return res; @@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_lds, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env, target_ulong ptr, uintptr_t retaddr) { - int res, page_index; + CPUTLBEntry *entry; + int res; target_ulong addr; int mmu_idx; TCGMemOpIdx oi; @@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_lds, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env, #endif addr = ptr; - page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); mmu_idx = CPU_MMU_INDEX; - if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ != + entry = tlb_entry(env, mmu_idx, addr); + if (unlikely(entry->ADDR_READ != (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) { oi = make_memop_idx(SHIFT, mmu_idx); res = (DATA_STYPE)glue(glue(helper_ret_ld, SRETSUFFIX), MMUSUFFIX)(env, addr, oi, retaddr); } else { - uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend; + uintptr_t hostaddr = addr + entry->addend; res = glue(glue(lds, SUFFIX), _p)((uint8_t *)hostaddr); } return res; @@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env, target_ulong ptr, RES_TYPE v, uintptr_t retaddr) { - int page_index; + CPUTLBEntry *entry; target_ulong addr; int mmu_idx; TCGMemOpIdx oi; @@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env, #endif addr = ptr; - page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); mmu_idx = CPU_MMU_INDEX; - if (unlikely(env->tlb_table[mmu_idx][page_index].addr_write != + entry = tlb_entry(env, mmu_idx, addr); + if (unlikely(entry->addr_write != (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) { oi = make_memop_idx(SHIFT, mmu_idx); glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(env, addr, v, oi, retaddr); } else { - uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend; + uintptr_t hostaddr = addr + entry->addend; glue(glue(st, SUFFIX), _p)((uint8_t *)hostaddr, v); } } diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data) { CPUArchState *env = cpu->env_ptr; target_ulong addr = (target_ulong) data.target_ptr; - int i; int mmu_idx; assert_cpu_is_self(cpu); @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data) } addr &= TARGET_PAGE_MASK; - i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); qemu_spin_lock(&env->tlb_lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { - tlb_flush_entry_locked(&env->tlb_table[mmu_idx][i], addr); + tlb_flush_entry_locked(tlb_entry(env, mmu_idx, addr), addr); tlb_flush_vtlb_page_locked(env, mmu_idx, addr); } qemu_spin_unlock(&env->tlb_lock); @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_by_mmuidx_async_work(CPUState *cpu, target_ulong addr_and_mmuidx = (target_ulong) data.target_ptr; target_ulong addr = addr_and_mmuidx & TARGET_PAGE_MASK; unsigned long mmu_idx_bitmap = addr_and_mmuidx & ALL_MMUIDX_BITS; - int page = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); int mmu_idx; assert_cpu_is_self(cpu); - tlb_debug("page:%d addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n", - page, addr, mmu_idx_bitmap); + tlb_debug("flush page addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n", + addr, mmu_idx_bitmap); qemu_spin_lock(&env->tlb_lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { if (test_bit(mmu_idx, &mmu_idx_bitmap)) { - tlb_flush_entry_locked(&env->tlb_table[mmu_idx][page], addr); + tlb_flush_entry_locked(tlb_entry(env, mmu_idx, addr), addr); tlb_flush_vtlb_page_locked(env, mmu_idx, addr); } } @@ -XXX,XX +XXX,XX @@ static inline void tlb_set_dirty1_locked(CPUTLBEntry *tlb_entry, void tlb_set_dirty(CPUState *cpu, target_ulong vaddr) { CPUArchState *env = cpu->env_ptr; - int i; int mmu_idx; assert_cpu_is_self(cpu); vaddr &= TARGET_PAGE_MASK; - i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); qemu_spin_lock(&env->tlb_lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { - tlb_set_dirty1_locked(&env->tlb_table[mmu_idx][i], vaddr); + tlb_set_dirty1_locked(tlb_entry(env, mmu_idx, vaddr), vaddr); } for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { @@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, iotlb = memory_region_section_get_iotlb(cpu, section, vaddr_page, paddr_page, xlat, prot, &address); - index = (vaddr_page >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - te = &env->tlb_table[mmu_idx][index]; + index = tlb_index(env, mmu_idx, vaddr_page); + te = tlb_entry(env, mmu_idx, vaddr_page); /* * Hold the TLB lock for the rest of the function. We could acquire/release @@ -XXX,XX +XXX,XX @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry, * repeat the MMU check here. This tlb_fill() call might * longjump out if this access should cause a guest exception. */ - int index; + CPUTLBEntry *entry; target_ulong tlb_addr; tlb_fill(cpu, addr, size, MMU_DATA_LOAD, mmu_idx, retaddr); - index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - tlb_addr = env->tlb_table[mmu_idx][index].addr_read; + entry = tlb_entry(env, mmu_idx, addr); + tlb_addr = entry->addr_read; if (!(tlb_addr & ~(TARGET_PAGE_MASK | TLB_RECHECK))) { /* RAM access */ - uintptr_t haddr = addr + env->tlb_table[mmu_idx][index].addend; + uintptr_t haddr = addr + entry->addend; return ldn_p((void *)haddr, size); } @@ -XXX,XX +XXX,XX @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry, * repeat the MMU check here. This tlb_fill() call might * longjump out if this access should cause a guest exception. */ - int index; + CPUTLBEntry *entry; target_ulong tlb_addr; tlb_fill(cpu, addr, size, MMU_DATA_STORE, mmu_idx, retaddr); - index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - tlb_addr = env->tlb_table[mmu_idx][index].addr_write; + entry = tlb_entry(env, mmu_idx, addr); + tlb_addr = entry->addr_write; if (!(tlb_addr & ~(TARGET_PAGE_MASK | TLB_RECHECK))) { /* RAM access */ - uintptr_t haddr = addr + env->tlb_table[mmu_idx][index].addend; + uintptr_t haddr = addr + entry->addend; stn_p((void *)haddr, size, val); return; @@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index, */ tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr) { - int mmu_idx, index; + uintptr_t mmu_idx = cpu_mmu_index(env, true); + uintptr_t index = tlb_index(env, mmu_idx, addr); + CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); void *p; - index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - mmu_idx = cpu_mmu_index(env, true); - if (unlikely(!tlb_hit(env->tlb_table[mmu_idx][index].addr_code, addr))) { + if (unlikely(!tlb_hit(entry->addr_code, addr))) { if (!VICTIM_TLB_HIT(addr_code, addr)) { tlb_fill(ENV_GET_CPU(env), addr, 0, MMU_INST_FETCH, mmu_idx, 0); } - assert(tlb_hit(env->tlb_table[mmu_idx][index].addr_code, addr)); + assert(tlb_hit(entry->addr_code, addr)); } - if (unlikely(env->tlb_table[mmu_idx][index].addr_code & - (TLB_RECHECK | TLB_MMIO))) { + if (unlikely(entry->addr_code & (TLB_RECHECK | TLB_MMIO))) { /* * Return -1 if we can't translate and execute from an entire * page of RAM here, which will cause us to execute by loading @@ -XXX,XX +XXX,XX @@ tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr) return -1; } - p = (void *)((uintptr_t)addr + env->tlb_table[mmu_idx][index].addend); + p = (void *)((uintptr_t)addr + entry->addend); return qemu_ram_addr_from_host_nofail(p); } @@ -XXX,XX +XXX,XX @@ tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr) void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, uintptr_t retaddr) { - int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write; + uintptr_t index = tlb_index(env, mmu_idx, addr); + CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); - if (!tlb_hit(tlb_addr, addr)) { + if (!tlb_hit(entry->addr_write, addr)) { /* TLB entry is for a different page */ if (!VICTIM_TLB_HIT(addr_write, addr)) { tlb_fill(ENV_GET_CPU(env), addr, size, MMU_DATA_STORE, @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, NotDirtyInfo *ndi) { size_t mmu_idx = get_mmuidx(oi); - size_t index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - CPUTLBEntry *tlbe = &env->tlb_table[mmu_idx][index]; + uintptr_t index = tlb_index(env, mmu_idx, addr); + CPUTLBEntry *tlbe = tlb_entry(env, mmu_idx, addr); target_ulong tlb_addr = tlbe->addr_write; TCGMemOp mop = get_memop(oi); int a_bits = get_alignment_bits(mop); -- 2.17.2 GCC7+ will no longer advertise support for 16-byte __atomic operations if only cmpxchg is supported, as for x86_64. Fortunately, x86_64 still has support for __sync_compare_and_swap_16 and we can make use of that. AArch64 does not have, nor ever has had such support, so open-code it. Reviewed-by: Emilio G. Cota <cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/atomic_template.h | 20 ++++- include/qemu/atomic128.h | 153 ++++++++++++++++++++++++++++++++++++ include/qemu/compiler.h | 11 +++ tcg/tcg.h | 16 ++-- accel/tcg/cputlb.c | 3 +- accel/tcg/user-exec.c | 5 +- configure | 19 +++++ 7 files changed, 213 insertions(+), 14 deletions(-) create mode 100644 include/qemu/atomic128.h diff --git a/accel/tcg/atomic_template.h b/accel/tcg/atomic_template.h index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/atomic_template.h +++ b/accel/tcg/atomic_template.h @@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, DATA_TYPE ret; ATOMIC_TRACE_RMW; +#if DATA_SIZE == 16 + ret = atomic16_cmpxchg(haddr, cmpv, newv); +#else ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv); +#endif ATOMIC_MMU_CLEANUP; return ret; } #if DATA_SIZE >= 16 +#if HAVE_ATOMIC128 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) { ATOMIC_MMU_DECLS; DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; ATOMIC_TRACE_LD; - __atomic_load(haddr, &val, __ATOMIC_RELAXED); + val = atomic16_read(haddr); ATOMIC_MMU_CLEANUP; return val; } @@ -XXX,XX +XXX,XX @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; ATOMIC_TRACE_ST; - __atomic_store(haddr, &val, __ATOMIC_RELAXED); + atomic16_set(haddr, val); ATOMIC_MMU_CLEANUP; } +#endif #else ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val EXTRA_ARGS) @@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, DATA_TYPE ret; ATOMIC_TRACE_RMW; +#if DATA_SIZE == 16 + ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv)); +#else ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); +#endif ATOMIC_MMU_CLEANUP; return BSWAP(ret); } #if DATA_SIZE >= 16 +#if HAVE_ATOMIC128 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) { ATOMIC_MMU_DECLS; DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; ATOMIC_TRACE_LD; - __atomic_load(haddr, &val, __ATOMIC_RELAXED); + val = atomic16_read(haddr); ATOMIC_MMU_CLEANUP; return BSWAP(val); } @@ -XXX,XX +XXX,XX @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ATOMIC_TRACE_ST; val = BSWAP(val); - __atomic_store(haddr, &val, __ATOMIC_RELAXED); + atomic16_set(haddr, val); ATOMIC_MMU_CLEANUP; } +#endif #else ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val EXTRA_ARGS) diff --git a/include/qemu/atomic128.h b/include/qemu/atomic128.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/include/qemu/atomic128.h @@ -XXX,XX +XXX,XX @@ +/* + * Simple interface for 128-bit atomic operations. + * + * Copyright (C) 2018 Linaro, Ltd. + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + * + * See docs/devel/atomics.txt for discussion about the guarantees each + * atomic primitive is meant to provide. + */ + +#ifndef QEMU_ATOMIC128_H +#define QEMU_ATOMIC128_H + +/* + * GCC is a house divided about supporting large atomic operations. + * + * For hosts that only have large compare-and-swap, a legalistic reading + * of the C++ standard means that one cannot implement __atomic_read on + * read-only memory, and thus all atomic operations must synchronize + * through libatomic. + * + * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80878 + * + * This interpretation is not especially helpful for QEMU. + * For softmmu, all RAM is always read/write from the hypervisor. + * For user-only, if the guest doesn't implement such an __atomic_read + * then the host need not worry about it either. + * + * Moreover, using libatomic is not an option, because its interface is + * built for std::atomic<T>, and requires that *all* accesses to such an + * object go through the library. In our case we do not have an object + * in the C/C++ sense, but a view of memory as seen by the guest. + * The guest may issue a large atomic operation and then access those + * pieces using word-sized accesses. From the hypervisor, we have no + * way to connect those two actions. + * + * Therefore, special case each platform. + */ + +#if defined(CONFIG_ATOMIC128) +static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) +{ + return atomic_cmpxchg__nocheck(ptr, cmp, new); +} +# define HAVE_CMPXCHG128 1 +#elif defined(CONFIG_CMPXCHG128) +static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) +{ + return __sync_val_compare_and_swap_16(ptr, cmp, new); +} +# define HAVE_CMPXCHG128 1 +#elif defined(__aarch64__) +/* Through gcc 8, aarch64 has no support for 128-bit at all. */ +static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) +{ + uint64_t cmpl = int128_getlo(cmp), cmph = int128_gethi(cmp); + uint64_t newl = int128_getlo(new), newh = int128_gethi(new); + uint64_t oldl, oldh; + uint32_t tmp; + + asm("0: ldaxp %[oldl], %[oldh], %[mem]\n\t" + "cmp %[oldl], %[cmpl]\n\t" + "ccmp %[oldh], %[cmph], #0, eq\n\t" + "b.ne 1f\n\t" + "stlxp %w[tmp], %[newl], %[newh], %[mem]\n\t" + "cbnz %w[tmp], 0b\n" + "1:" + : [mem] "+m"(*ptr), [tmp] "=&r"(tmp), + [oldl] "=&r"(oldl), [oldh] "=r"(oldh) + : [cmpl] "r"(cmpl), [cmph] "r"(cmph), + [newl] "r"(newl), [newh] "r"(newh) + : "memory", "cc"); + + return int128_make128(oldl, oldh); +} +# define HAVE_CMPXCHG128 1 +#else +/* Fallback definition that must be optimized away, or error. */ +Int128 QEMU_ERROR("unsupported atomic") + atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new); +# define HAVE_CMPXCHG128 0 +#endif /* Some definition for HAVE_CMPXCHG128 */ + + +#if defined(CONFIG_ATOMIC128) +static inline Int128 atomic16_read(Int128 *ptr) +{ + return atomic_read__nocheck(ptr); +} + +static inline void atomic16_set(Int128 *ptr, Int128 val) +{ + atomic_set__nocheck(ptr, val); +} + +# define HAVE_ATOMIC128 1 +#elif !defined(CONFIG_USER_ONLY) && defined(__aarch64__) +/* We can do better than cmpxchg for AArch64. */ +static inline Int128 atomic16_read(Int128 *ptr) +{ + uint64_t l, h; + uint32_t tmp; + + /* The load must be paired with the store to guarantee not tearing. */ + asm("0: ldxp %[l], %[h], %[mem]\n\t" + "stxp %w[tmp], %[l], %[h], %[mem]\n\t" + "cbnz %w[tmp], 0b" + : [mem] "+m"(*ptr), [tmp] "=r"(tmp), [l] "=r"(l), [h] "=r"(h)); + + return int128_make128(l, h); +} + +static inline void atomic16_set(Int128 *ptr, Int128 val) +{ + uint64_t l = int128_getlo(val), h = int128_gethi(val); + uint64_t t1, t2; + + /* Load into temporaries to acquire the exclusive access lock. */ + asm("0: ldxp %[t1], %[t2], %[mem]\n\t" + "stxp %w[t1], %[l], %[h], %[mem]\n\t" + "cbnz %w[t1], 0b" + : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2) + : [l] "r"(l), [h] "r"(h)); +} + +# define HAVE_ATOMIC128 1 +#elif !defined(CONFIG_USER_ONLY) && HAVE_CMPXCHG128 +static inline Int128 atomic16_read(Int128 *ptr) +{ + /* Maybe replace 0 with 0, returning the old value. */ + return atomic16_cmpxchg(ptr, 0, 0); +} + +static inline void atomic16_set(Int128 *ptr, Int128 val) +{ + Int128 old = *ptr, cmp; + do { + cmp = old; + old = atomic16_cmpxchg(ptr, cmp, val); + } while (old != cmp); +} + +# define HAVE_ATOMIC128 1 +#else +/* Fallback definitions that must be optimized away, or error. */ +Int128 QEMU_ERROR("unsupported atomic") atomic16_read(Int128 *ptr); +void QEMU_ERROR("unsupported atomic") atomic16_set(Int128 *ptr, Int128 val); +# define HAVE_ATOMIC128 0 +#endif /* Some definition for HAVE_ATOMIC128 */ + +#endif /* QEMU_ATOMIC128_H */ diff --git a/include/qemu/compiler.h b/include/qemu/compiler.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/compiler.h +++ b/include/qemu/compiler.h @@ -XXX,XX +XXX,XX @@ # define QEMU_FLATTEN #endif +/* + * If __attribute__((error)) is present, use it to produce an error at + * compile time. Otherwise, one must wait for the linker to diagnose + * the missing symbol. + */ +#if __has_attribute(error) +# define QEMU_ERROR(X) __attribute__((error(X))) +#else +# define QEMU_ERROR(X) +#endif + /* Implement C11 _Generic via GCC builtins. Example: * * QEMU_GENERIC(x, (float, sinf), (long double, sinl), sin) (x) diff --git a/tcg/tcg.h b/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.h +++ b/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ #include "qemu/queue.h" #include "tcg-mo.h" #include "tcg-target.h" +#include "qemu/int128.h" /* XXX: make safe guess about sizes */ #define MAX_OP_PER_INSTR 266 @@ -XXX,XX +XXX,XX @@ GEN_ATOMIC_HELPER_ALL(xchg) #undef GEN_ATOMIC_HELPER #endif /* CONFIG_SOFTMMU */ -#ifdef CONFIG_ATOMIC128 -#include "qemu/int128.h" - -/* These aren't really a "proper" helpers because TCG cannot manage Int128. - However, use the same format as the others, for use by the backends. */ +/* + * These aren't really a "proper" helpers because TCG cannot manage Int128. + * However, use the same format as the others, for use by the backends. + * + * The cmpxchg functions are only defined if HAVE_CMPXCHG128; + * the ld/st functions are only defined if HAVE_ATOMIC128, + * as defined by <qemu/atomic128.h>. + */ Int128 helper_atomic_cmpxchgo_le_mmu(CPUArchState *env, target_ulong addr, Int128 cmpv, Int128 newv, TCGMemOpIdx oi, uintptr_t retaddr); @@ -XXX,XX +XXX,XX @@ void helper_atomic_sto_le_mmu(CPUArchState *env, target_ulong addr, Int128 val, void helper_atomic_sto_be_mmu(CPUArchState *env, target_ulong addr, Int128 val, TCGMemOpIdx oi, uintptr_t retaddr); -#endif /* CONFIG_ATOMIC128 */ - #endif /* TCG_H */ diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ #include "exec/log.h" #include "exec/helper-proto.h" #include "qemu/atomic.h" +#include "qemu/atomic128.h" /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */ /* #define DEBUG_TLB */ @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, #include "atomic_template.h" #endif -#ifdef CONFIG_ATOMIC128 +#if HAVE_CMPXCHG128 || HAVE_ATOMIC128 #define DATA_SIZE 16 #include "atomic_template.h" #endif diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -XXX,XX +XXX,XX @@ #include "exec/cpu_ldst.h" #include "translate-all.h" #include "exec/helper-proto.h" +#include "qemu/atomic128.h" #undef EAX #undef ECX @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, /* The following is only callable from other helpers, and matches up with the softmmu version. */ -#ifdef CONFIG_ATOMIC128 +#if HAVE_ATOMIC128 || HAVE_CMPXCHG128 #undef EXTRA_ARGS #undef ATOMIC_NAME @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, #define DATA_SIZE 16 #include "atomic_template.h" -#endif /* CONFIG_ATOMIC128 */ +#endif diff --git a/configure b/configure index XXXXXXX..XXXXXXX 100755 --- a/configure +++ b/configure @@ -XXX,XX +XXX,XX @@ EOF fi fi +cmpxchg128=no +if test "$int128" = yes -a "$atomic128" = no; then + cat > $TMPC << EOF +int main(void) +{ + unsigned __int128 x = 0, y = 0; + __sync_val_compare_and_swap_16(&x, y, x); + return 0; +} +EOF + if compile_prog "" "" ; then + cmpxchg128=yes + fi +fi + ######################################### # See if 64-bit atomic operations are supported. # Note that without __atomic builtins, we can only @@ -XXX,XX +XXX,XX @@ if test "$atomic128" = "yes" ; then echo "CONFIG_ATOMIC128=y" >> $config_host_mak fi +if test "$cmpxchg128" = "yes" ; then + echo "CONFIG_CMPXCHG128=y" >> $config_host_mak +fi + if test "$atomic64" = "yes" ; then echo "CONFIG_ATOMIC64=y" >> $config_host_mak fi -- 2.17.2 Reviewed-by: Emilio G. Cota <cota@braap.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/i386/mem_helper.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/target/i386/mem_helper.c b/target/i386/mem_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/mem_helper.c +++ b/target/i386/mem_helper.c @@ -XXX,XX +XXX,XX @@ #include "exec/exec-all.h" #include "exec/cpu_ldst.h" #include "qemu/int128.h" +#include "qemu/atomic128.h" #include "tcg.h" void helper_cmpxchg8b_unlocked(CPUX86State *env, target_ulong a0) @@ -XXX,XX +XXX,XX @@ void helper_cmpxchg16b(CPUX86State *env, target_ulong a0) if ((a0 & 0xf) != 0) { raise_exception_ra(env, EXCP0D_GPF, ra); - } else { -#ifndef CONFIG_ATOMIC128 - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); -#else + } else if (HAVE_CMPXCHG128) { int eflags = cpu_cc_compute_all(env, CC_OP); Int128 cmpv = int128_make128(env->regs[R_EAX], env->regs[R_EDX]); @@ -XXX,XX +XXX,XX @@ void helper_cmpxchg16b(CPUX86State *env, target_ulong a0) eflags &= ~CC_Z; } CC_SRC = eflags; -#endif + } else { + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); } } #endif -- 2.17.2 Reviewed-by: Emilio G. Cota <cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/arm/helper-a64.c | 259 +++++++++++++++++++++------------------- 1 file changed, 133 insertions(+), 126 deletions(-) diff --git a/target/arm/helper-a64.c b/target/arm/helper-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-a64.c +++ b/target/arm/helper-a64.c @@ -XXX,XX +XXX,XX @@ #include "exec/exec-all.h" #include "exec/cpu_ldst.h" #include "qemu/int128.h" +#include "qemu/atomic128.h" #include "tcg.h" #include "fpu/softfloat.h" #include <zlib.h> /* For crc32 */ @@ -XXX,XX +XXX,XX @@ uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes) return crc32c(acc, buf, bytes) ^ 0xffffffff; } -/* Returns 0 on success; 1 otherwise. */ -static uint64_t do_paired_cmpxchg64_le(CPUARMState *env, uint64_t addr, - uint64_t new_lo, uint64_t new_hi, - bool parallel, uintptr_t ra) +uint64_t HELPER(paired_cmpxchg64_le)(CPUARMState *env, uint64_t addr, + uint64_t new_lo, uint64_t new_hi) { - Int128 oldv, cmpv, newv; + Int128 cmpv = int128_make128(env->exclusive_val, env->exclusive_high); + Int128 newv = int128_make128(new_lo, new_hi); + Int128 oldv; + uintptr_t ra = GETPC(); + uint64_t o0, o1; bool success; - cmpv = int128_make128(env->exclusive_val, env->exclusive_high); - newv = int128_make128(new_lo, new_hi); - - if (parallel) { -#ifndef CONFIG_ATOMIC128 - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); -#else - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); - oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra); - success = int128_eq(oldv, cmpv); -#endif - } else { - uint64_t o0, o1; - #ifdef CONFIG_USER_ONLY - /* ??? Enforce alignment. */ - uint64_t *haddr = g2h(addr); + /* ??? Enforce alignment. */ + uint64_t *haddr = g2h(addr); - helper_retaddr = ra; - o0 = ldq_le_p(haddr + 0); - o1 = ldq_le_p(haddr + 1); - oldv = int128_make128(o0, o1); + helper_retaddr = ra; + o0 = ldq_le_p(haddr + 0); + o1 = ldq_le_p(haddr + 1); + oldv = int128_make128(o0, o1); - success = int128_eq(oldv, cmpv); - if (success) { - stq_le_p(haddr + 0, int128_getlo(newv)); - stq_le_p(haddr + 1, int128_gethi(newv)); - } - helper_retaddr = 0; -#else - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi0 = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); - TCGMemOpIdx oi1 = make_memop_idx(MO_LEQ, mem_idx); - - o0 = helper_le_ldq_mmu(env, addr + 0, oi0, ra); - o1 = helper_le_ldq_mmu(env, addr + 8, oi1, ra); - oldv = int128_make128(o0, o1); - - success = int128_eq(oldv, cmpv); - if (success) { - helper_le_stq_mmu(env, addr + 0, int128_getlo(newv), oi1, ra); - helper_le_stq_mmu(env, addr + 8, int128_gethi(newv), oi1, ra); - } -#endif + success = int128_eq(oldv, cmpv); + if (success) { + stq_le_p(haddr + 0, int128_getlo(newv)); + stq_le_p(haddr + 1, int128_gethi(newv)); } + helper_retaddr = 0; +#else + int mem_idx = cpu_mmu_index(env, false); + TCGMemOpIdx oi0 = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); + TCGMemOpIdx oi1 = make_memop_idx(MO_LEQ, mem_idx); + + o0 = helper_le_ldq_mmu(env, addr + 0, oi0, ra); + o1 = helper_le_ldq_mmu(env, addr + 8, oi1, ra); + oldv = int128_make128(o0, o1); + + success = int128_eq(oldv, cmpv); + if (success) { + helper_le_stq_mmu(env, addr + 0, int128_getlo(newv), oi1, ra); + helper_le_stq_mmu(env, addr + 8, int128_gethi(newv), oi1, ra); + } +#endif return !success; } -uint64_t HELPER(paired_cmpxchg64_le)(CPUARMState *env, uint64_t addr, - uint64_t new_lo, uint64_t new_hi) -{ - return do_paired_cmpxchg64_le(env, addr, new_lo, new_hi, false, GETPC()); -} - uint64_t HELPER(paired_cmpxchg64_le_parallel)(CPUARMState *env, uint64_t addr, uint64_t new_lo, uint64_t new_hi) -{ - return do_paired_cmpxchg64_le(env, addr, new_lo, new_hi, true, GETPC()); -} - -static uint64_t do_paired_cmpxchg64_be(CPUARMState *env, uint64_t addr, - uint64_t new_lo, uint64_t new_hi, - bool parallel, uintptr_t ra) { Int128 oldv, cmpv, newv; + uintptr_t ra = GETPC(); bool success; + int mem_idx; + TCGMemOpIdx oi; - /* high and low need to be switched here because this is not actually a - * 128bit store but two doublewords stored consecutively - */ - cmpv = int128_make128(env->exclusive_high, env->exclusive_val); - newv = int128_make128(new_hi, new_lo); - - if (parallel) { -#ifndef CONFIG_ATOMIC128 + if (!HAVE_CMPXCHG128) { cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); -#else - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx); - oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra); - success = int128_eq(oldv, cmpv); -#endif - } else { - uint64_t o0, o1; - -#ifdef CONFIG_USER_ONLY - /* ??? Enforce alignment. */ - uint64_t *haddr = g2h(addr); - - helper_retaddr = ra; - o1 = ldq_be_p(haddr + 0); - o0 = ldq_be_p(haddr + 1); - oldv = int128_make128(o0, o1); - - success = int128_eq(oldv, cmpv); - if (success) { - stq_be_p(haddr + 0, int128_gethi(newv)); - stq_be_p(haddr + 1, int128_getlo(newv)); - } - helper_retaddr = 0; -#else - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi0 = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx); - TCGMemOpIdx oi1 = make_memop_idx(MO_BEQ, mem_idx); - - o1 = helper_be_ldq_mmu(env, addr + 0, oi0, ra); - o0 = helper_be_ldq_mmu(env, addr + 8, oi1, ra); - oldv = int128_make128(o0, o1); - - success = int128_eq(oldv, cmpv); - if (success) { - helper_be_stq_mmu(env, addr + 0, int128_gethi(newv), oi1, ra); - helper_be_stq_mmu(env, addr + 8, int128_getlo(newv), oi1, ra); - } -#endif } + mem_idx = cpu_mmu_index(env, false); + oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); + + cmpv = int128_make128(env->exclusive_val, env->exclusive_high); + newv = int128_make128(new_lo, new_hi); + oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra); + + success = int128_eq(oldv, cmpv); return !success; } uint64_t HELPER(paired_cmpxchg64_be)(CPUARMState *env, uint64_t addr, uint64_t new_lo, uint64_t new_hi) { - return do_paired_cmpxchg64_be(env, addr, new_lo, new_hi, false, GETPC()); + /* + * High and low need to be switched here because this is not actually a + * 128bit store but two doublewords stored consecutively + */ + Int128 cmpv = int128_make128(env->exclusive_val, env->exclusive_high); + Int128 newv = int128_make128(new_lo, new_hi); + Int128 oldv; + uintptr_t ra = GETPC(); + uint64_t o0, o1; + bool success; + +#ifdef CONFIG_USER_ONLY + /* ??? Enforce alignment. */ + uint64_t *haddr = g2h(addr); + + helper_retaddr = ra; + o1 = ldq_be_p(haddr + 0); + o0 = ldq_be_p(haddr + 1); + oldv = int128_make128(o0, o1); + + success = int128_eq(oldv, cmpv); + if (success) { + stq_be_p(haddr + 0, int128_gethi(newv)); + stq_be_p(haddr + 1, int128_getlo(newv)); + } + helper_retaddr = 0; +#else + int mem_idx = cpu_mmu_index(env, false); + TCGMemOpIdx oi0 = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx); + TCGMemOpIdx oi1 = make_memop_idx(MO_BEQ, mem_idx); + + o1 = helper_be_ldq_mmu(env, addr + 0, oi0, ra); + o0 = helper_be_ldq_mmu(env, addr + 8, oi1, ra); + oldv = int128_make128(o0, o1); + + success = int128_eq(oldv, cmpv); + if (success) { + helper_be_stq_mmu(env, addr + 0, int128_gethi(newv), oi1, ra); + helper_be_stq_mmu(env, addr + 8, int128_getlo(newv), oi1, ra); + } +#endif + + return !success; } uint64_t HELPER(paired_cmpxchg64_be_parallel)(CPUARMState *env, uint64_t addr, - uint64_t new_lo, uint64_t new_hi) + uint64_t new_lo, uint64_t new_hi) { - return do_paired_cmpxchg64_be(env, addr, new_lo, new_hi, true, GETPC()); + Int128 oldv, cmpv, newv; + uintptr_t ra = GETPC(); + bool success; + int mem_idx; + TCGMemOpIdx oi; + + if (!HAVE_CMPXCHG128) { + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); + } + + mem_idx = cpu_mmu_index(env, false); + oi = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx); + + /* + * High and low need to be switched here because this is not actually a + * 128bit store but two doublewords stored consecutively + */ + cmpv = int128_make128(env->exclusive_high, env->exclusive_val); + newv = int128_make128(new_hi, new_lo); + oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra); + + success = int128_eq(oldv, cmpv); + return !success; } /* Writes back the old data into Rs. */ void HELPER(casp_le_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr, uint64_t new_lo, uint64_t new_hi) { - uintptr_t ra = GETPC(); -#ifndef CONFIG_ATOMIC128 - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); -#else Int128 oldv, cmpv, newv; + uintptr_t ra = GETPC(); + int mem_idx; + TCGMemOpIdx oi; + + if (!HAVE_CMPXCHG128) { + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); + } + + mem_idx = cpu_mmu_index(env, false); + oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); cmpv = int128_make128(env->xregs[rs], env->xregs[rs + 1]); newv = int128_make128(new_lo, new_hi); - - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra); env->xregs[rs] = int128_getlo(oldv); env->xregs[rs + 1] = int128_gethi(oldv); -#endif } void HELPER(casp_be_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr, uint64_t new_hi, uint64_t new_lo) { - uintptr_t ra = GETPC(); -#ifndef CONFIG_ATOMIC128 - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); -#else Int128 oldv, cmpv, newv; + uintptr_t ra = GETPC(); + int mem_idx; + TCGMemOpIdx oi; + + if (!HAVE_CMPXCHG128) { + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); + } + + mem_idx = cpu_mmu_index(env, false); + oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); cmpv = int128_make128(env->xregs[rs + 1], env->xregs[rs]); newv = int128_make128(new_lo, new_hi); - - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra); env->xregs[rs + 1] = int128_getlo(oldv); env->xregs[rs] = int128_gethi(oldv); -#endif } /* -- 2.17.2 Reviewed-by: Emilio G. Cota <cota@braap.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/arm/helper-a64.c | 16 ++++------------ target/arm/translate-a64.c | 38 ++++++++++++++++++++++---------------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/target/arm/helper-a64.c b/target/arm/helper-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-a64.c +++ b/target/arm/helper-a64.c @@ -XXX,XX +XXX,XX @@ uint64_t HELPER(paired_cmpxchg64_le_parallel)(CPUARMState *env, uint64_t addr, int mem_idx; TCGMemOpIdx oi; - if (!HAVE_CMPXCHG128) { - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); - } + assert(HAVE_CMPXCHG128); mem_idx = cpu_mmu_index(env, false); oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); @@ -XXX,XX +XXX,XX @@ uint64_t HELPER(paired_cmpxchg64_be_parallel)(CPUARMState *env, uint64_t addr, int mem_idx; TCGMemOpIdx oi; - if (!HAVE_CMPXCHG128) { - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); - } + assert(HAVE_CMPXCHG128); mem_idx = cpu_mmu_index(env, false); oi = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx); @@ -XXX,XX +XXX,XX @@ void HELPER(casp_le_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr, int mem_idx; TCGMemOpIdx oi; - if (!HAVE_CMPXCHG128) { - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); - } + assert(HAVE_CMPXCHG128); mem_idx = cpu_mmu_index(env, false); oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); @@ -XXX,XX +XXX,XX @@ void HELPER(casp_be_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr, int mem_idx; TCGMemOpIdx oi; - if (!HAVE_CMPXCHG128) { - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); - } + assert(HAVE_CMPXCHG128); mem_idx = cpu_mmu_index(env, false); oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ #include "trace-tcg.h" #include "translate-a64.h" +#include "qemu/atomic128.h" static TCGv_i64 cpu_X[32]; static TCGv_i64 cpu_pc; @@ -XXX,XX +XXX,XX @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2, get_mem_index(s), MO_64 | MO_ALIGN | s->be_data); tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val); - } else if (s->be_data == MO_LE) { - if (tb_cflags(s->base.tb) & CF_PARALLEL) { + } else if (tb_cflags(s->base.tb) & CF_PARALLEL) { + if (!HAVE_CMPXCHG128) { + gen_helper_exit_atomic(cpu_env); + s->base.is_jmp = DISAS_NORETURN; + } else if (s->be_data == MO_LE) { gen_helper_paired_cmpxchg64_le_parallel(tmp, cpu_env, cpu_exclusive_addr, cpu_reg(s, rt), cpu_reg(s, rt2)); } else { - gen_helper_paired_cmpxchg64_le(tmp, cpu_env, cpu_exclusive_addr, - cpu_reg(s, rt), cpu_reg(s, rt2)); - } - } else { - if (tb_cflags(s->base.tb) & CF_PARALLEL) { gen_helper_paired_cmpxchg64_be_parallel(tmp, cpu_env, cpu_exclusive_addr, cpu_reg(s, rt), cpu_reg(s, rt2)); - } else { - gen_helper_paired_cmpxchg64_be(tmp, cpu_env, cpu_exclusive_addr, - cpu_reg(s, rt), cpu_reg(s, rt2)); } + } else if (s->be_data == MO_LE) { + gen_helper_paired_cmpxchg64_le(tmp, cpu_env, cpu_exclusive_addr, + cpu_reg(s, rt), cpu_reg(s, rt2)); + } else { + gen_helper_paired_cmpxchg64_be(tmp, cpu_env, cpu_exclusive_addr, + cpu_reg(s, rt), cpu_reg(s, rt2)); } } else { tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr, cpu_exclusive_val, @@ -XXX,XX +XXX,XX @@ static void gen_compare_and_swap_pair(DisasContext *s, int rs, int rt, } tcg_temp_free_i64(cmp); } else if (tb_cflags(s->base.tb) & CF_PARALLEL) { - TCGv_i32 tcg_rs = tcg_const_i32(rs); - - if (s->be_data == MO_LE) { - gen_helper_casp_le_parallel(cpu_env, tcg_rs, addr, t1, t2); + if (HAVE_CMPXCHG128) { + TCGv_i32 tcg_rs = tcg_const_i32(rs); + if (s->be_data == MO_LE) { + gen_helper_casp_le_parallel(cpu_env, tcg_rs, addr, t1, t2); + } else { + gen_helper_casp_be_parallel(cpu_env, tcg_rs, addr, t1, t2); + } + tcg_temp_free_i32(tcg_rs); } else { - gen_helper_casp_be_parallel(cpu_env, tcg_rs, addr, t1, t2); + gen_helper_exit_atomic(cpu_env); + s->base.is_jmp = DISAS_NORETURN; } - tcg_temp_free_i32(tcg_rs); } else { TCGv_i64 d1 = tcg_temp_new_i64(); TCGv_i64 d2 = tcg_temp_new_i64(); -- 2.17.2 Reviewed-by: Emilio G. Cota <cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/ppc/helper.h | 2 +- target/ppc/mem_helper.c | 33 ++++++++++-- target/ppc/translate.c | 115 +++++++++++++++++++++------------------- 3 files changed, 88 insertions(+), 62 deletions(-) diff --git a/target/ppc/helper.h b/target/ppc/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/helper.h +++ b/target/ppc/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_4(dscliq, void, env, fprp, fprp, i32) DEF_HELPER_1(tbegin, void, env) DEF_HELPER_FLAGS_1(fixup_thrm, TCG_CALL_NO_RWG, void, env) -#if defined(TARGET_PPC64) && defined(CONFIG_ATOMIC128) +#ifdef TARGET_PPC64 DEF_HELPER_FLAGS_3(lq_le_parallel, TCG_CALL_NO_WG, i64, env, tl, i32) DEF_HELPER_FLAGS_3(lq_be_parallel, TCG_CALL_NO_WG, i64, env, tl, i32) DEF_HELPER_FLAGS_5(stq_le_parallel, TCG_CALL_NO_WG, diff --git a/target/ppc/mem_helper.c b/target/ppc/mem_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/mem_helper.c +++ b/target/ppc/mem_helper.c @@ -XXX,XX +XXX,XX @@ #include "exec/cpu_ldst.h" #include "tcg.h" #include "internal.h" +#include "qemu/atomic128.h" //#define DEBUG_OP @@ -XXX,XX +XXX,XX @@ target_ulong helper_lscbx(CPUPPCState *env, target_ulong addr, uint32_t reg, return i; } -#if defined(TARGET_PPC64) && defined(CONFIG_ATOMIC128) +#ifdef TARGET_PPC64 uint64_t helper_lq_le_parallel(CPUPPCState *env, target_ulong addr, uint32_t opidx) { - Int128 ret = helper_atomic_ldo_le_mmu(env, addr, opidx, GETPC()); + Int128 ret; + + /* We will have raised EXCP_ATOMIC from the translator. */ + assert(HAVE_ATOMIC128); + ret = helper_atomic_ldo_le_mmu(env, addr, opidx, GETPC()); env->retxh = int128_gethi(ret); return int128_getlo(ret); } @@ -XXX,XX +XXX,XX @@ uint64_t helper_lq_le_parallel(CPUPPCState *env, target_ulong addr, uint64_t helper_lq_be_parallel(CPUPPCState *env, target_ulong addr, uint32_t opidx) { - Int128 ret = helper_atomic_ldo_be_mmu(env, addr, opidx, GETPC()); + Int128 ret; + + /* We will have raised EXCP_ATOMIC from the translator. */ + assert(HAVE_ATOMIC128); + ret = helper_atomic_ldo_be_mmu(env, addr, opidx, GETPC()); env->retxh = int128_gethi(ret); return int128_getlo(ret); } @@ -XXX,XX +XXX,XX @@ uint64_t helper_lq_be_parallel(CPUPPCState *env, target_ulong addr, void helper_stq_le_parallel(CPUPPCState *env, target_ulong addr, uint64_t lo, uint64_t hi, uint32_t opidx) { - Int128 val = int128_make128(lo, hi); + Int128 val; + + /* We will have raised EXCP_ATOMIC from the translator. */ + assert(HAVE_ATOMIC128); + val = int128_make128(lo, hi); helper_atomic_sto_le_mmu(env, addr, val, opidx, GETPC()); } void helper_stq_be_parallel(CPUPPCState *env, target_ulong addr, uint64_t lo, uint64_t hi, uint32_t opidx) { - Int128 val = int128_make128(lo, hi); + Int128 val; + + /* We will have raised EXCP_ATOMIC from the translator. */ + assert(HAVE_ATOMIC128); + val = int128_make128(lo, hi); helper_atomic_sto_be_mmu(env, addr, val, opidx, GETPC()); } @@ -XXX,XX +XXX,XX @@ uint32_t helper_stqcx_le_parallel(CPUPPCState *env, target_ulong addr, { bool success = false; + /* We will have raised EXCP_ATOMIC from the translator. */ + assert(HAVE_CMPXCHG128); + if (likely(addr == env->reserve_addr)) { Int128 oldv, cmpv, newv; @@ -XXX,XX +XXX,XX @@ uint32_t helper_stqcx_be_parallel(CPUPPCState *env, target_ulong addr, { bool success = false; + /* We will have raised EXCP_ATOMIC from the translator. */ + assert(HAVE_CMPXCHG128); + if (likely(addr == env->reserve_addr)) { Int128 oldv, cmpv, newv; diff --git a/target/ppc/translate.c b/target/ppc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/translate.c +++ b/target/ppc/translate.c @@ -XXX,XX +XXX,XX @@ #include "trace-tcg.h" #include "exec/translator.h" #include "exec/log.h" +#include "qemu/atomic128.h" #define CPU_SINGLE_STEP 0x1 @@ -XXX,XX +XXX,XX @@ static void gen_lq(DisasContext *ctx) hi = cpu_gpr[rd]; if (tb_cflags(ctx->base.tb) & CF_PARALLEL) { -#ifdef CONFIG_ATOMIC128 - TCGv_i32 oi = tcg_temp_new_i32(); - if (ctx->le_mode) { - tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ, ctx->mem_idx)); - gen_helper_lq_le_parallel(lo, cpu_env, EA, oi); + if (HAVE_ATOMIC128) { + TCGv_i32 oi = tcg_temp_new_i32(); + if (ctx->le_mode) { + tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ, ctx->mem_idx)); + gen_helper_lq_le_parallel(lo, cpu_env, EA, oi); + } else { + tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ, ctx->mem_idx)); + gen_helper_lq_be_parallel(lo, cpu_env, EA, oi); + } + tcg_temp_free_i32(oi); + tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh)); } else { - tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ, ctx->mem_idx)); - gen_helper_lq_be_parallel(lo, cpu_env, EA, oi); + /* Restart with exclusive lock. */ + gen_helper_exit_atomic(cpu_env); + ctx->base.is_jmp = DISAS_NORETURN; } - tcg_temp_free_i32(oi); - tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh)); -#else - /* Restart with exclusive lock. */ - gen_helper_exit_atomic(cpu_env); - ctx->base.is_jmp = DISAS_NORETURN; -#endif } else if (ctx->le_mode) { tcg_gen_qemu_ld_i64(lo, EA, ctx->mem_idx, MO_LEQ); gen_addr_add(ctx, EA, EA, 8); @@ -XXX,XX +XXX,XX @@ static void gen_std(DisasContext *ctx) hi = cpu_gpr[rs]; if (tb_cflags(ctx->base.tb) & CF_PARALLEL) { -#ifdef CONFIG_ATOMIC128 - TCGv_i32 oi = tcg_temp_new_i32(); - if (ctx->le_mode) { - tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ, ctx->mem_idx)); - gen_helper_stq_le_parallel(cpu_env, EA, lo, hi, oi); + if (HAVE_ATOMIC128) { + TCGv_i32 oi = tcg_temp_new_i32(); + if (ctx->le_mode) { + tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ, ctx->mem_idx)); + gen_helper_stq_le_parallel(cpu_env, EA, lo, hi, oi); + } else { + tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ, ctx->mem_idx)); + gen_helper_stq_be_parallel(cpu_env, EA, lo, hi, oi); + } + tcg_temp_free_i32(oi); } else { - tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ, ctx->mem_idx)); - gen_helper_stq_be_parallel(cpu_env, EA, lo, hi, oi); + /* Restart with exclusive lock. */ + gen_helper_exit_atomic(cpu_env); + ctx->base.is_jmp = DISAS_NORETURN; } - tcg_temp_free_i32(oi); -#else - /* Restart with exclusive lock. */ - gen_helper_exit_atomic(cpu_env); - ctx->base.is_jmp = DISAS_NORETURN; -#endif } else if (ctx->le_mode) { tcg_gen_qemu_st_i64(lo, EA, ctx->mem_idx, MO_LEQ); gen_addr_add(ctx, EA, EA, 8); @@ -XXX,XX +XXX,XX @@ static void gen_lqarx(DisasContext *ctx) hi = cpu_gpr[rd]; if (tb_cflags(ctx->base.tb) & CF_PARALLEL) { -#ifdef CONFIG_ATOMIC128 - TCGv_i32 oi = tcg_temp_new_i32(); - if (ctx->le_mode) { - tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ | MO_ALIGN_16, - ctx->mem_idx)); - gen_helper_lq_le_parallel(lo, cpu_env, EA, oi); + if (HAVE_ATOMIC128) { + TCGv_i32 oi = tcg_temp_new_i32(); + if (ctx->le_mode) { + tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ | MO_ALIGN_16, + ctx->mem_idx)); + gen_helper_lq_le_parallel(lo, cpu_env, EA, oi); + } else { + tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ | MO_ALIGN_16, + ctx->mem_idx)); + gen_helper_lq_be_parallel(lo, cpu_env, EA, oi); + } + tcg_temp_free_i32(oi); + tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh)); } else { - tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ | MO_ALIGN_16, - ctx->mem_idx)); - gen_helper_lq_be_parallel(lo, cpu_env, EA, oi); + /* Restart with exclusive lock. */ + gen_helper_exit_atomic(cpu_env); + ctx->base.is_jmp = DISAS_NORETURN; + tcg_temp_free(EA); + return; } - tcg_temp_free_i32(oi); - tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh)); -#else - /* Restart with exclusive lock. */ - gen_helper_exit_atomic(cpu_env); - ctx->base.is_jmp = DISAS_NORETURN; - tcg_temp_free(EA); - return; -#endif } else if (ctx->le_mode) { tcg_gen_qemu_ld_i64(lo, EA, ctx->mem_idx, MO_LEQ | MO_ALIGN_16); tcg_gen_mov_tl(cpu_reserve, EA); @@ -XXX,XX +XXX,XX @@ static void gen_stqcx_(DisasContext *ctx) hi = cpu_gpr[rs]; if (tb_cflags(ctx->base.tb) & CF_PARALLEL) { - TCGv_i32 oi = tcg_const_i32(DEF_MEMOP(MO_Q) | MO_ALIGN_16); -#ifdef CONFIG_ATOMIC128 - if (ctx->le_mode) { - gen_helper_stqcx_le_parallel(cpu_crf[0], cpu_env, EA, lo, hi, oi); + if (HAVE_CMPXCHG128) { + TCGv_i32 oi = tcg_const_i32(DEF_MEMOP(MO_Q) | MO_ALIGN_16); + if (ctx->le_mode) { + gen_helper_stqcx_le_parallel(cpu_crf[0], cpu_env, + EA, lo, hi, oi); + } else { + gen_helper_stqcx_be_parallel(cpu_crf[0], cpu_env, + EA, lo, hi, oi); + } + tcg_temp_free_i32(oi); } else { - gen_helper_stqcx_le_parallel(cpu_crf[0], cpu_env, EA, lo, hi, oi); + /* Restart with exclusive lock. */ + gen_helper_exit_atomic(cpu_env); + ctx->base.is_jmp = DISAS_NORETURN; } -#else - /* Restart with exclusive lock. */ - gen_helper_exit_atomic(cpu_env); - ctx->base.is_jmp = DISAS_NORETURN; -#endif tcg_temp_free(EA); - tcg_temp_free_i32(oi); } else { TCGLabel *lab_fail = gen_new_label(); TCGLabel *lab_over = gen_new_label(); -- 2.17.2 Reviewed-by: David Hildenbrand <david@redhat.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/s390x/mem_helper.c | 92 +++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 51 deletions(-) diff --git a/target/s390x/mem_helper.c b/target/s390x/mem_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/mem_helper.c +++ b/target/s390x/mem_helper.c @@ -XXX,XX +XXX,XX @@ #include "exec/exec-all.h" #include "exec/cpu_ldst.h" #include "qemu/int128.h" +#include "qemu/atomic128.h" #if !defined(CONFIG_USER_ONLY) #include "hw/s390x/storage-keys.h" @@ -XXX,XX +XXX,XX @@ static void do_cdsg(CPUS390XState *env, uint64_t addr, bool fail; if (parallel) { -#ifndef CONFIG_ATOMIC128 +#if !HAVE_CMPXCHG128 cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); #else int mem_idx = cpu_mmu_index(env, false); @@ -XXX,XX +XXX,XX @@ void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr, static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, uint64_t a2, bool parallel) { -#if !defined(CONFIG_USER_ONLY) || defined(CONFIG_ATOMIC128) uint32_t mem_idx = cpu_mmu_index(env, false); -#endif uintptr_t ra = GETPC(); uint32_t fc = extract32(env->regs[0], 0, 8); uint32_t sc = extract32(env->regs[0], 8, 8); @@ -XXX,XX +XXX,XX @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, probe_write(env, a2, 0, mem_idx, ra); #endif - /* Note that the compare-and-swap is atomic, and the store is atomic, but - the complete operation is not. Therefore we do not need to assert serial - context in order to implement this. That said, restart early if we can't - support either operation that is supposed to be atomic. */ + /* + * Note that the compare-and-swap is atomic, and the store is atomic, + * but the complete operation is not. Therefore we do not need to + * assert serial context in order to implement this. That said, + * restart early if we can't support either operation that is supposed + * to be atomic. + */ if (parallel) { - int mask = 0; -#if !defined(CONFIG_ATOMIC64) - mask = -8; -#elif !defined(CONFIG_ATOMIC128) - mask = -16; + uint32_t max = 2; +#ifdef CONFIG_ATOMIC64 + max = 3; #endif - if (((4 << fc) | (1 << sc)) & mask) { + if ((HAVE_CMPXCHG128 ? 0 : fc + 2 > max) || + (HAVE_ATOMIC128 ? 0 : sc > max)) { cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); } } @@ -XXX,XX +XXX,XX @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, Int128 cv = int128_make128(env->regs[r3 + 1], env->regs[r3]); Int128 ov; - if (parallel) { -#ifdef CONFIG_ATOMIC128 - TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); - ov = helper_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi, ra); - cc = !int128_eq(ov, cv); -#else - /* Note that we asserted !parallel above. */ - g_assert_not_reached(); -#endif - } else { + if (!parallel) { uint64_t oh = cpu_ldq_data_ra(env, a1 + 0, ra); uint64_t ol = cpu_ldq_data_ra(env, a1 + 8, ra); @@ -XXX,XX +XXX,XX @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, cpu_stq_data_ra(env, a1 + 0, int128_gethi(nv), ra); cpu_stq_data_ra(env, a1 + 8, int128_getlo(nv), ra); + } else if (HAVE_CMPXCHG128) { + TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); + ov = helper_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi, ra); + cc = !int128_eq(ov, cv); + } else { + /* Note that we asserted !parallel above. */ + g_assert_not_reached(); } env->regs[r3 + 0] = int128_gethi(ov); @@ -XXX,XX +XXX,XX @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, cpu_stq_data_ra(env, a2, svh, ra); break; case 4: - if (parallel) { -#ifdef CONFIG_ATOMIC128 + if (!parallel) { + cpu_stq_data_ra(env, a2 + 0, svh, ra); + cpu_stq_data_ra(env, a2 + 8, svl, ra); + } else if (HAVE_ATOMIC128) { TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); Int128 sv = int128_make128(svl, svh); helper_atomic_sto_be_mmu(env, a2, sv, oi, ra); -#else + } else { /* Note that we asserted !parallel above. */ g_assert_not_reached(); -#endif - } else { - cpu_stq_data_ra(env, a2 + 0, svh, ra); - cpu_stq_data_ra(env, a2 + 8, svl, ra); } break; default: @@ -XXX,XX +XXX,XX @@ static uint64_t do_lpq(CPUS390XState *env, uint64_t addr, bool parallel) uintptr_t ra = GETPC(); uint64_t hi, lo; - if (parallel) { -#ifndef CONFIG_ATOMIC128 - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); -#else + if (!parallel) { + check_alignment(env, addr, 16, ra); + hi = cpu_ldq_data_ra(env, addr + 0, ra); + lo = cpu_ldq_data_ra(env, addr + 8, ra); + } else if (HAVE_ATOMIC128) { int mem_idx = cpu_mmu_index(env, false); TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); Int128 v = helper_atomic_ldo_be_mmu(env, addr, oi, ra); hi = int128_gethi(v); lo = int128_getlo(v); -#endif } else { - check_alignment(env, addr, 16, ra); - - hi = cpu_ldq_data_ra(env, addr + 0, ra); - lo = cpu_ldq_data_ra(env, addr + 8, ra); + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); } env->retxl = lo; @@ -XXX,XX +XXX,XX @@ static void do_stpq(CPUS390XState *env, uint64_t addr, { uintptr_t ra = GETPC(); - if (parallel) { -#ifndef CONFIG_ATOMIC128 - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); -#else - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); - - Int128 v = int128_make128(low, high); - helper_atomic_sto_be_mmu(env, addr, v, oi, ra); -#endif - } else { + if (!parallel) { check_alignment(env, addr, 16, ra); - cpu_stq_data_ra(env, addr + 0, high, ra); cpu_stq_data_ra(env, addr + 8, low, ra); + } else if (HAVE_ATOMIC128) { + int mem_idx = cpu_mmu_index(env, false); + TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); + Int128 v = int128_make128(low, high); + helper_atomic_sto_be_mmu(env, addr, v, oi, ra); + } else { + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); } } -- 2.17.2 Reviewed-by: David Hildenbrand <david@redhat.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/s390x/mem_helper.c | 128 ++++++++++++++++++-------------------- 1 file changed, 61 insertions(+), 67 deletions(-) diff --git a/target/s390x/mem_helper.c b/target/s390x/mem_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/mem_helper.c +++ b/target/s390x/mem_helper.c @@ -XXX,XX +XXX,XX @@ uint32_t HELPER(trXX)(CPUS390XState *env, uint32_t r1, uint32_t r2, return cc; } -static void do_cdsg(CPUS390XState *env, uint64_t addr, - uint32_t r1, uint32_t r3, bool parallel) +void HELPER(cdsg)(CPUS390XState *env, uint64_t addr, + uint32_t r1, uint32_t r3) { uintptr_t ra = GETPC(); Int128 cmpv = int128_make128(env->regs[r1 + 1], env->regs[r1]); Int128 newv = int128_make128(env->regs[r3 + 1], env->regs[r3]); Int128 oldv; + uint64_t oldh, oldl; bool fail; - if (parallel) { -#if !HAVE_CMPXCHG128 - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); -#else - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); - oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra); - fail = !int128_eq(oldv, cmpv); -#endif - } else { - uint64_t oldh, oldl; + check_alignment(env, addr, 16, ra); - check_alignment(env, addr, 16, ra); + oldh = cpu_ldq_data_ra(env, addr + 0, ra); + oldl = cpu_ldq_data_ra(env, addr + 8, ra); - oldh = cpu_ldq_data_ra(env, addr + 0, ra); - oldl = cpu_ldq_data_ra(env, addr + 8, ra); - - oldv = int128_make128(oldl, oldh); - fail = !int128_eq(oldv, cmpv); - if (fail) { - newv = oldv; - } - - cpu_stq_data_ra(env, addr + 0, int128_gethi(newv), ra); - cpu_stq_data_ra(env, addr + 8, int128_getlo(newv), ra); + oldv = int128_make128(oldl, oldh); + fail = !int128_eq(oldv, cmpv); + if (fail) { + newv = oldv; } + cpu_stq_data_ra(env, addr + 0, int128_gethi(newv), ra); + cpu_stq_data_ra(env, addr + 8, int128_getlo(newv), ra); + env->cc_op = fail; env->regs[r1] = int128_gethi(oldv); env->regs[r1 + 1] = int128_getlo(oldv); } -void HELPER(cdsg)(CPUS390XState *env, uint64_t addr, - uint32_t r1, uint32_t r3) -{ - do_cdsg(env, addr, r1, r3, false); -} - void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr, uint32_t r1, uint32_t r3) { - do_cdsg(env, addr, r1, r3, true); + uintptr_t ra = GETPC(); + Int128 cmpv = int128_make128(env->regs[r1 + 1], env->regs[r1]); + Int128 newv = int128_make128(env->regs[r3 + 1], env->regs[r3]); + int mem_idx; + TCGMemOpIdx oi; + Int128 oldv; + bool fail; + + if (!HAVE_CMPXCHG128) { + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); + } + + mem_idx = cpu_mmu_index(env, false); + oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); + oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra); + fail = !int128_eq(oldv, cmpv); + + env->cc_op = fail; + env->regs[r1] = int128_gethi(oldv); + env->regs[r1 + 1] = int128_getlo(oldv); } static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, @@ -XXX,XX +XXX,XX @@ uint64_t HELPER(lra)(CPUS390XState *env, uint64_t addr) #endif /* load pair from quadword */ -static uint64_t do_lpq(CPUS390XState *env, uint64_t addr, bool parallel) +uint64_t HELPER(lpq)(CPUS390XState *env, uint64_t addr) { uintptr_t ra = GETPC(); uint64_t hi, lo; - if (!parallel) { - check_alignment(env, addr, 16, ra); - hi = cpu_ldq_data_ra(env, addr + 0, ra); - lo = cpu_ldq_data_ra(env, addr + 8, ra); - } else if (HAVE_ATOMIC128) { + check_alignment(env, addr, 16, ra); + hi = cpu_ldq_data_ra(env, addr + 0, ra); + lo = cpu_ldq_data_ra(env, addr + 8, ra); + + env->retxl = lo; + return hi; +} + +uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr) +{ + uintptr_t ra = GETPC(); + uint64_t hi, lo; + + if (HAVE_ATOMIC128) { int mem_idx = cpu_mmu_index(env, false); TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); Int128 v = helper_atomic_ldo_be_mmu(env, addr, oi, ra); @@ -XXX,XX +XXX,XX @@ static uint64_t do_lpq(CPUS390XState *env, uint64_t addr, bool parallel) return hi; } -uint64_t HELPER(lpq)(CPUS390XState *env, uint64_t addr) -{ - return do_lpq(env, addr, false); -} - -uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr) -{ - return do_lpq(env, addr, true); -} - /* store pair to quadword */ -static void do_stpq(CPUS390XState *env, uint64_t addr, - uint64_t low, uint64_t high, bool parallel) +void HELPER(stpq)(CPUS390XState *env, uint64_t addr, + uint64_t low, uint64_t high) { uintptr_t ra = GETPC(); - if (!parallel) { - check_alignment(env, addr, 16, ra); - cpu_stq_data_ra(env, addr + 0, high, ra); - cpu_stq_data_ra(env, addr + 8, low, ra); - } else if (HAVE_ATOMIC128) { + check_alignment(env, addr, 16, ra); + cpu_stq_data_ra(env, addr + 0, high, ra); + cpu_stq_data_ra(env, addr + 8, low, ra); +} + +void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr, + uint64_t low, uint64_t high) +{ + uintptr_t ra = GETPC(); + + if (HAVE_ATOMIC128) { int mem_idx = cpu_mmu_index(env, false); TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); Int128 v = int128_make128(low, high); @@ -XXX,XX +XXX,XX @@ static void do_stpq(CPUS390XState *env, uint64_t addr, } } -void HELPER(stpq)(CPUS390XState *env, uint64_t addr, - uint64_t low, uint64_t high) -{ - do_stpq(env, addr, low, high, false); -} - -void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr, - uint64_t low, uint64_t high) -{ - do_stpq(env, addr, low, high, true); -} - /* Execute instruction. This instruction executes an insn modified with the contents of r1. It does not change the executed instruction in memory; it does not change the program counter. -- 2.17.2 When op raises an exception, it may not have initialized the output temps that would be written back by wout or cout. Reviewed-by: David Hildenbrand <david@redhat.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/s390x/translate.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/target/s390x/translate.c b/target/s390x/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/translate.c +++ b/target/s390x/translate.c @@ -XXX,XX +XXX,XX @@ struct DisasInsn { const char *name; + /* Pre-process arguments before HELP_OP. */ void (*help_in1)(DisasContext *, DisasFields *, DisasOps *); void (*help_in2)(DisasContext *, DisasFields *, DisasOps *); void (*help_prep)(DisasContext *, DisasFields *, DisasOps *); + + /* + * Post-process output after HELP_OP. + * Note that these are not called if HELP_OP returns DISAS_NORETURN. + */ void (*help_wout)(DisasContext *, DisasFields *, DisasOps *); void (*help_cout)(DisasContext *, DisasOps *); + + /* Implement the operation itself. */ DisasJumpType (*help_op)(DisasContext *, DisasOps *); uint64_t data; @@ -XXX,XX +XXX,XX @@ static DisasJumpType translate_one(CPUS390XState *env, DisasContext *s) if (insn->help_op) { ret = insn->help_op(s, &o); } - if (insn->help_wout) { - insn->help_wout(s, &f, &o); - } - if (insn->help_cout) { - insn->help_cout(s, &o); + if (ret != DISAS_NORETURN) { + if (insn->help_wout) { + insn->help_wout(s, &f, &o); + } + if (insn->help_cout) { + insn->help_cout(s, &o); + } } /* Free any temporaries created by the helpers. */ -- 2.17.2 Reviewed-by: David Hildenbrand <david@redhat.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/s390x/mem_helper.c | 40 +++++++++++++++++++-------------------- target/s390x/translate.c | 25 +++++++++++++++++------- 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/target/s390x/mem_helper.c b/target/s390x/mem_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/mem_helper.c +++ b/target/s390x/mem_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr, Int128 oldv; bool fail; - if (!HAVE_CMPXCHG128) { - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); - } + assert(HAVE_CMPXCHG128); mem_idx = cpu_mmu_index(env, false); oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); @@ -XXX,XX +XXX,XX @@ uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr) { uintptr_t ra = GETPC(); uint64_t hi, lo; + int mem_idx; + TCGMemOpIdx oi; + Int128 v; - if (HAVE_ATOMIC128) { - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); - Int128 v = helper_atomic_ldo_be_mmu(env, addr, oi, ra); - hi = int128_gethi(v); - lo = int128_getlo(v); - } else { - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); - } + assert(HAVE_ATOMIC128); + + mem_idx = cpu_mmu_index(env, false); + oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); + v = helper_atomic_ldo_be_mmu(env, addr, oi, ra); + hi = int128_gethi(v); + lo = int128_getlo(v); env->retxl = lo; return hi; @@ -XXX,XX +XXX,XX @@ void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr, uint64_t low, uint64_t high) { uintptr_t ra = GETPC(); + int mem_idx; + TCGMemOpIdx oi; + Int128 v; - if (HAVE_ATOMIC128) { - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); - Int128 v = int128_make128(low, high); - helper_atomic_sto_be_mmu(env, addr, v, oi, ra); - } else { - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); - } + assert(HAVE_ATOMIC128); + + mem_idx = cpu_mmu_index(env, false); + oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); + v = int128_make128(low, high); + helper_atomic_sto_be_mmu(env, addr, v, oi, ra); } /* Execute instruction. This instruction executes an insn modified with diff --git a/target/s390x/translate.c b/target/s390x/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/translate.c +++ b/target/s390x/translate.c @@ -XXX,XX +XXX,XX @@ #include "trace-tcg.h" #include "exec/translator.h" #include "exec/log.h" +#include "qemu/atomic128.h" /* Information that (most) every instruction needs to manipulate. */ @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_cdsg(DisasContext *s, DisasOps *o) int r3 = get_field(s->fields, r3); int d2 = get_field(s->fields, d2); int b2 = get_field(s->fields, b2); + DisasJumpType ret = DISAS_NEXT; TCGv_i64 addr; TCGv_i32 t_r1, t_r3; @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_cdsg(DisasContext *s, DisasOps *o) addr = get_address(s, 0, b2, d2); t_r1 = tcg_const_i32(r1); t_r3 = tcg_const_i32(r3); - if (tb_cflags(s->base.tb) & CF_PARALLEL) { + if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { + gen_helper_cdsg(cpu_env, addr, t_r1, t_r3); + } else if (HAVE_CMPXCHG128) { gen_helper_cdsg_parallel(cpu_env, addr, t_r1, t_r3); } else { - gen_helper_cdsg(cpu_env, addr, t_r1, t_r3); + gen_helper_exit_atomic(cpu_env); + ret = DISAS_NORETURN; } tcg_temp_free_i64(addr); tcg_temp_free_i32(t_r1); tcg_temp_free_i32(t_r3); set_cc_static(s); - return DISAS_NEXT; + return ret; } static DisasJumpType op_csst(DisasContext *s, DisasOps *o) @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_lpd(DisasContext *s, DisasOps *o) static DisasJumpType op_lpq(DisasContext *s, DisasOps *o) { - if (tb_cflags(s->base.tb) & CF_PARALLEL) { + if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { + gen_helper_lpq(o->out, cpu_env, o->in2); + } else if (HAVE_ATOMIC128) { gen_helper_lpq_parallel(o->out, cpu_env, o->in2); } else { - gen_helper_lpq(o->out, cpu_env, o->in2); + gen_helper_exit_atomic(cpu_env); + return DISAS_NORETURN; } return_low128(o->out2); return DISAS_NEXT; @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_stmh(DisasContext *s, DisasOps *o) static DisasJumpType op_stpq(DisasContext *s, DisasOps *o) { - if (tb_cflags(s->base.tb) & CF_PARALLEL) { + if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { + gen_helper_stpq(cpu_env, o->in2, o->out2, o->out); + } else if (HAVE_ATOMIC128) { gen_helper_stpq_parallel(cpu_env, o->in2, o->out2, o->out); } else { - gen_helper_stpq(cpu_env, o->in2, o->out2, o->out); + gen_helper_exit_atomic(cpu_env); + return DISAS_NORETURN; } return DISAS_NEXT; } -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> Updates can come from other threads, so readers that do not take tlb_lock must use atomic_read to avoid undefined behaviour (UB). This completes the conversion to tlb_lock. This conversion results on average in no performance loss, as the following experiments (run on an Intel i7-6700K CPU @ 4.00GHz) show. 1. aarch64 bootup+shutdown test: - Before: Performance counter stats for 'taskset -c 0 ../img/aarch64/die.sh' (10 runs): 7487.087786 task-clock (msec) # 0.998 CPUs utilized ( +- 0.12% ) 31,574,905,303 cycles # 4.217 GHz ( +- 0.12% ) 57,097,908,812 instructions # 1.81 insns per cycle ( +- 0.08% ) 10,255,415,367 branches # 1369.747 M/sec ( +- 0.08% ) 173,278,962 branch-misses # 1.69% of all branches ( +- 0.18% ) 7.504481349 seconds time elapsed ( +- 0.14% ) - After: Performance counter stats for 'taskset -c 0 ../img/aarch64/die.sh' (10 runs): 7462.441328 task-clock (msec) # 0.998 CPUs utilized ( +- 0.07% ) 31,478,476,520 cycles # 4.218 GHz ( +- 0.07% ) 57,017,330,084 instructions # 1.81 insns per cycle ( +- 0.05% ) 10,251,929,667 branches # 1373.804 M/sec ( +- 0.05% ) 173,023,787 branch-misses # 1.69% of all branches ( +- 0.11% ) 7.474970463 seconds time elapsed ( +- 0.07% ) 2. SPEC06int: SPEC06int (test set) [Y axis: Speedup over master] 1.15 +-+----+------+------+------+------+------+-------+------+------+------+------+------+------+----+-+ | | 1.1 +-+.................................+++.............................+ tlb-lock-v2 (m+++x) +-+ | +++ | +++ tlb-lock-v3 (spinl|ck) | | +++ | | +++ +++ | | | 1.05 +-+....+++...........####.........|####.+++.|......|.....###....+++...........+++....###.........+-+ | ### ++#| # |# |# ***### +++### +++#+# | +++ | #|# ### | 1 +-+++***+#++++####+++#++#++++++++++#++#+*+*++#++++#+#+****+#++++###++++###++++###++++#+#++++#+#+++-+ | *+* # #++# *** # #### *** # * *++# ****+# *| * # ****|# |# # #|# #+# # # | 0.95 +-+..*.*.#....#..#.*|*..#...#..#.*|*..#.*.*..#.*|.*.#.*++*.#.*++*+#.****.#....#+#....#.#..++#.#..+-+ | * * # # # *|* # # # *|* # * * # *++* # * * # * * # * |* # ++# # # # *** # | | * * # ++# # *+* # # # *|* # * * # * * # * * # * * # *++* # **** # ++# # * * # | 0.9 +-+..*.*.#...|#..#.*.*..#.++#..#.*|*..#.*.*..#.*..*.#.*..*.#.*..*.#.*..*.#.*.|*.#...|#.#..*.*.#..+-+ | * * # *** # * * # |# # *+* # * * # * * # * * # * * # * * # *++* # |# # * * # | 0.85 +-+..*.*.#..*|*..#.*.*..#.***..#.*.*..#.*.*..#.*..*.#.*..*.#.*..*.#.*..*.#.*..*.#.****.#..*.*.#..+-+ | * * # *+* # * * # *|* # * * # * * # * * # * * # * * # * * # * * # * |* # * * # | | * * # * * # * * # *+* # * * # * * # * * # * * # * * # * * # * * # * |* # * * # | 0.8 +-+..*.*.#..*.*..#.*.*..#.*.*..#.*.*..#.*.*..#.*..*.#.*..*.#.*..*.#.*..*.#.*..*.#.*++*.#..*.*.#..+-+ | * * # * * # * * # * * # * * # * * # * * # * * # * * # * * # * * # * * # * * # | 0.75 +-+--***##--***###-***###-***###-***###-***###-****##-****##-****##-****##-****##-****##--***##--+-+ 400.perlben401.bzip2403.gcc429.m445.gob456.hmme45462.libqua464.h26471.omnet473483.xalancbmkgeomean png: https://imgur.com/a/BHzpPTW Notes: - tlb-lock-v2 corresponds to an implementation with a mutex. - tlb-lock-v3 corresponds to the current implementation, i.e. a spinlock and a single lock acquisition in tlb_set_page_with_attrs. Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181016153840.25877-1-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/softmmu_template.h | 12 ++++++------ include/exec/cpu_ldst.h | 11 ++++++++++- include/exec/cpu_ldst_template.h | 2 +- accel/tcg/cputlb.c | 19 +++++++++++++------ 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/accel/tcg/softmmu_template.h b/accel/tcg/softmmu_template.h index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/softmmu_template.h +++ b/accel/tcg/softmmu_template.h @@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, uintptr_t mmu_idx = get_mmuidx(oi); uintptr_t index = tlb_index(env, mmu_idx, addr); CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); - target_ulong tlb_addr = entry->addr_write; + target_ulong tlb_addr = tlb_addr_write(entry); unsigned a_bits = get_alignment_bits(get_memop(oi)); uintptr_t haddr; @@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); } - tlb_addr = entry->addr_write & ~TLB_INVALID_MASK; + tlb_addr = tlb_addr_write(entry) & ~TLB_INVALID_MASK; } /* Handle an IO access. */ @@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, cannot evict the first. */ page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK; entry2 = tlb_entry(env, mmu_idx, page2); - if (!tlb_hit_page(entry2->addr_write, page2) + if (!tlb_hit_page(tlb_addr_write(entry2), page2) && !VICTIM_TLB_HIT(addr_write, page2)) { tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); @@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, uintptr_t mmu_idx = get_mmuidx(oi); uintptr_t index = tlb_index(env, mmu_idx, addr); CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); - target_ulong tlb_addr = entry->addr_write; + target_ulong tlb_addr = tlb_addr_write(entry); unsigned a_bits = get_alignment_bits(get_memop(oi)); uintptr_t haddr; @@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); } - tlb_addr = entry->addr_write & ~TLB_INVALID_MASK; + tlb_addr = tlb_addr_write(entry) & ~TLB_INVALID_MASK; } /* Handle an IO access. */ @@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, cannot evict the first. */ page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK; entry2 = tlb_entry(env, mmu_idx, page2); - if (!tlb_hit_page(entry2->addr_write, page2) + if (!tlb_hit_page(tlb_addr_write(entry2), page2) && !VICTIM_TLB_HIT(addr_write, page2)) { tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu_ldst.h +++ b/include/exec/cpu_ldst.h @@ -XXX,XX +XXX,XX @@ extern __thread uintptr_t helper_retaddr; /* The memory helpers for tcg-generated code need tcg_target_long etc. */ #include "tcg.h" +static inline target_ulong tlb_addr_write(const CPUTLBEntry *entry) +{ +#if TCG_OVERSIZED_GUEST + return entry->addr_write; +#else + return atomic_read(&entry->addr_write); +#endif +} + /* Find the TLB index corresponding to the mmu_idx + address pair. */ static inline uintptr_t tlb_index(CPUArchState *env, uintptr_t mmu_idx, target_ulong addr) @@ -XXX,XX +XXX,XX @@ static inline void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr, tlb_addr = tlbentry->addr_read; break; case 1: - tlb_addr = tlbentry->addr_write; + tlb_addr = tlb_addr_write(tlbentry); break; case 2: tlb_addr = tlbentry->addr_code; diff --git a/include/exec/cpu_ldst_template.h b/include/exec/cpu_ldst_template.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu_ldst_template.h +++ b/include/exec/cpu_ldst_template.h @@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env, addr = ptr; mmu_idx = CPU_MMU_INDEX; entry = tlb_entry(env, mmu_idx, addr); - if (unlikely(entry->addr_write != + if (unlikely(tlb_addr_write(entry) != (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) { oi = make_memop_idx(SHIFT, mmu_idx); glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(env, addr, v, oi, diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ static inline bool tlb_hit_page_anyprot(CPUTLBEntry *tlb_entry, target_ulong page) { return tlb_hit_page(tlb_entry->addr_read, page) || - tlb_hit_page(tlb_entry->addr_write, page) || + tlb_hit_page(tlb_addr_write(tlb_entry), page) || tlb_hit_page(tlb_entry->addr_code, page); } @@ -XXX,XX +XXX,XX @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry, tlb_fill(cpu, addr, size, MMU_DATA_STORE, mmu_idx, retaddr); entry = tlb_entry(env, mmu_idx, addr); - tlb_addr = entry->addr_write; + tlb_addr = tlb_addr_write(entry); if (!(tlb_addr & ~(TARGET_PAGE_MASK | TLB_RECHECK))) { /* RAM access */ uintptr_t haddr = addr + entry->addend; @@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index, assert_cpu_is_self(ENV_GET_CPU(env)); for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) { CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx]; - target_ulong cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs); + target_ulong cmp; + + /* elt_ofs might correspond to .addr_write, so use atomic_read */ +#if TCG_OVERSIZED_GUEST + cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs); +#else + cmp = atomic_read((target_ulong *)((uintptr_t)vtlb + elt_ofs)); +#endif if (cmp == page) { /* Found entry in victim tlb, swap tlb and iotlb. */ @@ -XXX,XX +XXX,XX @@ void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, uintptr_t index = tlb_index(env, mmu_idx, addr); CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); - if (!tlb_hit(entry->addr_write, addr)) { + if (!tlb_hit(tlb_addr_write(entry), addr)) { /* TLB entry is for a different page */ if (!VICTIM_TLB_HIT(addr_write, addr)) { tlb_fill(ENV_GET_CPU(env), addr, size, MMU_DATA_STORE, @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, size_t mmu_idx = get_mmuidx(oi); uintptr_t index = tlb_index(env, mmu_idx, addr); CPUTLBEntry *tlbe = tlb_entry(env, mmu_idx, addr); - target_ulong tlb_addr = tlbe->addr_write; + target_ulong tlb_addr = tlb_addr_write(tlbe); TCGMemOp mop = get_memop(oi); int a_bits = get_alignment_bits(mop); int s_bits = mop & MO_SIZE; @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, tlb_fill(ENV_GET_CPU(env), addr, 1 << s_bits, MMU_DATA_STORE, mmu_idx, retaddr); } - tlb_addr = tlbe->addr_write & ~TLB_INVALID_MASK; + tlb_addr = tlb_addr_write(tlbe) & ~TLB_INVALID_MASK; } /* Notice an IO access or a needs-MMU-lookup access */ -- 2.17.2