The following changes since commit ff56877e911782dedc9a424233fd3f62369c258c: Merge remote-tracking branch 'remotes/kraxel/tags/vga-20181015-pull-request' into staging (2018-10-15 15:03:45 +0100) are available in the Git repository at: https://github.com/rth7680/qemu.git tags/pull-tcg-20181016 for you to fetch changes up to e3e9d1ea20c75718ce7c528c588a0a497f12f750: cputlb: read CPUTLBEntry.addr_write atomically (2018-10-16 10:04:27 -0700) ---------------------------------------------------------------- Queued tcg patches ---------------------------------------------------------------- Emilio G. Cota (10): tcg: access cpu->icount_decr.u16.high with atomics tcg: fix use of uninitialized variable under CONFIG_PROFILER tcg: plug holes in struct TCGProfile tcg: distribute tcg_time into TCG contexts target/alpha: remove tlb_flush from alpha_cpu_initfn target/unicore32: remove tlb_flush from uc32_init_fn exec: introduce tlb_init cputlb: fix assert_cpu_is_self macro cputlb: serialize tlb updates with env->tlb_lock cputlb: read CPUTLBEntry.addr_write atomically Richard Henderson (11): tcg: Implement CPU_LOG_TB_NOCHAIN during expansion tcg: Add tlb_index and tlb_entry helpers tcg: Split CONFIG_ATOMIC128 target/i386: Convert to HAVE_CMPXCHG128 target/arm: Convert to HAVE_CMPXCHG128 target/arm: Check HAVE_CMPXCHG128 at translate time target/ppc: Convert to HAVE_CMPXCHG128 and HAVE_ATOMIC128 target/s390x: Convert to HAVE_CMPXCHG128 and HAVE_ATOMIC128 target/s390x: Split do_cdsg, do_lpq, do_stpq target/s390x: Skip wout, cout helpers if op helper does not return target/s390x: Check HAVE_ATOMIC128 and HAVE_CMPXCHG128 at translate accel/tcg/atomic_template.h | 20 +++- accel/tcg/softmmu_template.h | 64 +++++----- include/exec/cpu-defs.h | 3 + include/exec/cpu_ldst.h | 30 ++++- include/exec/cpu_ldst_template.h | 25 ++-- include/exec/exec-all.h | 8 ++ include/qemu/atomic128.h | 155 ++++++++++++++++++++++++ include/qemu/timer.h | 1 - target/ppc/helper.h | 2 +- tcg/tcg.h | 20 ++-- accel/tcg/cpu-exec.c | 2 +- accel/tcg/cputlb.c | 235 +++++++++++++++++++----------------- accel/tcg/tcg-all.c | 2 +- accel/tcg/translate-all.c | 2 +- accel/tcg/user-exec.c | 5 +- cpus.c | 3 +- exec.c | 1 + monitor.c | 13 +- qom/cpu.c | 2 +- target/alpha/cpu.c | 1 - target/arm/helper-a64.c | 251 +++++++++++++++++++-------------------- target/arm/translate-a64.c | 38 +++--- target/i386/mem_helper.c | 9 +- target/ppc/mem_helper.c | 33 ++++- target/ppc/translate.c | 115 +++++++++--------- target/s390x/mem_helper.c | 202 +++++++++++++++---------------- target/s390x/translate.c | 45 +++++-- target/unicore32/cpu.c | 2 - tcg/tcg-op.c | 9 +- tcg/tcg.c | 25 +++- configure | 19 +++ 31 files changed, 830 insertions(+), 512 deletions(-) create mode 100644 include/qemu/atomic128.h Rather than test NOCHAIN before linking, do not emit the goto_tb opcode at all. We already do this for goto_ptr. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cpu-exec.c | 2 +- tcg/tcg-op.c | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ static inline TranslationBlock *tb_find(CPUState *cpu, } #endif /* See if we can patch the calling TB. */ - if (last_tb && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { + if (last_tb) { tb_add_jump(last_tb, tb_exit, tb); } return tb; diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -XXX,XX +XXX,XX @@ void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx) seen this numbered exit before, via tcg_gen_goto_tb. */ tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx)); #endif + /* When not chaining, exit without indicating a link. */ + if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { + val = 0; + } } else { /* This is an exit via the exitreq label. */ tcg_debug_assert(idx == TB_EXIT_REQUESTED); @@ -XXX,XX +XXX,XX @@ void tcg_gen_goto_tb(unsigned idx) tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0); tcg_ctx->goto_tb_issue_mask |= 1 << idx; #endif - tcg_gen_op1i(INDEX_op_goto_tb, idx); + /* When not chaining, we simply fall through to the "fallback" exit. */ + if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { + tcg_gen_op1i(INDEX_op_goto_tb, idx); + } } void tcg_gen_lookup_and_goto_ptr(void) -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> Consistently access u16.high with atomics to avoid undefined behaviour in MTTCG. Note that icount_decr.u16.low is only used in icount mode, so regular accesses to it are OK. Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181010144853.13005-2-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/tcg-all.c | 2 +- accel/tcg/translate-all.c | 2 +- qom/cpu.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/accel/tcg/tcg-all.c b/accel/tcg/tcg-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/tcg-all.c +++ b/accel/tcg/tcg-all.c @@ -XXX,XX +XXX,XX @@ static void tcg_handle_interrupt(CPUState *cpu, int mask) if (!qemu_cpu_is_self(cpu)) { qemu_cpu_kick(cpu); } else { - cpu->icount_decr.u16.high = -1; + atomic_set(&cpu->icount_decr.u16.high, -1); if (use_icount && !cpu->can_do_io && (mask & ~old_mask) != 0) { diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ void cpu_interrupt(CPUState *cpu, int mask) { g_assert(qemu_mutex_iothread_locked()); cpu->interrupt_request |= mask; - cpu->icount_decr.u16.high = -1; + atomic_set(&cpu->icount_decr.u16.high, -1); } /* diff --git a/qom/cpu.c b/qom/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/qom/cpu.c +++ b/qom/cpu.c @@ -XXX,XX +XXX,XX @@ static void cpu_common_reset(CPUState *cpu) cpu->mem_io_pc = 0; cpu->mem_io_vaddr = 0; cpu->icount_extra = 0; - cpu->icount_decr.u32 = 0; + atomic_set(&cpu->icount_decr.u32, 0); cpu->can_do_io = 1; cpu->exception_index = -1; cpu->crash_occurred = false; -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> We forgot to initialize n in commit 15fa08f845 ("tcg: Dynamically allocate TCGOps", 2017-12-29). Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181010144853.13005-3-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb) #ifdef CONFIG_PROFILER { - int n; + int n = 0; QTAILQ_FOREACH(op, &s->ops, link) { n++; -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> This plugs two 4-byte holes in 64-bit. Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181010144853.13005-4-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/tcg.h b/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.h +++ b/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ typedef struct TCGProfile { int64_t tb_count; int64_t op_count; /* total insn count */ int op_count_max; /* max insn per TB */ - int64_t temp_count; int temp_count_max; + int64_t temp_count; int64_t del_op_count; int64_t code_in_len; int64_t code_out_len; -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> When we implemented per-vCPU TCG contexts, we forgot to also distribute the tcg_time counter, which has remained as a global accessed without any serialization, leading to potentially missed counts. Fix it by distributing the field over the TCG contexts, embedding it into TCGProfile with a field called "cpu_exec_time", which is more descriptive than "tcg_time". Add a function to query this value directly, and for completeness, fill in the field in tcg_profile_snapshot, even though its callers do not use it. Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181010144853.13005-5-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/qemu/timer.h | 1 - tcg/tcg.h | 2 ++ cpus.c | 3 ++- monitor.c | 13 ++++++++++--- tcg/tcg.c | 23 +++++++++++++++++++++++ 5 files changed, 37 insertions(+), 5 deletions(-) diff --git a/include/qemu/timer.h b/include/qemu/timer.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/timer.h +++ b/include/qemu/timer.h @@ -XXX,XX +XXX,XX @@ static inline int64_t profile_getclock(void) return get_clock(); } -extern int64_t tcg_time; extern int64_t dev_time; #endif diff --git a/tcg/tcg.h b/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.h +++ b/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ typedef struct TCGOp { QEMU_BUILD_BUG_ON(NB_OPS > (1 << 8)); typedef struct TCGProfile { + int64_t cpu_exec_time; int64_t tb_count1; int64_t tb_count; int64_t op_count; /* total insn count */ @@ -XXX,XX +XXX,XX @@ int tcg_check_temp_count(void); #define tcg_check_temp_count() 0 #endif +int64_t tcg_cpu_exec_time(void); void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf); void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf); diff --git a/cpus.c b/cpus.c index XXXXXXX..XXXXXXX 100644 --- a/cpus.c +++ b/cpus.c @@ -XXX,XX +XXX,XX @@ static int tcg_cpu_exec(CPUState *cpu) ret = cpu_exec(cpu); cpu_exec_end(cpu); #ifdef CONFIG_PROFILER - tcg_time += profile_getclock() - ti; + atomic_set(&tcg_ctx->prof.cpu_exec_time, + tcg_ctx->prof.cpu_exec_time + profile_getclock() - ti); #endif return ret; } diff --git a/monitor.c b/monitor.c index XXXXXXX..XXXXXXX 100644 --- a/monitor.c +++ b/monitor.c @@ -XXX,XX +XXX,XX @@ #include "sysemu/cpus.h" #include "sysemu/iothread.h" #include "qemu/cutils.h" +#include "tcg/tcg.h" #if defined(TARGET_S390X) #include "hw/s390x/storage-keys.h" @@ -XXX,XX +XXX,XX @@ static void hmp_info_numa(Monitor *mon, const QDict *qdict) #ifdef CONFIG_PROFILER -int64_t tcg_time; int64_t dev_time; static void hmp_info_profile(Monitor *mon, const QDict *qdict) { + static int64_t last_cpu_exec_time; + int64_t cpu_exec_time; + int64_t delta; + + cpu_exec_time = tcg_cpu_exec_time(); + delta = cpu_exec_time - last_cpu_exec_time; + monitor_printf(mon, "async time %" PRId64 " (%0.3f)\n", dev_time, dev_time / (double)NANOSECONDS_PER_SECOND); monitor_printf(mon, "qemu time %" PRId64 " (%0.3f)\n", - tcg_time, tcg_time / (double)NANOSECONDS_PER_SECOND); - tcg_time = 0; + delta, delta / (double)NANOSECONDS_PER_SECOND); + last_cpu_exec_time = cpu_exec_time; dev_time = 0; } #else diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ /* Define to jump the ELF file used to communicate with GDB. */ #undef DEBUG_JIT +#include "qemu/error-report.h" #include "qemu/cutils.h" #include "qemu/host-utils.h" #include "qemu/timer.h" @@ -XXX,XX +XXX,XX @@ void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table) const TCGProfile *orig = &s->prof; if (counters) { + PROF_ADD(prof, orig, cpu_exec_time); PROF_ADD(prof, orig, tb_count1); PROF_ADD(prof, orig, tb_count); PROF_ADD(prof, orig, op_count); @@ -XXX,XX +XXX,XX @@ void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf) prof.table_op_count[i]); } } + +int64_t tcg_cpu_exec_time(void) +{ + unsigned int n_ctxs = atomic_read(&n_tcg_ctxs); + unsigned int i; + int64_t ret = 0; + + for (i = 0; i < n_ctxs; i++) { + const TCGContext *s = atomic_read(&tcg_ctxs[i]); + const TCGProfile *prof = &s->prof; + + ret += atomic_read(&prof->cpu_exec_time); + } + return ret; +} #else void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf) { cpu_fprintf(f, "[TCG profiler not compiled]\n"); } + +int64_t tcg_cpu_exec_time(void) +{ + error_report("%s: TCG profiler not compiled", __func__); + exit(EXIT_FAILURE); +} #endif -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> As far as I can tell tlb_flush does not need to be called this early. tlb_flush is eventually called after the CPU has been realized. This change paves the way to the introduction of tlb_init, which will be called from cpu_exec_realizefn. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181009174557.16125-2-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/alpha/cpu.c | 1 - 1 file changed, 1 deletion(-) diff --git a/target/alpha/cpu.c b/target/alpha/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/alpha/cpu.c +++ b/target/alpha/cpu.c @@ -XXX,XX +XXX,XX @@ static void alpha_cpu_initfn(Object *obj) CPUAlphaState *env = &cpu->env; cs->env_ptr = env; - tlb_flush(cs); env->lock_addr = -1; #if defined(CONFIG_USER_ONLY) -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> As far as I can tell tlb_flush does not need to be called this early. tlb_flush is eventually called after the CPU has been realized. This change paves the way to the introduction of tlb_init, which will be called from cpu_exec_realizefn. Cc: Guan Xuetao <gxt@mprc.pku.edu.cn> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181009174557.16125-3-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/unicore32/cpu.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/target/unicore32/cpu.c b/target/unicore32/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/unicore32/cpu.c +++ b/target/unicore32/cpu.c @@ -XXX,XX +XXX,XX @@ static void uc32_cpu_initfn(Object *obj) env->uncached_asr = ASR_MODE_PRIV; env->regs[31] = 0x03000000; #endif - - tlb_flush(cs); } static const VMStateDescription vmstate_uc32_cpu = { -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> Paves the way for the addition of a per-TLB lock. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181009174557.16125-4-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/exec-all.h | 8 ++++++++ accel/tcg/cputlb.c | 4 ++++ exec.c | 1 + 3 files changed, 13 insertions(+) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ void cpu_address_space_init(CPUState *cpu, int asidx, #if !defined(CONFIG_USER_ONLY) && defined(CONFIG_TCG) /* cputlb.c */ +/** + * tlb_init - initialize a CPU's TLB + * @cpu: CPU whose TLB should be initialized + */ +void tlb_init(CPUState *cpu); /** * tlb_flush_page: * @cpu: CPU whose TLB should be flushed @@ -XXX,XX +XXX,XX @@ void tlb_set_page(CPUState *cpu, target_ulong vaddr, void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, uintptr_t retaddr); #else +static inline void tlb_init(CPUState *cpu) +{ +} static inline void tlb_flush_page(CPUState *cpu, target_ulong addr) { } diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ QEMU_BUILD_BUG_ON(sizeof(target_ulong) > sizeof(run_on_cpu_data)); QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16); #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1) +void tlb_init(CPUState *cpu) +{ +} + /* flush_all_helper: run fn across all cpus * * If the wait flag is set then the src cpu's helper will be queued as diff --git a/exec.c b/exec.c index XXXXXXX..XXXXXXX 100644 --- a/exec.c +++ b/exec.c @@ -XXX,XX +XXX,XX @@ void cpu_exec_realizefn(CPUState *cpu, Error **errp) tcg_target_initialized = true; cc->tcg_initialize(); } + tlb_init(cpu); #ifndef CONFIG_USER_ONLY if (qdev_get_vmsd(DEVICE(cpu)) == NULL) { -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181009174557.16125-5-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cputlb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ } \ } while (0) -#define assert_cpu_is_self(this_cpu) do { \ +#define assert_cpu_is_self(cpu) do { \ if (DEBUG_TLB_GATE) { \ - g_assert(!cpu->created || qemu_cpu_is_self(cpu)); \ + g_assert(!(cpu)->created || qemu_cpu_is_self(cpu)); \ } \ } while (0) -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> Currently we rely on atomic operations for cross-CPU invalidations. There are two cases that these atomics miss: cross-CPU invalidations can race with either (1) vCPU threads flushing their TLB, which happens via memset, or (2) vCPUs calling tlb_reset_dirty on their TLB, which updates .addr_write with a regular store. This results in undefined behaviour, since we're mixing regular and atomic ops on concurrent accesses. Fix it by using tlb_lock, a per-vCPU lock. All updaters of tlb_table and the corresponding victim cache now hold the lock. The readers that do not hold tlb_lock must use atomic reads when reading .addr_write, since this field can be updated by other threads; the conversion to atomic reads is done in the next patch. Note that an alternative fix would be to expand the use of atomic ops. However, in the case of TLB flushes this would have a huge performance impact, since (1) TLB flushes can happen very frequently and (2) we currently use a full memory barrier to flush each TLB entry, and a TLB has many entries. Instead, acquiring the lock is barely slower than a full memory barrier since it is uncontended, and with a single lock acquisition we can flush the entire TLB. Tested-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181009174557.16125-6-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/cpu-defs.h | 3 + accel/tcg/cputlb.c | 155 ++++++++++++++++++++++------------------ 2 files changed, 87 insertions(+), 71 deletions(-) diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu-defs.h +++ b/include/exec/cpu-defs.h @@ -XXX,XX +XXX,XX @@ #endif #include "qemu/host-utils.h" +#include "qemu/thread.h" #include "qemu/queue.h" #ifdef CONFIG_TCG #include "tcg-target.h" @@ -XXX,XX +XXX,XX @@ typedef struct CPUIOTLBEntry { #define CPU_COMMON_TLB \ /* The meaning of the MMU modes is defined in the target code. */ \ + /* tlb_lock serializes updates to tlb_table and tlb_v_table */ \ + QemuSpin tlb_lock; \ CPUTLBEntry tlb_table[NB_MMU_MODES][CPU_TLB_SIZE]; \ CPUTLBEntry tlb_v_table[NB_MMU_MODES][CPU_VTLB_SIZE]; \ CPUIOTLBEntry iotlb[NB_MMU_MODES][CPU_TLB_SIZE]; \ diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16); void tlb_init(CPUState *cpu) { + CPUArchState *env = cpu->env_ptr; + + qemu_spin_init(&env->tlb_lock); } /* flush_all_helper: run fn across all cpus @@ -XXX,XX +XXX,XX @@ static void tlb_flush_nocheck(CPUState *cpu) atomic_set(&env->tlb_flush_count, env->tlb_flush_count + 1); tlb_debug("(count: %zu)\n", tlb_flush_count()); + /* + * tlb_table/tlb_v_table updates from any thread must hold tlb_lock. + * However, updates from the owner thread (as is the case here; see the + * above assert_cpu_is_self) do not need atomic_set because all reads + * that do not hold the lock are performed by the same owner thread. + */ + qemu_spin_lock(&env->tlb_lock); memset(env->tlb_table, -1, sizeof(env->tlb_table)); memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table)); + qemu_spin_unlock(&env->tlb_lock); + cpu_tb_jmp_cache_clear(cpu); env->vtlb_index = 0; @@ -XXX,XX +XXX,XX @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data) tlb_debug("start: mmu_idx:0x%04lx\n", mmu_idx_bitmask); + qemu_spin_lock(&env->tlb_lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { if (test_bit(mmu_idx, &mmu_idx_bitmask)) { @@ -XXX,XX +XXX,XX @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data) memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0])); } } + qemu_spin_unlock(&env->tlb_lock); cpu_tb_jmp_cache_clear(cpu); @@ -XXX,XX +XXX,XX @@ static inline bool tlb_hit_page_anyprot(CPUTLBEntry *tlb_entry, tlb_hit_page(tlb_entry->addr_code, page); } -static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong page) +/* Called with tlb_lock held */ +static inline void tlb_flush_entry_locked(CPUTLBEntry *tlb_entry, + target_ulong page) { if (tlb_hit_page_anyprot(tlb_entry, page)) { memset(tlb_entry, -1, sizeof(*tlb_entry)); } } -static inline void tlb_flush_vtlb_page(CPUArchState *env, int mmu_idx, - target_ulong page) +/* Called with tlb_lock held */ +static inline void tlb_flush_vtlb_page_locked(CPUArchState *env, int mmu_idx, + target_ulong page) { int k; + + assert_cpu_is_self(ENV_GET_CPU(env)); for (k = 0; k < CPU_VTLB_SIZE; k++) { - tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], page); + tlb_flush_entry_locked(&env->tlb_v_table[mmu_idx][k], page); } } @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data) addr &= TARGET_PAGE_MASK; i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); + qemu_spin_lock(&env->tlb_lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { - tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr); - tlb_flush_vtlb_page(env, mmu_idx, addr); + tlb_flush_entry_locked(&env->tlb_table[mmu_idx][i], addr); + tlb_flush_vtlb_page_locked(env, mmu_idx, addr); } + qemu_spin_unlock(&env->tlb_lock); tb_flush_jmp_cache(cpu, addr); } @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_by_mmuidx_async_work(CPUState *cpu, tlb_debug("page:%d addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n", page, addr, mmu_idx_bitmap); + qemu_spin_lock(&env->tlb_lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { if (test_bit(mmu_idx, &mmu_idx_bitmap)) { - tlb_flush_entry(&env->tlb_table[mmu_idx][page], addr); - tlb_flush_vtlb_page(env, mmu_idx, addr); + tlb_flush_entry_locked(&env->tlb_table[mmu_idx][page], addr); + tlb_flush_vtlb_page_locked(env, mmu_idx, addr); } } + qemu_spin_unlock(&env->tlb_lock); tb_flush_jmp_cache(cpu, addr); } @@ -XXX,XX +XXX,XX @@ void tlb_unprotect_code(ram_addr_t ram_addr) * most usual is detecting writes to code regions which may invalidate * generated code. * - * Because we want other vCPUs to respond to changes straight away we - * update the te->addr_write field atomically. If the TLB entry has - * been changed by the vCPU in the mean time we skip the update. + * Other vCPUs might be reading their TLBs during guest execution, so we update + * te->addr_write with atomic_set. We don't need to worry about this for + * oversized guests as MTTCG is disabled for them. * - * As this function uses atomic accesses we also need to ensure - * updates to tlb_entries follow the same access rules. We don't need - * to worry about this for oversized guests as MTTCG is disabled for - * them. + * Called with tlb_lock held. */ - -static void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start, - uintptr_t length) +static void tlb_reset_dirty_range_locked(CPUTLBEntry *tlb_entry, + uintptr_t start, uintptr_t length) { -#if TCG_OVERSIZED_GUEST uintptr_t addr = tlb_entry->addr_write; if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) { addr &= TARGET_PAGE_MASK; addr += tlb_entry->addend; if ((addr - start) < length) { +#if TCG_OVERSIZED_GUEST tlb_entry->addr_write |= TLB_NOTDIRTY; - } - } #else - /* paired with atomic_mb_set in tlb_set_page_with_attrs */ - uintptr_t orig_addr = atomic_mb_read(&tlb_entry->addr_write); - uintptr_t addr = orig_addr; - - if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) { - addr &= TARGET_PAGE_MASK; - addr += atomic_read(&tlb_entry->addend); - if ((addr - start) < length) { - uintptr_t notdirty_addr = orig_addr | TLB_NOTDIRTY; - atomic_cmpxchg(&tlb_entry->addr_write, orig_addr, notdirty_addr); + atomic_set(&tlb_entry->addr_write, + tlb_entry->addr_write | TLB_NOTDIRTY); +#endif } } -#endif } -/* For atomic correctness when running MTTCG we need to use the right - * primitives when copying entries */ -static inline void copy_tlb_helper(CPUTLBEntry *d, CPUTLBEntry *s, - bool atomic_set) +/* + * Called with tlb_lock held. + * Called only from the vCPU context, i.e. the TLB's owner thread. + */ +static inline void copy_tlb_helper_locked(CPUTLBEntry *d, const CPUTLBEntry *s) { -#if TCG_OVERSIZED_GUEST *d = *s; -#else - if (atomic_set) { - d->addr_read = s->addr_read; - d->addr_code = s->addr_code; - atomic_set(&d->addend, atomic_read(&s->addend)); - /* Pairs with flag setting in tlb_reset_dirty_range */ - atomic_mb_set(&d->addr_write, atomic_read(&s->addr_write)); - } else { - d->addr_read = s->addr_read; - d->addr_write = atomic_read(&s->addr_write); - d->addr_code = s->addr_code; - d->addend = atomic_read(&s->addend); - } -#endif } /* This is a cross vCPU call (i.e. another vCPU resetting the flags of - * the target vCPU). As such care needs to be taken that we don't - * dangerously race with another vCPU update. The only thing actually - * updated is the target TLB entry ->addr_write flags. + * the target vCPU). + * We must take tlb_lock to avoid racing with another vCPU update. The only + * thing actually updated is the target TLB entry ->addr_write flags. */ void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length) { @@ -XXX,XX +XXX,XX @@ void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length) int mmu_idx; env = cpu->env_ptr; + qemu_spin_lock(&env->tlb_lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { unsigned int i; for (i = 0; i < CPU_TLB_SIZE; i++) { - tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i], - start1, length); + tlb_reset_dirty_range_locked(&env->tlb_table[mmu_idx][i], start1, + length); } for (i = 0; i < CPU_VTLB_SIZE; i++) { - tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i], - start1, length); + tlb_reset_dirty_range_locked(&env->tlb_v_table[mmu_idx][i], start1, + length); } } + qemu_spin_unlock(&env->tlb_lock); } -static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr) +/* Called with tlb_lock held */ +static inline void tlb_set_dirty1_locked(CPUTLBEntry *tlb_entry, + target_ulong vaddr) { if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY)) { tlb_entry->addr_write = vaddr; @@ -XXX,XX +XXX,XX @@ void tlb_set_dirty(CPUState *cpu, target_ulong vaddr) vaddr &= TARGET_PAGE_MASK; i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); + qemu_spin_lock(&env->tlb_lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { - tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr); + tlb_set_dirty1_locked(&env->tlb_table[mmu_idx][i], vaddr); } for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { int k; for (k = 0; k < CPU_VTLB_SIZE; k++) { - tlb_set_dirty1(&env->tlb_v_table[mmu_idx][k], vaddr); + tlb_set_dirty1_locked(&env->tlb_v_table[mmu_idx][k], vaddr); } } + qemu_spin_unlock(&env->tlb_lock); } /* Our TLB does not support large pages, so remember the area covered by @@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat; } - /* Make sure there's no cached translation for the new page. */ - tlb_flush_vtlb_page(env, mmu_idx, vaddr_page); - code_address = address; iotlb = memory_region_section_get_iotlb(cpu, section, vaddr_page, paddr_page, xlat, prot, &address); @@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, index = (vaddr_page >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); te = &env->tlb_table[mmu_idx][index]; + /* + * Hold the TLB lock for the rest of the function. We could acquire/release + * the lock several times in the function, but it is faster to amortize the + * acquisition cost by acquiring it just once. Note that this leads to + * a longer critical section, but this is not a concern since the TLB lock + * is unlikely to be contended. + */ + qemu_spin_lock(&env->tlb_lock); + + /* Make sure there's no cached translation for the new page. */ + tlb_flush_vtlb_page_locked(env, mmu_idx, vaddr_page); + /* * Only evict the old entry to the victim tlb if it's for a * different page; otherwise just overwrite the stale data. @@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, CPUTLBEntry *tv = &env->tlb_v_table[mmu_idx][vidx]; /* Evict the old entry into the victim tlb. */ - copy_tlb_helper(tv, te, true); + copy_tlb_helper_locked(tv, te); env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index]; } @@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, } } - /* Pairs with flag setting in tlb_reset_dirty_range */ - copy_tlb_helper(te, &tn, true); - /* atomic_mb_set(&te->addr_write, write_address); */ + copy_tlb_helper_locked(te, &tn); + qemu_spin_unlock(&env->tlb_lock); } /* Add a new TLB entry, but without specifying the memory @@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index, size_t elt_ofs, target_ulong page) { size_t vidx; + + assert_cpu_is_self(ENV_GET_CPU(env)); for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) { CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx]; target_ulong cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs); @@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index, /* Found entry in victim tlb, swap tlb and iotlb. */ CPUTLBEntry tmptlb, *tlb = &env->tlb_table[mmu_idx][index]; - copy_tlb_helper(&tmptlb, tlb, false); - copy_tlb_helper(tlb, vtlb, true); - copy_tlb_helper(vtlb, &tmptlb, true); + qemu_spin_lock(&env->tlb_lock); + copy_tlb_helper_locked(&tmptlb, tlb); + copy_tlb_helper_locked(tlb, vtlb); + copy_tlb_helper_locked(vtlb, &tmptlb); + qemu_spin_unlock(&env->tlb_lock); CPUIOTLBEntry tmpio, *io = &env->iotlb[mmu_idx][index]; CPUIOTLBEntry *vio = &env->iotlb_v[mmu_idx][vidx]; -- 2.17.2 Isolate the computation of an index from an address into a helper before we change that function. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> [ cota: convert tlb_vaddr_to_host; use atomic_read on addr_write ] Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181009175129.17888-2-cota@braap.org> --- accel/tcg/softmmu_template.h | 64 +++++++++++++++++--------------- include/exec/cpu_ldst.h | 19 ++++++++-- include/exec/cpu_ldst_template.h | 25 +++++++------ accel/tcg/cputlb.c | 60 ++++++++++++++---------------- 4 files changed, 90 insertions(+), 78 deletions(-) diff --git a/accel/tcg/softmmu_template.h b/accel/tcg/softmmu_template.h index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/softmmu_template.h +++ b/accel/tcg/softmmu_template.h @@ -XXX,XX +XXX,XX @@ static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env, WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - unsigned mmu_idx = get_mmuidx(oi); - int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ; + uintptr_t mmu_idx = get_mmuidx(oi); + uintptr_t index = tlb_index(env, mmu_idx, addr); + CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); + target_ulong tlb_addr = entry->ADDR_READ; unsigned a_bits = get_alignment_bits(get_memop(oi)); uintptr_t haddr; DATA_TYPE res; @@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, READ_ACCESS_TYPE, mmu_idx, retaddr); } - tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ; + tlb_addr = entry->ADDR_READ; } /* Handle an IO access. */ @@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, return res; } - haddr = addr + env->tlb_table[mmu_idx][index].addend; + haddr = addr + entry->addend; #if DATA_SIZE == 1 res = glue(glue(ld, LSUFFIX), _p)((uint8_t *)haddr); #else @@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - unsigned mmu_idx = get_mmuidx(oi); - int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ; + uintptr_t mmu_idx = get_mmuidx(oi); + uintptr_t index = tlb_index(env, mmu_idx, addr); + CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); + target_ulong tlb_addr = entry->ADDR_READ; unsigned a_bits = get_alignment_bits(get_memop(oi)); uintptr_t haddr; DATA_TYPE res; @@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr, tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, READ_ACCESS_TYPE, mmu_idx, retaddr); } - tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ; + tlb_addr = entry->ADDR_READ; } /* Handle an IO access. */ @@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr, return res; } - haddr = addr + env->tlb_table[mmu_idx][index].addend; + haddr = addr + entry->addend; res = glue(glue(ld, LSUFFIX), _be_p)((uint8_t *)haddr); return res; } @@ -XXX,XX +XXX,XX @@ static inline void glue(io_write, SUFFIX)(CPUArchState *env, void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, TCGMemOpIdx oi, uintptr_t retaddr) { - unsigned mmu_idx = get_mmuidx(oi); - int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write; + uintptr_t mmu_idx = get_mmuidx(oi); + uintptr_t index = tlb_index(env, mmu_idx, addr); + CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); + target_ulong tlb_addr = entry->addr_write; unsigned a_bits = get_alignment_bits(get_memop(oi)); uintptr_t haddr; @@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); } - tlb_addr = env->tlb_table[mmu_idx][index].addr_write & ~TLB_INVALID_MASK; + tlb_addr = entry->addr_write & ~TLB_INVALID_MASK; } /* Handle an IO access. */ @@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, if (DATA_SIZE > 1 && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1 >= TARGET_PAGE_SIZE)) { - int i, index2; - target_ulong page2, tlb_addr2; + int i; + target_ulong page2; + CPUTLBEntry *entry2; do_unaligned_access: /* Ensure the second page is in the TLB. Note that the first page is already guaranteed to be filled, and that the second page cannot evict the first. */ page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK; - index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write; - if (!tlb_hit_page(tlb_addr2, page2) + entry2 = tlb_entry(env, mmu_idx, page2); + if (!tlb_hit_page(entry2->addr_write, page2) && !VICTIM_TLB_HIT(addr_write, page2)) { tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); @@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, return; } - haddr = addr + env->tlb_table[mmu_idx][index].addend; + haddr = addr + entry->addend; #if DATA_SIZE == 1 glue(glue(st, SUFFIX), _p)((uint8_t *)haddr, val); #else @@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, TCGMemOpIdx oi, uintptr_t retaddr) { - unsigned mmu_idx = get_mmuidx(oi); - int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write; + uintptr_t mmu_idx = get_mmuidx(oi); + uintptr_t index = tlb_index(env, mmu_idx, addr); + CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); + target_ulong tlb_addr = entry->addr_write; unsigned a_bits = get_alignment_bits(get_memop(oi)); uintptr_t haddr; @@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); } - tlb_addr = env->tlb_table[mmu_idx][index].addr_write & ~TLB_INVALID_MASK; + tlb_addr = entry->addr_write & ~TLB_INVALID_MASK; } /* Handle an IO access. */ @@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, if (DATA_SIZE > 1 && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1 >= TARGET_PAGE_SIZE)) { - int i, index2; - target_ulong page2, tlb_addr2; + int i; + target_ulong page2; + CPUTLBEntry *entry2; do_unaligned_access: /* Ensure the second page is in the TLB. Note that the first page is already guaranteed to be filled, and that the second page cannot evict the first. */ page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK; - index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write; - if (!tlb_hit_page(tlb_addr2, page2) + entry2 = tlb_entry(env, mmu_idx, page2); + if (!tlb_hit_page(entry2->addr_write, page2) && !VICTIM_TLB_HIT(addr_write, page2)) { tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); @@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, return; } - haddr = addr + env->tlb_table[mmu_idx][index].addend; + haddr = addr + entry->addend; glue(glue(st, SUFFIX), _be_p)((uint8_t *)haddr, val); } #endif /* DATA_SIZE > 1 */ diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu_ldst.h +++ b/include/exec/cpu_ldst.h @@ -XXX,XX +XXX,XX @@ extern __thread uintptr_t helper_retaddr; /* The memory helpers for tcg-generated code need tcg_target_long etc. */ #include "tcg.h" +/* Find the TLB index corresponding to the mmu_idx + address pair. */ +static inline uintptr_t tlb_index(CPUArchState *env, uintptr_t mmu_idx, + target_ulong addr) +{ + return (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); +} + +/* Find the TLB entry corresponding to the mmu_idx + address pair. */ +static inline CPUTLBEntry *tlb_entry(CPUArchState *env, uintptr_t mmu_idx, + target_ulong addr) +{ + return &env->tlb_table[mmu_idx][tlb_index(env, mmu_idx, addr)]; +} + #ifdef MMU_MODE0_SUFFIX #define CPU_MMU_INDEX 0 #define MEMSUFFIX MMU_MODE0_SUFFIX @@ -XXX,XX +XXX,XX @@ static inline void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr, #if defined(CONFIG_USER_ONLY) return g2h(addr); #else - int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - CPUTLBEntry *tlbentry = &env->tlb_table[mmu_idx][index]; + CPUTLBEntry *tlbentry = tlb_entry(env, mmu_idx, addr); abi_ptr tlb_addr; uintptr_t haddr; @@ -XXX,XX +XXX,XX @@ static inline void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr, return NULL; } - haddr = addr + env->tlb_table[mmu_idx][index].addend; + haddr = addr + tlbentry->addend; return (void *)haddr; #endif /* defined(CONFIG_USER_ONLY) */ } diff --git a/include/exec/cpu_ldst_template.h b/include/exec/cpu_ldst_template.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu_ldst_template.h +++ b/include/exec/cpu_ldst_template.h @@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_ld, USUFFIX), MEMSUFFIX), _ra)(CPUArchState *env, target_ulong ptr, uintptr_t retaddr) { - int page_index; + CPUTLBEntry *entry; RES_TYPE res; target_ulong addr; int mmu_idx; @@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_ld, USUFFIX), MEMSUFFIX), _ra)(CPUArchState *env, #endif addr = ptr; - page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); mmu_idx = CPU_MMU_INDEX; - if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ != + entry = tlb_entry(env, mmu_idx, addr); + if (unlikely(entry->ADDR_READ != (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) { oi = make_memop_idx(SHIFT, mmu_idx); res = glue(glue(helper_ret_ld, URETSUFFIX), MMUSUFFIX)(env, addr, oi, retaddr); } else { - uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend; + uintptr_t hostaddr = addr + entry->addend; res = glue(glue(ld, USUFFIX), _p)((uint8_t *)hostaddr); } return res; @@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_lds, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env, target_ulong ptr, uintptr_t retaddr) { - int res, page_index; + CPUTLBEntry *entry; + int res; target_ulong addr; int mmu_idx; TCGMemOpIdx oi; @@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_lds, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env, #endif addr = ptr; - page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); mmu_idx = CPU_MMU_INDEX; - if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ != + entry = tlb_entry(env, mmu_idx, addr); + if (unlikely(entry->ADDR_READ != (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) { oi = make_memop_idx(SHIFT, mmu_idx); res = (DATA_STYPE)glue(glue(helper_ret_ld, SRETSUFFIX), MMUSUFFIX)(env, addr, oi, retaddr); } else { - uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend; + uintptr_t hostaddr = addr + entry->addend; res = glue(glue(lds, SUFFIX), _p)((uint8_t *)hostaddr); } return res; @@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env, target_ulong ptr, RES_TYPE v, uintptr_t retaddr) { - int page_index; + CPUTLBEntry *entry; target_ulong addr; int mmu_idx; TCGMemOpIdx oi; @@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env, #endif addr = ptr; - page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); mmu_idx = CPU_MMU_INDEX; - if (unlikely(env->tlb_table[mmu_idx][page_index].addr_write != + entry = tlb_entry(env, mmu_idx, addr); + if (unlikely(entry->addr_write != (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) { oi = make_memop_idx(SHIFT, mmu_idx); glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(env, addr, v, oi, retaddr); } else { - uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend; + uintptr_t hostaddr = addr + entry->addend; glue(glue(st, SUFFIX), _p)((uint8_t *)hostaddr, v); } } diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data) { CPUArchState *env = cpu->env_ptr; target_ulong addr = (target_ulong) data.target_ptr; - int i; int mmu_idx; assert_cpu_is_self(cpu); @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data) } addr &= TARGET_PAGE_MASK; - i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); qemu_spin_lock(&env->tlb_lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { - tlb_flush_entry_locked(&env->tlb_table[mmu_idx][i], addr); + tlb_flush_entry_locked(tlb_entry(env, mmu_idx, addr), addr); tlb_flush_vtlb_page_locked(env, mmu_idx, addr); } qemu_spin_unlock(&env->tlb_lock); @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_by_mmuidx_async_work(CPUState *cpu, target_ulong addr_and_mmuidx = (target_ulong) data.target_ptr; target_ulong addr = addr_and_mmuidx & TARGET_PAGE_MASK; unsigned long mmu_idx_bitmap = addr_and_mmuidx & ALL_MMUIDX_BITS; - int page = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); int mmu_idx; assert_cpu_is_self(cpu); - tlb_debug("page:%d addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n", - page, addr, mmu_idx_bitmap); + tlb_debug("flush page addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n", + addr, mmu_idx_bitmap); qemu_spin_lock(&env->tlb_lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { if (test_bit(mmu_idx, &mmu_idx_bitmap)) { - tlb_flush_entry_locked(&env->tlb_table[mmu_idx][page], addr); + tlb_flush_entry_locked(tlb_entry(env, mmu_idx, addr), addr); tlb_flush_vtlb_page_locked(env, mmu_idx, addr); } } @@ -XXX,XX +XXX,XX @@ static inline void tlb_set_dirty1_locked(CPUTLBEntry *tlb_entry, void tlb_set_dirty(CPUState *cpu, target_ulong vaddr) { CPUArchState *env = cpu->env_ptr; - int i; int mmu_idx; assert_cpu_is_self(cpu); vaddr &= TARGET_PAGE_MASK; - i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); qemu_spin_lock(&env->tlb_lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { - tlb_set_dirty1_locked(&env->tlb_table[mmu_idx][i], vaddr); + tlb_set_dirty1_locked(tlb_entry(env, mmu_idx, vaddr), vaddr); } for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { @@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, iotlb = memory_region_section_get_iotlb(cpu, section, vaddr_page, paddr_page, xlat, prot, &address); - index = (vaddr_page >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - te = &env->tlb_table[mmu_idx][index]; + index = tlb_index(env, mmu_idx, vaddr_page); + te = tlb_entry(env, mmu_idx, vaddr_page); /* * Hold the TLB lock for the rest of the function. We could acquire/release @@ -XXX,XX +XXX,XX @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry, * repeat the MMU check here. This tlb_fill() call might * longjump out if this access should cause a guest exception. */ - int index; + CPUTLBEntry *entry; target_ulong tlb_addr; tlb_fill(cpu, addr, size, MMU_DATA_LOAD, mmu_idx, retaddr); - index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - tlb_addr = env->tlb_table[mmu_idx][index].addr_read; + entry = tlb_entry(env, mmu_idx, addr); + tlb_addr = entry->addr_read; if (!(tlb_addr & ~(TARGET_PAGE_MASK | TLB_RECHECK))) { /* RAM access */ - uintptr_t haddr = addr + env->tlb_table[mmu_idx][index].addend; + uintptr_t haddr = addr + entry->addend; return ldn_p((void *)haddr, size); } @@ -XXX,XX +XXX,XX @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry, * repeat the MMU check here. This tlb_fill() call might * longjump out if this access should cause a guest exception. */ - int index; + CPUTLBEntry *entry; target_ulong tlb_addr; tlb_fill(cpu, addr, size, MMU_DATA_STORE, mmu_idx, retaddr); - index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - tlb_addr = env->tlb_table[mmu_idx][index].addr_write; + entry = tlb_entry(env, mmu_idx, addr); + tlb_addr = entry->addr_write; if (!(tlb_addr & ~(TARGET_PAGE_MASK | TLB_RECHECK))) { /* RAM access */ - uintptr_t haddr = addr + env->tlb_table[mmu_idx][index].addend; + uintptr_t haddr = addr + entry->addend; stn_p((void *)haddr, size, val); return; @@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index, */ tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr) { - int mmu_idx, index; + uintptr_t mmu_idx = cpu_mmu_index(env, true); + uintptr_t index = tlb_index(env, mmu_idx, addr); + CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); void *p; - index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - mmu_idx = cpu_mmu_index(env, true); - if (unlikely(!tlb_hit(env->tlb_table[mmu_idx][index].addr_code, addr))) { + if (unlikely(!tlb_hit(entry->addr_code, addr))) { if (!VICTIM_TLB_HIT(addr_code, addr)) { tlb_fill(ENV_GET_CPU(env), addr, 0, MMU_INST_FETCH, mmu_idx, 0); } - assert(tlb_hit(env->tlb_table[mmu_idx][index].addr_code, addr)); + assert(tlb_hit(entry->addr_code, addr)); } - if (unlikely(env->tlb_table[mmu_idx][index].addr_code & - (TLB_RECHECK | TLB_MMIO))) { + if (unlikely(entry->addr_code & (TLB_RECHECK | TLB_MMIO))) { /* * Return -1 if we can't translate and execute from an entire * page of RAM here, which will cause us to execute by loading @@ -XXX,XX +XXX,XX @@ tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr) return -1; } - p = (void *)((uintptr_t)addr + env->tlb_table[mmu_idx][index].addend); + p = (void *)((uintptr_t)addr + entry->addend); return qemu_ram_addr_from_host_nofail(p); } @@ -XXX,XX +XXX,XX @@ tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr) void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, uintptr_t retaddr) { - int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write; + uintptr_t index = tlb_index(env, mmu_idx, addr); + CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); - if (!tlb_hit(tlb_addr, addr)) { + if (!tlb_hit(entry->addr_write, addr)) { /* TLB entry is for a different page */ if (!VICTIM_TLB_HIT(addr_write, addr)) { tlb_fill(ENV_GET_CPU(env), addr, size, MMU_DATA_STORE, @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, NotDirtyInfo *ndi) { size_t mmu_idx = get_mmuidx(oi); - size_t index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - CPUTLBEntry *tlbe = &env->tlb_table[mmu_idx][index]; + uintptr_t index = tlb_index(env, mmu_idx, addr); + CPUTLBEntry *tlbe = tlb_entry(env, mmu_idx, addr); target_ulong tlb_addr = tlbe->addr_write; TCGMemOp mop = get_memop(oi); int a_bits = get_alignment_bits(mop); -- 2.17.2 GCC7+ will no longer advertise support for 16-byte __atomic operations if only cmpxchg is supported, as for x86_64. Fortunately, x86_64 still has support for __sync_compare_and_swap_16 and we can make use of that. AArch64 does not have, nor ever has had such support, so open-code it. Reviewed-by: Emilio G. Cota <cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/atomic_template.h | 20 ++++- include/qemu/atomic128.h | 155 ++++++++++++++++++++++++++++++++++++ tcg/tcg.h | 16 ++-- accel/tcg/cputlb.c | 3 +- accel/tcg/user-exec.c | 5 +- configure | 19 +++++ 6 files changed, 204 insertions(+), 14 deletions(-) create mode 100644 include/qemu/atomic128.h diff --git a/accel/tcg/atomic_template.h b/accel/tcg/atomic_template.h index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/atomic_template.h +++ b/accel/tcg/atomic_template.h @@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, DATA_TYPE ret; ATOMIC_TRACE_RMW; +#if DATA_SIZE == 16 + ret = atomic16_cmpxchg(haddr, cmpv, newv); +#else ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv); +#endif ATOMIC_MMU_CLEANUP; return ret; } #if DATA_SIZE >= 16 +#if HAVE_ATOMIC128 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) { ATOMIC_MMU_DECLS; DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; ATOMIC_TRACE_LD; - __atomic_load(haddr, &val, __ATOMIC_RELAXED); + val = atomic16_read(haddr); ATOMIC_MMU_CLEANUP; return val; } @@ -XXX,XX +XXX,XX @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; ATOMIC_TRACE_ST; - __atomic_store(haddr, &val, __ATOMIC_RELAXED); + atomic16_set(haddr, val); ATOMIC_MMU_CLEANUP; } +#endif #else ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val EXTRA_ARGS) @@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, DATA_TYPE ret; ATOMIC_TRACE_RMW; +#if DATA_SIZE == 16 + ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv)); +#else ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); +#endif ATOMIC_MMU_CLEANUP; return BSWAP(ret); } #if DATA_SIZE >= 16 +#if HAVE_ATOMIC128 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) { ATOMIC_MMU_DECLS; DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; ATOMIC_TRACE_LD; - __atomic_load(haddr, &val, __ATOMIC_RELAXED); + val = atomic16_read(haddr); ATOMIC_MMU_CLEANUP; return BSWAP(val); } @@ -XXX,XX +XXX,XX @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ATOMIC_TRACE_ST; val = BSWAP(val); - __atomic_store(haddr, &val, __ATOMIC_RELAXED); + atomic16_set(haddr, val); ATOMIC_MMU_CLEANUP; } +#endif #else ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val EXTRA_ARGS) diff --git a/include/qemu/atomic128.h b/include/qemu/atomic128.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/include/qemu/atomic128.h @@ -XXX,XX +XXX,XX @@ +/* + * Simple interface for 128-bit atomic operations. + * + * Copyright (C) 2018 Linaro, Ltd. + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + * + * See docs/devel/atomics.txt for discussion about the guarantees each + * atomic primitive is meant to provide. + */ + +#ifndef QEMU_ATOMIC128_H +#define QEMU_ATOMIC128_H + +/* + * GCC is a house divided about supporting large atomic operations. + * + * For hosts that only have large compare-and-swap, a legalistic reading + * of the C++ standard means that one cannot implement __atomic_read on + * read-only memory, and thus all atomic operations must synchronize + * through libatomic. + * + * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80878 + * + * This interpretation is not especially helpful for QEMU. + * For softmmu, all RAM is always read/write from the hypervisor. + * For user-only, if the guest doesn't implement such an __atomic_read + * then the host need not worry about it either. + * + * Moreover, using libatomic is not an option, because its interface is + * built for std::atomic<T>, and requires that *all* accesses to such an + * object go through the library. In our case we do not have an object + * in the C/C++ sense, but a view of memory as seen by the guest. + * The guest may issue a large atomic operation and then access those + * pieces using word-sized accesses. From the hypervisor, we have no + * way to connect those two actions. + * + * Therefore, special case each platform. + */ + +#if defined(CONFIG_ATOMIC128) +static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) +{ + return atomic_cmpxchg__nocheck(ptr, cmp, new); +} +# define HAVE_CMPXCHG128 1 +#elif defined(CONFIG_CMPXCHG128) +static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) +{ + return __sync_val_compare_and_swap_16(ptr, cmp, new); +} +# define HAVE_CMPXCHG128 1 +#elif defined(__aarch64__) +/* Through gcc 8, aarch64 has no support for 128-bit at all. */ +static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) +{ + uint64_t cmpl = int128_getlo(cmp), cmph = int128_gethi(cmp); + uint64_t newl = int128_getlo(new), newh = int128_gethi(new); + uint64_t oldl, oldh; + uint32_t tmp; + + asm("0: ldaxp %[oldl], %[oldh], %[mem]\n\t" + "cmp %[oldl], %[cmpl]\n\t" + "ccmp %[oldh], %[cmph], #0, eq\n\t" + "b.ne 1f\n\t" + "stlxp %w[tmp], %[newl], %[newh], %[mem]\n\t" + "cbnz %w[tmp], 0b\n" + "1:" + : [mem] "+m"(*ptr), [tmp] "=&r"(tmp), + [oldl] "=&r"(oldl), [oldh] "=r"(oldh) + : [cmpl] "r"(cmpl), [cmph] "r"(cmph), + [newl] "r"(newl), [newh] "r"(newh) + : "memory", "cc"); + + return int128_make128(oldl, oldh); +} +# define HAVE_CMPXCHG128 1 +#else +/* Fallback definition that must be optimized away, or error. */ +Int128 __attribute__((error("unsupported atomic"))) + atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new); +# define HAVE_CMPXCHG128 0 +#endif /* Some definition for HAVE_CMPXCHG128 */ + + +#if defined(CONFIG_ATOMIC128) +static inline Int128 atomic16_read(Int128 *ptr) +{ + return atomic_read__nocheck(ptr); +} + +static inline void atomic16_set(Int128 *ptr, Int128 val) +{ + atomic_set__nocheck(ptr, val); +} + +# define HAVE_ATOMIC128 1 +#elif !defined(CONFIG_USER_ONLY) && defined(__aarch64__) +/* We can do better than cmpxchg for AArch64. */ +static inline Int128 atomic16_read(Int128 *ptr) +{ + uint64_t l, h; + uint32_t tmp; + + /* The load must be paired with the store to guarantee not tearing. */ + asm("0: ldxp %[l], %[h], %[mem]\n\t" + "stxp %w[tmp], %[l], %[h], %[mem]\n\t" + "cbnz %w[tmp], 0b" + : [mem] "+m"(*ptr), [tmp] "=r"(tmp), [l] "=r"(l), [h] "=r"(h)); + + return int128_make128(l, h); +} + +static inline void atomic16_set(Int128 *ptr, Int128 val) +{ + uint64_t l = int128_getlo(val), h = int128_gethi(val); + uint64_t t1, t2; + + /* Load into temporaries to acquire the exclusive access lock. */ + asm("0: ldxp %[t1], %[t2], %[mem]\n\t" + "stxp %w[t1], %[l], %[h], %[mem]\n\t" + "cbnz %w[t1], 0b" + : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2) + : [l] "r"(l), [h] "r"(h)); +} + +# define HAVE_ATOMIC128 1 +#elif !defined(CONFIG_USER_ONLY) && HAVE_CMPXCHG128 +static inline Int128 atomic16_read(Int128 *ptr) +{ + /* Maybe replace 0 with 0, returning the old value. */ + return atomic16_cmpxchg(ptr, 0, 0); +} + +static inline void atomic16_set(Int128 *ptr, Int128 val) +{ + Int128 old = *ptr, cmp; + do { + cmp = old; + old = atomic16_cmpxchg(ptr, cmp, val); + } while (old != cmp); +} + +# define HAVE_ATOMIC128 1 +#else +/* Fallback definitions that must be optimized away, or error. */ +Int128 __attribute__((error("unsupported atomic"))) + atomic16_read(Int128 *ptr); +void __attribute__((error("unsupported atomic"))) + atomic16_set(Int128 *ptr, Int128 val); +# define HAVE_ATOMIC128 0 +#endif /* Some definition for HAVE_ATOMIC128 */ + +#endif /* QEMU_ATOMIC128_H */ diff --git a/tcg/tcg.h b/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.h +++ b/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ #include "qemu/queue.h" #include "tcg-mo.h" #include "tcg-target.h" +#include "qemu/int128.h" /* XXX: make safe guess about sizes */ #define MAX_OP_PER_INSTR 266 @@ -XXX,XX +XXX,XX @@ GEN_ATOMIC_HELPER_ALL(xchg) #undef GEN_ATOMIC_HELPER #endif /* CONFIG_SOFTMMU */ -#ifdef CONFIG_ATOMIC128 -#include "qemu/int128.h" - -/* These aren't really a "proper" helpers because TCG cannot manage Int128. - However, use the same format as the others, for use by the backends. */ +/* + * These aren't really a "proper" helpers because TCG cannot manage Int128. + * However, use the same format as the others, for use by the backends. + * + * The cmpxchg functions are only defined if HAVE_CMPXCHG128; + * the ld/st functions are only defined if HAVE_ATOMIC128, + * as defined by <qemu/atomic128.h>. + */ Int128 helper_atomic_cmpxchgo_le_mmu(CPUArchState *env, target_ulong addr, Int128 cmpv, Int128 newv, TCGMemOpIdx oi, uintptr_t retaddr); @@ -XXX,XX +XXX,XX @@ void helper_atomic_sto_le_mmu(CPUArchState *env, target_ulong addr, Int128 val, void helper_atomic_sto_be_mmu(CPUArchState *env, target_ulong addr, Int128 val, TCGMemOpIdx oi, uintptr_t retaddr); -#endif /* CONFIG_ATOMIC128 */ - #endif /* TCG_H */ diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ #include "exec/log.h" #include "exec/helper-proto.h" #include "qemu/atomic.h" +#include "qemu/atomic128.h" /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */ /* #define DEBUG_TLB */ @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, #include "atomic_template.h" #endif -#ifdef CONFIG_ATOMIC128 +#if HAVE_CMPXCHG128 || HAVE_ATOMIC128 #define DATA_SIZE 16 #include "atomic_template.h" #endif diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -XXX,XX +XXX,XX @@ #include "exec/cpu_ldst.h" #include "translate-all.h" #include "exec/helper-proto.h" +#include "qemu/atomic128.h" #undef EAX #undef ECX @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, /* The following is only callable from other helpers, and matches up with the softmmu version. */ -#ifdef CONFIG_ATOMIC128 +#if HAVE_ATOMIC128 || HAVE_CMPXCHG128 #undef EXTRA_ARGS #undef ATOMIC_NAME @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, #define DATA_SIZE 16 #include "atomic_template.h" -#endif /* CONFIG_ATOMIC128 */ +#endif diff --git a/configure b/configure index XXXXXXX..XXXXXXX 100755 --- a/configure +++ b/configure @@ -XXX,XX +XXX,XX @@ EOF fi fi +cmpxchg128=no +if test "$int128" = yes -a "$atomic128" = no; then + cat > $TMPC << EOF +int main(void) +{ + unsigned __int128 x = 0, y = 0; + __sync_val_compare_and_swap_16(&x, y, x); + return 0; +} +EOF + if compile_prog "" "" ; then + cmpxchg128=yes + fi +fi + ######################################### # See if 64-bit atomic operations are supported. # Note that without __atomic builtins, we can only @@ -XXX,XX +XXX,XX @@ if test "$atomic128" = "yes" ; then echo "CONFIG_ATOMIC128=y" >> $config_host_mak fi +if test "$cmpxchg128" = "yes" ; then + echo "CONFIG_CMPXCHG128=y" >> $config_host_mak +fi + if test "$atomic64" = "yes" ; then echo "CONFIG_ATOMIC64=y" >> $config_host_mak fi -- 2.17.2 Reviewed-by: Emilio G. Cota <cota@braap.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/i386/mem_helper.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/target/i386/mem_helper.c b/target/i386/mem_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/mem_helper.c +++ b/target/i386/mem_helper.c @@ -XXX,XX +XXX,XX @@ #include "exec/exec-all.h" #include "exec/cpu_ldst.h" #include "qemu/int128.h" +#include "qemu/atomic128.h" #include "tcg.h" void helper_cmpxchg8b_unlocked(CPUX86State *env, target_ulong a0) @@ -XXX,XX +XXX,XX @@ void helper_cmpxchg16b(CPUX86State *env, target_ulong a0) if ((a0 & 0xf) != 0) { raise_exception_ra(env, EXCP0D_GPF, ra); - } else { -#ifndef CONFIG_ATOMIC128 - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); -#else + } else if (HAVE_CMPXCHG128) { int eflags = cpu_cc_compute_all(env, CC_OP); Int128 cmpv = int128_make128(env->regs[R_EAX], env->regs[R_EDX]); @@ -XXX,XX +XXX,XX @@ void helper_cmpxchg16b(CPUX86State *env, target_ulong a0) eflags &= ~CC_Z; } CC_SRC = eflags; -#endif + } else { + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); } } #endif -- 2.17.2 Reviewed-by: Emilio G. Cota <cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/arm/helper-a64.c | 259 +++++++++++++++++++++------------------- 1 file changed, 133 insertions(+), 126 deletions(-) diff --git a/target/arm/helper-a64.c b/target/arm/helper-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-a64.c +++ b/target/arm/helper-a64.c @@ -XXX,XX +XXX,XX @@ #include "exec/exec-all.h" #include "exec/cpu_ldst.h" #include "qemu/int128.h" +#include "qemu/atomic128.h" #include "tcg.h" #include "fpu/softfloat.h" #include <zlib.h> /* For crc32 */ @@ -XXX,XX +XXX,XX @@ uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes) return crc32c(acc, buf, bytes) ^ 0xffffffff; } -/* Returns 0 on success; 1 otherwise. */ -static uint64_t do_paired_cmpxchg64_le(CPUARMState *env, uint64_t addr, - uint64_t new_lo, uint64_t new_hi, - bool parallel, uintptr_t ra) +uint64_t HELPER(paired_cmpxchg64_le)(CPUARMState *env, uint64_t addr, + uint64_t new_lo, uint64_t new_hi) { - Int128 oldv, cmpv, newv; + Int128 cmpv = int128_make128(env->exclusive_val, env->exclusive_high); + Int128 newv = int128_make128(new_lo, new_hi); + Int128 oldv; + uintptr_t ra = GETPC(); + uint64_t o0, o1; bool success; - cmpv = int128_make128(env->exclusive_val, env->exclusive_high); - newv = int128_make128(new_lo, new_hi); - - if (parallel) { -#ifndef CONFIG_ATOMIC128 - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); -#else - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); - oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra); - success = int128_eq(oldv, cmpv); -#endif - } else { - uint64_t o0, o1; - #ifdef CONFIG_USER_ONLY - /* ??? Enforce alignment. */ - uint64_t *haddr = g2h(addr); + /* ??? Enforce alignment. */ + uint64_t *haddr = g2h(addr); - helper_retaddr = ra; - o0 = ldq_le_p(haddr + 0); - o1 = ldq_le_p(haddr + 1); - oldv = int128_make128(o0, o1); + helper_retaddr = ra; + o0 = ldq_le_p(haddr + 0); + o1 = ldq_le_p(haddr + 1); + oldv = int128_make128(o0, o1); - success = int128_eq(oldv, cmpv); - if (success) { - stq_le_p(haddr + 0, int128_getlo(newv)); - stq_le_p(haddr + 1, int128_gethi(newv)); - } - helper_retaddr = 0; -#else - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi0 = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); - TCGMemOpIdx oi1 = make_memop_idx(MO_LEQ, mem_idx); - - o0 = helper_le_ldq_mmu(env, addr + 0, oi0, ra); - o1 = helper_le_ldq_mmu(env, addr + 8, oi1, ra); - oldv = int128_make128(o0, o1); - - success = int128_eq(oldv, cmpv); - if (success) { - helper_le_stq_mmu(env, addr + 0, int128_getlo(newv), oi1, ra); - helper_le_stq_mmu(env, addr + 8, int128_gethi(newv), oi1, ra); - } -#endif + success = int128_eq(oldv, cmpv); + if (success) { + stq_le_p(haddr + 0, int128_getlo(newv)); + stq_le_p(haddr + 1, int128_gethi(newv)); } + helper_retaddr = 0; +#else + int mem_idx = cpu_mmu_index(env, false); + TCGMemOpIdx oi0 = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); + TCGMemOpIdx oi1 = make_memop_idx(MO_LEQ, mem_idx); + + o0 = helper_le_ldq_mmu(env, addr + 0, oi0, ra); + o1 = helper_le_ldq_mmu(env, addr + 8, oi1, ra); + oldv = int128_make128(o0, o1); + + success = int128_eq(oldv, cmpv); + if (success) { + helper_le_stq_mmu(env, addr + 0, int128_getlo(newv), oi1, ra); + helper_le_stq_mmu(env, addr + 8, int128_gethi(newv), oi1, ra); + } +#endif return !success; } -uint64_t HELPER(paired_cmpxchg64_le)(CPUARMState *env, uint64_t addr, - uint64_t new_lo, uint64_t new_hi) -{ - return do_paired_cmpxchg64_le(env, addr, new_lo, new_hi, false, GETPC()); -} - uint64_t HELPER(paired_cmpxchg64_le_parallel)(CPUARMState *env, uint64_t addr, uint64_t new_lo, uint64_t new_hi) -{ - return do_paired_cmpxchg64_le(env, addr, new_lo, new_hi, true, GETPC()); -} - -static uint64_t do_paired_cmpxchg64_be(CPUARMState *env, uint64_t addr, - uint64_t new_lo, uint64_t new_hi, - bool parallel, uintptr_t ra) { Int128 oldv, cmpv, newv; + uintptr_t ra = GETPC(); bool success; + int mem_idx; + TCGMemOpIdx oi; - /* high and low need to be switched here because this is not actually a - * 128bit store but two doublewords stored consecutively - */ - cmpv = int128_make128(env->exclusive_high, env->exclusive_val); - newv = int128_make128(new_hi, new_lo); - - if (parallel) { -#ifndef CONFIG_ATOMIC128 + if (!HAVE_CMPXCHG128) { cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); -#else - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx); - oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra); - success = int128_eq(oldv, cmpv); -#endif - } else { - uint64_t o0, o1; - -#ifdef CONFIG_USER_ONLY - /* ??? Enforce alignment. */ - uint64_t *haddr = g2h(addr); - - helper_retaddr = ra; - o1 = ldq_be_p(haddr + 0); - o0 = ldq_be_p(haddr + 1); - oldv = int128_make128(o0, o1); - - success = int128_eq(oldv, cmpv); - if (success) { - stq_be_p(haddr + 0, int128_gethi(newv)); - stq_be_p(haddr + 1, int128_getlo(newv)); - } - helper_retaddr = 0; -#else - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi0 = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx); - TCGMemOpIdx oi1 = make_memop_idx(MO_BEQ, mem_idx); - - o1 = helper_be_ldq_mmu(env, addr + 0, oi0, ra); - o0 = helper_be_ldq_mmu(env, addr + 8, oi1, ra); - oldv = int128_make128(o0, o1); - - success = int128_eq(oldv, cmpv); - if (success) { - helper_be_stq_mmu(env, addr + 0, int128_gethi(newv), oi1, ra); - helper_be_stq_mmu(env, addr + 8, int128_getlo(newv), oi1, ra); - } -#endif } + mem_idx = cpu_mmu_index(env, false); + oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); + + cmpv = int128_make128(env->exclusive_val, env->exclusive_high); + newv = int128_make128(new_lo, new_hi); + oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra); + + success = int128_eq(oldv, cmpv); return !success; } uint64_t HELPER(paired_cmpxchg64_be)(CPUARMState *env, uint64_t addr, uint64_t new_lo, uint64_t new_hi) { - return do_paired_cmpxchg64_be(env, addr, new_lo, new_hi, false, GETPC()); + /* + * High and low need to be switched here because this is not actually a + * 128bit store but two doublewords stored consecutively + */ + Int128 cmpv = int128_make128(env->exclusive_val, env->exclusive_high); + Int128 newv = int128_make128(new_lo, new_hi); + Int128 oldv; + uintptr_t ra = GETPC(); + uint64_t o0, o1; + bool success; + +#ifdef CONFIG_USER_ONLY + /* ??? Enforce alignment. */ + uint64_t *haddr = g2h(addr); + + helper_retaddr = ra; + o1 = ldq_be_p(haddr + 0); + o0 = ldq_be_p(haddr + 1); + oldv = int128_make128(o0, o1); + + success = int128_eq(oldv, cmpv); + if (success) { + stq_be_p(haddr + 0, int128_gethi(newv)); + stq_be_p(haddr + 1, int128_getlo(newv)); + } + helper_retaddr = 0; +#else + int mem_idx = cpu_mmu_index(env, false); + TCGMemOpIdx oi0 = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx); + TCGMemOpIdx oi1 = make_memop_idx(MO_BEQ, mem_idx); + + o1 = helper_be_ldq_mmu(env, addr + 0, oi0, ra); + o0 = helper_be_ldq_mmu(env, addr + 8, oi1, ra); + oldv = int128_make128(o0, o1); + + success = int128_eq(oldv, cmpv); + if (success) { + helper_be_stq_mmu(env, addr + 0, int128_gethi(newv), oi1, ra); + helper_be_stq_mmu(env, addr + 8, int128_getlo(newv), oi1, ra); + } +#endif + + return !success; } uint64_t HELPER(paired_cmpxchg64_be_parallel)(CPUARMState *env, uint64_t addr, - uint64_t new_lo, uint64_t new_hi) + uint64_t new_lo, uint64_t new_hi) { - return do_paired_cmpxchg64_be(env, addr, new_lo, new_hi, true, GETPC()); + Int128 oldv, cmpv, newv; + uintptr_t ra = GETPC(); + bool success; + int mem_idx; + TCGMemOpIdx oi; + + if (!HAVE_CMPXCHG128) { + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); + } + + mem_idx = cpu_mmu_index(env, false); + oi = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx); + + /* + * High and low need to be switched here because this is not actually a + * 128bit store but two doublewords stored consecutively + */ + cmpv = int128_make128(env->exclusive_high, env->exclusive_val); + newv = int128_make128(new_hi, new_lo); + oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra); + + success = int128_eq(oldv, cmpv); + return !success; } /* Writes back the old data into Rs. */ void HELPER(casp_le_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr, uint64_t new_lo, uint64_t new_hi) { - uintptr_t ra = GETPC(); -#ifndef CONFIG_ATOMIC128 - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); -#else Int128 oldv, cmpv, newv; + uintptr_t ra = GETPC(); + int mem_idx; + TCGMemOpIdx oi; + + if (!HAVE_CMPXCHG128) { + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); + } + + mem_idx = cpu_mmu_index(env, false); + oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); cmpv = int128_make128(env->xregs[rs], env->xregs[rs + 1]); newv = int128_make128(new_lo, new_hi); - - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra); env->xregs[rs] = int128_getlo(oldv); env->xregs[rs + 1] = int128_gethi(oldv); -#endif } void HELPER(casp_be_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr, uint64_t new_hi, uint64_t new_lo) { - uintptr_t ra = GETPC(); -#ifndef CONFIG_ATOMIC128 - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); -#else Int128 oldv, cmpv, newv; + uintptr_t ra = GETPC(); + int mem_idx; + TCGMemOpIdx oi; + + if (!HAVE_CMPXCHG128) { + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); + } + + mem_idx = cpu_mmu_index(env, false); + oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); cmpv = int128_make128(env->xregs[rs + 1], env->xregs[rs]); newv = int128_make128(new_lo, new_hi); - - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra); env->xregs[rs + 1] = int128_getlo(oldv); env->xregs[rs] = int128_gethi(oldv); -#endif } /* -- 2.17.2 Reviewed-by: Emilio G. Cota <cota@braap.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/arm/helper-a64.c | 16 ++++------------ target/arm/translate-a64.c | 38 ++++++++++++++++++++++---------------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/target/arm/helper-a64.c b/target/arm/helper-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-a64.c +++ b/target/arm/helper-a64.c @@ -XXX,XX +XXX,XX @@ uint64_t HELPER(paired_cmpxchg64_le_parallel)(CPUARMState *env, uint64_t addr, int mem_idx; TCGMemOpIdx oi; - if (!HAVE_CMPXCHG128) { - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); - } + assert(HAVE_CMPXCHG128); mem_idx = cpu_mmu_index(env, false); oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); @@ -XXX,XX +XXX,XX @@ uint64_t HELPER(paired_cmpxchg64_be_parallel)(CPUARMState *env, uint64_t addr, int mem_idx; TCGMemOpIdx oi; - if (!HAVE_CMPXCHG128) { - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); - } + assert(HAVE_CMPXCHG128); mem_idx = cpu_mmu_index(env, false); oi = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx); @@ -XXX,XX +XXX,XX @@ void HELPER(casp_le_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr, int mem_idx; TCGMemOpIdx oi; - if (!HAVE_CMPXCHG128) { - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); - } + assert(HAVE_CMPXCHG128); mem_idx = cpu_mmu_index(env, false); oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); @@ -XXX,XX +XXX,XX @@ void HELPER(casp_be_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr, int mem_idx; TCGMemOpIdx oi; - if (!HAVE_CMPXCHG128) { - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); - } + assert(HAVE_CMPXCHG128); mem_idx = cpu_mmu_index(env, false); oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ #include "trace-tcg.h" #include "translate-a64.h" +#include "qemu/atomic128.h" static TCGv_i64 cpu_X[32]; static TCGv_i64 cpu_pc; @@ -XXX,XX +XXX,XX @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2, get_mem_index(s), MO_64 | MO_ALIGN | s->be_data); tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val); - } else if (s->be_data == MO_LE) { - if (tb_cflags(s->base.tb) & CF_PARALLEL) { + } else if (tb_cflags(s->base.tb) & CF_PARALLEL) { + if (!HAVE_CMPXCHG128) { + gen_helper_exit_atomic(cpu_env); + s->base.is_jmp = DISAS_NORETURN; + } else if (s->be_data == MO_LE) { gen_helper_paired_cmpxchg64_le_parallel(tmp, cpu_env, cpu_exclusive_addr, cpu_reg(s, rt), cpu_reg(s, rt2)); } else { - gen_helper_paired_cmpxchg64_le(tmp, cpu_env, cpu_exclusive_addr, - cpu_reg(s, rt), cpu_reg(s, rt2)); - } - } else { - if (tb_cflags(s->base.tb) & CF_PARALLEL) { gen_helper_paired_cmpxchg64_be_parallel(tmp, cpu_env, cpu_exclusive_addr, cpu_reg(s, rt), cpu_reg(s, rt2)); - } else { - gen_helper_paired_cmpxchg64_be(tmp, cpu_env, cpu_exclusive_addr, - cpu_reg(s, rt), cpu_reg(s, rt2)); } + } else if (s->be_data == MO_LE) { + gen_helper_paired_cmpxchg64_le(tmp, cpu_env, cpu_exclusive_addr, + cpu_reg(s, rt), cpu_reg(s, rt2)); + } else { + gen_helper_paired_cmpxchg64_be(tmp, cpu_env, cpu_exclusive_addr, + cpu_reg(s, rt), cpu_reg(s, rt2)); } } else { tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr, cpu_exclusive_val, @@ -XXX,XX +XXX,XX @@ static void gen_compare_and_swap_pair(DisasContext *s, int rs, int rt, } tcg_temp_free_i64(cmp); } else if (tb_cflags(s->base.tb) & CF_PARALLEL) { - TCGv_i32 tcg_rs = tcg_const_i32(rs); - - if (s->be_data == MO_LE) { - gen_helper_casp_le_parallel(cpu_env, tcg_rs, addr, t1, t2); + if (HAVE_CMPXCHG128) { + TCGv_i32 tcg_rs = tcg_const_i32(rs); + if (s->be_data == MO_LE) { + gen_helper_casp_le_parallel(cpu_env, tcg_rs, addr, t1, t2); + } else { + gen_helper_casp_be_parallel(cpu_env, tcg_rs, addr, t1, t2); + } + tcg_temp_free_i32(tcg_rs); } else { - gen_helper_casp_be_parallel(cpu_env, tcg_rs, addr, t1, t2); + gen_helper_exit_atomic(cpu_env); + s->base.is_jmp = DISAS_NORETURN; } - tcg_temp_free_i32(tcg_rs); } else { TCGv_i64 d1 = tcg_temp_new_i64(); TCGv_i64 d2 = tcg_temp_new_i64(); -- 2.17.2 Reviewed-by: Emilio G. Cota <cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/ppc/helper.h | 2 +- target/ppc/mem_helper.c | 33 ++++++++++-- target/ppc/translate.c | 115 +++++++++++++++++++++------------------- 3 files changed, 88 insertions(+), 62 deletions(-) diff --git a/target/ppc/helper.h b/target/ppc/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/helper.h +++ b/target/ppc/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_4(dscliq, void, env, fprp, fprp, i32) DEF_HELPER_1(tbegin, void, env) DEF_HELPER_FLAGS_1(fixup_thrm, TCG_CALL_NO_RWG, void, env) -#if defined(TARGET_PPC64) && defined(CONFIG_ATOMIC128) +#ifdef TARGET_PPC64 DEF_HELPER_FLAGS_3(lq_le_parallel, TCG_CALL_NO_WG, i64, env, tl, i32) DEF_HELPER_FLAGS_3(lq_be_parallel, TCG_CALL_NO_WG, i64, env, tl, i32) DEF_HELPER_FLAGS_5(stq_le_parallel, TCG_CALL_NO_WG, diff --git a/target/ppc/mem_helper.c b/target/ppc/mem_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/mem_helper.c +++ b/target/ppc/mem_helper.c @@ -XXX,XX +XXX,XX @@ #include "exec/cpu_ldst.h" #include "tcg.h" #include "internal.h" +#include "qemu/atomic128.h" //#define DEBUG_OP @@ -XXX,XX +XXX,XX @@ target_ulong helper_lscbx(CPUPPCState *env, target_ulong addr, uint32_t reg, return i; } -#if defined(TARGET_PPC64) && defined(CONFIG_ATOMIC128) +#ifdef TARGET_PPC64 uint64_t helper_lq_le_parallel(CPUPPCState *env, target_ulong addr, uint32_t opidx) { - Int128 ret = helper_atomic_ldo_le_mmu(env, addr, opidx, GETPC()); + Int128 ret; + + /* We will have raised EXCP_ATOMIC from the translator. */ + assert(HAVE_ATOMIC128); + ret = helper_atomic_ldo_le_mmu(env, addr, opidx, GETPC()); env->retxh = int128_gethi(ret); return int128_getlo(ret); } @@ -XXX,XX +XXX,XX @@ uint64_t helper_lq_le_parallel(CPUPPCState *env, target_ulong addr, uint64_t helper_lq_be_parallel(CPUPPCState *env, target_ulong addr, uint32_t opidx) { - Int128 ret = helper_atomic_ldo_be_mmu(env, addr, opidx, GETPC()); + Int128 ret; + + /* We will have raised EXCP_ATOMIC from the translator. */ + assert(HAVE_ATOMIC128); + ret = helper_atomic_ldo_be_mmu(env, addr, opidx, GETPC()); env->retxh = int128_gethi(ret); return int128_getlo(ret); } @@ -XXX,XX +XXX,XX @@ uint64_t helper_lq_be_parallel(CPUPPCState *env, target_ulong addr, void helper_stq_le_parallel(CPUPPCState *env, target_ulong addr, uint64_t lo, uint64_t hi, uint32_t opidx) { - Int128 val = int128_make128(lo, hi); + Int128 val; + + /* We will have raised EXCP_ATOMIC from the translator. */ + assert(HAVE_ATOMIC128); + val = int128_make128(lo, hi); helper_atomic_sto_le_mmu(env, addr, val, opidx, GETPC()); } void helper_stq_be_parallel(CPUPPCState *env, target_ulong addr, uint64_t lo, uint64_t hi, uint32_t opidx) { - Int128 val = int128_make128(lo, hi); + Int128 val; + + /* We will have raised EXCP_ATOMIC from the translator. */ + assert(HAVE_ATOMIC128); + val = int128_make128(lo, hi); helper_atomic_sto_be_mmu(env, addr, val, opidx, GETPC()); } @@ -XXX,XX +XXX,XX @@ uint32_t helper_stqcx_le_parallel(CPUPPCState *env, target_ulong addr, { bool success = false; + /* We will have raised EXCP_ATOMIC from the translator. */ + assert(HAVE_CMPXCHG128); + if (likely(addr == env->reserve_addr)) { Int128 oldv, cmpv, newv; @@ -XXX,XX +XXX,XX @@ uint32_t helper_stqcx_be_parallel(CPUPPCState *env, target_ulong addr, { bool success = false; + /* We will have raised EXCP_ATOMIC from the translator. */ + assert(HAVE_CMPXCHG128); + if (likely(addr == env->reserve_addr)) { Int128 oldv, cmpv, newv; diff --git a/target/ppc/translate.c b/target/ppc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/translate.c +++ b/target/ppc/translate.c @@ -XXX,XX +XXX,XX @@ #include "trace-tcg.h" #include "exec/translator.h" #include "exec/log.h" +#include "qemu/atomic128.h" #define CPU_SINGLE_STEP 0x1 @@ -XXX,XX +XXX,XX @@ static void gen_lq(DisasContext *ctx) hi = cpu_gpr[rd]; if (tb_cflags(ctx->base.tb) & CF_PARALLEL) { -#ifdef CONFIG_ATOMIC128 - TCGv_i32 oi = tcg_temp_new_i32(); - if (ctx->le_mode) { - tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ, ctx->mem_idx)); - gen_helper_lq_le_parallel(lo, cpu_env, EA, oi); + if (HAVE_ATOMIC128) { + TCGv_i32 oi = tcg_temp_new_i32(); + if (ctx->le_mode) { + tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ, ctx->mem_idx)); + gen_helper_lq_le_parallel(lo, cpu_env, EA, oi); + } else { + tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ, ctx->mem_idx)); + gen_helper_lq_be_parallel(lo, cpu_env, EA, oi); + } + tcg_temp_free_i32(oi); + tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh)); } else { - tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ, ctx->mem_idx)); - gen_helper_lq_be_parallel(lo, cpu_env, EA, oi); + /* Restart with exclusive lock. */ + gen_helper_exit_atomic(cpu_env); + ctx->base.is_jmp = DISAS_NORETURN; } - tcg_temp_free_i32(oi); - tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh)); -#else - /* Restart with exclusive lock. */ - gen_helper_exit_atomic(cpu_env); - ctx->base.is_jmp = DISAS_NORETURN; -#endif } else if (ctx->le_mode) { tcg_gen_qemu_ld_i64(lo, EA, ctx->mem_idx, MO_LEQ); gen_addr_add(ctx, EA, EA, 8); @@ -XXX,XX +XXX,XX @@ static void gen_std(DisasContext *ctx) hi = cpu_gpr[rs]; if (tb_cflags(ctx->base.tb) & CF_PARALLEL) { -#ifdef CONFIG_ATOMIC128 - TCGv_i32 oi = tcg_temp_new_i32(); - if (ctx->le_mode) { - tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ, ctx->mem_idx)); - gen_helper_stq_le_parallel(cpu_env, EA, lo, hi, oi); + if (HAVE_ATOMIC128) { + TCGv_i32 oi = tcg_temp_new_i32(); + if (ctx->le_mode) { + tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ, ctx->mem_idx)); + gen_helper_stq_le_parallel(cpu_env, EA, lo, hi, oi); + } else { + tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ, ctx->mem_idx)); + gen_helper_stq_be_parallel(cpu_env, EA, lo, hi, oi); + } + tcg_temp_free_i32(oi); } else { - tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ, ctx->mem_idx)); - gen_helper_stq_be_parallel(cpu_env, EA, lo, hi, oi); + /* Restart with exclusive lock. */ + gen_helper_exit_atomic(cpu_env); + ctx->base.is_jmp = DISAS_NORETURN; } - tcg_temp_free_i32(oi); -#else - /* Restart with exclusive lock. */ - gen_helper_exit_atomic(cpu_env); - ctx->base.is_jmp = DISAS_NORETURN; -#endif } else if (ctx->le_mode) { tcg_gen_qemu_st_i64(lo, EA, ctx->mem_idx, MO_LEQ); gen_addr_add(ctx, EA, EA, 8); @@ -XXX,XX +XXX,XX @@ static void gen_lqarx(DisasContext *ctx) hi = cpu_gpr[rd]; if (tb_cflags(ctx->base.tb) & CF_PARALLEL) { -#ifdef CONFIG_ATOMIC128 - TCGv_i32 oi = tcg_temp_new_i32(); - if (ctx->le_mode) { - tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ | MO_ALIGN_16, - ctx->mem_idx)); - gen_helper_lq_le_parallel(lo, cpu_env, EA, oi); + if (HAVE_ATOMIC128) { + TCGv_i32 oi = tcg_temp_new_i32(); + if (ctx->le_mode) { + tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ | MO_ALIGN_16, + ctx->mem_idx)); + gen_helper_lq_le_parallel(lo, cpu_env, EA, oi); + } else { + tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ | MO_ALIGN_16, + ctx->mem_idx)); + gen_helper_lq_be_parallel(lo, cpu_env, EA, oi); + } + tcg_temp_free_i32(oi); + tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh)); } else { - tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ | MO_ALIGN_16, - ctx->mem_idx)); - gen_helper_lq_be_parallel(lo, cpu_env, EA, oi); + /* Restart with exclusive lock. */ + gen_helper_exit_atomic(cpu_env); + ctx->base.is_jmp = DISAS_NORETURN; + tcg_temp_free(EA); + return; } - tcg_temp_free_i32(oi); - tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh)); -#else - /* Restart with exclusive lock. */ - gen_helper_exit_atomic(cpu_env); - ctx->base.is_jmp = DISAS_NORETURN; - tcg_temp_free(EA); - return; -#endif } else if (ctx->le_mode) { tcg_gen_qemu_ld_i64(lo, EA, ctx->mem_idx, MO_LEQ | MO_ALIGN_16); tcg_gen_mov_tl(cpu_reserve, EA); @@ -XXX,XX +XXX,XX @@ static void gen_stqcx_(DisasContext *ctx) hi = cpu_gpr[rs]; if (tb_cflags(ctx->base.tb) & CF_PARALLEL) { - TCGv_i32 oi = tcg_const_i32(DEF_MEMOP(MO_Q) | MO_ALIGN_16); -#ifdef CONFIG_ATOMIC128 - if (ctx->le_mode) { - gen_helper_stqcx_le_parallel(cpu_crf[0], cpu_env, EA, lo, hi, oi); + if (HAVE_CMPXCHG128) { + TCGv_i32 oi = tcg_const_i32(DEF_MEMOP(MO_Q) | MO_ALIGN_16); + if (ctx->le_mode) { + gen_helper_stqcx_le_parallel(cpu_crf[0], cpu_env, + EA, lo, hi, oi); + } else { + gen_helper_stqcx_be_parallel(cpu_crf[0], cpu_env, + EA, lo, hi, oi); + } + tcg_temp_free_i32(oi); } else { - gen_helper_stqcx_le_parallel(cpu_crf[0], cpu_env, EA, lo, hi, oi); + /* Restart with exclusive lock. */ + gen_helper_exit_atomic(cpu_env); + ctx->base.is_jmp = DISAS_NORETURN; } -#else - /* Restart with exclusive lock. */ - gen_helper_exit_atomic(cpu_env); - ctx->base.is_jmp = DISAS_NORETURN; -#endif tcg_temp_free(EA); - tcg_temp_free_i32(oi); } else { TCGLabel *lab_fail = gen_new_label(); TCGLabel *lab_over = gen_new_label(); -- 2.17.2 Reviewed-by: David Hildenbrand <david@redhat.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/s390x/mem_helper.c | 92 +++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 51 deletions(-) diff --git a/target/s390x/mem_helper.c b/target/s390x/mem_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/mem_helper.c +++ b/target/s390x/mem_helper.c @@ -XXX,XX +XXX,XX @@ #include "exec/exec-all.h" #include "exec/cpu_ldst.h" #include "qemu/int128.h" +#include "qemu/atomic128.h" #if !defined(CONFIG_USER_ONLY) #include "hw/s390x/storage-keys.h" @@ -XXX,XX +XXX,XX @@ static void do_cdsg(CPUS390XState *env, uint64_t addr, bool fail; if (parallel) { -#ifndef CONFIG_ATOMIC128 +#if !HAVE_CMPXCHG128 cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); #else int mem_idx = cpu_mmu_index(env, false); @@ -XXX,XX +XXX,XX @@ void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr, static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, uint64_t a2, bool parallel) { -#if !defined(CONFIG_USER_ONLY) || defined(CONFIG_ATOMIC128) uint32_t mem_idx = cpu_mmu_index(env, false); -#endif uintptr_t ra = GETPC(); uint32_t fc = extract32(env->regs[0], 0, 8); uint32_t sc = extract32(env->regs[0], 8, 8); @@ -XXX,XX +XXX,XX @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, probe_write(env, a2, 0, mem_idx, ra); #endif - /* Note that the compare-and-swap is atomic, and the store is atomic, but - the complete operation is not. Therefore we do not need to assert serial - context in order to implement this. That said, restart early if we can't - support either operation that is supposed to be atomic. */ + /* + * Note that the compare-and-swap is atomic, and the store is atomic, + * but the complete operation is not. Therefore we do not need to + * assert serial context in order to implement this. That said, + * restart early if we can't support either operation that is supposed + * to be atomic. + */ if (parallel) { - int mask = 0; -#if !defined(CONFIG_ATOMIC64) - mask = -8; -#elif !defined(CONFIG_ATOMIC128) - mask = -16; + uint32_t max = 2; +#ifdef CONFIG_ATOMIC64 + max = 3; #endif - if (((4 << fc) | (1 << sc)) & mask) { + if ((HAVE_CMPXCHG128 ? 0 : fc + 2 > max) || + (HAVE_ATOMIC128 ? 0 : sc > max)) { cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); } } @@ -XXX,XX +XXX,XX @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, Int128 cv = int128_make128(env->regs[r3 + 1], env->regs[r3]); Int128 ov; - if (parallel) { -#ifdef CONFIG_ATOMIC128 - TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); - ov = helper_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi, ra); - cc = !int128_eq(ov, cv); -#else - /* Note that we asserted !parallel above. */ - g_assert_not_reached(); -#endif - } else { + if (!parallel) { uint64_t oh = cpu_ldq_data_ra(env, a1 + 0, ra); uint64_t ol = cpu_ldq_data_ra(env, a1 + 8, ra); @@ -XXX,XX +XXX,XX @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, cpu_stq_data_ra(env, a1 + 0, int128_gethi(nv), ra); cpu_stq_data_ra(env, a1 + 8, int128_getlo(nv), ra); + } else if (HAVE_CMPXCHG128) { + TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); + ov = helper_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi, ra); + cc = !int128_eq(ov, cv); + } else { + /* Note that we asserted !parallel above. */ + g_assert_not_reached(); } env->regs[r3 + 0] = int128_gethi(ov); @@ -XXX,XX +XXX,XX @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, cpu_stq_data_ra(env, a2, svh, ra); break; case 4: - if (parallel) { -#ifdef CONFIG_ATOMIC128 + if (!parallel) { + cpu_stq_data_ra(env, a2 + 0, svh, ra); + cpu_stq_data_ra(env, a2 + 8, svl, ra); + } else if (HAVE_ATOMIC128) { TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); Int128 sv = int128_make128(svl, svh); helper_atomic_sto_be_mmu(env, a2, sv, oi, ra); -#else + } else { /* Note that we asserted !parallel above. */ g_assert_not_reached(); -#endif - } else { - cpu_stq_data_ra(env, a2 + 0, svh, ra); - cpu_stq_data_ra(env, a2 + 8, svl, ra); } break; default: @@ -XXX,XX +XXX,XX @@ static uint64_t do_lpq(CPUS390XState *env, uint64_t addr, bool parallel) uintptr_t ra = GETPC(); uint64_t hi, lo; - if (parallel) { -#ifndef CONFIG_ATOMIC128 - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); -#else + if (!parallel) { + check_alignment(env, addr, 16, ra); + hi = cpu_ldq_data_ra(env, addr + 0, ra); + lo = cpu_ldq_data_ra(env, addr + 8, ra); + } else if (HAVE_ATOMIC128) { int mem_idx = cpu_mmu_index(env, false); TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); Int128 v = helper_atomic_ldo_be_mmu(env, addr, oi, ra); hi = int128_gethi(v); lo = int128_getlo(v); -#endif } else { - check_alignment(env, addr, 16, ra); - - hi = cpu_ldq_data_ra(env, addr + 0, ra); - lo = cpu_ldq_data_ra(env, addr + 8, ra); + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); } env->retxl = lo; @@ -XXX,XX +XXX,XX @@ static void do_stpq(CPUS390XState *env, uint64_t addr, { uintptr_t ra = GETPC(); - if (parallel) { -#ifndef CONFIG_ATOMIC128 - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); -#else - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); - - Int128 v = int128_make128(low, high); - helper_atomic_sto_be_mmu(env, addr, v, oi, ra); -#endif - } else { + if (!parallel) { check_alignment(env, addr, 16, ra); - cpu_stq_data_ra(env, addr + 0, high, ra); cpu_stq_data_ra(env, addr + 8, low, ra); + } else if (HAVE_ATOMIC128) { + int mem_idx = cpu_mmu_index(env, false); + TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); + Int128 v = int128_make128(low, high); + helper_atomic_sto_be_mmu(env, addr, v, oi, ra); + } else { + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); } } -- 2.17.2 Reviewed-by: David Hildenbrand <david@redhat.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/s390x/mem_helper.c | 128 ++++++++++++++++++-------------------- 1 file changed, 61 insertions(+), 67 deletions(-) diff --git a/target/s390x/mem_helper.c b/target/s390x/mem_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/mem_helper.c +++ b/target/s390x/mem_helper.c @@ -XXX,XX +XXX,XX @@ uint32_t HELPER(trXX)(CPUS390XState *env, uint32_t r1, uint32_t r2, return cc; } -static void do_cdsg(CPUS390XState *env, uint64_t addr, - uint32_t r1, uint32_t r3, bool parallel) +void HELPER(cdsg)(CPUS390XState *env, uint64_t addr, + uint32_t r1, uint32_t r3) { uintptr_t ra = GETPC(); Int128 cmpv = int128_make128(env->regs[r1 + 1], env->regs[r1]); Int128 newv = int128_make128(env->regs[r3 + 1], env->regs[r3]); Int128 oldv; + uint64_t oldh, oldl; bool fail; - if (parallel) { -#if !HAVE_CMPXCHG128 - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); -#else - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); - oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra); - fail = !int128_eq(oldv, cmpv); -#endif - } else { - uint64_t oldh, oldl; + check_alignment(env, addr, 16, ra); - check_alignment(env, addr, 16, ra); + oldh = cpu_ldq_data_ra(env, addr + 0, ra); + oldl = cpu_ldq_data_ra(env, addr + 8, ra); - oldh = cpu_ldq_data_ra(env, addr + 0, ra); - oldl = cpu_ldq_data_ra(env, addr + 8, ra); - - oldv = int128_make128(oldl, oldh); - fail = !int128_eq(oldv, cmpv); - if (fail) { - newv = oldv; - } - - cpu_stq_data_ra(env, addr + 0, int128_gethi(newv), ra); - cpu_stq_data_ra(env, addr + 8, int128_getlo(newv), ra); + oldv = int128_make128(oldl, oldh); + fail = !int128_eq(oldv, cmpv); + if (fail) { + newv = oldv; } + cpu_stq_data_ra(env, addr + 0, int128_gethi(newv), ra); + cpu_stq_data_ra(env, addr + 8, int128_getlo(newv), ra); + env->cc_op = fail; env->regs[r1] = int128_gethi(oldv); env->regs[r1 + 1] = int128_getlo(oldv); } -void HELPER(cdsg)(CPUS390XState *env, uint64_t addr, - uint32_t r1, uint32_t r3) -{ - do_cdsg(env, addr, r1, r3, false); -} - void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr, uint32_t r1, uint32_t r3) { - do_cdsg(env, addr, r1, r3, true); + uintptr_t ra = GETPC(); + Int128 cmpv = int128_make128(env->regs[r1 + 1], env->regs[r1]); + Int128 newv = int128_make128(env->regs[r3 + 1], env->regs[r3]); + int mem_idx; + TCGMemOpIdx oi; + Int128 oldv; + bool fail; + + if (!HAVE_CMPXCHG128) { + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); + } + + mem_idx = cpu_mmu_index(env, false); + oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); + oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra); + fail = !int128_eq(oldv, cmpv); + + env->cc_op = fail; + env->regs[r1] = int128_gethi(oldv); + env->regs[r1 + 1] = int128_getlo(oldv); } static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, @@ -XXX,XX +XXX,XX @@ uint64_t HELPER(lra)(CPUS390XState *env, uint64_t addr) #endif /* load pair from quadword */ -static uint64_t do_lpq(CPUS390XState *env, uint64_t addr, bool parallel) +uint64_t HELPER(lpq)(CPUS390XState *env, uint64_t addr) { uintptr_t ra = GETPC(); uint64_t hi, lo; - if (!parallel) { - check_alignment(env, addr, 16, ra); - hi = cpu_ldq_data_ra(env, addr + 0, ra); - lo = cpu_ldq_data_ra(env, addr + 8, ra); - } else if (HAVE_ATOMIC128) { + check_alignment(env, addr, 16, ra); + hi = cpu_ldq_data_ra(env, addr + 0, ra); + lo = cpu_ldq_data_ra(env, addr + 8, ra); + + env->retxl = lo; + return hi; +} + +uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr) +{ + uintptr_t ra = GETPC(); + uint64_t hi, lo; + + if (HAVE_ATOMIC128) { int mem_idx = cpu_mmu_index(env, false); TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); Int128 v = helper_atomic_ldo_be_mmu(env, addr, oi, ra); @@ -XXX,XX +XXX,XX @@ static uint64_t do_lpq(CPUS390XState *env, uint64_t addr, bool parallel) return hi; } -uint64_t HELPER(lpq)(CPUS390XState *env, uint64_t addr) -{ - return do_lpq(env, addr, false); -} - -uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr) -{ - return do_lpq(env, addr, true); -} - /* store pair to quadword */ -static void do_stpq(CPUS390XState *env, uint64_t addr, - uint64_t low, uint64_t high, bool parallel) +void HELPER(stpq)(CPUS390XState *env, uint64_t addr, + uint64_t low, uint64_t high) { uintptr_t ra = GETPC(); - if (!parallel) { - check_alignment(env, addr, 16, ra); - cpu_stq_data_ra(env, addr + 0, high, ra); - cpu_stq_data_ra(env, addr + 8, low, ra); - } else if (HAVE_ATOMIC128) { + check_alignment(env, addr, 16, ra); + cpu_stq_data_ra(env, addr + 0, high, ra); + cpu_stq_data_ra(env, addr + 8, low, ra); +} + +void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr, + uint64_t low, uint64_t high) +{ + uintptr_t ra = GETPC(); + + if (HAVE_ATOMIC128) { int mem_idx = cpu_mmu_index(env, false); TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); Int128 v = int128_make128(low, high); @@ -XXX,XX +XXX,XX @@ static void do_stpq(CPUS390XState *env, uint64_t addr, } } -void HELPER(stpq)(CPUS390XState *env, uint64_t addr, - uint64_t low, uint64_t high) -{ - do_stpq(env, addr, low, high, false); -} - -void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr, - uint64_t low, uint64_t high) -{ - do_stpq(env, addr, low, high, true); -} - /* Execute instruction. This instruction executes an insn modified with the contents of r1. It does not change the executed instruction in memory; it does not change the program counter. -- 2.17.2 When op raises an exception, it may not have initialized the output temps that would be written back by wout or cout. Reviewed-by: David Hildenbrand <david@redhat.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/s390x/translate.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/target/s390x/translate.c b/target/s390x/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/translate.c +++ b/target/s390x/translate.c @@ -XXX,XX +XXX,XX @@ struct DisasInsn { const char *name; + /* Pre-process arguments before HELP_OP. */ void (*help_in1)(DisasContext *, DisasFields *, DisasOps *); void (*help_in2)(DisasContext *, DisasFields *, DisasOps *); void (*help_prep)(DisasContext *, DisasFields *, DisasOps *); + + /* + * Post-process output after HELP_OP. + * Note that these are not called if HELP_OP returns DISAS_NORETURN. + */ void (*help_wout)(DisasContext *, DisasFields *, DisasOps *); void (*help_cout)(DisasContext *, DisasOps *); + + /* Implement the operation itself. */ DisasJumpType (*help_op)(DisasContext *, DisasOps *); uint64_t data; @@ -XXX,XX +XXX,XX @@ static DisasJumpType translate_one(CPUS390XState *env, DisasContext *s) if (insn->help_op) { ret = insn->help_op(s, &o); } - if (insn->help_wout) { - insn->help_wout(s, &f, &o); - } - if (insn->help_cout) { - insn->help_cout(s, &o); + if (ret != DISAS_NORETURN) { + if (insn->help_wout) { + insn->help_wout(s, &f, &o); + } + if (insn->help_cout) { + insn->help_cout(s, &o); + } } /* Free any temporaries created by the helpers. */ -- 2.17.2 Reviewed-by: David Hildenbrand <david@redhat.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/s390x/mem_helper.c | 40 +++++++++++++++++++-------------------- target/s390x/translate.c | 25 +++++++++++++++++------- 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/target/s390x/mem_helper.c b/target/s390x/mem_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/mem_helper.c +++ b/target/s390x/mem_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr, Int128 oldv; bool fail; - if (!HAVE_CMPXCHG128) { - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); - } + assert(HAVE_CMPXCHG128); mem_idx = cpu_mmu_index(env, false); oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); @@ -XXX,XX +XXX,XX @@ uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr) { uintptr_t ra = GETPC(); uint64_t hi, lo; + int mem_idx; + TCGMemOpIdx oi; + Int128 v; - if (HAVE_ATOMIC128) { - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); - Int128 v = helper_atomic_ldo_be_mmu(env, addr, oi, ra); - hi = int128_gethi(v); - lo = int128_getlo(v); - } else { - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); - } + assert(HAVE_ATOMIC128); + + mem_idx = cpu_mmu_index(env, false); + oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); + v = helper_atomic_ldo_be_mmu(env, addr, oi, ra); + hi = int128_gethi(v); + lo = int128_getlo(v); env->retxl = lo; return hi; @@ -XXX,XX +XXX,XX @@ void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr, uint64_t low, uint64_t high) { uintptr_t ra = GETPC(); + int mem_idx; + TCGMemOpIdx oi; + Int128 v; - if (HAVE_ATOMIC128) { - int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); - Int128 v = int128_make128(low, high); - helper_atomic_sto_be_mmu(env, addr, v, oi, ra); - } else { - cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); - } + assert(HAVE_ATOMIC128); + + mem_idx = cpu_mmu_index(env, false); + oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); + v = int128_make128(low, high); + helper_atomic_sto_be_mmu(env, addr, v, oi, ra); } /* Execute instruction. This instruction executes an insn modified with diff --git a/target/s390x/translate.c b/target/s390x/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/translate.c +++ b/target/s390x/translate.c @@ -XXX,XX +XXX,XX @@ #include "trace-tcg.h" #include "exec/translator.h" #include "exec/log.h" +#include "qemu/atomic128.h" /* Information that (most) every instruction needs to manipulate. */ @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_cdsg(DisasContext *s, DisasOps *o) int r3 = get_field(s->fields, r3); int d2 = get_field(s->fields, d2); int b2 = get_field(s->fields, b2); + DisasJumpType ret = DISAS_NEXT; TCGv_i64 addr; TCGv_i32 t_r1, t_r3; @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_cdsg(DisasContext *s, DisasOps *o) addr = get_address(s, 0, b2, d2); t_r1 = tcg_const_i32(r1); t_r3 = tcg_const_i32(r3); - if (tb_cflags(s->base.tb) & CF_PARALLEL) { + if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { + gen_helper_cdsg(cpu_env, addr, t_r1, t_r3); + } else if (HAVE_CMPXCHG128) { gen_helper_cdsg_parallel(cpu_env, addr, t_r1, t_r3); } else { - gen_helper_cdsg(cpu_env, addr, t_r1, t_r3); + gen_helper_exit_atomic(cpu_env); + ret = DISAS_NORETURN; } tcg_temp_free_i64(addr); tcg_temp_free_i32(t_r1); tcg_temp_free_i32(t_r3); set_cc_static(s); - return DISAS_NEXT; + return ret; } static DisasJumpType op_csst(DisasContext *s, DisasOps *o) @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_lpd(DisasContext *s, DisasOps *o) static DisasJumpType op_lpq(DisasContext *s, DisasOps *o) { - if (tb_cflags(s->base.tb) & CF_PARALLEL) { + if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { + gen_helper_lpq(o->out, cpu_env, o->in2); + } else if (HAVE_ATOMIC128) { gen_helper_lpq_parallel(o->out, cpu_env, o->in2); } else { - gen_helper_lpq(o->out, cpu_env, o->in2); + gen_helper_exit_atomic(cpu_env); + return DISAS_NORETURN; } return_low128(o->out2); return DISAS_NEXT; @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_stmh(DisasContext *s, DisasOps *o) static DisasJumpType op_stpq(DisasContext *s, DisasOps *o) { - if (tb_cflags(s->base.tb) & CF_PARALLEL) { + if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { + gen_helper_stpq(cpu_env, o->in2, o->out2, o->out); + } else if (HAVE_ATOMIC128) { gen_helper_stpq_parallel(cpu_env, o->in2, o->out2, o->out); } else { - gen_helper_stpq(cpu_env, o->in2, o->out2, o->out); + gen_helper_exit_atomic(cpu_env); + return DISAS_NORETURN; } return DISAS_NEXT; } -- 2.17.2 From: "Emilio G. Cota" <cota@braap.org> Updates can come from other threads, so readers that do not take tlb_lock must use atomic_read to avoid undefined behaviour (UB). This completes the conversion to tlb_lock. This conversion results on average in no performance loss, as the following experiments (run on an Intel i7-6700K CPU @ 4.00GHz) show. 1. aarch64 bootup+shutdown test: - Before: Performance counter stats for 'taskset -c 0 ../img/aarch64/die.sh' (10 runs): 7487.087786 task-clock (msec) # 0.998 CPUs utilized ( +- 0.12% ) 31,574,905,303 cycles # 4.217 GHz ( +- 0.12% ) 57,097,908,812 instructions # 1.81 insns per cycle ( +- 0.08% ) 10,255,415,367 branches # 1369.747 M/sec ( +- 0.08% ) 173,278,962 branch-misses # 1.69% of all branches ( +- 0.18% ) 7.504481349 seconds time elapsed ( +- 0.14% ) - After: Performance counter stats for 'taskset -c 0 ../img/aarch64/die.sh' (10 runs): 7462.441328 task-clock (msec) # 0.998 CPUs utilized ( +- 0.07% ) 31,478,476,520 cycles # 4.218 GHz ( +- 0.07% ) 57,017,330,084 instructions # 1.81 insns per cycle ( +- 0.05% ) 10,251,929,667 branches # 1373.804 M/sec ( +- 0.05% ) 173,023,787 branch-misses # 1.69% of all branches ( +- 0.11% ) 7.474970463 seconds time elapsed ( +- 0.07% ) 2. SPEC06int: SPEC06int (test set) [Y axis: Speedup over master] 1.15 +-+----+------+------+------+------+------+-------+------+------+------+------+------+------+----+-+ | | 1.1 +-+.................................+++.............................+ tlb-lock-v2 (m+++x) +-+ | +++ | +++ tlb-lock-v3 (spinl|ck) | | +++ | | +++ +++ | | | 1.05 +-+....+++...........####.........|####.+++.|......|.....###....+++...........+++....###.........+-+ | ### ++#| # |# |# ***### +++### +++#+# | +++ | #|# ### | 1 +-+++***+#++++####+++#++#++++++++++#++#+*+*++#++++#+#+****+#++++###++++###++++###++++#+#++++#+#+++-+ | *+* # #++# *** # #### *** # * *++# ****+# *| * # ****|# |# # #|# #+# # # | 0.95 +-+..*.*.#....#..#.*|*..#...#..#.*|*..#.*.*..#.*|.*.#.*++*.#.*++*+#.****.#....#+#....#.#..++#.#..+-+ | * * # # # *|* # # # *|* # * * # *++* # * * # * * # * |* # ++# # # # *** # | | * * # ++# # *+* # # # *|* # * * # * * # * * # * * # *++* # **** # ++# # * * # | 0.9 +-+..*.*.#...|#..#.*.*..#.++#..#.*|*..#.*.*..#.*..*.#.*..*.#.*..*.#.*..*.#.*.|*.#...|#.#..*.*.#..+-+ | * * # *** # * * # |# # *+* # * * # * * # * * # * * # * * # *++* # |# # * * # | 0.85 +-+..*.*.#..*|*..#.*.*..#.***..#.*.*..#.*.*..#.*..*.#.*..*.#.*..*.#.*..*.#.*..*.#.****.#..*.*.#..+-+ | * * # *+* # * * # *|* # * * # * * # * * # * * # * * # * * # * * # * |* # * * # | | * * # * * # * * # *+* # * * # * * # * * # * * # * * # * * # * * # * |* # * * # | 0.8 +-+..*.*.#..*.*..#.*.*..#.*.*..#.*.*..#.*.*..#.*..*.#.*..*.#.*..*.#.*..*.#.*..*.#.*++*.#..*.*.#..+-+ | * * # * * # * * # * * # * * # * * # * * # * * # * * # * * # * * # * * # * * # | 0.75 +-+--***##--***###-***###-***###-***###-***###-****##-****##-****##-****##-****##-****##--***##--+-+ 400.perlben401.bzip2403.gcc429.m445.gob456.hmme45462.libqua464.h26471.omnet473483.xalancbmkgeomean png: https://imgur.com/a/BHzpPTW Notes: - tlb-lock-v2 corresponds to an implementation with a mutex. - tlb-lock-v3 corresponds to the current implementation, i.e. a spinlock and a single lock acquisition in tlb_set_page_with_attrs. Signed-off-by: Emilio G. Cota <cota@braap.org> Message-Id: <20181016153840.25877-1-cota@braap.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/softmmu_template.h | 12 ++++++------ include/exec/cpu_ldst.h | 11 ++++++++++- include/exec/cpu_ldst_template.h | 2 +- accel/tcg/cputlb.c | 19 +++++++++++++------ 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/accel/tcg/softmmu_template.h b/accel/tcg/softmmu_template.h index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/softmmu_template.h +++ b/accel/tcg/softmmu_template.h @@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, uintptr_t mmu_idx = get_mmuidx(oi); uintptr_t index = tlb_index(env, mmu_idx, addr); CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); - target_ulong tlb_addr = entry->addr_write; + target_ulong tlb_addr = tlb_addr_write(entry); unsigned a_bits = get_alignment_bits(get_memop(oi)); uintptr_t haddr; @@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); } - tlb_addr = entry->addr_write & ~TLB_INVALID_MASK; + tlb_addr = tlb_addr_write(entry) & ~TLB_INVALID_MASK; } /* Handle an IO access. */ @@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, cannot evict the first. */ page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK; entry2 = tlb_entry(env, mmu_idx, page2); - if (!tlb_hit_page(entry2->addr_write, page2) + if (!tlb_hit_page(tlb_addr_write(entry2), page2) && !VICTIM_TLB_HIT(addr_write, page2)) { tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); @@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, uintptr_t mmu_idx = get_mmuidx(oi); uintptr_t index = tlb_index(env, mmu_idx, addr); CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); - target_ulong tlb_addr = entry->addr_write; + target_ulong tlb_addr = tlb_addr_write(entry); unsigned a_bits = get_alignment_bits(get_memop(oi)); uintptr_t haddr; @@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); } - tlb_addr = entry->addr_write & ~TLB_INVALID_MASK; + tlb_addr = tlb_addr_write(entry) & ~TLB_INVALID_MASK; } /* Handle an IO access. */ @@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, cannot evict the first. */ page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK; entry2 = tlb_entry(env, mmu_idx, page2); - if (!tlb_hit_page(entry2->addr_write, page2) + if (!tlb_hit_page(tlb_addr_write(entry2), page2) && !VICTIM_TLB_HIT(addr_write, page2)) { tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu_ldst.h +++ b/include/exec/cpu_ldst.h @@ -XXX,XX +XXX,XX @@ extern __thread uintptr_t helper_retaddr; /* The memory helpers for tcg-generated code need tcg_target_long etc. */ #include "tcg.h" +static inline target_ulong tlb_addr_write(const CPUTLBEntry *entry) +{ +#if TCG_OVERSIZED_GUEST + return entry->addr_write; +#else + return atomic_read(&entry->addr_write); +#endif +} + /* Find the TLB index corresponding to the mmu_idx + address pair. */ static inline uintptr_t tlb_index(CPUArchState *env, uintptr_t mmu_idx, target_ulong addr) @@ -XXX,XX +XXX,XX @@ static inline void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr, tlb_addr = tlbentry->addr_read; break; case 1: - tlb_addr = tlbentry->addr_write; + tlb_addr = tlb_addr_write(tlbentry); break; case 2: tlb_addr = tlbentry->addr_code; diff --git a/include/exec/cpu_ldst_template.h b/include/exec/cpu_ldst_template.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu_ldst_template.h +++ b/include/exec/cpu_ldst_template.h @@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env, addr = ptr; mmu_idx = CPU_MMU_INDEX; entry = tlb_entry(env, mmu_idx, addr); - if (unlikely(entry->addr_write != + if (unlikely(tlb_addr_write(entry) != (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) { oi = make_memop_idx(SHIFT, mmu_idx); glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(env, addr, v, oi, diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ static inline bool tlb_hit_page_anyprot(CPUTLBEntry *tlb_entry, target_ulong page) { return tlb_hit_page(tlb_entry->addr_read, page) || - tlb_hit_page(tlb_entry->addr_write, page) || + tlb_hit_page(tlb_addr_write(tlb_entry), page) || tlb_hit_page(tlb_entry->addr_code, page); } @@ -XXX,XX +XXX,XX @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry, tlb_fill(cpu, addr, size, MMU_DATA_STORE, mmu_idx, retaddr); entry = tlb_entry(env, mmu_idx, addr); - tlb_addr = entry->addr_write; + tlb_addr = tlb_addr_write(entry); if (!(tlb_addr & ~(TARGET_PAGE_MASK | TLB_RECHECK))) { /* RAM access */ uintptr_t haddr = addr + entry->addend; @@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index, assert_cpu_is_self(ENV_GET_CPU(env)); for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) { CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx]; - target_ulong cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs); + target_ulong cmp; + + /* elt_ofs might correspond to .addr_write, so use atomic_read */ +#if TCG_OVERSIZED_GUEST + cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs); +#else + cmp = atomic_read((target_ulong *)((uintptr_t)vtlb + elt_ofs)); +#endif if (cmp == page) { /* Found entry in victim tlb, swap tlb and iotlb. */ @@ -XXX,XX +XXX,XX @@ void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, uintptr_t index = tlb_index(env, mmu_idx, addr); CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); - if (!tlb_hit(entry->addr_write, addr)) { + if (!tlb_hit(tlb_addr_write(entry), addr)) { /* TLB entry is for a different page */ if (!VICTIM_TLB_HIT(addr_write, addr)) { tlb_fill(ENV_GET_CPU(env), addr, size, MMU_DATA_STORE, @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, size_t mmu_idx = get_mmuidx(oi); uintptr_t index = tlb_index(env, mmu_idx, addr); CPUTLBEntry *tlbe = tlb_entry(env, mmu_idx, addr); - target_ulong tlb_addr = tlbe->addr_write; + target_ulong tlb_addr = tlb_addr_write(tlbe); TCGMemOp mop = get_memop(oi); int a_bits = get_alignment_bits(mop); int s_bits = mop & MO_SIZE; @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, tlb_fill(ENV_GET_CPU(env), addr, 1 << s_bits, MMU_DATA_STORE, mmu_idx, retaddr); } - tlb_addr = tlbe->addr_write & ~TLB_INVALID_MASK; + tlb_addr = tlb_addr_write(tlbe) & ~TLB_INVALID_MASK; } /* Notice an IO access or a needs-MMU-lookup access */ -- 2.17.2 The following changes since commit 77f3804ab7ed94b471a14acb260e5aeacf26193f: Merge remote-tracking branch 'remotes/kevin/tags/for-upstream' into staging (2021-02-02 16:47:51 +0000) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20210202 for you to fetch changes up to 0c823e596877a30fd6c17a1ae9f98218a53055ea: tcg: Remove TCG_TARGET_CON_SET_H (2021-02-02 12:12:43 -1000) ---------------------------------------------------------------- TCG backend constraints cleanup ---------------------------------------------------------------- Richard Henderson (24): tcg/tci: Drop L and S constraints tcg/tci: Remove TCG_TARGET_HAS_* ifdefs tcg/i386: Move constraint type check to tcg_target_const_match tcg/i386: Tidy register constraint definitions tcg/i386: Split out target constraints to tcg-target-con-str.h tcg/arm: Split out target constraints to tcg-target-con-str.h tcg/aarch64: Split out target constraints to tcg-target-con-str.h tcg/ppc: Split out target constraints to tcg-target-con-str.h tcg/tci: Split out target constraints to tcg-target-con-str.h tcg/mips: Split out target constraints to tcg-target-con-str.h tcg/riscv: Split out target constraints to tcg-target-con-str.h tcg/s390: Split out target constraints to tcg-target-con-str.h tcg/sparc: Split out target constraints to tcg-target-con-str.h tcg: Remove TCG_TARGET_CON_STR_H tcg/i386: Split out constraint sets to tcg-target-con-set.h tcg/aarch64: Split out constraint sets to tcg-target-con-set.h tcg/arm: Split out constraint sets to tcg-target-con-set.h tcg/mips: Split out constraint sets to tcg-target-con-set.h tcg/ppc: Split out constraint sets to tcg-target-con-set.h tcg/riscv: Split out constraint sets to tcg-target-con-set.h tcg/s390: Split out constraint sets to tcg-target-con-set.h tcg/sparc: Split out constraint sets to tcg-target-con-set.h tcg/tci: Split out constraint sets to tcg-target-con-set.h tcg: Remove TCG_TARGET_CON_SET_H tcg/aarch64/tcg-target-con-set.h | 36 ++++ tcg/aarch64/tcg-target-con-str.h | 24 +++ tcg/arm/tcg-target-con-set.h | 35 ++++ tcg/arm/tcg-target-con-str.h | 22 +++ tcg/i386/tcg-target-con-set.h | 55 ++++++ tcg/i386/tcg-target-con-str.h | 33 ++++ tcg/mips/tcg-target-con-set.h | 36 ++++ tcg/mips/tcg-target-con-str.h | 24 +++ tcg/ppc/tcg-target-con-set.h | 42 +++++ tcg/ppc/tcg-target-con-str.h | 30 ++++ tcg/riscv/tcg-target-con-set.h | 30 ++++ tcg/riscv/tcg-target-con-str.h | 21 +++ tcg/s390/tcg-target-con-set.h | 29 ++++ tcg/s390/tcg-target-con-str.h | 28 +++ tcg/sparc/tcg-target-con-set.h | 32 ++++ tcg/sparc/tcg-target-con-str.h | 23 +++ tcg/sparc/tcg-target.h | 4 - tcg/tci/tcg-target-con-set.h | 25 +++ tcg/tci/tcg-target-con-str.h | 11 ++ tcg/tcg.c | 136 +++++++++++++-- tcg/aarch64/tcg-target.c.inc | 137 ++++----------- tcg/arm/tcg-target.c.inc | 168 ++++++------------ tcg/i386/tcg-target.c.inc | 317 +++++++++++----------------------- tcg/mips/tcg-target.c.inc | 173 ++++++------------- tcg/ppc/tcg-target.c.inc | 209 ++++++++--------------- tcg/riscv/tcg-target.c.inc | 135 ++++----------- tcg/s390/tcg-target.c.inc | 174 +++++++------------ tcg/sparc/tcg-target.c.inc | 156 ++++++----------- tcg/tci/tcg-target.c.inc | 359 ++++++++++++++------------------------- 29 files changed, 1244 insertions(+), 1260 deletions(-) create mode 100644 tcg/aarch64/tcg-target-con-set.h create mode 100644 tcg/aarch64/tcg-target-con-str.h create mode 100644 tcg/arm/tcg-target-con-set.h create mode 100644 tcg/arm/tcg-target-con-str.h create mode 100644 tcg/i386/tcg-target-con-set.h create mode 100644 tcg/i386/tcg-target-con-str.h create mode 100644 tcg/mips/tcg-target-con-set.h create mode 100644 tcg/mips/tcg-target-con-str.h create mode 100644 tcg/ppc/tcg-target-con-set.h create mode 100644 tcg/ppc/tcg-target-con-str.h create mode 100644 tcg/riscv/tcg-target-con-set.h create mode 100644 tcg/riscv/tcg-target-con-str.h create mode 100644 tcg/s390/tcg-target-con-set.h create mode 100644 tcg/s390/tcg-target-con-str.h create mode 100644 tcg/sparc/tcg-target-con-set.h create mode 100644 tcg/sparc/tcg-target-con-str.h create mode 100644 tcg/tci/tcg-target-con-set.h create mode 100644 tcg/tci/tcg-target-con-str.h These are identical to the 'r' constraint. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci/tcg-target.c.inc | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ # define R64 "r" #endif #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS -# define L "L", "L" -# define S "S", "S" +# define L "r", "r" +# define S "r", "r" #else -# define L "L" -# define S "S" +# define L "r" +# define S "r" #endif /* TODO: documentation. */ @@ -XXX,XX +XXX,XX @@ static const char *target_parse_constraint(TCGArgConstraint *ct, { switch (*ct_str++) { case 'r': - case 'L': /* qemu_ld constraint */ - case 'S': /* qemu_st constraint */ ct->regs = BIT(TCG_TARGET_NB_REGS) - 1; break; default: -- 2.25.1 The opcodes always exist, regardless of whether or not they are enabled. Remove the unnecessary ifdefs. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci/tcg-target.c.inc | 82 ---------------------------------------- 1 file changed, 82 deletions(-) diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef tcg_target_op_defs[] = { { INDEX_op_add_i32, { R, RI, RI } }, { INDEX_op_sub_i32, { R, RI, RI } }, { INDEX_op_mul_i32, { R, RI, RI } }, -#if TCG_TARGET_HAS_div_i32 { INDEX_op_div_i32, { R, R, R } }, { INDEX_op_divu_i32, { R, R, R } }, { INDEX_op_rem_i32, { R, R, R } }, { INDEX_op_remu_i32, { R, R, R } }, -#elif TCG_TARGET_HAS_div2_i32 - { INDEX_op_div2_i32, { R, R, "0", "1", R } }, - { INDEX_op_divu2_i32, { R, R, "0", "1", R } }, -#endif /* TODO: Does R, RI, RI result in faster code than R, R, RI? If both operands are constants, we can optimize. */ { INDEX_op_and_i32, { R, RI, RI } }, -#if TCG_TARGET_HAS_andc_i32 { INDEX_op_andc_i32, { R, RI, RI } }, -#endif -#if TCG_TARGET_HAS_eqv_i32 { INDEX_op_eqv_i32, { R, RI, RI } }, -#endif -#if TCG_TARGET_HAS_nand_i32 { INDEX_op_nand_i32, { R, RI, RI } }, -#endif -#if TCG_TARGET_HAS_nor_i32 { INDEX_op_nor_i32, { R, RI, RI } }, -#endif { INDEX_op_or_i32, { R, RI, RI } }, -#if TCG_TARGET_HAS_orc_i32 { INDEX_op_orc_i32, { R, RI, RI } }, -#endif { INDEX_op_xor_i32, { R, RI, RI } }, { INDEX_op_shl_i32, { R, RI, RI } }, { INDEX_op_shr_i32, { R, RI, RI } }, { INDEX_op_sar_i32, { R, RI, RI } }, -#if TCG_TARGET_HAS_rot_i32 { INDEX_op_rotl_i32, { R, RI, RI } }, { INDEX_op_rotr_i32, { R, RI, RI } }, -#endif -#if TCG_TARGET_HAS_deposit_i32 { INDEX_op_deposit_i32, { R, "0", R } }, -#endif { INDEX_op_brcond_i32, { R, RI } }, { INDEX_op_setcond_i32, { R, R, RI } }, -#if TCG_TARGET_REG_BITS == 64 { INDEX_op_setcond_i64, { R, R, RI } }, -#endif /* TCG_TARGET_REG_BITS == 64 */ -#if TCG_TARGET_REG_BITS == 32 /* TODO: Support R, R, R, R, RI, RI? Will it be faster? */ { INDEX_op_add2_i32, { R, R, R, R, R, R } }, { INDEX_op_sub2_i32, { R, R, R, R, R, R } }, { INDEX_op_brcond2_i32, { R, R, RI, RI } }, { INDEX_op_mulu2_i32, { R, R, R, R } }, { INDEX_op_setcond2_i32, { R, R, R, RI, RI } }, -#endif -#if TCG_TARGET_HAS_not_i32 { INDEX_op_not_i32, { R, R } }, -#endif -#if TCG_TARGET_HAS_neg_i32 { INDEX_op_neg_i32, { R, R } }, -#endif -#if TCG_TARGET_REG_BITS == 64 { INDEX_op_ld8u_i64, { R, R } }, { INDEX_op_ld8s_i64, { R, R } }, { INDEX_op_ld16u_i64, { R, R } }, @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef tcg_target_op_defs[] = { { INDEX_op_add_i64, { R, RI, RI } }, { INDEX_op_sub_i64, { R, RI, RI } }, { INDEX_op_mul_i64, { R, RI, RI } }, -#if TCG_TARGET_HAS_div_i64 { INDEX_op_div_i64, { R, R, R } }, { INDEX_op_divu_i64, { R, R, R } }, { INDEX_op_rem_i64, { R, R, R } }, { INDEX_op_remu_i64, { R, R, R } }, -#elif TCG_TARGET_HAS_div2_i64 - { INDEX_op_div2_i64, { R, R, "0", "1", R } }, - { INDEX_op_divu2_i64, { R, R, "0", "1", R } }, -#endif { INDEX_op_and_i64, { R, RI, RI } }, -#if TCG_TARGET_HAS_andc_i64 { INDEX_op_andc_i64, { R, RI, RI } }, -#endif -#if TCG_TARGET_HAS_eqv_i64 { INDEX_op_eqv_i64, { R, RI, RI } }, -#endif -#if TCG_TARGET_HAS_nand_i64 { INDEX_op_nand_i64, { R, RI, RI } }, -#endif -#if TCG_TARGET_HAS_nor_i64 { INDEX_op_nor_i64, { R, RI, RI } }, -#endif { INDEX_op_or_i64, { R, RI, RI } }, -#if TCG_TARGET_HAS_orc_i64 { INDEX_op_orc_i64, { R, RI, RI } }, -#endif { INDEX_op_xor_i64, { R, RI, RI } }, { INDEX_op_shl_i64, { R, RI, RI } }, { INDEX_op_shr_i64, { R, RI, RI } }, { INDEX_op_sar_i64, { R, RI, RI } }, -#if TCG_TARGET_HAS_rot_i64 { INDEX_op_rotl_i64, { R, RI, RI } }, { INDEX_op_rotr_i64, { R, RI, RI } }, -#endif -#if TCG_TARGET_HAS_deposit_i64 { INDEX_op_deposit_i64, { R, "0", R } }, -#endif { INDEX_op_brcond_i64, { R, RI } }, -#if TCG_TARGET_HAS_ext8s_i64 { INDEX_op_ext8s_i64, { R, R } }, -#endif -#if TCG_TARGET_HAS_ext16s_i64 { INDEX_op_ext16s_i64, { R, R } }, -#endif -#if TCG_TARGET_HAS_ext32s_i64 { INDEX_op_ext32s_i64, { R, R } }, -#endif -#if TCG_TARGET_HAS_ext8u_i64 { INDEX_op_ext8u_i64, { R, R } }, -#endif -#if TCG_TARGET_HAS_ext16u_i64 { INDEX_op_ext16u_i64, { R, R } }, -#endif -#if TCG_TARGET_HAS_ext32u_i64 { INDEX_op_ext32u_i64, { R, R } }, -#endif { INDEX_op_ext_i32_i64, { R, R } }, { INDEX_op_extu_i32_i64, { R, R } }, -#if TCG_TARGET_HAS_bswap16_i64 { INDEX_op_bswap16_i64, { R, R } }, -#endif -#if TCG_TARGET_HAS_bswap32_i64 { INDEX_op_bswap32_i64, { R, R } }, -#endif -#if TCG_TARGET_HAS_bswap64_i64 { INDEX_op_bswap64_i64, { R, R } }, -#endif -#if TCG_TARGET_HAS_not_i64 { INDEX_op_not_i64, { R, R } }, -#endif -#if TCG_TARGET_HAS_neg_i64 { INDEX_op_neg_i64, { R, R } }, -#endif -#endif /* TCG_TARGET_REG_BITS == 64 */ { INDEX_op_qemu_ld_i32, { R, L } }, { INDEX_op_qemu_ld_i64, { R64, L } }, @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef tcg_target_op_defs[] = { { INDEX_op_qemu_st_i32, { R, S } }, { INDEX_op_qemu_st_i64, { R64, S } }, -#if TCG_TARGET_HAS_ext8s_i32 { INDEX_op_ext8s_i32, { R, R } }, -#endif -#if TCG_TARGET_HAS_ext16s_i32 { INDEX_op_ext16s_i32, { R, R } }, -#endif -#if TCG_TARGET_HAS_ext8u_i32 { INDEX_op_ext8u_i32, { R, R } }, -#endif -#if TCG_TARGET_HAS_ext16u_i32 { INDEX_op_ext16u_i32, { R, R } }, -#endif -#if TCG_TARGET_HAS_bswap16_i32 { INDEX_op_bswap16_i32, { R, R } }, -#endif -#if TCG_TARGET_HAS_bswap32_i32 { INDEX_op_bswap32_i32, { R, R } }, -#endif { INDEX_op_mb, { } }, { -1 }, -- 2.25.1 Rather than check the type when filling in the constraint, check it when matching the constant. This removes the only use of the type argument to target_parse_constraint. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const char *target_parse_constraint(TCGArgConstraint *ct, break; case 'e': - ct->ct |= (type == TCG_TYPE_I32 ? TCG_CT_CONST : TCG_CT_CONST_S32); + ct->ct |= TCG_CT_CONST_S32; break; case 'Z': - ct->ct |= (type == TCG_TYPE_I32 ? TCG_CT_CONST : TCG_CT_CONST_U32); + ct->ct |= TCG_CT_CONST_U32; break; case 'I': - ct->ct |= (type == TCG_TYPE_I32 ? TCG_CT_CONST : TCG_CT_CONST_I32); + ct->ct |= TCG_CT_CONST_I32; break; default: @@ -XXX,XX +XXX,XX @@ static inline int tcg_target_const_match(tcg_target_long val, TCGType type, if (ct & TCG_CT_CONST) { return 1; } - if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) { - return 1; - } - if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) { - return 1; - } - if ((ct & TCG_CT_CONST_I32) && ~val == (int32_t)~val) { - return 1; + if (type == TCG_TYPE_I32) { + if (ct & (TCG_CT_CONST_S32 | TCG_CT_CONST_U32 | TCG_CT_CONST_I32)) { + return 1; + } + } else { + if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) { + return 1; + } + if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) { + return 1; + } + if ((ct & TCG_CT_CONST_I32) && ~val == (int32_t)~val) { + return 1; + } } if ((ct & TCG_CT_CONST_WSZ) && val == (type == TCG_TYPE_I32 ? 32 : 64)) { return 1; -- 2.25.1 Create symbolic constants for all low-byte-addressable and second-byte-addressable registers. Create a symbol for the registers that need reserving for softmmu. There is no functional change for 's', as this letter is only used for i386. The BYTEL name is correct for the action we wish from the constraint. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 40 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const int tcg_target_call_oarg_regs[] = { # define TCG_REG_L1 TCG_REG_EDX #endif +#define ALL_BYTEH_REGS 0x0000000fu +#if TCG_TARGET_REG_BITS == 64 +# define ALL_GENERAL_REGS 0x0000ffffu +# define ALL_VECTOR_REGS 0xffff0000u +# define ALL_BYTEL_REGS ALL_GENERAL_REGS +#else +# define ALL_GENERAL_REGS 0x000000ffu +# define ALL_VECTOR_REGS 0x00ff0000u +# define ALL_BYTEL_REGS ALL_BYTEH_REGS +#endif +#ifdef CONFIG_SOFTMMU +# define SOFTMMU_RESERVE_REGS ((1 << TCG_REG_L0) | (1 << TCG_REG_L1)) +#else +# define SOFTMMU_RESERVE_REGS 0 +#endif + /* The host compiler should supply <cpuid.h> to enable runtime features detection, as we're not going to go so far as our own inline assembly. If not available, default values will be assumed. */ @@ -XXX,XX +XXX,XX @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, return true; } -#if TCG_TARGET_REG_BITS == 64 -#define ALL_GENERAL_REGS 0x0000ffffu -#define ALL_VECTOR_REGS 0xffff0000u -#else -#define ALL_GENERAL_REGS 0x000000ffu -#define ALL_VECTOR_REGS 0x00ff0000u -#endif - /* parse target specific constraints */ static const char *target_parse_constraint(TCGArgConstraint *ct, const char *ct_str, TCGType type) @@ -XXX,XX +XXX,XX @@ static const char *target_parse_constraint(TCGArgConstraint *ct, break; case 'q': /* A register that can be used as a byte operand. */ - ct->regs = TCG_TARGET_REG_BITS == 64 ? 0xffff : 0xf; + ct->regs |= ALL_BYTEL_REGS; break; case 'Q': /* A register with an addressable second byte (e.g. %ah). */ - ct->regs = 0xf; + ct->regs |= ALL_BYTEH_REGS; break; case 'r': /* A general register. */ @@ -XXX,XX +XXX,XX @@ static const char *target_parse_constraint(TCGArgConstraint *ct, case 'L': /* qemu_ld/st data+address constraint */ - ct->regs = TCG_TARGET_REG_BITS == 64 ? 0xffff : 0xff; -#ifdef CONFIG_SOFTMMU - tcg_regset_reset_reg(ct->regs, TCG_REG_L0); - tcg_regset_reset_reg(ct->regs, TCG_REG_L1); -#endif + ct->regs |= ALL_GENERAL_REGS & ~SOFTMMU_RESERVE_REGS; break; case 's': /* qemu_st8_i32 data constraint */ - ct->regs = 0xf; -#ifdef CONFIG_SOFTMMU - tcg_regset_reset_reg(ct->regs, TCG_REG_L0); - tcg_regset_reset_reg(ct->regs, TCG_REG_L1); -#endif + ct->regs |= ALL_BYTEL_REGS & ~SOFTMMU_RESERVE_REGS; break; case 'e': -- 2.25.1 This eliminates the target-specific function target_parse_constraint and folds it into the single caller, process_op_defs. Since this is done directly into the switch statement, duplicates are compilation errors rather than silently ignored at runtime. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target-con-str.h | 33 +++++++++++++++++ tcg/i386/tcg-target.h | 1 + tcg/tcg.c | 33 ++++++++++++++--- tcg/i386/tcg-target.c.inc | 69 ----------------------------------- 4 files changed, 62 insertions(+), 74 deletions(-) create mode 100644 tcg/i386/tcg-target-con-str.h diff --git a/tcg/i386/tcg-target-con-str.h b/tcg/i386/tcg-target-con-str.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/i386/tcg-target-con-str.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define i386 target-specific operand constraints. + * Copyright (c) 2021 Linaro + * + */ + +/* + * Define constraint letters for register sets: + * REGS(letter, register_mask) + */ +REGS('a', 1u << TCG_REG_EAX) +REGS('b', 1u << TCG_REG_EBX) +REGS('c', 1u << TCG_REG_ECX) +REGS('d', 1u << TCG_REG_EDX) +REGS('S', 1u << TCG_REG_ESI) +REGS('D', 1u << TCG_REG_EDI) + +REGS('r', ALL_GENERAL_REGS) +REGS('x', ALL_VECTOR_REGS) +REGS('q', ALL_BYTEL_REGS) /* regs that can be used as a byte operand */ +REGS('Q', ALL_BYTEH_REGS) /* regs with a second byte (e.g. %ah) */ +REGS('L', ALL_GENERAL_REGS & ~SOFTMMU_RESERVE_REGS) /* qemu_ld/st */ +REGS('s', ALL_BYTEL_REGS & ~SOFTMMU_RESERVE_REGS) /* qemu_st8_i32 data */ + +/* + * Define constraint letters for constants: + * CONST(letter, TCG_CT_CONST_* bit set) + */ +CONST('e', TCG_CT_CONST_S32) +CONST('I', TCG_CT_CONST_I32) +CONST('W', TCG_CT_CONST_WSZ) +CONST('Z', TCG_CT_CONST_U32) diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_CON_STR_H #endif diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void tcg_register_jit_int(const void *buf, size_t size, __attribute__((unused)); /* Forward declarations for functions declared and used in tcg-target.c.inc. */ +#ifndef TCG_TARGET_CON_STR_H static const char *target_parse_constraint(TCGArgConstraint *ct, const char *ct_str, TCGType type); +#endif static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, intptr_t arg2); static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); @@ -XXX,XX +XXX,XX @@ static void process_op_defs(TCGContext *s) for (op = 0; op < NB_OPS; op++) { TCGOpDef *def = &tcg_op_defs[op]; const TCGTargetOpDef *tdefs; - TCGType type; int i, nb_args; if (def->flags & TCG_OPF_NOT_PRESENT) { @@ -XXX,XX +XXX,XX @@ static void process_op_defs(TCGContext *s) /* Missing TCGTargetOpDef entry. */ tcg_debug_assert(tdefs != NULL); - type = (def->flags & TCG_OPF_64BIT ? TCG_TYPE_I64 : TCG_TYPE_I32); for (i = 0; i < nb_args; i++) { const char *ct_str = tdefs->args_ct_str[i]; /* Incomplete TCGTargetOpDef entry. */ @@ -XXX,XX +XXX,XX @@ static void process_op_defs(TCGContext *s) def->args_ct[i].ct |= TCG_CT_CONST; ct_str++; break; + +#ifdef TCG_TARGET_CON_STR_H + /* Include all of the target-specific constraints. */ + +#undef CONST +#define CONST(CASE, MASK) \ + case CASE: def->args_ct[i].ct |= MASK; ct_str++; break; +#define REGS(CASE, MASK) \ + case CASE: def->args_ct[i].regs |= MASK; ct_str++; break; + +#include "tcg-target-con-str.h" + +#undef REGS +#undef CONST default: - ct_str = target_parse_constraint(&def->args_ct[i], - ct_str, type); /* Typo in TCGTargetOpDef constraint. */ - tcg_debug_assert(ct_str != NULL); + g_assert_not_reached(); +#else + default: + { + TCGType type = (def->flags & TCG_OPF_64BIT + ? TCG_TYPE_I64 : TCG_TYPE_I32); + ct_str = target_parse_constraint(&def->args_ct[i], + ct_str, type); + /* Typo in TCGTargetOpDef constraint. */ + tcg_debug_assert(ct_str != NULL); + } +#endif } } } diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, return true; } -/* parse target specific constraints */ -static const char *target_parse_constraint(TCGArgConstraint *ct, - const char *ct_str, TCGType type) -{ - switch(*ct_str++) { - case 'a': - tcg_regset_set_reg(ct->regs, TCG_REG_EAX); - break; - case 'b': - tcg_regset_set_reg(ct->regs, TCG_REG_EBX); - break; - case 'c': - tcg_regset_set_reg(ct->regs, TCG_REG_ECX); - break; - case 'd': - tcg_regset_set_reg(ct->regs, TCG_REG_EDX); - break; - case 'S': - tcg_regset_set_reg(ct->regs, TCG_REG_ESI); - break; - case 'D': - tcg_regset_set_reg(ct->regs, TCG_REG_EDI); - break; - case 'q': - /* A register that can be used as a byte operand. */ - ct->regs |= ALL_BYTEL_REGS; - break; - case 'Q': - /* A register with an addressable second byte (e.g. %ah). */ - ct->regs |= ALL_BYTEH_REGS; - break; - case 'r': - /* A general register. */ - ct->regs |= ALL_GENERAL_REGS; - break; - case 'W': - /* With TZCNT/LZCNT, we can have operand-size as an input. */ - ct->ct |= TCG_CT_CONST_WSZ; - break; - case 'x': - /* A vector register. */ - ct->regs |= ALL_VECTOR_REGS; - break; - - case 'L': - /* qemu_ld/st data+address constraint */ - ct->regs |= ALL_GENERAL_REGS & ~SOFTMMU_RESERVE_REGS; - break; - case 's': - /* qemu_st8_i32 data constraint */ - ct->regs |= ALL_BYTEL_REGS & ~SOFTMMU_RESERVE_REGS; - break; - - case 'e': - ct->ct |= TCG_CT_CONST_S32; - break; - case 'Z': - ct->ct |= TCG_CT_CONST_U32; - break; - case 'I': - ct->ct |= TCG_CT_CONST_I32; - break; - - default: - return NULL; - } - return ct_str; -} - /* test if a constant matches the constraint */ static inline int tcg_target_const_match(tcg_target_long val, TCGType type, const TCGArgConstraint *arg_ct) -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/arm/tcg-target-con-str.h | 22 +++++++++++ tcg/arm/tcg-target.h | 1 + tcg/arm/tcg-target.c.inc | 74 +++++++++--------------------------- 3 files changed, 41 insertions(+), 56 deletions(-) create mode 100644 tcg/arm/tcg-target-con-str.h diff --git a/tcg/arm/tcg-target-con-str.h b/tcg/arm/tcg-target-con-str.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/arm/tcg-target-con-str.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define Arm target-specific operand constraints. + * Copyright (c) 2021 Linaro + */ + +/* + * Define constraint letters for register sets: + * REGS(letter, register_mask) + */ +REGS('r', ALL_GENERAL_REGS) +REGS('l', ALL_QLOAD_REGS) +REGS('s', ALL_QSTORE_REGS) + +/* + * Define constraint letters for constants: + * CONST(letter, TCG_CT_CONST_* bit set) + */ +CONST('I', TCG_CT_CONST_ARM) +CONST('K', TCG_CT_CONST_INV) +CONST('N', TCG_CT_CONST_NEG) +CONST('Z', TCG_CT_CONST_ZERO) diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_CON_STR_H #endif diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, #define TCG_CT_CONST_NEG 0x400 #define TCG_CT_CONST_ZERO 0x800 -/* parse target specific constraints */ -static const char *target_parse_constraint(TCGArgConstraint *ct, - const char *ct_str, TCGType type) -{ - switch (*ct_str++) { - case 'I': - ct->ct |= TCG_CT_CONST_ARM; - break; - case 'K': - ct->ct |= TCG_CT_CONST_INV; - break; - case 'N': /* The gcc constraint letter is L, already used here. */ - ct->ct |= TCG_CT_CONST_NEG; - break; - case 'Z': - ct->ct |= TCG_CT_CONST_ZERO; - break; +#define ALL_GENERAL_REGS 0xffffu - case 'r': - ct->regs = 0xffff; - break; - - /* qemu_ld address */ - case 'l': - ct->regs = 0xffff; +/* + * r0-r2 will be overwritten when reading the tlb entry (softmmu only) + * and r0-r1 doing the byte swapping, so don't use these. + * r3 is removed for softmmu to avoid clashes with helper arguments. + */ #ifdef CONFIG_SOFTMMU - /* r0-r2,lr will be overwritten when reading the tlb entry, - so don't use these. */ - tcg_regset_reset_reg(ct->regs, TCG_REG_R0); - tcg_regset_reset_reg(ct->regs, TCG_REG_R1); - tcg_regset_reset_reg(ct->regs, TCG_REG_R2); - tcg_regset_reset_reg(ct->regs, TCG_REG_R3); - tcg_regset_reset_reg(ct->regs, TCG_REG_R14); +#define ALL_QLOAD_REGS \ + (ALL_GENERAL_REGS & ~((1 << TCG_REG_R0) | (1 << TCG_REG_R1) | \ + (1 << TCG_REG_R2) | (1 << TCG_REG_R3) | \ + (1 << TCG_REG_R14))) +#define ALL_QSTORE_REGS \ + (ALL_GENERAL_REGS & ~((1 << TCG_REG_R0) | (1 << TCG_REG_R1) | \ + (1 << TCG_REG_R2) | (1 << TCG_REG_R14) | \ + ((TARGET_LONG_BITS == 64) << TCG_REG_R3))) +#else +#define ALL_QLOAD_REGS ALL_GENERAL_REGS +#define ALL_QSTORE_REGS \ + (ALL_GENERAL_REGS & ~((1 << TCG_REG_R0) | (1 << TCG_REG_R1))) #endif - break; - - /* qemu_st address & data */ - case 's': - ct->regs = 0xffff; - /* r0-r2 will be overwritten when reading the tlb entry (softmmu only) - and r0-r1 doing the byte swapping, so don't use these. */ - tcg_regset_reset_reg(ct->regs, TCG_REG_R0); - tcg_regset_reset_reg(ct->regs, TCG_REG_R1); -#if defined(CONFIG_SOFTMMU) - /* Avoid clashes with registers being used for helper args */ - tcg_regset_reset_reg(ct->regs, TCG_REG_R2); -#if TARGET_LONG_BITS == 64 - /* Avoid clashes with registers being used for helper args */ - tcg_regset_reset_reg(ct->regs, TCG_REG_R3); -#endif - tcg_regset_reset_reg(ct->regs, TCG_REG_R14); -#endif - break; - - default: - return NULL; - } - return ct_str; -} static inline uint32_t rotl(uint32_t val, int n) { -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target-con-str.h | 24 +++++++++++++++ tcg/aarch64/tcg-target.h | 1 + tcg/aarch64/tcg-target.c.inc | 51 +++++--------------------------- 3 files changed, 33 insertions(+), 43 deletions(-) create mode 100644 tcg/aarch64/tcg-target-con-str.h diff --git a/tcg/aarch64/tcg-target-con-str.h b/tcg/aarch64/tcg-target-con-str.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/aarch64/tcg-target-con-str.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Define AArch64 target-specific operand constraints. + * Copyright (c) 2021 Linaro + */ + +/* + * Define constraint letters for register sets: + * REGS(letter, register_mask) + */ +REGS('r', ALL_GENERAL_REGS) +REGS('l', ALL_QLDST_REGS) +REGS('w', ALL_VECTOR_REGS) + +/* + * Define constraint letters for constants: + * CONST(letter, TCG_CT_CONST_* bit set) + */ +CONST('A', TCG_CT_CONST_AIMM) +CONST('L', TCG_CT_CONST_LIMM) +CONST('M', TCG_CT_CONST_MONE) +CONST('O', TCG_CT_CONST_ORRI) +CONST('N', TCG_CT_CONST_ANDI) +CONST('Z', TCG_CT_CONST_ZERO) diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_CON_STR_H #endif /* AARCH64_TCG_TARGET_H */ diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, #define TCG_CT_CONST_ORRI 0x1000 #define TCG_CT_CONST_ANDI 0x2000 -/* parse target specific constraints */ -static const char *target_parse_constraint(TCGArgConstraint *ct, - const char *ct_str, TCGType type) -{ - switch (*ct_str++) { - case 'r': /* general registers */ - ct->regs |= 0xffffffffu; - break; - case 'w': /* advsimd registers */ - ct->regs |= 0xffffffff00000000ull; - break; - case 'l': /* qemu_ld / qemu_st address, data_reg */ - ct->regs = 0xffffffffu; +#define ALL_GENERAL_REGS 0xffffffffu +#define ALL_VECTOR_REGS 0xffffffff00000000ull + #ifdef CONFIG_SOFTMMU - /* x0 and x1 will be overwritten when reading the tlb entry, - and x2, and x3 for helper args, better to avoid using them. */ - tcg_regset_reset_reg(ct->regs, TCG_REG_X0); - tcg_regset_reset_reg(ct->regs, TCG_REG_X1); - tcg_regset_reset_reg(ct->regs, TCG_REG_X2); - tcg_regset_reset_reg(ct->regs, TCG_REG_X3); +#define ALL_QLDST_REGS \ + (ALL_GENERAL_REGS & ~((1 << TCG_REG_X0) | (1 << TCG_REG_X1) | \ + (1 << TCG_REG_X2) | (1 << TCG_REG_X3))) +#else +#define ALL_QLDST_REGS ALL_GENERAL_REGS #endif - break; - case 'A': /* Valid for arithmetic immediate (positive or negative). */ - ct->ct |= TCG_CT_CONST_AIMM; - break; - case 'L': /* Valid for logical immediate. */ - ct->ct |= TCG_CT_CONST_LIMM; - break; - case 'M': /* minus one */ - ct->ct |= TCG_CT_CONST_MONE; - break; - case 'O': /* vector orr/bic immediate */ - ct->ct |= TCG_CT_CONST_ORRI; - break; - case 'N': /* vector orr/bic immediate, inverted */ - ct->ct |= TCG_CT_CONST_ANDI; - break; - case 'Z': /* zero */ - ct->ct |= TCG_CT_CONST_ZERO; - break; - default: - return NULL; - } - return ct_str; -} /* Match a constant valid for addition (12-bit, optionally shifted). */ static inline bool is_aimm(uint64_t val) -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/ppc/tcg-target-con-str.h | 30 +++++++++++++++ tcg/ppc/tcg-target.h | 1 + tcg/ppc/tcg-target.c.inc | 73 ++++++++---------------------------- 3 files changed, 46 insertions(+), 58 deletions(-) create mode 100644 tcg/ppc/tcg-target-con-str.h diff --git a/tcg/ppc/tcg-target-con-str.h b/tcg/ppc/tcg-target-con-str.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/ppc/tcg-target-con-str.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define PowerPC target-specific operand constraints. + * Copyright (c) 2021 Linaro + */ + +/* + * Define constraint letters for register sets: + * REGS(letter, register_mask) + */ +REGS('r', ALL_GENERAL_REGS) +REGS('v', ALL_VECTOR_REGS) +REGS('A', 1u << TCG_REG_R3) +REGS('B', 1u << TCG_REG_R4) +REGS('C', 1u << TCG_REG_R5) +REGS('D', 1u << TCG_REG_R6) +REGS('L', ALL_QLOAD_REGS) +REGS('S', ALL_QSTORE_REGS) + +/* + * Define constraint letters for constants: + * CONST(letter, TCG_CT_CONST_* bit set) + */ +CONST('I', TCG_CT_CONST_S16) +CONST('J', TCG_CT_CONST_U16) +CONST('M', TCG_CT_CONST_MONE) +CONST('T', TCG_CT_CONST_S32) +CONST('U', TCG_CT_CONST_U32) +CONST('W', TCG_CT_CONST_WSZ) +CONST('Z', TCG_CT_CONST_ZERO) diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_CON_STR_H #endif diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ #define TCG_CT_CONST_MONE 0x2000 #define TCG_CT_CONST_WSZ 0x4000 +#define ALL_GENERAL_REGS 0xffffffffu +#define ALL_VECTOR_REGS 0xffffffff00000000ull + +#ifdef CONFIG_SOFTMMU +#define ALL_QLOAD_REGS \ + (ALL_GENERAL_REGS & \ + ~((1 << TCG_REG_R3) | (1 << TCG_REG_R4) | (1 << TCG_REG_R5))) +#define ALL_QSTORE_REGS \ + (ALL_GENERAL_REGS & ~((1 << TCG_REG_R3) | (1 << TCG_REG_R4) | \ + (1 << TCG_REG_R5) | (1 << TCG_REG_R6))) +#else +#define ALL_QLOAD_REGS (ALL_GENERAL_REGS & ~(1 << TCG_REG_R3)) +#define ALL_QSTORE_REGS ALL_QLOAD_REGS +#endif + TCGPowerISA have_isa; static bool have_isel; bool have_altivec; @@ -XXX,XX +XXX,XX @@ static bool reloc_pc14(tcg_insn_unit *src_rw, const tcg_insn_unit *target) return false; } -/* parse target specific constraints */ -static const char *target_parse_constraint(TCGArgConstraint *ct, - const char *ct_str, TCGType type) -{ - switch (*ct_str++) { - case 'A': case 'B': case 'C': case 'D': - tcg_regset_set_reg(ct->regs, 3 + ct_str[0] - 'A'); - break; - case 'r': - ct->regs = 0xffffffff; - break; - case 'v': - ct->regs = 0xffffffff00000000ull; - break; - case 'L': /* qemu_ld constraint */ - ct->regs = 0xffffffff; - tcg_regset_reset_reg(ct->regs, TCG_REG_R3); -#ifdef CONFIG_SOFTMMU - tcg_regset_reset_reg(ct->regs, TCG_REG_R4); - tcg_regset_reset_reg(ct->regs, TCG_REG_R5); -#endif - break; - case 'S': /* qemu_st constraint */ - ct->regs = 0xffffffff; - tcg_regset_reset_reg(ct->regs, TCG_REG_R3); -#ifdef CONFIG_SOFTMMU - tcg_regset_reset_reg(ct->regs, TCG_REG_R4); - tcg_regset_reset_reg(ct->regs, TCG_REG_R5); - tcg_regset_reset_reg(ct->regs, TCG_REG_R6); -#endif - break; - case 'I': - ct->ct |= TCG_CT_CONST_S16; - break; - case 'J': - ct->ct |= TCG_CT_CONST_U16; - break; - case 'M': - ct->ct |= TCG_CT_CONST_MONE; - break; - case 'T': - ct->ct |= TCG_CT_CONST_S32; - break; - case 'U': - ct->ct |= TCG_CT_CONST_U32; - break; - case 'W': - ct->ct |= TCG_CT_CONST_WSZ; - break; - case 'Z': - ct->ct |= TCG_CT_CONST_ZERO; - break; - default: - return NULL; - } - return ct_str; -} - /* test if a constant matches the constraint */ static int tcg_target_const_match(tcg_target_long val, TCGType type, const TCGArgConstraint *arg_ct) -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci/tcg-target-con-str.h | 11 +++++++++++ tcg/tci/tcg-target.h | 2 ++ tcg/tci/tcg-target.c.inc | 14 -------------- 3 files changed, 13 insertions(+), 14 deletions(-) create mode 100644 tcg/tci/tcg-target-con-str.h diff --git a/tcg/tci/tcg-target-con-str.h b/tcg/tci/tcg-target-con-str.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/tci/tcg-target-con-str.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define TCI target-specific operand constraints. + * Copyright (c) 2021 Linaro + */ + +/* + * Define constraint letters for register sets: + * REGS(letter, register_mask) + */ +REGS('r', MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, /* no need to flush icache explicitly */ } +#define TCG_TARGET_CON_STR_H + #endif /* TCG_TARGET_H */ diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, return true; } -/* Parse target specific constraints. */ -static const char *target_parse_constraint(TCGArgConstraint *ct, - const char *ct_str, TCGType type) -{ - switch (*ct_str++) { - case 'r': - ct->regs = BIT(TCG_TARGET_NB_REGS) - 1; - break; - default: - return NULL; - } - return ct_str; -} - #if defined(CONFIG_DEBUG_TCG_INTERPRETER) /* Show current bytecode. Used by tcg interpreter. */ void tci_disas(uint8_t opc) -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target-con-str.h | 24 +++++++++++ tcg/mips/tcg-target.h | 1 + tcg/mips/tcg-target.c.inc | 77 ++++++++++------------------------- 3 files changed, 46 insertions(+), 56 deletions(-) create mode 100644 tcg/mips/tcg-target-con-str.h diff --git a/tcg/mips/tcg-target-con-str.h b/tcg/mips/tcg-target-con-str.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/mips/tcg-target-con-str.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define MIPS target-specific operand constraints. + * Copyright (c) 2021 Linaro + */ + +/* + * Define constraint letters for register sets: + * REGS(letter, register_mask) + */ +REGS('r', ALL_GENERAL_REGS) +REGS('L', ALL_QLOAD_REGS) +REGS('S', ALL_QSTORE_REGS) + +/* + * Define constraint letters for constants: + * CONST(letter, TCG_CT_CONST_* bit set) + */ +CONST('I', TCG_CT_CONST_U16) +CONST('J', TCG_CT_CONST_S16) +CONST('K', TCG_CT_CONST_P2M1) +CONST('N', TCG_CT_CONST_N16) +CONST('W', TCG_CT_CONST_WSZ) +CONST('Z', TCG_CT_CONST_ZERO) diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #ifdef CONFIG_SOFTMMU #define TCG_TARGET_NEED_LDST_LABELS #endif +#define TCG_TARGET_CON_STR_H #endif diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, #define TCG_CT_CONST_N16 0x1000 /* "Negatable" 16-bit: -32767 - 32767 */ #define TCG_CT_CONST_WSZ 0x2000 /* word size */ +#define ALL_GENERAL_REGS 0xffffffffu +#define NOA0_REGS (ALL_GENERAL_REGS & ~(1 << TCG_REG_A0)) + +#ifdef CONFIG_SOFTMMU +#define ALL_QLOAD_REGS \ + (NOA0_REGS & ~((TCG_TARGET_REG_BITS < TARGET_LONG_BITS) << TCG_REG_A2)) +#define ALL_QSTORE_REGS \ + (NOA0_REGS & ~(TCG_TARGET_REG_BITS < TARGET_LONG_BITS \ + ? (1 << TCG_REG_A2) | (1 << TCG_REG_A3) \ + : (1 << TCG_REG_A1))) +#else +#define ALL_QLOAD_REGS NOA0_REGS +#define ALL_QSTORE_REGS NOA0_REGS +#endif + + static inline bool is_p2m1(tcg_target_long val) { return val && ((val + 1) & val) == 0; } -/* parse target specific constraints */ -static const char *target_parse_constraint(TCGArgConstraint *ct, - const char *ct_str, TCGType type) -{ - switch(*ct_str++) { - case 'r': - ct->regs = 0xffffffff; - break; - case 'L': /* qemu_ld input arg constraint */ - ct->regs = 0xffffffff; - tcg_regset_reset_reg(ct->regs, TCG_REG_A0); -#if defined(CONFIG_SOFTMMU) - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { - tcg_regset_reset_reg(ct->regs, TCG_REG_A2); - } -#endif - break; - case 'S': /* qemu_st constraint */ - ct->regs = 0xffffffff; - tcg_regset_reset_reg(ct->regs, TCG_REG_A0); -#if defined(CONFIG_SOFTMMU) - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { - tcg_regset_reset_reg(ct->regs, TCG_REG_A2); - tcg_regset_reset_reg(ct->regs, TCG_REG_A3); - } else { - tcg_regset_reset_reg(ct->regs, TCG_REG_A1); - } -#endif - break; - case 'I': - ct->ct |= TCG_CT_CONST_U16; - break; - case 'J': - ct->ct |= TCG_CT_CONST_S16; - break; - case 'K': - ct->ct |= TCG_CT_CONST_P2M1; - break; - case 'N': - ct->ct |= TCG_CT_CONST_N16; - break; - case 'W': - ct->ct |= TCG_CT_CONST_WSZ; - break; - case 'Z': - /* We are cheating a bit here, using the fact that the register - ZERO is also the register number 0. Hence there is no need - to check for const_args in each instruction. */ - ct->ct |= TCG_CT_CONST_ZERO; - break; - default: - return NULL; - } - return ct_str; -} - /* test if a constant matches the constraint */ static inline int tcg_target_const_match(tcg_target_long val, TCGType type, const TCGArgConstraint *arg_ct) @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGArg a0, a1, a2; int c2; + /* + * Note that many operands use the constraint set "rZ". + * We make use of the fact that 0 is the ZERO register, + * and hence such cases need not check for const_args. + */ a0 = args[0]; a1 = args[1]; a2 = args[2]; -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target-con-str.h | 21 ++++++++++++++ tcg/riscv/tcg-target.h | 1 + tcg/riscv/tcg-target.c.inc | 52 +++++++++------------------------- 3 files changed, 35 insertions(+), 39 deletions(-) create mode 100644 tcg/riscv/tcg-target-con-str.h diff --git a/tcg/riscv/tcg-target-con-str.h b/tcg/riscv/tcg-target-con-str.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/riscv/tcg-target-con-str.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define RISC-V target-specific operand constraints. + * Copyright (c) 2021 Linaro + */ + +/* + * Define constraint letters for register sets: + * REGS(letter, register_mask) + */ +REGS('r', ALL_GENERAL_REGS) +REGS('L', ALL_GENERAL_REGS & ~SOFTMMU_RESERVE_REGS) + +/* + * Define constraint letters for constants: + * CONST(letter, TCG_CT_CONST_* bit set) + */ +CONST('I', TCG_CT_CONST_S12) +CONST('N', TCG_CT_CONST_N12) +CONST('M', TCG_CT_CONST_M12) +CONST('Z', TCG_CT_CONST_ZERO) diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_POOL_LABELS #define TCG_TARGET_HAS_MEMORY_BSWAP 0 +#define TCG_TARGET_CON_STR_H #endif diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const int tcg_target_call_oarg_regs[] = { #define TCG_CT_CONST_N12 0x400 #define TCG_CT_CONST_M12 0x800 +#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32) +/* + * For softmmu, we need to avoid conflicts with the first 5 + * argument registers to call the helper. Some of these are + * also used for the tlb lookup. + */ +#ifdef CONFIG_SOFTMMU +#define SOFTMMU_RESERVE_REGS MAKE_64BIT_MASK(TCG_REG_A0, 5) +#else +#define SOFTMMU_RESERVE_REGS 0 +#endif + + static inline tcg_target_long sextreg(tcg_target_long val, int pos, int len) { if (TCG_TARGET_REG_BITS == 32) { @@ -XXX,XX +XXX,XX @@ static inline tcg_target_long sextreg(tcg_target_long val, int pos, int len) } } -/* parse target specific constraints */ -static const char *target_parse_constraint(TCGArgConstraint *ct, - const char *ct_str, TCGType type) -{ - switch (*ct_str++) { - case 'r': - ct->regs = 0xffffffff; - break; - case 'L': - /* qemu_ld/qemu_st constraint */ - ct->regs = 0xffffffff; - /* qemu_ld/qemu_st uses TCG_REG_TMP0 */ -#if defined(CONFIG_SOFTMMU) - tcg_regset_reset_reg(ct->regs, tcg_target_call_iarg_regs[0]); - tcg_regset_reset_reg(ct->regs, tcg_target_call_iarg_regs[1]); - tcg_regset_reset_reg(ct->regs, tcg_target_call_iarg_regs[2]); - tcg_regset_reset_reg(ct->regs, tcg_target_call_iarg_regs[3]); - tcg_regset_reset_reg(ct->regs, tcg_target_call_iarg_regs[4]); -#endif - break; - case 'I': - ct->ct |= TCG_CT_CONST_S12; - break; - case 'N': - ct->ct |= TCG_CT_CONST_N12; - break; - case 'M': - ct->ct |= TCG_CT_CONST_M12; - break; - case 'Z': - /* we can use a zero immediate as a zero register argument. */ - ct->ct |= TCG_CT_CONST_ZERO; - break; - default: - return NULL; - } - return ct_str; -} - /* test if a constant matches the constraint */ static int tcg_target_const_match(tcg_target_long val, TCGType type, const TCGArgConstraint *arg_ct) -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/s390/tcg-target-con-str.h | 28 ++++++++++++++++++ tcg/s390/tcg-target.h | 1 + tcg/s390/tcg-target.c.inc | 53 +++++++++-------------------------- 3 files changed, 42 insertions(+), 40 deletions(-) create mode 100644 tcg/s390/tcg-target-con-str.h diff --git a/tcg/s390/tcg-target-con-str.h b/tcg/s390/tcg-target-con-str.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/s390/tcg-target-con-str.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define S390 target-specific operand constraints. + * Copyright (c) 2021 Linaro + */ + +/* + * Define constraint letters for register sets: + * REGS(letter, register_mask) + */ +REGS('r', ALL_GENERAL_REGS) +REGS('L', ALL_GENERAL_REGS & ~SOFTMMU_RESERVE_REGS) +/* + * A (single) even/odd pair for division. + * TODO: Add something to the register allocator to allow + * this kind of regno+1 pairing to be done more generally. + */ +REGS('a', 1u << TCG_REG_R2) +REGS('b', 1u << TCG_REG_R3) + +/* + * Define constraint letters for constants: + * CONST(letter, TCG_CT_CONST_* bit set) + */ +CONST('A', TCG_CT_CONST_S33) +CONST('I', TCG_CT_CONST_S16) +CONST('J', TCG_CT_CONST_S32) +CONST('Z', TCG_CT_CONST_ZERO) diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.h +++ b/tcg/s390/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_CON_STR_H #endif diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.c.inc +++ b/tcg/s390/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ #define TCG_CT_CONST_S33 0x400 #define TCG_CT_CONST_ZERO 0x800 +#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 16) +/* + * For softmmu, we need to avoid conflicts with the first 3 + * argument registers to perform the tlb lookup, and to call + * the helper function. + */ +#ifdef CONFIG_SOFTMMU +#define SOFTMMU_RESERVE_REGS MAKE_64BIT_MASK(TCG_REG_R2, 3) +#else +#define SOFTMMU_RESERVE_REGS 0 +#endif + + /* Several places within the instruction set 0 means "no register" rather than TCG_REG_R0. */ #define TCG_REG_NONE 0 @@ -XXX,XX +XXX,XX @@ static bool patch_reloc(tcg_insn_unit *src_rw, int type, return false; } -/* parse target specific constraints */ -static const char *target_parse_constraint(TCGArgConstraint *ct, - const char *ct_str, TCGType type) -{ - switch (*ct_str++) { - case 'r': /* all registers */ - ct->regs = 0xffff; - break; - case 'L': /* qemu_ld/st constraint */ - ct->regs = 0xffff; - tcg_regset_reset_reg(ct->regs, TCG_REG_R2); - tcg_regset_reset_reg(ct->regs, TCG_REG_R3); - tcg_regset_reset_reg(ct->regs, TCG_REG_R4); - break; - case 'a': /* force R2 for division */ - ct->regs = 0; - tcg_regset_set_reg(ct->regs, TCG_REG_R2); - break; - case 'b': /* force R3 for division */ - ct->regs = 0; - tcg_regset_set_reg(ct->regs, TCG_REG_R3); - break; - case 'A': - ct->ct |= TCG_CT_CONST_S33; - break; - case 'I': - ct->ct |= TCG_CT_CONST_S16; - break; - case 'J': - ct->ct |= TCG_CT_CONST_S32; - break; - case 'Z': - ct->ct |= TCG_CT_CONST_ZERO; - break; - default: - return NULL; - } - return ct_str; -} - /* Test if a constant matches the constraint. */ static int tcg_target_const_match(tcg_target_long val, TCGType type, const TCGArgConstraint *arg_ct) -- 2.25.1 Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc/tcg-target-con-str.h | 23 ++++++++++ tcg/sparc/tcg-target.h | 5 +-- tcg/sparc/tcg-target.c.inc | 81 +++++++++++++--------------------- 3 files changed, 55 insertions(+), 54 deletions(-) create mode 100644 tcg/sparc/tcg-target-con-str.h diff --git a/tcg/sparc/tcg-target-con-str.h b/tcg/sparc/tcg-target-con-str.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/sparc/tcg-target-con-str.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define Sparc target-specific operand constraints. + * Copyright (c) 2021 Linaro + */ + +/* + * Define constraint letters for register sets: + * REGS(letter, register_mask) + */ +REGS('r', ALL_GENERAL_REGS) +REGS('R', ALL_GENERAL_REGS64) +REGS('s', ALL_QLDST_REGS) +REGS('S', ALL_QLDST_REGS64) +REGS('A', TARGET_LONG_BITS == 64 ? ALL_QLDST_REGS64 : ALL_QLDST_REGS) + +/* + * Define constraint letters for constants: + * CONST(letter, TCG_CT_CONST_* bit set) + */ +CONST('I', TCG_CT_CONST_S11) +CONST('J', TCG_CT_CONST_S13) +CONST('Z', TCG_CT_CONST_ZERO) diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.h +++ b/tcg/sparc/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { TCG_REG_I7, } TCGReg; -#define TCG_CT_CONST_S11 0x100 -#define TCG_CT_CONST_S13 0x200 -#define TCG_CT_CONST_ZERO 0x400 - /* used for function call generation */ #define TCG_REG_CALL_STACK TCG_REG_O6 @@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions; void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_CON_STR_H #endif diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { # define SPARC64 0 #endif -/* Note that sparcv8plus can only hold 64 bit quantities in %g and %o - registers. These are saved manually by the kernel in full 64-bit - slots. The %i and %l registers are saved by the register window - mechanism, which only allocates space for 32 bits. Given that this - window spill/fill can happen on any signal, we must consider the - high bits of the %i and %l registers garbage at all times. */ -#if SPARC64 -# define ALL_64 0xffffffffu +#define TCG_CT_CONST_S11 0x100 +#define TCG_CT_CONST_S13 0x200 +#define TCG_CT_CONST_ZERO 0x400 + +/* + * For softmmu, we need to avoid conflicts with the first 3 + * argument registers to perform the tlb lookup, and to call + * the helper function. + */ +#ifdef CONFIG_SOFTMMU +#define SOFTMMU_RESERVE_REGS MAKE_64BIT_MASK(TCG_REG_O0, 3) #else -# define ALL_64 0xffffu +#define SOFTMMU_RESERVE_REGS 0 #endif +/* + * Note that sparcv8plus can only hold 64 bit quantities in %g and %o + * registers. These are saved manually by the kernel in full 64-bit + * slots. The %i and %l registers are saved by the register window + * mechanism, which only allocates space for 32 bits. Given that this + * window spill/fill can happen on any signal, we must consider the + * high bits of the %i and %l registers garbage at all times. + */ +#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32) +#if SPARC64 +# define ALL_GENERAL_REGS64 ALL_GENERAL_REGS +#else +# define ALL_GENERAL_REGS64 MAKE_64BIT_MASK(0, 16) +#endif +#define ALL_QLDST_REGS (ALL_GENERAL_REGS & ~SOFTMMU_RESERVE_REGS) +#define ALL_QLDST_REGS64 (ALL_GENERAL_REGS64 & ~SOFTMMU_RESERVE_REGS) + /* Define some temporary registers. T2 is used for constant generation. */ #define TCG_REG_T1 TCG_REG_G1 #define TCG_REG_T2 TCG_REG_O7 @@ -XXX,XX +XXX,XX @@ static bool patch_reloc(tcg_insn_unit *src_rw, int type, return true; } -/* parse target specific constraints */ -static const char *target_parse_constraint(TCGArgConstraint *ct, - const char *ct_str, TCGType type) -{ - switch (*ct_str++) { - case 'r': - ct->regs = 0xffffffff; - break; - case 'R': - ct->regs = ALL_64; - break; - case 'A': /* qemu_ld/st address constraint */ - ct->regs = TARGET_LONG_BITS == 64 ? ALL_64 : 0xffffffff; - reserve_helpers: - tcg_regset_reset_reg(ct->regs, TCG_REG_O0); - tcg_regset_reset_reg(ct->regs, TCG_REG_O1); - tcg_regset_reset_reg(ct->regs, TCG_REG_O2); - break; - case 's': /* qemu_st data 32-bit constraint */ - ct->regs = 0xffffffff; - goto reserve_helpers; - case 'S': /* qemu_st data 64-bit constraint */ - ct->regs = ALL_64; - goto reserve_helpers; - case 'I': - ct->ct |= TCG_CT_CONST_S11; - break; - case 'J': - ct->ct |= TCG_CT_CONST_S13; - break; - case 'Z': - ct->ct |= TCG_CT_CONST_ZERO; - break; - default: - return NULL; - } - return ct_str; -} - /* test if a constant matches the constraint */ static inline int tcg_target_const_match(tcg_target_long val, TCGType type, const TCGArgConstraint *arg_ct) @@ -XXX,XX +XXX,XX @@ static void tcg_target_init(TCGContext *s) } #endif - tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffff; - tcg_target_available_regs[TCG_TYPE_I64] = ALL_64; + tcg_target_available_regs[TCG_TYPE_I32] = ALL_GENERAL_REGS; + tcg_target_available_regs[TCG_TYPE_I64] = ALL_GENERAL_REGS64; tcg_target_call_clobber_regs = 0; tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_G1); -- 2.25.1 All backends have now been converted to tcg-target-con-str.h, so we can remove the fallback code. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target.h | 1 - tcg/arm/tcg-target.h | 1 - tcg/i386/tcg-target.h | 1 - tcg/mips/tcg-target.h | 1 - tcg/ppc/tcg-target.h | 1 - tcg/riscv/tcg-target.h | 1 - tcg/s390/tcg-target.h | 1 - tcg/sparc/tcg-target.h | 1 - tcg/tci/tcg-target.h | 2 -- tcg/tcg.c | 16 ---------------- 10 files changed, 26 deletions(-) diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_CON_STR_H #endif /* AARCH64_TCG_TARGET_H */ diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_CON_STR_H #endif diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_CON_STR_H #endif diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #ifdef CONFIG_SOFTMMU #define TCG_TARGET_NEED_LDST_LABELS #endif -#define TCG_TARGET_CON_STR_H #endif diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_CON_STR_H #endif diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_POOL_LABELS #define TCG_TARGET_HAS_MEMORY_BSWAP 0 -#define TCG_TARGET_CON_STR_H #endif diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.h +++ b/tcg/s390/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_CON_STR_H #endif diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.h +++ b/tcg/sparc/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions; void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_CON_STR_H #endif diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, /* no need to flush icache explicitly */ } -#define TCG_TARGET_CON_STR_H - #endif /* TCG_TARGET_H */ diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void tcg_register_jit_int(const void *buf, size_t size, __attribute__((unused)); /* Forward declarations for functions declared and used in tcg-target.c.inc. */ -#ifndef TCG_TARGET_CON_STR_H -static const char *target_parse_constraint(TCGArgConstraint *ct, - const char *ct_str, TCGType type); -#endif static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, intptr_t arg2); static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); @@ -XXX,XX +XXX,XX @@ static void process_op_defs(TCGContext *s) ct_str++; break; -#ifdef TCG_TARGET_CON_STR_H /* Include all of the target-specific constraints. */ #undef CONST @@ -XXX,XX +XXX,XX @@ static void process_op_defs(TCGContext *s) default: /* Typo in TCGTargetOpDef constraint. */ g_assert_not_reached(); -#else - default: - { - TCGType type = (def->flags & TCG_OPF_64BIT - ? TCG_TYPE_I64 : TCG_TYPE_I32); - ct_str = target_parse_constraint(&def->args_ct[i], - ct_str, type); - /* Typo in TCGTargetOpDef constraint. */ - tcg_debug_assert(ct_str != NULL); - } -#endif } } } -- 2.25.1 This exports the constraint sets from tcg_target_op_def to a place we will be able to manipulate more in future. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target-con-set.h | 55 ++++++++++ tcg/i386/tcg-target.h | 1 + tcg/tcg.c | 119 +++++++++++++++++++++ tcg/i386/tcg-target.c.inc | 194 ++++++++++++---------------------- 4 files changed, 242 insertions(+), 127 deletions(-) create mode 100644 tcg/i386/tcg-target-con-set.h diff --git a/tcg/i386/tcg-target-con-set.h b/tcg/i386/tcg-target-con-set.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/i386/tcg-target-con-set.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define i386 target-specific constraint sets. + * Copyright (c) 2021 Linaro + */ + +/* + * C_On_Im(...) defines a constraint set with <n> outputs and <m> inputs. + * Each operand should be a sequence of constraint letters as defined by + * tcg-target-con-str.h; the constraint combination is inclusive or. + * + * C_N1_Im(...) defines a constraint set with 1 output and <m> inputs, + * except that the output must use a new register. + */ +C_O0_I1(r) +C_O0_I2(L, L) +C_O0_I2(qi, r) +C_O0_I2(re, r) +C_O0_I2(ri, r) +C_O0_I2(r, re) +C_O0_I2(s, L) +C_O0_I2(x, r) +C_O0_I3(L, L, L) +C_O0_I3(s, L, L) +C_O0_I4(L, L, L, L) +C_O0_I4(r, r, ri, ri) +C_O1_I1(r, 0) +C_O1_I1(r, L) +C_O1_I1(r, q) +C_O1_I1(r, r) +C_O1_I1(x, r) +C_O1_I1(x, x) +C_O1_I2(Q, 0, Q) +C_O1_I2(q, r, re) +C_O1_I2(r, 0, ci) +C_O1_I2(r, 0, r) +C_O1_I2(r, 0, re) +C_O1_I2(r, 0, reZ) +C_O1_I2(r, 0, ri) +C_O1_I2(r, 0, rI) +C_O1_I2(r, L, L) +C_O1_I2(r, r, re) +C_O1_I2(r, r, ri) +C_O1_I2(r, r, rI) +C_O1_I2(x, x, x) +C_N1_I2(r, r, r) +C_N1_I2(r, r, rW) +C_O1_I3(x, x, x, x) +C_O1_I4(r, r, re, r, 0) +C_O1_I4(r, r, r, ri, ri) +C_O2_I1(r, r, L) +C_O2_I2(a, d, a, r) +C_O2_I2(r, r, L, L) +C_O2_I3(a, d, 0, 1, r) +C_O2_I4(r, r, 0, 1, re, re) diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_CON_SET_H #endif diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ /* Forward declarations for functions declared in tcg-target.c.inc and used here. */ static void tcg_target_init(TCGContext *s); +#ifndef TCG_TARGET_CON_SET_H static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode); +#endif static void tcg_target_qemu_prologue(TCGContext *s); static bool patch_reloc(tcg_insn_unit *code_ptr, int type, intptr_t value, intptr_t addend); @@ -XXX,XX +XXX,XX @@ static void set_jmp_reset_offset(TCGContext *s, int which) s->tb_jmp_reset_offset[which] = tcg_current_code_size(s); } +#ifdef TCG_TARGET_CON_SET_H +#define C_PFX1(P, A) P##A +#define C_PFX2(P, A, B) P##A##_##B +#define C_PFX3(P, A, B, C) P##A##_##B##_##C +#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D +#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E +#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F + +/* Define an enumeration for the various combinations. */ + +#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1), +#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2), +#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3), +#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4), + +#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1), +#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2), +#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3), +#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4), + +#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2), + +#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1), +#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2), +#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3), +#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4), + +typedef enum { +#include "tcg-target-con-set.h" +} TCGConstraintSetIndex; + +static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode); + +#undef C_O0_I1 +#undef C_O0_I2 +#undef C_O0_I3 +#undef C_O0_I4 +#undef C_O1_I1 +#undef C_O1_I2 +#undef C_O1_I3 +#undef C_O1_I4 +#undef C_N1_I2 +#undef C_O2_I1 +#undef C_O2_I2 +#undef C_O2_I3 +#undef C_O2_I4 + +/* Put all of the constraint sets into an array, indexed by the enum. */ + +#define C_O0_I1(I1) { .args_ct_str = { #I1 } }, +#define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } }, +#define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } }, +#define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } }, + +#define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } }, +#define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } }, +#define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } }, +#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } }, + +#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } }, + +#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } }, +#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } }, +#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } }, +#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } }, + +static const TCGTargetOpDef constraint_sets[] = { +#include "tcg-target-con-set.h" +}; + + +#undef C_O0_I1 +#undef C_O0_I2 +#undef C_O0_I3 +#undef C_O0_I4 +#undef C_O1_I1 +#undef C_O1_I2 +#undef C_O1_I3 +#undef C_O1_I4 +#undef C_N1_I2 +#undef C_O2_I1 +#undef C_O2_I2 +#undef C_O2_I3 +#undef C_O2_I4 + +/* Expand the enumerator to be returned from tcg_target_op_def(). */ + +#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1) +#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2) +#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3) +#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4) + +#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1) +#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2) +#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3) +#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4) + +#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2) + +#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1) +#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2) +#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3) +#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4) + +#endif /* TCG_TARGET_CON_SET_H */ + #include "tcg-target.c.inc" /* compare a pointer @ptr and a tb_tc @s */ @@ -XXX,XX +XXX,XX @@ static void process_op_defs(TCGContext *s) continue; } +#ifdef TCG_TARGET_CON_SET_H + /* + * Macro magic should make it impossible, but double-check that + * the array index is in range. Since the signness of an enum + * is implementation defined, force the result to unsigned. + */ + unsigned con_set = tcg_target_op_def(op); + tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets)); + tdefs = &constraint_sets[con_set]; +#else tdefs = tcg_target_op_def(op); /* Missing TCGTargetOpDef entry. */ tcg_debug_assert(tdefs != NULL); +#endif for (i = 0; i < nb_args; i++) { const char *ct_str = tdefs->args_ct_str[i]; diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, } } -static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) +static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) { - static const TCGTargetOpDef r = { .args_ct_str = { "r" } }; - static const TCGTargetOpDef ri_r = { .args_ct_str = { "ri", "r" } }; - static const TCGTargetOpDef re_r = { .args_ct_str = { "re", "r" } }; - static const TCGTargetOpDef qi_r = { .args_ct_str = { "qi", "r" } }; - static const TCGTargetOpDef r_r = { .args_ct_str = { "r", "r" } }; - static const TCGTargetOpDef r_q = { .args_ct_str = { "r", "q" } }; - static const TCGTargetOpDef r_re = { .args_ct_str = { "r", "re" } }; - static const TCGTargetOpDef r_0 = { .args_ct_str = { "r", "0" } }; - static const TCGTargetOpDef r_r_ri = { .args_ct_str = { "r", "r", "ri" } }; - static const TCGTargetOpDef r_r_re = { .args_ct_str = { "r", "r", "re" } }; - static const TCGTargetOpDef r_0_r = { .args_ct_str = { "r", "0", "r" } }; - static const TCGTargetOpDef r_0_re = { .args_ct_str = { "r", "0", "re" } }; - static const TCGTargetOpDef r_0_ci = { .args_ct_str = { "r", "0", "ci" } }; - static const TCGTargetOpDef r_L = { .args_ct_str = { "r", "L" } }; - static const TCGTargetOpDef L_L = { .args_ct_str = { "L", "L" } }; - static const TCGTargetOpDef s_L = { .args_ct_str = { "s", "L" } }; - static const TCGTargetOpDef r_L_L = { .args_ct_str = { "r", "L", "L" } }; - static const TCGTargetOpDef r_r_L = { .args_ct_str = { "r", "r", "L" } }; - static const TCGTargetOpDef L_L_L = { .args_ct_str = { "L", "L", "L" } }; - static const TCGTargetOpDef s_L_L = { .args_ct_str = { "s", "L", "L" } }; - static const TCGTargetOpDef r_r_L_L - = { .args_ct_str = { "r", "r", "L", "L" } }; - static const TCGTargetOpDef L_L_L_L - = { .args_ct_str = { "L", "L", "L", "L" } }; - static const TCGTargetOpDef x_x = { .args_ct_str = { "x", "x" } }; - static const TCGTargetOpDef x_x_x = { .args_ct_str = { "x", "x", "x" } }; - static const TCGTargetOpDef x_x_x_x - = { .args_ct_str = { "x", "x", "x", "x" } }; - static const TCGTargetOpDef x_r = { .args_ct_str = { "x", "r" } }; - switch (op) { case INDEX_op_goto_ptr: - return &r; + return C_O0_I1(r); case INDEX_op_ld8u_i32: case INDEX_op_ld8u_i64: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_ld32u_i64: case INDEX_op_ld32s_i64: case INDEX_op_ld_i64: - return &r_r; + return C_O1_I1(r, r); case INDEX_op_st8_i32: case INDEX_op_st8_i64: - return &qi_r; + return C_O0_I2(qi, r); + case INDEX_op_st16_i32: case INDEX_op_st16_i64: case INDEX_op_st_i32: case INDEX_op_st32_i64: - return &ri_r; + return C_O0_I2(ri, r); + case INDEX_op_st_i64: - return &re_r; + return C_O0_I2(re, r); case INDEX_op_add_i32: case INDEX_op_add_i64: - return &r_r_re; + return C_O1_I2(r, r, re); + case INDEX_op_sub_i32: case INDEX_op_sub_i64: case INDEX_op_mul_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_or_i64: case INDEX_op_xor_i32: case INDEX_op_xor_i64: - return &r_0_re; + return C_O1_I2(r, 0, re); case INDEX_op_and_i32: case INDEX_op_and_i64: - { - static const TCGTargetOpDef and - = { .args_ct_str = { "r", "0", "reZ" } }; - return &and; - } - break; + return C_O1_I2(r, 0, reZ); + case INDEX_op_andc_i32: case INDEX_op_andc_i64: - { - static const TCGTargetOpDef andc - = { .args_ct_str = { "r", "r", "rI" } }; - return &andc; - } - break; + return C_O1_I2(r, r, rI); case INDEX_op_shl_i32: case INDEX_op_shl_i64: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_shr_i64: case INDEX_op_sar_i32: case INDEX_op_sar_i64: - return have_bmi2 ? &r_r_ri : &r_0_ci; + return have_bmi2 ? C_O1_I2(r, r, ri) : C_O1_I2(r, 0, ci); + case INDEX_op_rotl_i32: case INDEX_op_rotl_i64: case INDEX_op_rotr_i32: case INDEX_op_rotr_i64: - return &r_0_ci; + return C_O1_I2(r, 0, ci); case INDEX_op_brcond_i32: case INDEX_op_brcond_i64: - return &r_re; + return C_O0_I2(r, re); case INDEX_op_bswap16_i32: case INDEX_op_bswap16_i64: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_not_i32: case INDEX_op_not_i64: case INDEX_op_extrh_i64_i32: - return &r_0; + return C_O1_I1(r, 0); case INDEX_op_ext8s_i32: case INDEX_op_ext8s_i64: case INDEX_op_ext8u_i32: case INDEX_op_ext8u_i64: - return &r_q; + return C_O1_I1(r, q); + case INDEX_op_ext16s_i32: case INDEX_op_ext16s_i64: case INDEX_op_ext16u_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_sextract_i32: case INDEX_op_ctpop_i32: case INDEX_op_ctpop_i64: - return &r_r; + return C_O1_I1(r, r); + case INDEX_op_extract2_i32: case INDEX_op_extract2_i64: - return &r_0_r; + return C_O1_I2(r, 0, r); case INDEX_op_deposit_i32: case INDEX_op_deposit_i64: - { - static const TCGTargetOpDef dep - = { .args_ct_str = { "Q", "0", "Q" } }; - return &dep; - } + return C_O1_I2(Q, 0, Q); + case INDEX_op_setcond_i32: case INDEX_op_setcond_i64: - { - static const TCGTargetOpDef setc - = { .args_ct_str = { "q", "r", "re" } }; - return &setc; - } + return C_O1_I2(q, r, re); + case INDEX_op_movcond_i32: case INDEX_op_movcond_i64: - { - static const TCGTargetOpDef movc - = { .args_ct_str = { "r", "r", "re", "r", "0" } }; - return &movc; - } + return C_O1_I4(r, r, re, r, 0); + case INDEX_op_div2_i32: case INDEX_op_div2_i64: case INDEX_op_divu2_i32: case INDEX_op_divu2_i64: - { - static const TCGTargetOpDef div2 - = { .args_ct_str = { "a", "d", "0", "1", "r" } }; - return &div2; - } + return C_O2_I3(a, d, 0, 1, r); + case INDEX_op_mulu2_i32: case INDEX_op_mulu2_i64: case INDEX_op_muls2_i32: case INDEX_op_muls2_i64: - { - static const TCGTargetOpDef mul2 - = { .args_ct_str = { "a", "d", "a", "r" } }; - return &mul2; - } + return C_O2_I2(a, d, a, r); + case INDEX_op_add2_i32: case INDEX_op_add2_i64: case INDEX_op_sub2_i32: case INDEX_op_sub2_i64: - { - static const TCGTargetOpDef arith2 - = { .args_ct_str = { "r", "r", "0", "1", "re", "re" } }; - return &arith2; - } + return C_O2_I4(r, r, 0, 1, re, re); + case INDEX_op_ctz_i32: case INDEX_op_ctz_i64: - { - static const TCGTargetOpDef ctz[2] = { - { .args_ct_str = { "&r", "r", "r" } }, - { .args_ct_str = { "&r", "r", "rW" } }, - }; - return &ctz[have_bmi1]; - } + return have_bmi1 ? C_N1_I2(r, r, rW) : C_N1_I2(r, r, r); + case INDEX_op_clz_i32: case INDEX_op_clz_i64: - { - static const TCGTargetOpDef clz[2] = { - { .args_ct_str = { "&r", "r", "r" } }, - { .args_ct_str = { "&r", "r", "rW" } }, - }; - return &clz[have_lzcnt]; - } + return have_lzcnt ? C_N1_I2(r, r, rW) : C_N1_I2(r, r, r); case INDEX_op_qemu_ld_i32: - return TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &r_L : &r_L_L; + return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS + ? C_O1_I1(r, L) : C_O1_I2(r, L, L)); + case INDEX_op_qemu_st_i32: - return TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &L_L : &L_L_L; + return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS + ? C_O0_I2(L, L) : C_O0_I3(L, L, L)); case INDEX_op_qemu_st8_i32: - return TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &s_L : &s_L_L; + return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS + ? C_O0_I2(s, L) : C_O0_I3(s, L, L)); + case INDEX_op_qemu_ld_i64: - return (TCG_TARGET_REG_BITS == 64 ? &r_L - : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &r_r_L - : &r_r_L_L); + return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) + : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O2_I1(r, r, L) + : C_O2_I2(r, r, L, L)); + case INDEX_op_qemu_st_i64: - return (TCG_TARGET_REG_BITS == 64 ? &L_L - : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &L_L_L - : &L_L_L_L); + return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(L, L) + : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O0_I3(L, L, L) + : C_O0_I4(L, L, L, L)); case INDEX_op_brcond2_i32: - { - static const TCGTargetOpDef b2 - = { .args_ct_str = { "r", "r", "ri", "ri" } }; - return &b2; - } + return C_O0_I4(r, r, ri, ri); + case INDEX_op_setcond2_i32: - { - static const TCGTargetOpDef s2 - = { .args_ct_str = { "r", "r", "r", "ri", "ri" } }; - return &s2; - } + return C_O1_I4(r, r, r, ri, ri); case INDEX_op_ld_vec: - case INDEX_op_st_vec: case INDEX_op_dupm_vec: - return &x_r; + return C_O1_I1(x, r); + + case INDEX_op_st_vec: + return C_O0_I2(x, r); case INDEX_op_add_vec: case INDEX_op_sub_vec: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) #if TCG_TARGET_REG_BITS == 32 case INDEX_op_dup2_vec: #endif - return &x_x_x; + return C_O1_I2(x, x, x); + case INDEX_op_abs_vec: case INDEX_op_dup_vec: case INDEX_op_shli_vec: case INDEX_op_shri_vec: case INDEX_op_sari_vec: case INDEX_op_x86_psrldq_vec: - return &x_x; + return C_O1_I1(x, x); + case INDEX_op_x86_vpblendvb_vec: - return &x_x_x_x; + return C_O1_I3(x, x, x, x); default: - break; + g_assert_not_reached(); } - return NULL; } int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target-con-set.h | 36 +++++++++++++ tcg/aarch64/tcg-target.h | 1 + tcg/aarch64/tcg-target.c.inc | 86 +++++++++++--------------------- 3 files changed, 65 insertions(+), 58 deletions(-) create mode 100644 tcg/aarch64/tcg-target-con-set.h diff --git a/tcg/aarch64/tcg-target-con-set.h b/tcg/aarch64/tcg-target-con-set.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/aarch64/tcg-target-con-set.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Define AArch64 target-specific constraint sets. + * Copyright (c) 2021 Linaro + */ + +/* + * C_On_Im(...) defines a constraint set with <n> outputs and <m> inputs. + * Each operand should be a sequence of constraint letters as defined by + * tcg-target-con-str.h; the constraint combination is inclusive or. + */ +C_O0_I1(r) +C_O0_I2(lZ, l) +C_O0_I2(r, rA) +C_O0_I2(rZ, r) +C_O0_I2(w, r) +C_O1_I1(r, l) +C_O1_I1(r, r) +C_O1_I1(w, r) +C_O1_I1(w, w) +C_O1_I1(w, wr) +C_O1_I2(r, 0, rZ) +C_O1_I2(r, r, r) +C_O1_I2(r, r, rA) +C_O1_I2(r, r, rAL) +C_O1_I2(r, r, ri) +C_O1_I2(r, r, rL) +C_O1_I2(r, rZ, rZ) +C_O1_I2(w, 0, w) +C_O1_I2(w, w, w) +C_O1_I2(w, w, wN) +C_O1_I2(w, w, wO) +C_O1_I2(w, w, wZ) +C_O1_I3(w, w, w, w) +C_O1_I4(r, r, rA, rZ, rZ) +C_O2_I4(r, r, rZ, rZ, rA, rMZ) diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_CON_SET_H #endif /* AARCH64_TCG_TARGET_H */ diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, va_end(va); } -static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) +static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) { - static const TCGTargetOpDef r = { .args_ct_str = { "r" } }; - static const TCGTargetOpDef r_r = { .args_ct_str = { "r", "r" } }; - static const TCGTargetOpDef w_w = { .args_ct_str = { "w", "w" } }; - static const TCGTargetOpDef w_r = { .args_ct_str = { "w", "r" } }; - static const TCGTargetOpDef w_wr = { .args_ct_str = { "w", "wr" } }; - static const TCGTargetOpDef r_l = { .args_ct_str = { "r", "l" } }; - static const TCGTargetOpDef r_rA = { .args_ct_str = { "r", "rA" } }; - static const TCGTargetOpDef rZ_r = { .args_ct_str = { "rZ", "r" } }; - static const TCGTargetOpDef lZ_l = { .args_ct_str = { "lZ", "l" } }; - static const TCGTargetOpDef r_r_r = { .args_ct_str = { "r", "r", "r" } }; - static const TCGTargetOpDef w_w_w = { .args_ct_str = { "w", "w", "w" } }; - static const TCGTargetOpDef w_0_w = { .args_ct_str = { "w", "0", "w" } }; - static const TCGTargetOpDef w_w_wO = { .args_ct_str = { "w", "w", "wO" } }; - static const TCGTargetOpDef w_w_wN = { .args_ct_str = { "w", "w", "wN" } }; - static const TCGTargetOpDef w_w_wZ = { .args_ct_str = { "w", "w", "wZ" } }; - static const TCGTargetOpDef r_r_ri = { .args_ct_str = { "r", "r", "ri" } }; - static const TCGTargetOpDef r_r_rA = { .args_ct_str = { "r", "r", "rA" } }; - static const TCGTargetOpDef r_r_rL = { .args_ct_str = { "r", "r", "rL" } }; - static const TCGTargetOpDef r_r_rAL - = { .args_ct_str = { "r", "r", "rAL" } }; - static const TCGTargetOpDef dep - = { .args_ct_str = { "r", "0", "rZ" } }; - static const TCGTargetOpDef ext2 - = { .args_ct_str = { "r", "rZ", "rZ" } }; - static const TCGTargetOpDef movc - = { .args_ct_str = { "r", "r", "rA", "rZ", "rZ" } }; - static const TCGTargetOpDef add2 - = { .args_ct_str = { "r", "r", "rZ", "rZ", "rA", "rMZ" } }; - static const TCGTargetOpDef w_w_w_w - = { .args_ct_str = { "w", "w", "w", "w" } }; - switch (op) { case INDEX_op_goto_ptr: - return &r; + return C_O0_I1(r); case INDEX_op_ld8u_i32: case INDEX_op_ld8s_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_extract_i64: case INDEX_op_sextract_i32: case INDEX_op_sextract_i64: - return &r_r; + return C_O1_I1(r, r); case INDEX_op_st8_i32: case INDEX_op_st16_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_st16_i64: case INDEX_op_st32_i64: case INDEX_op_st_i64: - return &rZ_r; + return C_O0_I2(rZ, r); case INDEX_op_add_i32: case INDEX_op_add_i64: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_sub_i64: case INDEX_op_setcond_i32: case INDEX_op_setcond_i64: - return &r_r_rA; + return C_O1_I2(r, r, rA); case INDEX_op_mul_i32: case INDEX_op_mul_i64: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_remu_i64: case INDEX_op_muluh_i64: case INDEX_op_mulsh_i64: - return &r_r_r; + return C_O1_I2(r, r, r); case INDEX_op_and_i32: case INDEX_op_and_i64: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_orc_i64: case INDEX_op_eqv_i32: case INDEX_op_eqv_i64: - return &r_r_rL; + return C_O1_I2(r, r, rL); case INDEX_op_shl_i32: case INDEX_op_shr_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_sar_i64: case INDEX_op_rotl_i64: case INDEX_op_rotr_i64: - return &r_r_ri; + return C_O1_I2(r, r, ri); case INDEX_op_clz_i32: case INDEX_op_ctz_i32: case INDEX_op_clz_i64: case INDEX_op_ctz_i64: - return &r_r_rAL; + return C_O1_I2(r, r, rAL); case INDEX_op_brcond_i32: case INDEX_op_brcond_i64: - return &r_rA; + return C_O0_I2(r, rA); case INDEX_op_movcond_i32: case INDEX_op_movcond_i64: - return &movc; + return C_O1_I4(r, r, rA, rZ, rZ); case INDEX_op_qemu_ld_i32: case INDEX_op_qemu_ld_i64: - return &r_l; + return C_O1_I1(r, l); case INDEX_op_qemu_st_i32: case INDEX_op_qemu_st_i64: - return &lZ_l; + return C_O0_I2(lZ, l); case INDEX_op_deposit_i32: case INDEX_op_deposit_i64: - return &dep; + return C_O1_I2(r, 0, rZ); case INDEX_op_extract2_i32: case INDEX_op_extract2_i64: - return &ext2; + return C_O1_I2(r, rZ, rZ); case INDEX_op_add2_i32: case INDEX_op_add2_i64: case INDEX_op_sub2_i32: case INDEX_op_sub2_i64: - return &add2; + return C_O2_I4(r, r, rZ, rZ, rA, rMZ); case INDEX_op_add_vec: case INDEX_op_sub_vec: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_shrv_vec: case INDEX_op_sarv_vec: case INDEX_op_aa64_sshl_vec: - return &w_w_w; + return C_O1_I2(w, w, w); case INDEX_op_not_vec: case INDEX_op_neg_vec: case INDEX_op_abs_vec: case INDEX_op_shli_vec: case INDEX_op_shri_vec: case INDEX_op_sari_vec: - return &w_w; + return C_O1_I1(w, w); case INDEX_op_ld_vec: - case INDEX_op_st_vec: case INDEX_op_dupm_vec: - return &w_r; + return C_O1_I1(w, r); + case INDEX_op_st_vec: + return C_O0_I2(w, r); case INDEX_op_dup_vec: - return &w_wr; + return C_O1_I1(w, wr); case INDEX_op_or_vec: case INDEX_op_andc_vec: - return &w_w_wO; + return C_O1_I2(w, w, wO); case INDEX_op_and_vec: case INDEX_op_orc_vec: - return &w_w_wN; + return C_O1_I2(w, w, wN); case INDEX_op_cmp_vec: - return &w_w_wZ; + return C_O1_I2(w, w, wZ); case INDEX_op_bitsel_vec: - return &w_w_w_w; + return C_O1_I3(w, w, w, w); case INDEX_op_aa64_sli_vec: - return &w_0_w; + return C_O1_I2(w, 0, w); default: - return NULL; + g_assert_not_reached(); } } -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/arm/tcg-target-con-set.h | 35 ++++++++++++++ tcg/arm/tcg-target.h | 1 + tcg/arm/tcg-target.c.inc | 94 ++++++++++++------------------------ 3 files changed, 68 insertions(+), 62 deletions(-) create mode 100644 tcg/arm/tcg-target-con-set.h diff --git a/tcg/arm/tcg-target-con-set.h b/tcg/arm/tcg-target-con-set.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/arm/tcg-target-con-set.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define Arm target-specific constraint sets. + * Copyright (c) 2021 Linaro + */ + +/* + * C_On_Im(...) defines a constraint set with <n> outputs and <m> inputs. + * Each operand should be a sequence of constraint letters as defined by + * tcg-target-con-str.h; the constraint combination is inclusive or. + */ +C_O0_I1(r) +C_O0_I2(r, r) +C_O0_I2(r, rIN) +C_O0_I2(s, s) +C_O0_I3(s, s, s) +C_O0_I4(r, r, rI, rI) +C_O0_I4(s, s, s, s) +C_O1_I1(r, l) +C_O1_I1(r, r) +C_O1_I2(r, 0, rZ) +C_O1_I2(r, l, l) +C_O1_I2(r, r, r) +C_O1_I2(r, r, rI) +C_O1_I2(r, r, rIK) +C_O1_I2(r, r, rIN) +C_O1_I2(r, r, ri) +C_O1_I2(r, rZ, rZ) +C_O1_I4(r, r, r, rI, rI) +C_O1_I4(r, r, rIN, rIK, 0) +C_O2_I1(r, r, l) +C_O2_I2(r, r, l, l) +C_O2_I2(r, r, r, r) +C_O2_I4(r, r, r, r, rIN, rIK) +C_O2_I4(r, r, rI, rI, rIN, rIK) diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_CON_SET_H #endif diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, } } -static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) +static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) { - static const TCGTargetOpDef r = { .args_ct_str = { "r" } }; - static const TCGTargetOpDef r_r = { .args_ct_str = { "r", "r" } }; - static const TCGTargetOpDef s_s = { .args_ct_str = { "s", "s" } }; - static const TCGTargetOpDef r_l = { .args_ct_str = { "r", "l" } }; - static const TCGTargetOpDef r_r_r = { .args_ct_str = { "r", "r", "r" } }; - static const TCGTargetOpDef r_r_l = { .args_ct_str = { "r", "r", "l" } }; - static const TCGTargetOpDef r_l_l = { .args_ct_str = { "r", "l", "l" } }; - static const TCGTargetOpDef s_s_s = { .args_ct_str = { "s", "s", "s" } }; - static const TCGTargetOpDef r_r_ri = { .args_ct_str = { "r", "r", "ri" } }; - static const TCGTargetOpDef r_r_rI = { .args_ct_str = { "r", "r", "rI" } }; - static const TCGTargetOpDef r_r_rIN - = { .args_ct_str = { "r", "r", "rIN" } }; - static const TCGTargetOpDef r_r_rIK - = { .args_ct_str = { "r", "r", "rIK" } }; - static const TCGTargetOpDef r_r_r_r - = { .args_ct_str = { "r", "r", "r", "r" } }; - static const TCGTargetOpDef r_r_l_l - = { .args_ct_str = { "r", "r", "l", "l" } }; - static const TCGTargetOpDef s_s_s_s - = { .args_ct_str = { "s", "s", "s", "s" } }; - static const TCGTargetOpDef br - = { .args_ct_str = { "r", "rIN" } }; - static const TCGTargetOpDef ext2 - = { .args_ct_str = { "r", "rZ", "rZ" } }; - static const TCGTargetOpDef dep - = { .args_ct_str = { "r", "0", "rZ" } }; - static const TCGTargetOpDef movc - = { .args_ct_str = { "r", "r", "rIN", "rIK", "0" } }; - static const TCGTargetOpDef add2 - = { .args_ct_str = { "r", "r", "r", "r", "rIN", "rIK" } }; - static const TCGTargetOpDef sub2 - = { .args_ct_str = { "r", "r", "rI", "rI", "rIN", "rIK" } }; - static const TCGTargetOpDef br2 - = { .args_ct_str = { "r", "r", "rI", "rI" } }; - static const TCGTargetOpDef setc2 - = { .args_ct_str = { "r", "r", "r", "rI", "rI" } }; - switch (op) { case INDEX_op_goto_ptr: - return &r; + return C_O0_I1(r); case INDEX_op_ld8u_i32: case INDEX_op_ld8s_i32: case INDEX_op_ld16u_i32: case INDEX_op_ld16s_i32: case INDEX_op_ld_i32: - case INDEX_op_st8_i32: - case INDEX_op_st16_i32: - case INDEX_op_st_i32: case INDEX_op_neg_i32: case INDEX_op_not_i32: case INDEX_op_bswap16_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_ext16u_i32: case INDEX_op_extract_i32: case INDEX_op_sextract_i32: - return &r_r; + return C_O1_I1(r, r); + + case INDEX_op_st8_i32: + case INDEX_op_st16_i32: + case INDEX_op_st_i32: + return C_O0_I2(r, r); case INDEX_op_add_i32: case INDEX_op_sub_i32: case INDEX_op_setcond_i32: - return &r_r_rIN; + return C_O1_I2(r, r, rIN); + case INDEX_op_and_i32: case INDEX_op_andc_i32: case INDEX_op_clz_i32: case INDEX_op_ctz_i32: - return &r_r_rIK; + return C_O1_I2(r, r, rIK); + case INDEX_op_mul_i32: case INDEX_op_div_i32: case INDEX_op_divu_i32: - return &r_r_r; + return C_O1_I2(r, r, r); + case INDEX_op_mulu2_i32: case INDEX_op_muls2_i32: - return &r_r_r_r; + return C_O2_I2(r, r, r, r); + case INDEX_op_or_i32: case INDEX_op_xor_i32: - return &r_r_rI; + return C_O1_I2(r, r, rI); + case INDEX_op_shl_i32: case INDEX_op_shr_i32: case INDEX_op_sar_i32: case INDEX_op_rotl_i32: case INDEX_op_rotr_i32: - return &r_r_ri; + return C_O1_I2(r, r, ri); case INDEX_op_brcond_i32: - return &br; + return C_O0_I2(r, rIN); case INDEX_op_deposit_i32: - return &dep; + return C_O1_I2(r, 0, rZ); case INDEX_op_extract2_i32: - return &ext2; + return C_O1_I2(r, rZ, rZ); case INDEX_op_movcond_i32: - return &movc; + return C_O1_I4(r, r, rIN, rIK, 0); case INDEX_op_add2_i32: - return &add2; + return C_O2_I4(r, r, r, r, rIN, rIK); case INDEX_op_sub2_i32: - return &sub2; + return C_O2_I4(r, r, rI, rI, rIN, rIK); case INDEX_op_brcond2_i32: - return &br2; + return C_O0_I4(r, r, rI, rI); case INDEX_op_setcond2_i32: - return &setc2; + return C_O1_I4(r, r, r, rI, rI); case INDEX_op_qemu_ld_i32: - return TARGET_LONG_BITS == 32 ? &r_l : &r_l_l; + return TARGET_LONG_BITS == 32 ? C_O1_I1(r, l) : C_O1_I2(r, l, l); case INDEX_op_qemu_ld_i64: - return TARGET_LONG_BITS == 32 ? &r_r_l : &r_r_l_l; + return TARGET_LONG_BITS == 32 ? C_O2_I1(r, r, l) : C_O2_I2(r, r, l, l); case INDEX_op_qemu_st_i32: - return TARGET_LONG_BITS == 32 ? &s_s : &s_s_s; + return TARGET_LONG_BITS == 32 ? C_O0_I2(s, s) : C_O0_I3(s, s, s); case INDEX_op_qemu_st_i64: - return TARGET_LONG_BITS == 32 ? &s_s_s : &s_s_s_s; + return TARGET_LONG_BITS == 32 ? C_O0_I3(s, s, s) : C_O0_I4(s, s, s, s); default: - return NULL; + g_assert_not_reached(); } } -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target-con-set.h | 36 +++++++++++++ tcg/mips/tcg-target.h | 1 + tcg/mips/tcg-target.c.inc | 96 +++++++++++------------------------ 3 files changed, 66 insertions(+), 67 deletions(-) create mode 100644 tcg/mips/tcg-target-con-set.h diff --git a/tcg/mips/tcg-target-con-set.h b/tcg/mips/tcg-target-con-set.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/mips/tcg-target-con-set.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define MIPS target-specific constraint sets. + * Copyright (c) 2021 Linaro + */ + +/* + * C_On_Im(...) defines a constraint set with <n> outputs and <m> inputs. + * Each operand should be a sequence of constraint letters as defined by + * tcg-target-con-str.h; the constraint combination is inclusive or. + */ +C_O0_I1(r) +C_O0_I2(rZ, r) +C_O0_I2(rZ, rZ) +C_O0_I2(SZ, S) +C_O0_I3(SZ, S, S) +C_O0_I3(SZ, SZ, S) +C_O0_I4(rZ, rZ, rZ, rZ) +C_O0_I4(SZ, SZ, S, S) +C_O1_I1(r, L) +C_O1_I1(r, r) +C_O1_I2(r, 0, rZ) +C_O1_I2(r, L, L) +C_O1_I2(r, r, ri) +C_O1_I2(r, r, rI) +C_O1_I2(r, r, rIK) +C_O1_I2(r, r, rJ) +C_O1_I2(r, r, rWZ) +C_O1_I2(r, rZ, rN) +C_O1_I2(r, rZ, rZ) +C_O1_I4(r, rZ, rZ, rZ, 0) +C_O1_I4(r, rZ, rZ, rZ, rZ) +C_O2_I1(r, r, L) +C_O2_I2(r, r, L, L) +C_O2_I2(r, r, r, r) +C_O2_I4(r, r, rZ, rZ, rN, rN) diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #ifdef CONFIG_SOFTMMU #define TCG_TARGET_NEED_LDST_LABELS #endif +#define TCG_TARGET_CON_SET_H #endif diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, } } -static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) +static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) { - static const TCGTargetOpDef r = { .args_ct_str = { "r" } }; - static const TCGTargetOpDef r_r = { .args_ct_str = { "r", "r" } }; - static const TCGTargetOpDef r_L = { .args_ct_str = { "r", "L" } }; - static const TCGTargetOpDef rZ_r = { .args_ct_str = { "rZ", "r" } }; - static const TCGTargetOpDef SZ_S = { .args_ct_str = { "SZ", "S" } }; - static const TCGTargetOpDef rZ_rZ = { .args_ct_str = { "rZ", "rZ" } }; - static const TCGTargetOpDef r_r_L = { .args_ct_str = { "r", "r", "L" } }; - static const TCGTargetOpDef r_L_L = { .args_ct_str = { "r", "L", "L" } }; - static const TCGTargetOpDef r_r_ri = { .args_ct_str = { "r", "r", "ri" } }; - static const TCGTargetOpDef r_r_rI = { .args_ct_str = { "r", "r", "rI" } }; - static const TCGTargetOpDef r_r_rJ = { .args_ct_str = { "r", "r", "rJ" } }; - static const TCGTargetOpDef SZ_S_S = { .args_ct_str = { "SZ", "S", "S" } }; - static const TCGTargetOpDef SZ_SZ_S - = { .args_ct_str = { "SZ", "SZ", "S" } }; - static const TCGTargetOpDef SZ_SZ_S_S - = { .args_ct_str = { "SZ", "SZ", "S", "S" } }; - static const TCGTargetOpDef r_rZ_rN - = { .args_ct_str = { "r", "rZ", "rN" } }; - static const TCGTargetOpDef r_rZ_rZ - = { .args_ct_str = { "r", "rZ", "rZ" } }; - static const TCGTargetOpDef r_r_rIK - = { .args_ct_str = { "r", "r", "rIK" } }; - static const TCGTargetOpDef r_r_rWZ - = { .args_ct_str = { "r", "r", "rWZ" } }; - static const TCGTargetOpDef r_r_r_r - = { .args_ct_str = { "r", "r", "r", "r" } }; - static const TCGTargetOpDef r_r_L_L - = { .args_ct_str = { "r", "r", "L", "L" } }; - static const TCGTargetOpDef dep - = { .args_ct_str = { "r", "0", "rZ" } }; - static const TCGTargetOpDef movc - = { .args_ct_str = { "r", "rZ", "rZ", "rZ", "0" } }; - static const TCGTargetOpDef movc_r6 - = { .args_ct_str = { "r", "rZ", "rZ", "rZ", "rZ" } }; - static const TCGTargetOpDef add2 - = { .args_ct_str = { "r", "r", "rZ", "rZ", "rN", "rN" } }; - static const TCGTargetOpDef br2 - = { .args_ct_str = { "rZ", "rZ", "rZ", "rZ" } }; - static const TCGTargetOpDef setc2 - = { .args_ct_str = { "r", "rZ", "rZ", "rZ", "rZ" } }; - switch (op) { case INDEX_op_goto_ptr: - return &r; + return C_O0_I1(r); case INDEX_op_ld8u_i32: case INDEX_op_ld8s_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_extrl_i64_i32: case INDEX_op_extrh_i64_i32: case INDEX_op_extract_i64: - return &r_r; + return C_O1_I1(r, r); case INDEX_op_st8_i32: case INDEX_op_st16_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_st16_i64: case INDEX_op_st32_i64: case INDEX_op_st_i64: - return &rZ_r; + return C_O0_I2(rZ, r); case INDEX_op_add_i32: case INDEX_op_add_i64: - return &r_r_rJ; + return C_O1_I2(r, r, rJ); case INDEX_op_sub_i32: case INDEX_op_sub_i64: - return &r_rZ_rN; + return C_O1_I2(r, rZ, rN); case INDEX_op_mul_i32: case INDEX_op_mulsh_i32: case INDEX_op_muluh_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_remu_i64: case INDEX_op_nor_i64: case INDEX_op_setcond_i64: - return &r_rZ_rZ; + return C_O1_I2(r, rZ, rZ); case INDEX_op_muls2_i32: case INDEX_op_mulu2_i32: case INDEX_op_muls2_i64: case INDEX_op_mulu2_i64: - return &r_r_r_r; + return C_O2_I2(r, r, r, r); case INDEX_op_and_i32: case INDEX_op_and_i64: - return &r_r_rIK; + return C_O1_I2(r, r, rIK); case INDEX_op_or_i32: case INDEX_op_xor_i32: case INDEX_op_or_i64: case INDEX_op_xor_i64: - return &r_r_rI; + return C_O1_I2(r, r, rI); case INDEX_op_shl_i32: case INDEX_op_shr_i32: case INDEX_op_sar_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_sar_i64: case INDEX_op_rotr_i64: case INDEX_op_rotl_i64: - return &r_r_ri; + return C_O1_I2(r, r, ri); case INDEX_op_clz_i32: case INDEX_op_clz_i64: - return &r_r_rWZ; + return C_O1_I2(r, r, rWZ); case INDEX_op_deposit_i32: case INDEX_op_deposit_i64: - return &dep; + return C_O1_I2(r, 0, rZ); case INDEX_op_brcond_i32: case INDEX_op_brcond_i64: - return &rZ_rZ; + return C_O0_I2(rZ, rZ); case INDEX_op_movcond_i32: case INDEX_op_movcond_i64: - return use_mips32r6_instructions ? &movc_r6 : &movc; - + return (use_mips32r6_instructions + ? C_O1_I4(r, rZ, rZ, rZ, rZ) + : C_O1_I4(r, rZ, rZ, rZ, 0)); case INDEX_op_add2_i32: case INDEX_op_sub2_i32: - return &add2; + return C_O2_I4(r, r, rZ, rZ, rN, rN); case INDEX_op_setcond2_i32: - return &setc2; + return C_O1_I4(r, rZ, rZ, rZ, rZ); case INDEX_op_brcond2_i32: - return &br2; + return C_O0_I4(rZ, rZ, rZ, rZ); case INDEX_op_qemu_ld_i32: return (TCG_TARGET_REG_BITS == 64 || TARGET_LONG_BITS == 32 - ? &r_L : &r_L_L); + ? C_O1_I1(r, L) : C_O1_I2(r, L, L)); case INDEX_op_qemu_st_i32: return (TCG_TARGET_REG_BITS == 64 || TARGET_LONG_BITS == 32 - ? &SZ_S : &SZ_S_S); + ? C_O0_I2(SZ, S) : C_O0_I3(SZ, S, S)); case INDEX_op_qemu_ld_i64: - return (TCG_TARGET_REG_BITS == 64 ? &r_L - : TARGET_LONG_BITS == 32 ? &r_r_L : &r_r_L_L); + return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) + : TARGET_LONG_BITS == 32 ? C_O2_I1(r, r, L) + : C_O2_I2(r, r, L, L)); case INDEX_op_qemu_st_i64: - return (TCG_TARGET_REG_BITS == 64 ? &SZ_S - : TARGET_LONG_BITS == 32 ? &SZ_SZ_S : &SZ_SZ_S_S); + return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(SZ, S) + : TARGET_LONG_BITS == 32 ? C_O0_I3(SZ, SZ, S) + : C_O0_I4(SZ, SZ, S, S)); default: - return NULL; + g_assert_not_reached(); } } -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/ppc/tcg-target-con-set.h | 42 +++++++++++ tcg/ppc/tcg-target.h | 1 + tcg/ppc/tcg-target.c.inc | 136 +++++++++++++++-------------------- 3 files changed, 99 insertions(+), 80 deletions(-) create mode 100644 tcg/ppc/tcg-target-con-set.h diff --git a/tcg/ppc/tcg-target-con-set.h b/tcg/ppc/tcg-target-con-set.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/ppc/tcg-target-con-set.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define PowerPC target-specific constraint sets. + * Copyright (c) 2021 Linaro + */ + +/* + * C_On_Im(...) defines a constraint set with <n> outputs and <m> inputs. + * Each operand should be a sequence of constraint letters as defined by + * tcg-target-con-str.h; the constraint combination is inclusive or. + */ +C_O0_I1(r) +C_O0_I2(r, r) +C_O0_I2(r, ri) +C_O0_I2(S, S) +C_O0_I2(v, r) +C_O0_I3(S, S, S) +C_O0_I4(r, r, ri, ri) +C_O0_I4(S, S, S, S) +C_O1_I1(r, L) +C_O1_I1(r, r) +C_O1_I1(v, r) +C_O1_I1(v, v) +C_O1_I1(v, vr) +C_O1_I2(r, 0, rZ) +C_O1_I2(r, L, L) +C_O1_I2(r, rI, ri) +C_O1_I2(r, rI, rT) +C_O1_I2(r, r, r) +C_O1_I2(r, r, ri) +C_O1_I2(r, r, rI) +C_O1_I2(r, r, rT) +C_O1_I2(r, r, rU) +C_O1_I2(r, r, rZW) +C_O1_I2(v, v, v) +C_O1_I3(v, v, v, v) +C_O1_I4(r, r, ri, rZ, rZ) +C_O1_I4(r, r, r, ri, ri) +C_O2_I1(L, L, L) +C_O2_I2(L, L, L, L) +C_O2_I4(r, r, rI, rZM, r, r) +C_O2_I4(r, r, r, r, rI, rZM) diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_CON_SET_H #endif diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, va_end(va); } -static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) +static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) { - static const TCGTargetOpDef r = { .args_ct_str = { "r" } }; - static const TCGTargetOpDef r_r = { .args_ct_str = { "r", "r" } }; - static const TCGTargetOpDef r_L = { .args_ct_str = { "r", "L" } }; - static const TCGTargetOpDef S_S = { .args_ct_str = { "S", "S" } }; - static const TCGTargetOpDef r_ri = { .args_ct_str = { "r", "ri" } }; - static const TCGTargetOpDef r_r_r = { .args_ct_str = { "r", "r", "r" } }; - static const TCGTargetOpDef r_L_L = { .args_ct_str = { "r", "L", "L" } }; - static const TCGTargetOpDef L_L_L = { .args_ct_str = { "L", "L", "L" } }; - static const TCGTargetOpDef S_S_S = { .args_ct_str = { "S", "S", "S" } }; - static const TCGTargetOpDef r_r_ri = { .args_ct_str = { "r", "r", "ri" } }; - static const TCGTargetOpDef r_r_rI = { .args_ct_str = { "r", "r", "rI" } }; - static const TCGTargetOpDef r_r_rT = { .args_ct_str = { "r", "r", "rT" } }; - static const TCGTargetOpDef r_r_rU = { .args_ct_str = { "r", "r", "rU" } }; - static const TCGTargetOpDef r_rI_ri - = { .args_ct_str = { "r", "rI", "ri" } }; - static const TCGTargetOpDef r_rI_rT - = { .args_ct_str = { "r", "rI", "rT" } }; - static const TCGTargetOpDef r_r_rZW - = { .args_ct_str = { "r", "r", "rZW" } }; - static const TCGTargetOpDef L_L_L_L - = { .args_ct_str = { "L", "L", "L", "L" } }; - static const TCGTargetOpDef S_S_S_S - = { .args_ct_str = { "S", "S", "S", "S" } }; - static const TCGTargetOpDef movc - = { .args_ct_str = { "r", "r", "ri", "rZ", "rZ" } }; - static const TCGTargetOpDef dep - = { .args_ct_str = { "r", "0", "rZ" } }; - static const TCGTargetOpDef br2 - = { .args_ct_str = { "r", "r", "ri", "ri" } }; - static const TCGTargetOpDef setc2 - = { .args_ct_str = { "r", "r", "r", "ri", "ri" } }; - static const TCGTargetOpDef add2 - = { .args_ct_str = { "r", "r", "r", "r", "rI", "rZM" } }; - static const TCGTargetOpDef sub2 - = { .args_ct_str = { "r", "r", "rI", "rZM", "r", "r" } }; - static const TCGTargetOpDef v_r = { .args_ct_str = { "v", "r" } }; - static const TCGTargetOpDef v_vr = { .args_ct_str = { "v", "vr" } }; - static const TCGTargetOpDef v_v = { .args_ct_str = { "v", "v" } }; - static const TCGTargetOpDef v_v_v = { .args_ct_str = { "v", "v", "v" } }; - static const TCGTargetOpDef v_v_v_v - = { .args_ct_str = { "v", "v", "v", "v" } }; - switch (op) { case INDEX_op_goto_ptr: - return &r; + return C_O0_I1(r); case INDEX_op_ld8u_i32: case INDEX_op_ld8s_i32: case INDEX_op_ld16u_i32: case INDEX_op_ld16s_i32: case INDEX_op_ld_i32: - case INDEX_op_st8_i32: - case INDEX_op_st16_i32: - case INDEX_op_st_i32: case INDEX_op_ctpop_i32: case INDEX_op_neg_i32: case INDEX_op_not_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_ld32u_i64: case INDEX_op_ld32s_i64: case INDEX_op_ld_i64: - case INDEX_op_st8_i64: - case INDEX_op_st16_i64: - case INDEX_op_st32_i64: - case INDEX_op_st_i64: case INDEX_op_ctpop_i64: case INDEX_op_neg_i64: case INDEX_op_not_i64: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_bswap32_i64: case INDEX_op_bswap64_i64: case INDEX_op_extract_i64: - return &r_r; + return C_O1_I1(r, r); + + case INDEX_op_st8_i32: + case INDEX_op_st16_i32: + case INDEX_op_st_i32: + case INDEX_op_st8_i64: + case INDEX_op_st16_i64: + case INDEX_op_st32_i64: + case INDEX_op_st_i64: + return C_O0_I2(r, r); case INDEX_op_add_i32: case INDEX_op_and_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_rotl_i64: case INDEX_op_rotr_i64: case INDEX_op_setcond_i64: - return &r_r_ri; + return C_O1_I2(r, r, ri); + case INDEX_op_mul_i32: case INDEX_op_mul_i64: - return &r_r_rI; + return C_O1_I2(r, r, rI); + case INDEX_op_div_i32: case INDEX_op_divu_i32: case INDEX_op_nand_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_divu_i64: case INDEX_op_mulsh_i64: case INDEX_op_muluh_i64: - return &r_r_r; + return C_O1_I2(r, r, r); + case INDEX_op_sub_i32: - return &r_rI_ri; + return C_O1_I2(r, rI, ri); case INDEX_op_add_i64: - return &r_r_rT; + return C_O1_I2(r, r, rT); case INDEX_op_or_i64: case INDEX_op_xor_i64: - return &r_r_rU; + return C_O1_I2(r, r, rU); case INDEX_op_sub_i64: - return &r_rI_rT; + return C_O1_I2(r, rI, rT); case INDEX_op_clz_i32: case INDEX_op_ctz_i32: case INDEX_op_clz_i64: case INDEX_op_ctz_i64: - return &r_r_rZW; + return C_O1_I2(r, r, rZW); case INDEX_op_brcond_i32: case INDEX_op_brcond_i64: - return &r_ri; + return C_O0_I2(r, ri); case INDEX_op_movcond_i32: case INDEX_op_movcond_i64: - return &movc; + return C_O1_I4(r, r, ri, rZ, rZ); case INDEX_op_deposit_i32: case INDEX_op_deposit_i64: - return &dep; + return C_O1_I2(r, 0, rZ); case INDEX_op_brcond2_i32: - return &br2; + return C_O0_I4(r, r, ri, ri); case INDEX_op_setcond2_i32: - return &setc2; + return C_O1_I4(r, r, r, ri, ri); case INDEX_op_add2_i64: case INDEX_op_add2_i32: - return &add2; + return C_O2_I4(r, r, r, r, rI, rZM); case INDEX_op_sub2_i64: case INDEX_op_sub2_i32: - return &sub2; + return C_O2_I4(r, r, rI, rZM, r, r); case INDEX_op_qemu_ld_i32: return (TCG_TARGET_REG_BITS == 64 || TARGET_LONG_BITS == 32 - ? &r_L : &r_L_L); + ? C_O1_I1(r, L) + : C_O1_I2(r, L, L)); + case INDEX_op_qemu_st_i32: return (TCG_TARGET_REG_BITS == 64 || TARGET_LONG_BITS == 32 - ? &S_S : &S_S_S); + ? C_O0_I2(S, S) + : C_O0_I3(S, S, S)); + case INDEX_op_qemu_ld_i64: - return (TCG_TARGET_REG_BITS == 64 ? &r_L - : TARGET_LONG_BITS == 32 ? &L_L_L : &L_L_L_L); + return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) + : TARGET_LONG_BITS == 32 ? C_O2_I1(L, L, L) + : C_O2_I2(L, L, L, L)); + case INDEX_op_qemu_st_i64: - return (TCG_TARGET_REG_BITS == 64 ? &S_S - : TARGET_LONG_BITS == 32 ? &S_S_S : &S_S_S_S); + return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(S, S) + : TARGET_LONG_BITS == 32 ? C_O0_I3(S, S, S) + : C_O0_I4(S, S, S, S)); case INDEX_op_add_vec: case INDEX_op_sub_vec: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_ppc_mulou_vec: case INDEX_op_ppc_pkum_vec: case INDEX_op_dup2_vec: - return &v_v_v; + return C_O1_I2(v, v, v); + case INDEX_op_not_vec: case INDEX_op_neg_vec: - return &v_v; + return C_O1_I1(v, v); + case INDEX_op_dup_vec: - return have_isa_3_00 ? &v_vr : &v_v; + return have_isa_3_00 ? C_O1_I1(v, vr) : C_O1_I1(v, v); + case INDEX_op_ld_vec: - case INDEX_op_st_vec: case INDEX_op_dupm_vec: - return &v_r; + return C_O1_I1(v, r); + + case INDEX_op_st_vec: + return C_O0_I2(v, r); + case INDEX_op_bitsel_vec: case INDEX_op_ppc_msum_vec: - return &v_v_v_v; + return C_O1_I3(v, v, v, v); default: - return NULL; + g_assert_not_reached(); } } -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target-con-set.h | 30 ++++++++++++ tcg/riscv/tcg-target.h | 1 + tcg/riscv/tcg-target.c.inc | 83 ++++++++++------------------------ 3 files changed, 54 insertions(+), 60 deletions(-) create mode 100644 tcg/riscv/tcg-target-con-set.h diff --git a/tcg/riscv/tcg-target-con-set.h b/tcg/riscv/tcg-target-con-set.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/riscv/tcg-target-con-set.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define RISC-V target-specific constraint sets. + * Copyright (c) 2021 Linaro + */ + +/* + * C_On_Im(...) defines a constraint set with <n> outputs and <m> inputs. + * Each operand should be a sequence of constraint letters as defined by + * tcg-target-con-str.h; the constraint combination is inclusive or. + */ +C_O0_I1(r) +C_O0_I2(LZ, L) +C_O0_I2(rZ, r) +C_O0_I2(rZ, rZ) +C_O0_I3(LZ, L, L) +C_O0_I3(LZ, LZ, L) +C_O0_I4(LZ, LZ, L, L) +C_O0_I4(rZ, rZ, rZ, rZ) +C_O1_I1(r, L) +C_O1_I1(r, r) +C_O1_I2(r, L, L) +C_O1_I2(r, r, ri) +C_O1_I2(r, r, rI) +C_O1_I2(r, rZ, rN) +C_O1_I2(r, rZ, rZ) +C_O1_I4(r, rZ, rZ, rZ, rZ) +C_O2_I1(r, r, L) +C_O2_I2(r, r, L, L) +C_O2_I4(r, r, rZ, rZ, rM, rM) diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_POOL_LABELS #define TCG_TARGET_HAS_MEMORY_BSWAP 0 +#define TCG_TARGET_CON_SET_H #endif diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, } } -static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) +static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) { - static const TCGTargetOpDef r - = { .args_ct_str = { "r" } }; - static const TCGTargetOpDef r_r - = { .args_ct_str = { "r", "r" } }; - static const TCGTargetOpDef rZ_r - = { .args_ct_str = { "rZ", "r" } }; - static const TCGTargetOpDef rZ_rZ - = { .args_ct_str = { "rZ", "rZ" } }; - static const TCGTargetOpDef rZ_rZ_rZ_rZ - = { .args_ct_str = { "rZ", "rZ", "rZ", "rZ" } }; - static const TCGTargetOpDef r_r_ri - = { .args_ct_str = { "r", "r", "ri" } }; - static const TCGTargetOpDef r_r_rI - = { .args_ct_str = { "r", "r", "rI" } }; - static const TCGTargetOpDef r_rZ_rN - = { .args_ct_str = { "r", "rZ", "rN" } }; - static const TCGTargetOpDef r_rZ_rZ - = { .args_ct_str = { "r", "rZ", "rZ" } }; - static const TCGTargetOpDef r_rZ_rZ_rZ_rZ - = { .args_ct_str = { "r", "rZ", "rZ", "rZ", "rZ" } }; - static const TCGTargetOpDef r_L - = { .args_ct_str = { "r", "L" } }; - static const TCGTargetOpDef r_r_L - = { .args_ct_str = { "r", "r", "L" } }; - static const TCGTargetOpDef r_L_L - = { .args_ct_str = { "r", "L", "L" } }; - static const TCGTargetOpDef r_r_L_L - = { .args_ct_str = { "r", "r", "L", "L" } }; - static const TCGTargetOpDef LZ_L - = { .args_ct_str = { "LZ", "L" } }; - static const TCGTargetOpDef LZ_L_L - = { .args_ct_str = { "LZ", "L", "L" } }; - static const TCGTargetOpDef LZ_LZ_L - = { .args_ct_str = { "LZ", "LZ", "L" } }; - static const TCGTargetOpDef LZ_LZ_L_L - = { .args_ct_str = { "LZ", "LZ", "L", "L" } }; - static const TCGTargetOpDef r_r_rZ_rZ_rM_rM - = { .args_ct_str = { "r", "r", "rZ", "rZ", "rM", "rM" } }; - switch (op) { case INDEX_op_goto_ptr: - return &r; + return C_O0_I1(r); case INDEX_op_ld8u_i32: case INDEX_op_ld8s_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_extrl_i64_i32: case INDEX_op_extrh_i64_i32: case INDEX_op_ext_i32_i64: - return &r_r; + return C_O1_I1(r, r); case INDEX_op_st8_i32: case INDEX_op_st16_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_st16_i64: case INDEX_op_st32_i64: case INDEX_op_st_i64: - return &rZ_r; + return C_O0_I2(rZ, r); case INDEX_op_add_i32: case INDEX_op_and_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_and_i64: case INDEX_op_or_i64: case INDEX_op_xor_i64: - return &r_r_rI; + return C_O1_I2(r, r, rI); case INDEX_op_sub_i32: case INDEX_op_sub_i64: - return &r_rZ_rN; + return C_O1_I2(r, rZ, rN); case INDEX_op_mul_i32: case INDEX_op_mulsh_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_rem_i64: case INDEX_op_remu_i64: case INDEX_op_setcond_i64: - return &r_rZ_rZ; + return C_O1_I2(r, rZ, rZ); case INDEX_op_shl_i32: case INDEX_op_shr_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_shl_i64: case INDEX_op_shr_i64: case INDEX_op_sar_i64: - return &r_r_ri; + return C_O1_I2(r, r, ri); case INDEX_op_brcond_i32: case INDEX_op_brcond_i64: - return &rZ_rZ; + return C_O0_I2(rZ, rZ); case INDEX_op_add2_i32: case INDEX_op_add2_i64: case INDEX_op_sub2_i32: case INDEX_op_sub2_i64: - return &r_r_rZ_rZ_rM_rM; + return C_O2_I4(r, r, rZ, rZ, rM, rM); case INDEX_op_brcond2_i32: - return &rZ_rZ_rZ_rZ; + return C_O0_I4(rZ, rZ, rZ, rZ); case INDEX_op_setcond2_i32: - return &r_rZ_rZ_rZ_rZ; + return C_O1_I4(r, rZ, rZ, rZ, rZ); case INDEX_op_qemu_ld_i32: - return TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &r_L : &r_L_L; + return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS + ? C_O1_I1(r, L) : C_O1_I2(r, L, L)); case INDEX_op_qemu_st_i32: - return TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &LZ_L : &LZ_L_L; + return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS + ? C_O0_I2(LZ, L) : C_O0_I3(LZ, L, L)); case INDEX_op_qemu_ld_i64: - return TCG_TARGET_REG_BITS == 64 ? &r_L - : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &r_r_L - : &r_r_L_L; + return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) + : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O2_I1(r, r, L) + : C_O2_I2(r, r, L, L)); case INDEX_op_qemu_st_i64: - return TCG_TARGET_REG_BITS == 64 ? &LZ_L - : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &LZ_LZ_L - : &LZ_LZ_L_L; + return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(LZ, L) + : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O0_I3(LZ, LZ, L) + : C_O0_I4(LZ, LZ, L, L)); default: - return NULL; + g_assert_not_reached(); } } -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/s390/tcg-target-con-set.h | 29 ++++++++ tcg/s390/tcg-target.h | 1 + tcg/s390/tcg-target.c.inc | 121 ++++++++++++++-------------------- 3 files changed, 81 insertions(+), 70 deletions(-) create mode 100644 tcg/s390/tcg-target-con-set.h diff --git a/tcg/s390/tcg-target-con-set.h b/tcg/s390/tcg-target-con-set.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/s390/tcg-target-con-set.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define S390 target-specific constraint sets. + * Copyright (c) 2021 Linaro + */ + +/* + * C_On_Im(...) defines a constraint set with <n> outputs and <m> inputs. + * Each operand should be a sequence of constraint letters as defined by + * tcg-target-con-str.h; the constraint combination is inclusive or. + */ +C_O0_I1(r) +C_O0_I2(L, L) +C_O0_I2(r, r) +C_O0_I2(r, ri) +C_O1_I1(r, L) +C_O1_I1(r, r) +C_O1_I2(r, 0, ri) +C_O1_I2(r, 0, rI) +C_O1_I2(r, 0, rJ) +C_O1_I2(r, r, ri) +C_O1_I2(r, rZ, r) +C_O1_I4(r, r, ri, r, 0) +C_O1_I4(r, r, ri, rI, 0) +C_O2_I2(b, a, 0, r) +C_O2_I3(b, a, 0, 1, r) +C_O2_I4(r, r, 0, 1, rA, r) +C_O2_I4(r, r, 0, 1, ri, r) +C_O2_I4(r, r, 0, 1, r, r) diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.h +++ b/tcg/s390/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_CON_SET_H #endif diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.c.inc +++ b/tcg/s390/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, } } -static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) +static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) { - static const TCGTargetOpDef r = { .args_ct_str = { "r" } }; - static const TCGTargetOpDef r_r = { .args_ct_str = { "r", "r" } }; - static const TCGTargetOpDef r_L = { .args_ct_str = { "r", "L" } }; - static const TCGTargetOpDef L_L = { .args_ct_str = { "L", "L" } }; - static const TCGTargetOpDef r_ri = { .args_ct_str = { "r", "ri" } }; - static const TCGTargetOpDef r_r_ri = { .args_ct_str = { "r", "r", "ri" } }; - static const TCGTargetOpDef r_0_ri = { .args_ct_str = { "r", "0", "ri" } }; - static const TCGTargetOpDef r_0_rI = { .args_ct_str = { "r", "0", "rI" } }; - static const TCGTargetOpDef r_0_rJ = { .args_ct_str = { "r", "0", "rJ" } }; - static const TCGTargetOpDef a2_r - = { .args_ct_str = { "r", "r", "0", "1", "r", "r" } }; - static const TCGTargetOpDef a2_ri - = { .args_ct_str = { "r", "r", "0", "1", "ri", "r" } }; - static const TCGTargetOpDef a2_rA - = { .args_ct_str = { "r", "r", "0", "1", "rA", "r" } }; - switch (op) { case INDEX_op_goto_ptr: - return &r; + return C_O0_I1(r); case INDEX_op_ld8u_i32: case INDEX_op_ld8u_i64: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_ld32u_i64: case INDEX_op_ld32s_i64: case INDEX_op_ld_i64: + return C_O1_I1(r, r); + case INDEX_op_st8_i32: case INDEX_op_st8_i64: case INDEX_op_st16_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_st_i32: case INDEX_op_st32_i64: case INDEX_op_st_i64: - return &r_r; + return C_O0_I2(r, r); case INDEX_op_add_i32: case INDEX_op_add_i64: - return &r_r_ri; + case INDEX_op_shl_i64: + case INDEX_op_shr_i64: + case INDEX_op_sar_i64: + case INDEX_op_rotl_i32: + case INDEX_op_rotl_i64: + case INDEX_op_rotr_i32: + case INDEX_op_rotr_i64: + case INDEX_op_clz_i64: + case INDEX_op_setcond_i32: + case INDEX_op_setcond_i64: + return C_O1_I2(r, r, ri); + case INDEX_op_sub_i32: case INDEX_op_sub_i64: case INDEX_op_and_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_or_i64: case INDEX_op_xor_i32: case INDEX_op_xor_i64: - return (s390_facilities & FACILITY_DISTINCT_OPS ? &r_r_ri : &r_0_ri); + return (s390_facilities & FACILITY_DISTINCT_OPS + ? C_O1_I2(r, r, ri) + : C_O1_I2(r, 0, ri)); case INDEX_op_mul_i32: /* If we have the general-instruction-extensions, then we have MULTIPLY SINGLE IMMEDIATE with a signed 32-bit, otherwise we have only MULTIPLY HALFWORD IMMEDIATE, with a signed 16-bit. */ - return (s390_facilities & FACILITY_GEN_INST_EXT ? &r_0_ri : &r_0_rI); + return (s390_facilities & FACILITY_GEN_INST_EXT + ? C_O1_I2(r, 0, ri) + : C_O1_I2(r, 0, rI)); + case INDEX_op_mul_i64: - return (s390_facilities & FACILITY_GEN_INST_EXT ? &r_0_rJ : &r_0_rI); + return (s390_facilities & FACILITY_GEN_INST_EXT + ? C_O1_I2(r, 0, rJ) + : C_O1_I2(r, 0, rI)); case INDEX_op_shl_i32: case INDEX_op_shr_i32: case INDEX_op_sar_i32: - return (s390_facilities & FACILITY_DISTINCT_OPS ? &r_r_ri : &r_0_ri); - - case INDEX_op_shl_i64: - case INDEX_op_shr_i64: - case INDEX_op_sar_i64: - return &r_r_ri; - - case INDEX_op_rotl_i32: - case INDEX_op_rotl_i64: - case INDEX_op_rotr_i32: - case INDEX_op_rotr_i64: - return &r_r_ri; + return (s390_facilities & FACILITY_DISTINCT_OPS + ? C_O1_I2(r, r, ri) + : C_O1_I2(r, 0, ri)); case INDEX_op_brcond_i32: case INDEX_op_brcond_i64: - return &r_ri; + return C_O0_I2(r, ri); case INDEX_op_bswap16_i32: case INDEX_op_bswap16_i64: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_extu_i32_i64: case INDEX_op_extract_i32: case INDEX_op_extract_i64: - return &r_r; - - case INDEX_op_clz_i64: - case INDEX_op_setcond_i32: - case INDEX_op_setcond_i64: - return &r_r_ri; + return C_O1_I1(r, r); case INDEX_op_qemu_ld_i32: case INDEX_op_qemu_ld_i64: - return &r_L; + return C_O1_I1(r, L); case INDEX_op_qemu_st_i64: case INDEX_op_qemu_st_i32: - return &L_L; + return C_O0_I2(L, L); case INDEX_op_deposit_i32: case INDEX_op_deposit_i64: - { - static const TCGTargetOpDef dep - = { .args_ct_str = { "r", "rZ", "r" } }; - return &dep; - } + return C_O1_I2(r, rZ, r); + case INDEX_op_movcond_i32: case INDEX_op_movcond_i64: - { - static const TCGTargetOpDef movc - = { .args_ct_str = { "r", "r", "ri", "r", "0" } }; - static const TCGTargetOpDef movc_l - = { .args_ct_str = { "r", "r", "ri", "rI", "0" } }; - return (s390_facilities & FACILITY_LOAD_ON_COND2 ? &movc_l : &movc); - } + return (s390_facilities & FACILITY_LOAD_ON_COND2 + ? C_O1_I4(r, r, ri, rI, 0) + : C_O1_I4(r, r, ri, r, 0)); + case INDEX_op_div2_i32: case INDEX_op_div2_i64: case INDEX_op_divu2_i32: case INDEX_op_divu2_i64: - { - static const TCGTargetOpDef div2 - = { .args_ct_str = { "b", "a", "0", "1", "r" } }; - return &div2; - } + return C_O2_I3(b, a, 0, 1, r); + case INDEX_op_mulu2_i64: - { - static const TCGTargetOpDef mul2 - = { .args_ct_str = { "b", "a", "0", "r" } }; - return &mul2; - } + return C_O2_I2(b, a, 0, r); case INDEX_op_add2_i32: case INDEX_op_sub2_i32: - return (s390_facilities & FACILITY_EXT_IMM ? &a2_ri : &a2_r); + return (s390_facilities & FACILITY_EXT_IMM + ? C_O2_I4(r, r, 0, 1, ri, r) + : C_O2_I4(r, r, 0, 1, r, r)); + case INDEX_op_add2_i64: case INDEX_op_sub2_i64: - return (s390_facilities & FACILITY_EXT_IMM ? &a2_rA : &a2_r); + return (s390_facilities & FACILITY_EXT_IMM + ? C_O2_I4(r, r, 0, 1, rA, r) + : C_O2_I4(r, r, 0, 1, r, r)); default: - break; + g_assert_not_reached(); } - return NULL; } static void query_s390_facilities(void) -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc/tcg-target-con-set.h | 32 +++++++++++++++ tcg/sparc/tcg-target.h | 1 + tcg/sparc/tcg-target.c.inc | 75 +++++++++++----------------------- 3 files changed, 56 insertions(+), 52 deletions(-) create mode 100644 tcg/sparc/tcg-target-con-set.h diff --git a/tcg/sparc/tcg-target-con-set.h b/tcg/sparc/tcg-target-con-set.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/sparc/tcg-target-con-set.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define Sparc target-specific constraint sets. + * Copyright (c) 2021 Linaro + */ + +/* + * C_On_Im(...) defines a constraint set with <n> outputs and <m> inputs. + * Each operand should be a sequence of constraint letters as defined by + * tcg-target-con-str.h; the constraint combination is inclusive or. + */ +C_O0_I1(r) +C_O0_I2(rZ, r) +C_O0_I2(RZ, r) +C_O0_I2(rZ, rJ) +C_O0_I2(RZ, RJ) +C_O0_I2(sZ, A) +C_O0_I2(SZ, A) +C_O1_I1(r, A) +C_O1_I1(R, A) +C_O1_I1(r, r) +C_O1_I1(r, R) +C_O1_I1(R, r) +C_O1_I1(R, R) +C_O1_I2(R, R, R) +C_O1_I2(r, rZ, rJ) +C_O1_I2(R, RZ, RJ) +C_O1_I4(r, rZ, rJ, rI, 0) +C_O1_I4(R, RZ, RJ, RI, 0) +C_O2_I2(r, r, rZ, rJ) +C_O2_I4(R, R, RZ, RZ, RJ, RI) +C_O2_I4(r, r, rZ, rZ, rJ, rJ) diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.h +++ b/tcg/sparc/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions; void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_POOL_LABELS +#define TCG_TARGET_CON_SET_H #endif diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, } } -static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) +static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) { - static const TCGTargetOpDef r = { .args_ct_str = { "r" } }; - static const TCGTargetOpDef r_r = { .args_ct_str = { "r", "r" } }; - static const TCGTargetOpDef R_r = { .args_ct_str = { "R", "r" } }; - static const TCGTargetOpDef r_R = { .args_ct_str = { "r", "R" } }; - static const TCGTargetOpDef R_R = { .args_ct_str = { "R", "R" } }; - static const TCGTargetOpDef r_A = { .args_ct_str = { "r", "A" } }; - static const TCGTargetOpDef R_A = { .args_ct_str = { "R", "A" } }; - static const TCGTargetOpDef rZ_r = { .args_ct_str = { "rZ", "r" } }; - static const TCGTargetOpDef RZ_r = { .args_ct_str = { "RZ", "r" } }; - static const TCGTargetOpDef sZ_A = { .args_ct_str = { "sZ", "A" } }; - static const TCGTargetOpDef SZ_A = { .args_ct_str = { "SZ", "A" } }; - static const TCGTargetOpDef rZ_rJ = { .args_ct_str = { "rZ", "rJ" } }; - static const TCGTargetOpDef RZ_RJ = { .args_ct_str = { "RZ", "RJ" } }; - static const TCGTargetOpDef R_R_R = { .args_ct_str = { "R", "R", "R" } }; - static const TCGTargetOpDef r_rZ_rJ - = { .args_ct_str = { "r", "rZ", "rJ" } }; - static const TCGTargetOpDef R_RZ_RJ - = { .args_ct_str = { "R", "RZ", "RJ" } }; - static const TCGTargetOpDef r_r_rZ_rJ - = { .args_ct_str = { "r", "r", "rZ", "rJ" } }; - static const TCGTargetOpDef movc_32 - = { .args_ct_str = { "r", "rZ", "rJ", "rI", "0" } }; - static const TCGTargetOpDef movc_64 - = { .args_ct_str = { "R", "RZ", "RJ", "RI", "0" } }; - static const TCGTargetOpDef add2_32 - = { .args_ct_str = { "r", "r", "rZ", "rZ", "rJ", "rJ" } }; - static const TCGTargetOpDef add2_64 - = { .args_ct_str = { "R", "R", "RZ", "RZ", "RJ", "RI" } }; - switch (op) { case INDEX_op_goto_ptr: - return &r; + return C_O0_I1(r); case INDEX_op_ld8u_i32: case INDEX_op_ld8s_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_ld_i32: case INDEX_op_neg_i32: case INDEX_op_not_i32: - return &r_r; + return C_O1_I1(r, r); case INDEX_op_st8_i32: case INDEX_op_st16_i32: case INDEX_op_st_i32: - return &rZ_r; + return C_O0_I2(rZ, r); case INDEX_op_add_i32: case INDEX_op_mul_i32: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_shr_i32: case INDEX_op_sar_i32: case INDEX_op_setcond_i32: - return &r_rZ_rJ; + return C_O1_I2(r, rZ, rJ); case INDEX_op_brcond_i32: - return &rZ_rJ; + return C_O0_I2(rZ, rJ); case INDEX_op_movcond_i32: - return &movc_32; + return C_O1_I4(r, rZ, rJ, rI, 0); case INDEX_op_add2_i32: case INDEX_op_sub2_i32: - return &add2_32; + return C_O2_I4(r, r, rZ, rZ, rJ, rJ); case INDEX_op_mulu2_i32: case INDEX_op_muls2_i32: - return &r_r_rZ_rJ; + return C_O2_I2(r, r, rZ, rJ); case INDEX_op_ld8u_i64: case INDEX_op_ld8s_i64: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_ld_i64: case INDEX_op_ext_i32_i64: case INDEX_op_extu_i32_i64: - return &R_r; + return C_O1_I1(R, r); case INDEX_op_st8_i64: case INDEX_op_st16_i64: case INDEX_op_st32_i64: case INDEX_op_st_i64: - return &RZ_r; + return C_O0_I2(RZ, r); case INDEX_op_add_i64: case INDEX_op_mul_i64: @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) case INDEX_op_shr_i64: case INDEX_op_sar_i64: case INDEX_op_setcond_i64: - return &R_RZ_RJ; + return C_O1_I2(R, RZ, RJ); case INDEX_op_neg_i64: case INDEX_op_not_i64: case INDEX_op_ext32s_i64: case INDEX_op_ext32u_i64: - return &R_R; + return C_O1_I1(R, R); case INDEX_op_extrl_i64_i32: case INDEX_op_extrh_i64_i32: - return &r_R; + return C_O1_I1(r, R); case INDEX_op_brcond_i64: - return &RZ_RJ; + return C_O0_I2(RZ, RJ); case INDEX_op_movcond_i64: - return &movc_64; + return C_O1_I4(R, RZ, RJ, RI, 0); case INDEX_op_add2_i64: case INDEX_op_sub2_i64: - return &add2_64; + return C_O2_I4(R, R, RZ, RZ, RJ, RI); case INDEX_op_muluh_i64: - return &R_R_R; + return C_O1_I2(R, R, R); case INDEX_op_qemu_ld_i32: - return &r_A; + return C_O1_I1(r, A); case INDEX_op_qemu_ld_i64: - return &R_A; + return C_O1_I1(R, A); case INDEX_op_qemu_st_i32: - return &sZ_A; + return C_O0_I2(sZ, A); case INDEX_op_qemu_st_i64: - return &SZ_A; + return C_O0_I2(SZ, A); default: - return NULL; + g_assert_not_reached(); } } -- 2.25.1 This requires finishing the conversion to tcg_target_op_def. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci/tcg-target-con-set.h | 25 ++++ tcg/tci/tcg-target.h | 2 + tcg/tci/tcg-target.c.inc | 279 +++++++++++++++++------------------ 3 files changed, 161 insertions(+), 145 deletions(-) create mode 100644 tcg/tci/tcg-target-con-set.h diff --git a/tcg/tci/tcg-target-con-set.h b/tcg/tci/tcg-target-con-set.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tcg/tci/tcg-target-con-set.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: MIT */ +/* + * TCI target-specific constraint sets. + * Copyright (c) 2021 Linaro + */ + +/* + * C_On_Im(...) defines a constraint set with <n> outputs and <m> inputs. + * Each operand should be a sequence of constraint letters as defined by + * tcg-target-con-str.h; the constraint combination is inclusive or. + */ +C_O0_I2(r, r) +C_O0_I2(r, ri) +C_O0_I3(r, r, r) +C_O0_I4(r, r, ri, ri) +C_O0_I4(r, r, r, r) +C_O1_I1(r, r) +C_O1_I2(r, 0, r) +C_O1_I2(r, ri, ri) +C_O1_I2(r, r, r) +C_O1_I2(r, r, ri) +C_O1_I4(r, r, r, ri, ri) +C_O2_I1(r, r, r) +C_O2_I2(r, r, r, r) +C_O2_I4(r, r, r, r, r, r) diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, /* no need to flush icache explicitly */ } +#define TCG_TARGET_CON_SET_H + #endif /* TCG_TARGET_H */ diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ /* Bitfield n...m (in 32 bit value). */ #define BITS(n, m) (((0xffffffffU << (31 - n)) >> (31 - n + m)) << m) -/* Macros used in tcg_target_op_defs. */ -#define R "r" -#define RI "ri" -#if TCG_TARGET_REG_BITS == 32 -# define R64 "r", "r" -#else -# define R64 "r" -#endif -#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS -# define L "r", "r" -# define S "r", "r" -#else -# define L "r" -# define S "r" -#endif - -/* TODO: documentation. */ -static const TCGTargetOpDef tcg_target_op_defs[] = { - { INDEX_op_exit_tb, { NULL } }, - { INDEX_op_goto_tb, { NULL } }, - { INDEX_op_br, { NULL } }, - - { INDEX_op_ld8u_i32, { R, R } }, - { INDEX_op_ld8s_i32, { R, R } }, - { INDEX_op_ld16u_i32, { R, R } }, - { INDEX_op_ld16s_i32, { R, R } }, - { INDEX_op_ld_i32, { R, R } }, - { INDEX_op_st8_i32, { R, R } }, - { INDEX_op_st16_i32, { R, R } }, - { INDEX_op_st_i32, { R, R } }, - - { INDEX_op_add_i32, { R, RI, RI } }, - { INDEX_op_sub_i32, { R, RI, RI } }, - { INDEX_op_mul_i32, { R, RI, RI } }, - { INDEX_op_div_i32, { R, R, R } }, - { INDEX_op_divu_i32, { R, R, R } }, - { INDEX_op_rem_i32, { R, R, R } }, - { INDEX_op_remu_i32, { R, R, R } }, - /* TODO: Does R, RI, RI result in faster code than R, R, RI? - If both operands are constants, we can optimize. */ - { INDEX_op_and_i32, { R, RI, RI } }, - { INDEX_op_andc_i32, { R, RI, RI } }, - { INDEX_op_eqv_i32, { R, RI, RI } }, - { INDEX_op_nand_i32, { R, RI, RI } }, - { INDEX_op_nor_i32, { R, RI, RI } }, - { INDEX_op_or_i32, { R, RI, RI } }, - { INDEX_op_orc_i32, { R, RI, RI } }, - { INDEX_op_xor_i32, { R, RI, RI } }, - { INDEX_op_shl_i32, { R, RI, RI } }, - { INDEX_op_shr_i32, { R, RI, RI } }, - { INDEX_op_sar_i32, { R, RI, RI } }, - { INDEX_op_rotl_i32, { R, RI, RI } }, - { INDEX_op_rotr_i32, { R, RI, RI } }, - { INDEX_op_deposit_i32, { R, "0", R } }, - - { INDEX_op_brcond_i32, { R, RI } }, - - { INDEX_op_setcond_i32, { R, R, RI } }, - { INDEX_op_setcond_i64, { R, R, RI } }, - - /* TODO: Support R, R, R, R, RI, RI? Will it be faster? */ - { INDEX_op_add2_i32, { R, R, R, R, R, R } }, - { INDEX_op_sub2_i32, { R, R, R, R, R, R } }, - { INDEX_op_brcond2_i32, { R, R, RI, RI } }, - { INDEX_op_mulu2_i32, { R, R, R, R } }, - { INDEX_op_setcond2_i32, { R, R, R, RI, RI } }, - - { INDEX_op_not_i32, { R, R } }, - { INDEX_op_neg_i32, { R, R } }, - - { INDEX_op_ld8u_i64, { R, R } }, - { INDEX_op_ld8s_i64, { R, R } }, - { INDEX_op_ld16u_i64, { R, R } }, - { INDEX_op_ld16s_i64, { R, R } }, - { INDEX_op_ld32u_i64, { R, R } }, - { INDEX_op_ld32s_i64, { R, R } }, - { INDEX_op_ld_i64, { R, R } }, - - { INDEX_op_st8_i64, { R, R } }, - { INDEX_op_st16_i64, { R, R } }, - { INDEX_op_st32_i64, { R, R } }, - { INDEX_op_st_i64, { R, R } }, - - { INDEX_op_add_i64, { R, RI, RI } }, - { INDEX_op_sub_i64, { R, RI, RI } }, - { INDEX_op_mul_i64, { R, RI, RI } }, - { INDEX_op_div_i64, { R, R, R } }, - { INDEX_op_divu_i64, { R, R, R } }, - { INDEX_op_rem_i64, { R, R, R } }, - { INDEX_op_remu_i64, { R, R, R } }, - { INDEX_op_and_i64, { R, RI, RI } }, - { INDEX_op_andc_i64, { R, RI, RI } }, - { INDEX_op_eqv_i64, { R, RI, RI } }, - { INDEX_op_nand_i64, { R, RI, RI } }, - { INDEX_op_nor_i64, { R, RI, RI } }, - { INDEX_op_or_i64, { R, RI, RI } }, - { INDEX_op_orc_i64, { R, RI, RI } }, - { INDEX_op_xor_i64, { R, RI, RI } }, - { INDEX_op_shl_i64, { R, RI, RI } }, - { INDEX_op_shr_i64, { R, RI, RI } }, - { INDEX_op_sar_i64, { R, RI, RI } }, - { INDEX_op_rotl_i64, { R, RI, RI } }, - { INDEX_op_rotr_i64, { R, RI, RI } }, - { INDEX_op_deposit_i64, { R, "0", R } }, - { INDEX_op_brcond_i64, { R, RI } }, - - { INDEX_op_ext8s_i64, { R, R } }, - { INDEX_op_ext16s_i64, { R, R } }, - { INDEX_op_ext32s_i64, { R, R } }, - { INDEX_op_ext8u_i64, { R, R } }, - { INDEX_op_ext16u_i64, { R, R } }, - { INDEX_op_ext32u_i64, { R, R } }, - { INDEX_op_ext_i32_i64, { R, R } }, - { INDEX_op_extu_i32_i64, { R, R } }, - { INDEX_op_bswap16_i64, { R, R } }, - { INDEX_op_bswap32_i64, { R, R } }, - { INDEX_op_bswap64_i64, { R, R } }, - { INDEX_op_not_i64, { R, R } }, - { INDEX_op_neg_i64, { R, R } }, - - { INDEX_op_qemu_ld_i32, { R, L } }, - { INDEX_op_qemu_ld_i64, { R64, L } }, - - { INDEX_op_qemu_st_i32, { R, S } }, - { INDEX_op_qemu_st_i64, { R64, S } }, - - { INDEX_op_ext8s_i32, { R, R } }, - { INDEX_op_ext16s_i32, { R, R } }, - { INDEX_op_ext8u_i32, { R, R } }, - { INDEX_op_ext16u_i32, { R, R } }, - - { INDEX_op_bswap16_i32, { R, R } }, - { INDEX_op_bswap32_i32, { R, R } }, - - { INDEX_op_mb, { } }, - { -1 }, -}; - -static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) +static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) { - int i, n = ARRAY_SIZE(tcg_target_op_defs); + switch (op) { + case INDEX_op_ld8u_i32: + case INDEX_op_ld8s_i32: + case INDEX_op_ld16u_i32: + case INDEX_op_ld16s_i32: + case INDEX_op_ld_i32: + case INDEX_op_ld8u_i64: + case INDEX_op_ld8s_i64: + case INDEX_op_ld16u_i64: + case INDEX_op_ld16s_i64: + case INDEX_op_ld32u_i64: + case INDEX_op_ld32s_i64: + case INDEX_op_ld_i64: + case INDEX_op_not_i32: + case INDEX_op_not_i64: + case INDEX_op_neg_i32: + case INDEX_op_neg_i64: + case INDEX_op_ext8s_i32: + case INDEX_op_ext8s_i64: + case INDEX_op_ext16s_i32: + case INDEX_op_ext16s_i64: + case INDEX_op_ext8u_i32: + case INDEX_op_ext8u_i64: + case INDEX_op_ext16u_i32: + case INDEX_op_ext16u_i64: + case INDEX_op_ext32s_i64: + case INDEX_op_ext32u_i64: + case INDEX_op_ext_i32_i64: + case INDEX_op_extu_i32_i64: + case INDEX_op_bswap16_i32: + case INDEX_op_bswap16_i64: + case INDEX_op_bswap32_i32: + case INDEX_op_bswap32_i64: + case INDEX_op_bswap64_i64: + return C_O1_I1(r, r); - for (i = 0; i < n; ++i) { - if (tcg_target_op_defs[i].op == op) { - return &tcg_target_op_defs[i]; - } + case INDEX_op_st8_i32: + case INDEX_op_st16_i32: + case INDEX_op_st_i32: + case INDEX_op_st8_i64: + case INDEX_op_st16_i64: + case INDEX_op_st32_i64: + case INDEX_op_st_i64: + return C_O0_I2(r, r); + + case INDEX_op_div_i32: + case INDEX_op_div_i64: + case INDEX_op_divu_i32: + case INDEX_op_divu_i64: + case INDEX_op_rem_i32: + case INDEX_op_rem_i64: + case INDEX_op_remu_i32: + case INDEX_op_remu_i64: + return C_O1_I2(r, r, r); + + case INDEX_op_add_i32: + case INDEX_op_add_i64: + case INDEX_op_sub_i32: + case INDEX_op_sub_i64: + case INDEX_op_mul_i32: + case INDEX_op_mul_i64: + case INDEX_op_and_i32: + case INDEX_op_and_i64: + case INDEX_op_andc_i32: + case INDEX_op_andc_i64: + case INDEX_op_eqv_i32: + case INDEX_op_eqv_i64: + case INDEX_op_nand_i32: + case INDEX_op_nand_i64: + case INDEX_op_nor_i32: + case INDEX_op_nor_i64: + case INDEX_op_or_i32: + case INDEX_op_or_i64: + case INDEX_op_orc_i32: + case INDEX_op_orc_i64: + case INDEX_op_xor_i32: + case INDEX_op_xor_i64: + case INDEX_op_shl_i32: + case INDEX_op_shl_i64: + case INDEX_op_shr_i32: + case INDEX_op_shr_i64: + case INDEX_op_sar_i32: + case INDEX_op_sar_i64: + case INDEX_op_rotl_i32: + case INDEX_op_rotl_i64: + case INDEX_op_rotr_i32: + case INDEX_op_rotr_i64: + /* TODO: Does R, RI, RI result in faster code than R, R, RI? */ + return C_O1_I2(r, ri, ri); + + case INDEX_op_deposit_i32: + case INDEX_op_deposit_i64: + return C_O1_I2(r, 0, r); + + case INDEX_op_brcond_i32: + case INDEX_op_brcond_i64: + return C_O0_I2(r, ri); + + case INDEX_op_setcond_i32: + case INDEX_op_setcond_i64: + return C_O1_I2(r, r, ri); + +#if TCG_TARGET_REG_BITS == 32 + /* TODO: Support R, R, R, R, RI, RI? Will it be faster? */ + case INDEX_op_add2_i32: + case INDEX_op_sub2_i32: + return C_O2_I4(r, r, r, r, r, r); + case INDEX_op_brcond2_i32: + return C_O0_I4(r, r, ri, ri); + case INDEX_op_mulu2_i32: + return C_O2_I2(r, r, r, r); + case INDEX_op_setcond2_i32: + return C_O1_I4(r, r, r, ri, ri); +#endif + + case INDEX_op_qemu_ld_i32: + return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS + ? C_O1_I1(r, r) + : C_O1_I2(r, r, r)); + case INDEX_op_qemu_ld_i64: + return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, r) + : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O2_I1(r, r, r) + : C_O2_I2(r, r, r, r)); + case INDEX_op_qemu_st_i32: + return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS + ? C_O0_I2(r, r) + : C_O0_I3(r, r, r)); + case INDEX_op_qemu_st_i64: + return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(r, r) + : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O0_I3(r, r, r) + : C_O0_I4(r, r, r, r)); + + default: + g_assert_not_reached(); } - return NULL; } static const int tcg_target_reg_alloc_order[] = { -- 2.25.1 All backends have now been converted to tcg-target-con-set.h, so we can remove the fallback code. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target.h | 1 - tcg/arm/tcg-target.h | 1 - tcg/i386/tcg-target.h | 1 - tcg/mips/tcg-target.h | 1 - tcg/ppc/tcg-target.h | 1 - tcg/riscv/tcg-target.h | 1 - tcg/s390/tcg-target.h | 1 - tcg/sparc/tcg-target.h | 1 - tcg/tci/tcg-target.h | 2 -- tcg/tcg.c | 12 ------------ 10 files changed, 22 deletions(-) diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_CON_SET_H #endif /* AARCH64_TCG_TARGET_H */ diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_CON_SET_H #endif diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_CON_SET_H #endif diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #ifdef CONFIG_SOFTMMU #define TCG_TARGET_NEED_LDST_LABELS #endif -#define TCG_TARGET_CON_SET_H #endif diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_CON_SET_H #endif diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_POOL_LABELS #define TCG_TARGET_HAS_MEMORY_BSWAP 0 -#define TCG_TARGET_CON_SET_H #endif diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390/tcg-target.h +++ b/tcg/s390/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, #define TCG_TARGET_NEED_LDST_LABELS #endif #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_CON_SET_H #endif diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.h +++ b/tcg/sparc/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions; void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_CON_SET_H #endif diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, /* no need to flush icache explicitly */ } -#define TCG_TARGET_CON_SET_H - #endif /* TCG_TARGET_H */ diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ /* Forward declarations for functions declared in tcg-target.c.inc and used here. */ static void tcg_target_init(TCGContext *s); -#ifndef TCG_TARGET_CON_SET_H -static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode); -#endif static void tcg_target_qemu_prologue(TCGContext *s); static bool patch_reloc(tcg_insn_unit *code_ptr, int type, intptr_t value, intptr_t addend); @@ -XXX,XX +XXX,XX @@ static void set_jmp_reset_offset(TCGContext *s, int which) s->tb_jmp_reset_offset[which] = tcg_current_code_size(s); } -#ifdef TCG_TARGET_CON_SET_H #define C_PFX1(P, A) P##A #define C_PFX2(P, A, B) P##A##_##B #define C_PFX3(P, A, B, C) P##A##_##B##_##C @@ -XXX,XX +XXX,XX @@ static const TCGTargetOpDef constraint_sets[] = { #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3) #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4) -#endif /* TCG_TARGET_CON_SET_H */ - #include "tcg-target.c.inc" /* compare a pointer @ptr and a tb_tc @s */ @@ -XXX,XX +XXX,XX @@ static void process_op_defs(TCGContext *s) continue; } -#ifdef TCG_TARGET_CON_SET_H /* * Macro magic should make it impossible, but double-check that * the array index is in range. Since the signness of an enum @@ -XXX,XX +XXX,XX @@ static void process_op_defs(TCGContext *s) unsigned con_set = tcg_target_op_def(op); tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets)); tdefs = &constraint_sets[con_set]; -#else - tdefs = tcg_target_op_def(op); - /* Missing TCGTargetOpDef entry. */ - tcg_debug_assert(tdefs != NULL); -#endif for (i = 0; i < nb_args; i++) { const char *ct_str = tdefs->args_ct_str[i]; -- 2.25.1