TCG patch queue, plus one target/sh4 patch that Yoshinori Sato asked me to process. r~ The following changes since commit efbf38d73e5dcc4d5f8b98c6e7a12be1f3b91745: Merge tag 'for-upstream' of git://repo.or.cz/qemu/kevin into staging (2022-10-03 15:06:07 -0400) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20221004 for you to fetch changes up to ab419fd8a035a65942de4e63effcd55ccbf1a9fe: target/sh4: Fix TB_FLAG_UNALIGN (2022-10-04 12:33:05 -0700) ---------------------------------------------------------------- Cache CPUClass for use in hot code paths. Add CPUTLBEntryFull, probe_access_full, tlb_set_page_full. Add generic support for TARGET_TB_PCREL. tcg/ppc: Optimize 26-bit jumps using STQ for POWER 2.07 target/sh4: Fix TB_FLAG_UNALIGN ---------------------------------------------------------------- Alex Bennée (3): cpu: cache CPUClass in CPUState for hot code paths hw/core/cpu-sysemu: used cached class in cpu_asidx_from_attrs cputlb: used cached CPUClass in our hot-paths Leandro Lupori (1): tcg/ppc: Optimize 26-bit jumps Richard Henderson (16): accel/tcg: Rename CPUIOTLBEntry to CPUTLBEntryFull accel/tcg: Drop addr member from SavedIOTLB accel/tcg: Suppress auto-invalidate in probe_access_internal accel/tcg: Introduce probe_access_full accel/tcg: Introduce tlb_set_page_full include/exec: Introduce TARGET_PAGE_ENTRY_EXTRA accel/tcg: Remove PageDesc code_bitmap accel/tcg: Use bool for page_find_alloc accel/tcg: Use DisasContextBase in plugin_gen_tb_start accel/tcg: Do not align tb->page_addr[0] accel/tcg: Inline tb_flush_jmp_cache include/hw/core: Create struct CPUJumpCache hw/core: Add CPUClass.get_pc accel/tcg: Introduce tb_pc and log_pc accel/tcg: Introduce TARGET_TB_PCREL target/sh4: Fix TB_FLAG_UNALIGN accel/tcg/internal.h | 10 ++ accel/tcg/tb-hash.h | 1 + accel/tcg/tb-jmp-cache.h | 65 ++++++++ include/exec/cpu-common.h | 1 + include/exec/cpu-defs.h | 48 ++++-- include/exec/exec-all.h | 75 ++++++++- include/exec/plugin-gen.h | 7 +- include/hw/core/cpu.h | 28 ++-- include/qemu/typedefs.h | 2 + include/tcg/tcg.h | 2 +- target/sh4/cpu.h | 56 ++++--- accel/stubs/tcg-stub.c | 4 + accel/tcg/cpu-exec.c | 80 +++++----- accel/tcg/cputlb.c | 259 ++++++++++++++++++-------------- accel/tcg/plugin-gen.c | 22 +-- accel/tcg/translate-all.c | 214 ++++++++++++-------------- accel/tcg/translator.c | 2 +- cpu.c | 9 +- hw/core/cpu-common.c | 3 +- hw/core/cpu-sysemu.c | 5 +- linux-user/sh4/signal.c | 6 +- plugins/core.c | 2 +- target/alpha/cpu.c | 9 ++ target/arm/cpu.c | 17 ++- target/arm/mte_helper.c | 14 +- target/arm/sve_helper.c | 4 +- target/arm/translate-a64.c | 2 +- target/avr/cpu.c | 10 +- target/cris/cpu.c | 8 + target/hexagon/cpu.c | 10 +- target/hppa/cpu.c | 12 +- target/i386/cpu.c | 9 ++ target/i386/tcg/tcg-cpu.c | 2 +- target/loongarch/cpu.c | 11 +- target/m68k/cpu.c | 8 + target/microblaze/cpu.c | 10 +- target/mips/cpu.c | 8 + target/mips/tcg/exception.c | 2 +- target/mips/tcg/sysemu/special_helper.c | 2 +- target/nios2/cpu.c | 9 ++ target/openrisc/cpu.c | 10 +- target/ppc/cpu_init.c | 8 + target/riscv/cpu.c | 17 ++- target/rx/cpu.c | 10 +- target/s390x/cpu.c | 8 + target/s390x/tcg/mem_helper.c | 4 - target/sh4/cpu.c | 18 ++- target/sh4/helper.c | 6 +- target/sh4/translate.c | 90 +++++------ target/sparc/cpu.c | 10 +- target/tricore/cpu.c | 11 +- target/xtensa/cpu.c | 8 + tcg/tcg.c | 8 +- trace/control-target.c | 2 +- tcg/ppc/tcg-target.c.inc | 119 +++++++++++---- 55 files changed, 915 insertions(+), 462 deletions(-) create mode 100644 accel/tcg/tb-jmp-cache.h From: Alex Bennée <alex.bennee@linaro.org> The class cast checkers are quite expensive and always on (unlike the dynamic case who's checks are gated by CONFIG_QOM_CAST_DEBUG). To avoid the overhead of repeatedly checking something which should never change we cache the CPUClass reference for use in the hot code paths. Signed-off-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20220811151413.3350684-3-alex.bennee@linaro.org> Signed-off-by: Cédric Le Goater <clg@kaod.org> Message-Id: <20220923084803.498337-3-clg@kaod.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/hw/core/cpu.h | 9 +++++++++ cpu.c | 9 ++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/core/cpu.h +++ b/include/hw/core/cpu.h @@ -XXX,XX +XXX,XX @@ typedef int (*WriteCoreDumpFunction)(const void *buf, size_t size, */ #define CPU(obj) ((CPUState *)(obj)) +/* + * The class checkers bring in CPU_GET_CLASS() which is potentially + * expensive given the eventual call to + * object_class_dynamic_cast_assert(). Because of this the CPUState + * has a cached value for the class in cs->cc which is set up in + * cpu_exec_realizefn() for use in hot code paths. + */ typedef struct CPUClass CPUClass; DECLARE_CLASS_CHECKERS(CPUClass, CPU, TYPE_CPU) @@ -XXX,XX +XXX,XX @@ struct qemu_work_item; struct CPUState { /*< private >*/ DeviceState parent_obj; + /* cache to avoid expensive CPU_GET_CLASS */ + CPUClass *cc; /*< public >*/ int nr_cores; diff --git a/cpu.c b/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/cpu.c +++ b/cpu.c @@ -XXX,XX +XXX,XX @@ const VMStateDescription vmstate_cpu_common = { void cpu_exec_realizefn(CPUState *cpu, Error **errp) { -#ifndef CONFIG_USER_ONLY - CPUClass *cc = CPU_GET_CLASS(cpu); -#endif + /* cache the cpu class for the hotpath */ + cpu->cc = CPU_GET_CLASS(cpu); cpu_list_add(cpu); if (!accel_cpu_realizefn(cpu, errp)) { @@ -XXX,XX +XXX,XX @@ void cpu_exec_realizefn(CPUState *cpu, Error **errp) if (qdev_get_vmsd(DEVICE(cpu)) == NULL) { vmstate_register(NULL, cpu->cpu_index, &vmstate_cpu_common, cpu); } - if (cc->sysemu_ops->legacy_vmsd != NULL) { - vmstate_register(NULL, cpu->cpu_index, cc->sysemu_ops->legacy_vmsd, cpu); + if (cpu->cc->sysemu_ops->legacy_vmsd != NULL) { + vmstate_register(NULL, cpu->cpu_index, cpu->cc->sysemu_ops->legacy_vmsd, cpu); } #endif /* CONFIG_USER_ONLY */ } -- 2.34.1 From: Alex Bennée <alex.bennee@linaro.org> This is a heavily used function so lets avoid the cost of CPU_GET_CLASS. On the romulus-bmc run it has a modest effect: Before: 36.812 s ± 0.506 s After: 35.912 s ± 0.168 s Signed-off-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20220811151413.3350684-4-alex.bennee@linaro.org> Signed-off-by: Cédric Le Goater <clg@kaod.org> Message-Id: <20220923084803.498337-4-clg@kaod.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- hw/core/cpu-sysemu.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hw/core/cpu-sysemu.c b/hw/core/cpu-sysemu.c index XXXXXXX..XXXXXXX 100644 --- a/hw/core/cpu-sysemu.c +++ b/hw/core/cpu-sysemu.c @@ -XXX,XX +XXX,XX @@ hwaddr cpu_get_phys_page_debug(CPUState *cpu, vaddr addr) int cpu_asidx_from_attrs(CPUState *cpu, MemTxAttrs attrs) { - CPUClass *cc = CPU_GET_CLASS(cpu); int ret = 0; - if (cc->sysemu_ops->asidx_from_attrs) { - ret = cc->sysemu_ops->asidx_from_attrs(cpu, attrs); + if (cpu->cc->sysemu_ops->asidx_from_attrs) { + ret = cpu->cc->sysemu_ops->asidx_from_attrs(cpu, attrs); assert(ret < cpu->num_ases && ret >= 0); } return ret; -- 2.34.1 From: Alex Bennée <alex.bennee@linaro.org> Before: 35.912 s ± 0.168 s After: 35.565 s ± 0.087 s Signed-off-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20220811151413.3350684-5-alex.bennee@linaro.org> Signed-off-by: Cédric Le Goater <clg@kaod.org> Message-Id: <20220923084803.498337-5-clg@kaod.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cputlb.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ void tlb_set_page(CPUState *cpu, target_ulong vaddr, static void tlb_fill(CPUState *cpu, target_ulong addr, int size, MMUAccessType access_type, int mmu_idx, uintptr_t retaddr) { - CPUClass *cc = CPU_GET_CLASS(cpu); bool ok; /* * This is not a probe, so only valid return is success; failure * should result in exception + longjmp to the cpu loop. */ - ok = cc->tcg_ops->tlb_fill(cpu, addr, size, - access_type, mmu_idx, false, retaddr); + ok = cpu->cc->tcg_ops->tlb_fill(cpu, addr, size, + access_type, mmu_idx, false, retaddr); assert(ok); } @@ -XXX,XX +XXX,XX @@ static inline void cpu_unaligned_access(CPUState *cpu, vaddr addr, MMUAccessType access_type, int mmu_idx, uintptr_t retaddr) { - CPUClass *cc = CPU_GET_CLASS(cpu); - - cc->tcg_ops->do_unaligned_access(cpu, addr, access_type, mmu_idx, retaddr); + cpu->cc->tcg_ops->do_unaligned_access(cpu, addr, access_type, + mmu_idx, retaddr); } static inline void cpu_transaction_failed(CPUState *cpu, hwaddr physaddr, @@ -XXX,XX +XXX,XX @@ static int probe_access_internal(CPUArchState *env, target_ulong addr, if (!tlb_hit_page(tlb_addr, page_addr)) { if (!victim_tlb_hit(env, mmu_idx, index, elt_ofs, page_addr)) { CPUState *cs = env_cpu(env); - CPUClass *cc = CPU_GET_CLASS(cs); - if (!cc->tcg_ops->tlb_fill(cs, addr, fault_size, access_type, - mmu_idx, nonfault, retaddr)) { + if (!cs->cc->tcg_ops->tlb_fill(cs, addr, fault_size, access_type, + mmu_idx, nonfault, retaddr)) { /* Non-faulting page table read failed. */ *phost = NULL; return TLB_INVALID_MASK; -- 2.34.1 This structure will shortly contain more than just data for accessing MMIO. Rename the 'addr' member to 'xlat_section' to more clearly indicate its purpose. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/cpu-defs.h | 22 ++++---- accel/tcg/cputlb.c | 102 +++++++++++++++++++------------------ target/arm/mte_helper.c | 14 ++--- target/arm/sve_helper.c | 4 +- target/arm/translate-a64.c | 2 +- 5 files changed, 73 insertions(+), 71 deletions(-) diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu-defs.h +++ b/include/exec/cpu-defs.h @@ -XXX,XX +XXX,XX @@ typedef uint64_t target_ulong; # endif # endif +/* Minimalized TLB entry for use by TCG fast path. */ typedef struct CPUTLBEntry { /* bit TARGET_LONG_BITS to TARGET_PAGE_BITS : virtual address bit TARGET_PAGE_BITS-1..4 : Nonzero for accesses that should not @@ -XXX,XX +XXX,XX @@ typedef struct CPUTLBEntry { QEMU_BUILD_BUG_ON(sizeof(CPUTLBEntry) != (1 << CPU_TLB_ENTRY_BITS)); -/* The IOTLB is not accessed directly inline by generated TCG code, - * so the CPUIOTLBEntry layout is not as critical as that of the - * CPUTLBEntry. (This is also why we don't want to combine the two - * structs into one.) +/* + * The full TLB entry, which is not accessed by generated TCG code, + * so the layout is not as critical as that of CPUTLBEntry. This is + * also why we don't want to combine the two structs. */ -typedef struct CPUIOTLBEntry { +typedef struct CPUTLBEntryFull { /* - * @addr contains: + * @xlat_section contains: * - in the lower TARGET_PAGE_BITS, a physical section number * - with the lower TARGET_PAGE_BITS masked off, an offset which * must be added to the virtual address to obtain: @@ -XXX,XX +XXX,XX @@ typedef struct CPUIOTLBEntry { * number is PHYS_SECTION_NOTDIRTY or PHYS_SECTION_ROM) * + the offset within the target MemoryRegion (otherwise) */ - hwaddr addr; + hwaddr xlat_section; MemTxAttrs attrs; -} CPUIOTLBEntry; +} CPUTLBEntryFull; /* * Data elements that are per MMU mode, minus the bits accessed by @@ -XXX,XX +XXX,XX @@ typedef struct CPUTLBDesc { size_t vindex; /* The tlb victim table, in two parts. */ CPUTLBEntry vtable[CPU_VTLB_SIZE]; - CPUIOTLBEntry viotlb[CPU_VTLB_SIZE]; - /* The iotlb. */ - CPUIOTLBEntry *iotlb; + CPUTLBEntryFull vfulltlb[CPU_VTLB_SIZE]; + CPUTLBEntryFull *fulltlb; } CPUTLBDesc; /* diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ static void tlb_mmu_resize_locked(CPUTLBDesc *desc, CPUTLBDescFast *fast, } g_free(fast->table); - g_free(desc->iotlb); + g_free(desc->fulltlb); tlb_window_reset(desc, now, 0); /* desc->n_used_entries is cleared by the caller */ fast->mask = (new_size - 1) << CPU_TLB_ENTRY_BITS; fast->table = g_try_new(CPUTLBEntry, new_size); - desc->iotlb = g_try_new(CPUIOTLBEntry, new_size); + desc->fulltlb = g_try_new(CPUTLBEntryFull, new_size); /* * If the allocations fail, try smaller sizes. We just freed some @@ -XXX,XX +XXX,XX @@ static void tlb_mmu_resize_locked(CPUTLBDesc *desc, CPUTLBDescFast *fast, * allocations to fail though, so we progressively reduce the allocation * size, aborting if we cannot even allocate the smallest TLB we support. */ - while (fast->table == NULL || desc->iotlb == NULL) { + while (fast->table == NULL || desc->fulltlb == NULL) { if (new_size == (1 << CPU_TLB_DYN_MIN_BITS)) { error_report("%s: %s", __func__, strerror(errno)); abort(); @@ -XXX,XX +XXX,XX @@ static void tlb_mmu_resize_locked(CPUTLBDesc *desc, CPUTLBDescFast *fast, fast->mask = (new_size - 1) << CPU_TLB_ENTRY_BITS; g_free(fast->table); - g_free(desc->iotlb); + g_free(desc->fulltlb); fast->table = g_try_new(CPUTLBEntry, new_size); - desc->iotlb = g_try_new(CPUIOTLBEntry, new_size); + desc->fulltlb = g_try_new(CPUTLBEntryFull, new_size); } } @@ -XXX,XX +XXX,XX @@ static void tlb_mmu_init(CPUTLBDesc *desc, CPUTLBDescFast *fast, int64_t now) desc->n_used_entries = 0; fast->mask = (n_entries - 1) << CPU_TLB_ENTRY_BITS; fast->table = g_new(CPUTLBEntry, n_entries); - desc->iotlb = g_new(CPUIOTLBEntry, n_entries); + desc->fulltlb = g_new(CPUTLBEntryFull, n_entries); tlb_mmu_flush_locked(desc, fast); } @@ -XXX,XX +XXX,XX @@ void tlb_destroy(CPUState *cpu) CPUTLBDescFast *fast = &env_tlb(env)->f[i]; g_free(fast->table); - g_free(desc->iotlb); + g_free(desc->fulltlb); } } @@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, /* Evict the old entry into the victim tlb. */ copy_tlb_helper_locked(tv, te); - desc->viotlb[vidx] = desc->iotlb[index]; + desc->vfulltlb[vidx] = desc->fulltlb[index]; tlb_n_used_entries_dec(env, mmu_idx); } @@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, * subtract here is that of the page base, and not the same as the * vaddr we add back in io_readx()/io_writex()/get_page_addr_code(). */ - desc->iotlb[index].addr = iotlb - vaddr_page; - desc->iotlb[index].attrs = attrs; + desc->fulltlb[index].xlat_section = iotlb - vaddr_page; + desc->fulltlb[index].attrs = attrs; /* Now calculate the new entry */ tn.addend = addend - vaddr_page; @@ -XXX,XX +XXX,XX @@ static inline void cpu_transaction_failed(CPUState *cpu, hwaddr physaddr, } } -static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry, +static uint64_t io_readx(CPUArchState *env, CPUTLBEntryFull *full, int mmu_idx, target_ulong addr, uintptr_t retaddr, MMUAccessType access_type, MemOp op) { @@ -XXX,XX +XXX,XX @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry, bool locked = false; MemTxResult r; - section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs); + section = iotlb_to_section(cpu, full->xlat_section, full->attrs); mr = section->mr; - mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr; + mr_offset = (full->xlat_section & TARGET_PAGE_MASK) + addr; cpu->mem_io_pc = retaddr; if (!cpu->can_do_io) { cpu_io_recompile(cpu, retaddr); @@ -XXX,XX +XXX,XX @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry, qemu_mutex_lock_iothread(); locked = true; } - r = memory_region_dispatch_read(mr, mr_offset, &val, op, iotlbentry->attrs); + r = memory_region_dispatch_read(mr, mr_offset, &val, op, full->attrs); if (r != MEMTX_OK) { hwaddr physaddr = mr_offset + section->offset_within_address_space - section->offset_within_region; cpu_transaction_failed(cpu, physaddr, addr, memop_size(op), access_type, - mmu_idx, iotlbentry->attrs, r, retaddr); + mmu_idx, full->attrs, r, retaddr); } if (locked) { qemu_mutex_unlock_iothread(); @@ -XXX,XX +XXX,XX @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry, } /* - * Save a potentially trashed IOTLB entry for later lookup by plugin. - * This is read by tlb_plugin_lookup if the iotlb entry doesn't match + * Save a potentially trashed CPUTLBEntryFull for later lookup by plugin. + * This is read by tlb_plugin_lookup if the fulltlb entry doesn't match * because of the side effect of io_writex changing memory layout. */ static void save_iotlb_data(CPUState *cs, hwaddr addr, @@ -XXX,XX +XXX,XX @@ static void save_iotlb_data(CPUState *cs, hwaddr addr, #endif } -static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry, +static void io_writex(CPUArchState *env, CPUTLBEntryFull *full, int mmu_idx, uint64_t val, target_ulong addr, uintptr_t retaddr, MemOp op) { @@ -XXX,XX +XXX,XX @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry, bool locked = false; MemTxResult r; - section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs); + section = iotlb_to_section(cpu, full->xlat_section, full->attrs); mr = section->mr; - mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr; + mr_offset = (full->xlat_section & TARGET_PAGE_MASK) + addr; if (!cpu->can_do_io) { cpu_io_recompile(cpu, retaddr); } @@ -XXX,XX +XXX,XX @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry, * The memory_region_dispatch may trigger a flush/resize * so for plugins we save the iotlb_data just in case. */ - save_iotlb_data(cpu, iotlbentry->addr, section, mr_offset); + save_iotlb_data(cpu, full->xlat_section, section, mr_offset); if (!qemu_mutex_iothread_locked()) { qemu_mutex_lock_iothread(); locked = true; } - r = memory_region_dispatch_write(mr, mr_offset, val, op, iotlbentry->attrs); + r = memory_region_dispatch_write(mr, mr_offset, val, op, full->attrs); if (r != MEMTX_OK) { hwaddr physaddr = mr_offset + section->offset_within_address_space - section->offset_within_region; cpu_transaction_failed(cpu, physaddr, addr, memop_size(op), - MMU_DATA_STORE, mmu_idx, iotlbentry->attrs, r, + MMU_DATA_STORE, mmu_idx, full->attrs, r, retaddr); } if (locked) { @@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index, copy_tlb_helper_locked(vtlb, &tmptlb); qemu_spin_unlock(&env_tlb(env)->c.lock); - CPUIOTLBEntry tmpio, *io = &env_tlb(env)->d[mmu_idx].iotlb[index]; - CPUIOTLBEntry *vio = &env_tlb(env)->d[mmu_idx].viotlb[vidx]; - tmpio = *io; *io = *vio; *vio = tmpio; + CPUTLBEntryFull *f1 = &env_tlb(env)->d[mmu_idx].fulltlb[index]; + CPUTLBEntryFull *f2 = &env_tlb(env)->d[mmu_idx].vfulltlb[vidx]; + CPUTLBEntryFull tmpf; + tmpf = *f1; *f1 = *f2; *f2 = tmpf; return true; } } @@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index, (ADDR) & TARGET_PAGE_MASK) static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size, - CPUIOTLBEntry *iotlbentry, uintptr_t retaddr) + CPUTLBEntryFull *full, uintptr_t retaddr) { - ram_addr_t ram_addr = mem_vaddr + iotlbentry->addr; + ram_addr_t ram_addr = mem_vaddr + full->xlat_section; trace_memory_notdirty_write_access(mem_vaddr, ram_addr, size); @@ -XXX,XX +XXX,XX @@ int probe_access_flags(CPUArchState *env, target_ulong addr, /* Handle clean RAM pages. */ if (unlikely(flags & TLB_NOTDIRTY)) { uintptr_t index = tlb_index(env, mmu_idx, addr); - CPUIOTLBEntry *iotlbentry = &env_tlb(env)->d[mmu_idx].iotlb[index]; + CPUTLBEntryFull *full = &env_tlb(env)->d[mmu_idx].fulltlb[index]; - notdirty_write(env_cpu(env), addr, 1, iotlbentry, retaddr); + notdirty_write(env_cpu(env), addr, 1, full, retaddr); flags &= ~TLB_NOTDIRTY; } @@ -XXX,XX +XXX,XX @@ void *probe_access(CPUArchState *env, target_ulong addr, int size, if (unlikely(flags & (TLB_NOTDIRTY | TLB_WATCHPOINT))) { uintptr_t index = tlb_index(env, mmu_idx, addr); - CPUIOTLBEntry *iotlbentry = &env_tlb(env)->d[mmu_idx].iotlb[index]; + CPUTLBEntryFull *full = &env_tlb(env)->d[mmu_idx].fulltlb[index]; /* Handle watchpoints. */ if (flags & TLB_WATCHPOINT) { int wp_access = (access_type == MMU_DATA_STORE ? BP_MEM_WRITE : BP_MEM_READ); cpu_check_watchpoint(env_cpu(env), addr, size, - iotlbentry->attrs, wp_access, retaddr); + full->attrs, wp_access, retaddr); } /* Handle clean RAM pages. */ if (flags & TLB_NOTDIRTY) { - notdirty_write(env_cpu(env), addr, 1, iotlbentry, retaddr); + notdirty_write(env_cpu(env), addr, 1, full, retaddr); } } @@ -XXX,XX +XXX,XX @@ tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, * should have just filled the TLB. The one corner case is io_writex * which can cause TLB flushes and potential resizing of the TLBs * losing the information we need. In those cases we need to recover - * data from a copy of the iotlbentry. As long as this always occurs + * data from a copy of the CPUTLBEntryFull. As long as this always occurs * from the same thread (which a mem callback will be) this is safe. */ @@ -XXX,XX +XXX,XX @@ bool tlb_plugin_lookup(CPUState *cpu, target_ulong addr, int mmu_idx, if (likely(tlb_hit(tlb_addr, addr))) { /* We must have an iotlb entry for MMIO */ if (tlb_addr & TLB_MMIO) { - CPUIOTLBEntry *iotlbentry; - iotlbentry = &env_tlb(env)->d[mmu_idx].iotlb[index]; + CPUTLBEntryFull *full; + full = &env_tlb(env)->d[mmu_idx].fulltlb[index]; data->is_io = true; - data->v.io.section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs); - data->v.io.offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr; + data->v.io.section = + iotlb_to_section(cpu, full->xlat_section, full->attrs); + data->v.io.offset = (full->xlat_section & TARGET_PAGE_MASK) + addr; } else { data->is_io = false; data->v.ram.hostaddr = (void *)((uintptr_t)addr + tlbe->addend); @@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, if (unlikely(tlb_addr & TLB_NOTDIRTY)) { notdirty_write(env_cpu(env), addr, size, - &env_tlb(env)->d[mmu_idx].iotlb[index], retaddr); + &env_tlb(env)->d[mmu_idx].fulltlb[index], retaddr); } return hostaddr; @@ -XXX,XX +XXX,XX @@ load_helper(CPUArchState *env, target_ulong addr, MemOpIdx oi, /* Handle anything that isn't just a straight memory access. */ if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) { - CPUIOTLBEntry *iotlbentry; + CPUTLBEntryFull *full; bool need_swap; /* For anything that is unaligned, recurse through full_load. */ @@ -XXX,XX +XXX,XX @@ load_helper(CPUArchState *env, target_ulong addr, MemOpIdx oi, goto do_unaligned_access; } - iotlbentry = &env_tlb(env)->d[mmu_idx].iotlb[index]; + full = &env_tlb(env)->d[mmu_idx].fulltlb[index]; /* Handle watchpoints. */ if (unlikely(tlb_addr & TLB_WATCHPOINT)) { /* On watchpoint hit, this will longjmp out. */ cpu_check_watchpoint(env_cpu(env), addr, size, - iotlbentry->attrs, BP_MEM_READ, retaddr); + full->attrs, BP_MEM_READ, retaddr); } need_swap = size > 1 && (tlb_addr & TLB_BSWAP); /* Handle I/O access. */ if (likely(tlb_addr & TLB_MMIO)) { - return io_readx(env, iotlbentry, mmu_idx, addr, retaddr, + return io_readx(env, full, mmu_idx, addr, retaddr, access_type, op ^ (need_swap * MO_BSWAP)); } @@ -XXX,XX +XXX,XX @@ store_helper_unaligned(CPUArchState *env, target_ulong addr, uint64_t val, */ if (unlikely(tlb_addr & TLB_WATCHPOINT)) { cpu_check_watchpoint(env_cpu(env), addr, size - size2, - env_tlb(env)->d[mmu_idx].iotlb[index].attrs, + env_tlb(env)->d[mmu_idx].fulltlb[index].attrs, BP_MEM_WRITE, retaddr); } if (unlikely(tlb_addr2 & TLB_WATCHPOINT)) { cpu_check_watchpoint(env_cpu(env), page2, size2, - env_tlb(env)->d[mmu_idx].iotlb[index2].attrs, + env_tlb(env)->d[mmu_idx].fulltlb[index2].attrs, BP_MEM_WRITE, retaddr); } @@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val, /* Handle anything that isn't just a straight memory access. */ if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) { - CPUIOTLBEntry *iotlbentry; + CPUTLBEntryFull *full; bool need_swap; /* For anything that is unaligned, recurse through byte stores. */ @@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val, goto do_unaligned_access; } - iotlbentry = &env_tlb(env)->d[mmu_idx].iotlb[index]; + full = &env_tlb(env)->d[mmu_idx].fulltlb[index]; /* Handle watchpoints. */ if (unlikely(tlb_addr & TLB_WATCHPOINT)) { /* On watchpoint hit, this will longjmp out. */ cpu_check_watchpoint(env_cpu(env), addr, size, - iotlbentry->attrs, BP_MEM_WRITE, retaddr); + full->attrs, BP_MEM_WRITE, retaddr); } need_swap = size > 1 && (tlb_addr & TLB_BSWAP); /* Handle I/O access. */ if (tlb_addr & TLB_MMIO) { - io_writex(env, iotlbentry, mmu_idx, val, addr, retaddr, + io_writex(env, full, mmu_idx, val, addr, retaddr, op ^ (need_swap * MO_BSWAP)); return; } @@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val, /* Handle clean RAM pages. */ if (tlb_addr & TLB_NOTDIRTY) { - notdirty_write(env_cpu(env), addr, size, iotlbentry, retaddr); + notdirty_write(env_cpu(env), addr, size, full, retaddr); } haddr = (void *)((uintptr_t)addr + entry->addend); diff --git a/target/arm/mte_helper.c b/target/arm/mte_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/mte_helper.c +++ b/target/arm/mte_helper.c @@ -XXX,XX +XXX,XX @@ static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx, return tags + index; #else uintptr_t index; - CPUIOTLBEntry *iotlbentry; + CPUTLBEntryFull *full; int in_page, flags; ram_addr_t ptr_ra; hwaddr ptr_paddr, tag_paddr, xlat; @@ -XXX,XX +XXX,XX @@ static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx, assert(!(flags & TLB_INVALID_MASK)); /* - * Find the iotlbentry for ptr. This *must* be present in the TLB + * Find the CPUTLBEntryFull for ptr. This *must* be present in the TLB * because we just found the mapping. * TODO: Perhaps there should be a cputlb helper that returns a * matching tlb entry + iotlb entry. @@ -XXX,XX +XXX,XX @@ static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx, g_assert(tlb_hit(comparator, ptr)); } # endif - iotlbentry = &env_tlb(env)->d[ptr_mmu_idx].iotlb[index]; + full = &env_tlb(env)->d[ptr_mmu_idx].fulltlb[index]; /* If the virtual page MemAttr != Tagged, access unchecked. */ - if (!arm_tlb_mte_tagged(&iotlbentry->attrs)) { + if (!arm_tlb_mte_tagged(&full->attrs)) { return NULL; } @@ -XXX,XX +XXX,XX @@ static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx, int wp = ptr_access == MMU_DATA_LOAD ? BP_MEM_READ : BP_MEM_WRITE; assert(ra != 0); cpu_check_watchpoint(env_cpu(env), ptr, ptr_size, - iotlbentry->attrs, wp, ra); + full->attrs, wp, ra); } /* @@ -XXX,XX +XXX,XX @@ static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx, tag_paddr = ptr_paddr >> (LOG2_TAG_GRANULE + 1); /* Look up the address in tag space. */ - tag_asi = iotlbentry->attrs.secure ? ARMASIdx_TagS : ARMASIdx_TagNS; + tag_asi = full->attrs.secure ? ARMASIdx_TagS : ARMASIdx_TagNS; tag_as = cpu_get_address_space(env_cpu(env), tag_asi); mr = address_space_translate(tag_as, tag_paddr, &xlat, NULL, tag_access == MMU_DATA_STORE, - iotlbentry->attrs); + full->attrs); /* * Note that @mr will never be NULL. If there is nothing in the address diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ bool sve_probe_page(SVEHostPage *info, bool nofault, CPUARMState *env, g_assert(tlb_hit(comparator, addr)); # endif - CPUIOTLBEntry *iotlbentry = &env_tlb(env)->d[mmu_idx].iotlb[index]; - info->attrs = iotlbentry->attrs; + CPUTLBEntryFull *full = &env_tlb(env)->d[mmu_idx].fulltlb[index]; + info->attrs = full->attrs; } #endif diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static bool is_guarded_page(CPUARMState *env, DisasContext *s) * table entry even for that case. */ return (tlb_hit(entry->addr_code, addr) && - arm_tlb_bti_gp(&env_tlb(env)->d[mmu_idx].iotlb[index].attrs)); + arm_tlb_bti_gp(&env_tlb(env)->d[mmu_idx].fulltlb[index].attrs)); #endif } -- 2.34.1 This field is only written, not read; remove it. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/hw/core/cpu.h | 1 - accel/tcg/cputlb.c | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/core/cpu.h +++ b/include/hw/core/cpu.h @@ -XXX,XX +XXX,XX @@ struct CPUWatchpoint { * the memory regions get moved around by io_writex. */ typedef struct SavedIOTLB { - hwaddr addr; MemoryRegionSection *section; hwaddr mr_offset; } SavedIOTLB; diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ static uint64_t io_readx(CPUArchState *env, CPUTLBEntryFull *full, * This is read by tlb_plugin_lookup if the fulltlb entry doesn't match * because of the side effect of io_writex changing memory layout. */ -static void save_iotlb_data(CPUState *cs, hwaddr addr, - MemoryRegionSection *section, hwaddr mr_offset) +static void save_iotlb_data(CPUState *cs, MemoryRegionSection *section, + hwaddr mr_offset) { #ifdef CONFIG_PLUGIN SavedIOTLB *saved = &cs->saved_iotlb; - saved->addr = addr; saved->section = section; saved->mr_offset = mr_offset; #endif @@ -XXX,XX +XXX,XX @@ static void io_writex(CPUArchState *env, CPUTLBEntryFull *full, * The memory_region_dispatch may trigger a flush/resize * so for plugins we save the iotlb_data just in case. */ - save_iotlb_data(cpu, full->xlat_section, section, mr_offset); + save_iotlb_data(cpu, section, mr_offset); if (!qemu_mutex_iothread_locked()) { qemu_mutex_lock_iothread(); -- 2.34.1 When PAGE_WRITE_INV is set when calling tlb_set_page, we immediately set TLB_INVALID_MASK in order to force tlb_fill to be called on the next lookup. Here in probe_access_internal, we have just called tlb_fill and eliminated true misses, thus the lookup must be valid. This allows us to remove a warning comment from s390x. There doesn't seem to be a reason to change the code though. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: David Hildenbrand <david@redhat.com> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cputlb.c | 10 +++++++++- target/s390x/tcg/mem_helper.c | 4 ---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ static int probe_access_internal(CPUArchState *env, target_ulong addr, } tlb_addr = tlb_read_ofs(entry, elt_ofs); + flags = TLB_FLAGS_MASK; page_addr = addr & TARGET_PAGE_MASK; if (!tlb_hit_page(tlb_addr, page_addr)) { if (!victim_tlb_hit(env, mmu_idx, index, elt_ofs, page_addr)) { @@ -XXX,XX +XXX,XX @@ static int probe_access_internal(CPUArchState *env, target_ulong addr, /* TLB resize via tlb_fill may have moved the entry. */ entry = tlb_entry(env, mmu_idx, addr); + + /* + * With PAGE_WRITE_INV, we set TLB_INVALID_MASK immediately, + * to force the next access through tlb_fill. We've just + * called tlb_fill, so we know that this entry *is* valid. + */ + flags &= ~TLB_INVALID_MASK; } tlb_addr = tlb_read_ofs(entry, elt_ofs); } - flags = tlb_addr & TLB_FLAGS_MASK; + flags &= tlb_addr; /* Fold all "mmio-like" bits into TLB_MMIO. This is not RAM. */ if (unlikely(flags & ~(TLB_WATCHPOINT | TLB_NOTDIRTY))) { diff --git a/target/s390x/tcg/mem_helper.c b/target/s390x/tcg/mem_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/tcg/mem_helper.c +++ b/target/s390x/tcg/mem_helper.c @@ -XXX,XX +XXX,XX @@ static int s390_probe_access(CPUArchState *env, target_ulong addr, int size, #else int flags; - /* - * For !CONFIG_USER_ONLY, we cannot rely on TLB_INVALID_MASK or haddr==NULL - * to detect if there was an exception during tlb_fill(). - */ env->tlb_fill_exc = 0; flags = probe_access_flags(env, addr, access_type, mmu_idx, nonfault, phost, ra); -- 2.34.1 Add an interface to return the CPUTLBEntryFull struct that goes with the lookup. The result is not intended to be valid across multiple lookups, so the user must use the results immediately. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/exec-all.h | 15 +++++++++++++ include/qemu/typedefs.h | 1 + accel/tcg/cputlb.c | 47 +++++++++++++++++++++++++---------------- 3 files changed, 45 insertions(+), 18 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ int probe_access_flags(CPUArchState *env, target_ulong addr, MMUAccessType access_type, int mmu_idx, bool nonfault, void **phost, uintptr_t retaddr); +#ifndef CONFIG_USER_ONLY +/** + * probe_access_full: + * Like probe_access_flags, except also return into @pfull. + * + * The CPUTLBEntryFull structure returned via @pfull is transient + * and must be consumed or copied immediately, before any further + * access or changes to TLB @mmu_idx. + */ +int probe_access_full(CPUArchState *env, target_ulong addr, + MMUAccessType access_type, int mmu_idx, + bool nonfault, void **phost, + CPUTLBEntryFull **pfull, uintptr_t retaddr); +#endif + #define CODE_GEN_ALIGN 16 /* must be >= of the size of a icache line */ /* Estimated block size for TB allocation. */ diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/typedefs.h +++ b/include/qemu/typedefs.h @@ -XXX,XX +XXX,XX @@ typedef struct ConfidentialGuestSupport ConfidentialGuestSupport; typedef struct CPUAddressSpace CPUAddressSpace; typedef struct CPUArchState CPUArchState; typedef struct CPUState CPUState; +typedef struct CPUTLBEntryFull CPUTLBEntryFull; typedef struct DeviceListener DeviceListener; typedef struct DeviceState DeviceState; typedef struct DirtyBitmapSnapshot DirtyBitmapSnapshot; diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size, static int probe_access_internal(CPUArchState *env, target_ulong addr, int fault_size, MMUAccessType access_type, int mmu_idx, bool nonfault, - void **phost, uintptr_t retaddr) + void **phost, CPUTLBEntryFull **pfull, + uintptr_t retaddr) { uintptr_t index = tlb_index(env, mmu_idx, addr); CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); @@ -XXX,XX +XXX,XX @@ static int probe_access_internal(CPUArchState *env, target_ulong addr, mmu_idx, nonfault, retaddr)) { /* Non-faulting page table read failed. */ *phost = NULL; + *pfull = NULL; return TLB_INVALID_MASK; } /* TLB resize via tlb_fill may have moved the entry. */ + index = tlb_index(env, mmu_idx, addr); entry = tlb_entry(env, mmu_idx, addr); /* @@ -XXX,XX +XXX,XX @@ static int probe_access_internal(CPUArchState *env, target_ulong addr, } flags &= tlb_addr; + *pfull = &env_tlb(env)->d[mmu_idx].fulltlb[index]; + /* Fold all "mmio-like" bits into TLB_MMIO. This is not RAM. */ if (unlikely(flags & ~(TLB_WATCHPOINT | TLB_NOTDIRTY))) { *phost = NULL; @@ -XXX,XX +XXX,XX @@ static int probe_access_internal(CPUArchState *env, target_ulong addr, return flags; } -int probe_access_flags(CPUArchState *env, target_ulong addr, - MMUAccessType access_type, int mmu_idx, - bool nonfault, void **phost, uintptr_t retaddr) +int probe_access_full(CPUArchState *env, target_ulong addr, + MMUAccessType access_type, int mmu_idx, + bool nonfault, void **phost, CPUTLBEntryFull **pfull, + uintptr_t retaddr) { - int flags; - - flags = probe_access_internal(env, addr, 0, access_type, mmu_idx, - nonfault, phost, retaddr); + int flags = probe_access_internal(env, addr, 0, access_type, mmu_idx, + nonfault, phost, pfull, retaddr); /* Handle clean RAM pages. */ if (unlikely(flags & TLB_NOTDIRTY)) { - uintptr_t index = tlb_index(env, mmu_idx, addr); - CPUTLBEntryFull *full = &env_tlb(env)->d[mmu_idx].fulltlb[index]; - - notdirty_write(env_cpu(env), addr, 1, full, retaddr); + notdirty_write(env_cpu(env), addr, 1, *pfull, retaddr); flags &= ~TLB_NOTDIRTY; } return flags; } +int probe_access_flags(CPUArchState *env, target_ulong addr, + MMUAccessType access_type, int mmu_idx, + bool nonfault, void **phost, uintptr_t retaddr) +{ + CPUTLBEntryFull *full; + + return probe_access_full(env, addr, access_type, mmu_idx, + nonfault, phost, &full, retaddr); +} + void *probe_access(CPUArchState *env, target_ulong addr, int size, MMUAccessType access_type, int mmu_idx, uintptr_t retaddr) { + CPUTLBEntryFull *full; void *host; int flags; g_assert(-(addr | TARGET_PAGE_MASK) >= size); flags = probe_access_internal(env, addr, size, access_type, mmu_idx, - false, &host, retaddr); + false, &host, &full, retaddr); /* Per the interface, size == 0 merely faults the access. */ if (size == 0) { @@ -XXX,XX +XXX,XX @@ void *probe_access(CPUArchState *env, target_ulong addr, int size, } if (unlikely(flags & (TLB_NOTDIRTY | TLB_WATCHPOINT))) { - uintptr_t index = tlb_index(env, mmu_idx, addr); - CPUTLBEntryFull *full = &env_tlb(env)->d[mmu_idx].fulltlb[index]; - /* Handle watchpoints. */ if (flags & TLB_WATCHPOINT) { int wp_access = (access_type == MMU_DATA_STORE @@ -XXX,XX +XXX,XX @@ void *probe_access(CPUArchState *env, target_ulong addr, int size, void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr, MMUAccessType access_type, int mmu_idx) { + CPUTLBEntryFull *full; void *host; int flags; flags = probe_access_internal(env, addr, 0, access_type, - mmu_idx, true, &host, 0); + mmu_idx, true, &host, &full, 0); /* No combination of flags are expected by the caller. */ return flags ? NULL : host; @@ -XXX,XX +XXX,XX @@ void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr, tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, void **hostp) { + CPUTLBEntryFull *full; void *p; (void)probe_access_internal(env, addr, 1, MMU_INST_FETCH, - cpu_mmu_index(env, true), false, &p, 0); + cpu_mmu_index(env, true), false, &p, &full, 0); if (p == NULL) { return -1; } -- 2.34.1 Now that we have collected all of the page data into CPUTLBEntryFull, provide an interface to record that all in one go, instead of using 4 arguments. This interface allows CPUTLBEntryFull to be extended without having to change the number of arguments. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/cpu-defs.h | 14 +++++++++++ include/exec/exec-all.h | 22 ++++++++++++++++++ accel/tcg/cputlb.c | 51 ++++++++++++++++++++++++++--------------- 3 files changed, 69 insertions(+), 18 deletions(-) diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu-defs.h +++ b/include/exec/cpu-defs.h @@ -XXX,XX +XXX,XX @@ typedef struct CPUTLBEntryFull { * + the offset within the target MemoryRegion (otherwise) */ hwaddr xlat_section; + + /* + * @phys_addr contains the physical address in the address space + * given by cpu_asidx_from_attrs(cpu, @attrs). + */ + hwaddr phys_addr; + + /* @attrs contains the memory transaction attributes for the page. */ MemTxAttrs attrs; + + /* @prot contains the complete protections for the page. */ + uint8_t prot; + + /* @lg_page_size contains the log2 of the page size. */ + uint8_t lg_page_size; } CPUTLBEntryFull; /* diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *cpu, uint16_t idxmap, unsigned bits); +/** + * tlb_set_page_full: + * @cpu: CPU context + * @mmu_idx: mmu index of the tlb to modify + * @vaddr: virtual address of the entry to add + * @full: the details of the tlb entry + * + * Add an entry to @cpu tlb index @mmu_idx. All of the fields of + * @full must be filled, except for xlat_section, and constitute + * the complete description of the translated page. + * + * This is generally called by the target tlb_fill function after + * having performed a successful page table walk to find the physical + * address and attributes for the translation. + * + * At most one entry for a given virtual address is permitted. Only a + * single TARGET_PAGE_SIZE region is mapped; @full->lg_page_size is only + * used by tlb_flush_page. + */ +void tlb_set_page_full(CPUState *cpu, int mmu_idx, target_ulong vaddr, + CPUTLBEntryFull *full); + /** * tlb_set_page_with_attrs: * @cpu: CPU to add this TLB entry for diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ static void tlb_add_large_page(CPUArchState *env, int mmu_idx, env_tlb(env)->d[mmu_idx].large_page_mask = lp_mask; } -/* Add a new TLB entry. At most one entry for a given virtual address +/* + * Add a new TLB entry. At most one entry for a given virtual address * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the * supplied size is only used by tlb_flush_page. * * Called from TCG-generated code, which is under an RCU read-side * critical section. */ -void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, - hwaddr paddr, MemTxAttrs attrs, int prot, - int mmu_idx, target_ulong size) +void tlb_set_page_full(CPUState *cpu, int mmu_idx, + target_ulong vaddr, CPUTLBEntryFull *full) { CPUArchState *env = cpu->env_ptr; CPUTLB *tlb = env_tlb(env); @@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, CPUTLBEntry *te, tn; hwaddr iotlb, xlat, sz, paddr_page; target_ulong vaddr_page; - int asidx = cpu_asidx_from_attrs(cpu, attrs); - int wp_flags; + int asidx, wp_flags, prot; bool is_ram, is_romd; assert_cpu_is_self(cpu); - if (size <= TARGET_PAGE_SIZE) { + if (full->lg_page_size <= TARGET_PAGE_BITS) { sz = TARGET_PAGE_SIZE; } else { - tlb_add_large_page(env, mmu_idx, vaddr, size); - sz = size; + sz = (hwaddr)1 << full->lg_page_size; + tlb_add_large_page(env, mmu_idx, vaddr, sz); } vaddr_page = vaddr & TARGET_PAGE_MASK; - paddr_page = paddr & TARGET_PAGE_MASK; + paddr_page = full->phys_addr & TARGET_PAGE_MASK; + prot = full->prot; + asidx = cpu_asidx_from_attrs(cpu, full->attrs); section = address_space_translate_for_iotlb(cpu, asidx, paddr_page, - &xlat, &sz, attrs, &prot); + &xlat, &sz, full->attrs, &prot); assert(sz >= TARGET_PAGE_SIZE); tlb_debug("vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx " prot=%x idx=%d\n", - vaddr, paddr, prot, mmu_idx); + vaddr, full->phys_addr, prot, mmu_idx); address = vaddr_page; - if (size < TARGET_PAGE_SIZE) { + if (full->lg_page_size < TARGET_PAGE_BITS) { /* Repeat the MMU check and TLB fill on every access. */ address |= TLB_INVALID_MASK; } - if (attrs.byte_swap) { + if (full->attrs.byte_swap) { address |= TLB_BSWAP; } @@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, * subtract here is that of the page base, and not the same as the * vaddr we add back in io_readx()/io_writex()/get_page_addr_code(). */ + desc->fulltlb[index] = *full; desc->fulltlb[index].xlat_section = iotlb - vaddr_page; - desc->fulltlb[index].attrs = attrs; + desc->fulltlb[index].phys_addr = paddr_page; + desc->fulltlb[index].prot = prot; /* Now calculate the new entry */ tn.addend = addend - vaddr_page; @@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, qemu_spin_unlock(&tlb->c.lock); } -/* Add a new TLB entry, but without specifying the memory - * transaction attributes to be used. - */ +void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, + hwaddr paddr, MemTxAttrs attrs, int prot, + int mmu_idx, target_ulong size) +{ + CPUTLBEntryFull full = { + .phys_addr = paddr, + .attrs = attrs, + .prot = prot, + .lg_page_size = ctz64(size) + }; + + assert(is_power_of_2(size)); + tlb_set_page_full(cpu, mmu_idx, vaddr, &full); +} + void tlb_set_page(CPUState *cpu, target_ulong vaddr, hwaddr paddr, int prot, int mmu_idx, target_ulong size) -- 2.34.1 Allow the target to cache items from the guest page tables. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/cpu-defs.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu-defs.h +++ b/include/exec/cpu-defs.h @@ -XXX,XX +XXX,XX @@ typedef struct CPUTLBEntryFull { /* @lg_page_size contains the log2 of the page size. */ uint8_t lg_page_size; + + /* + * Allow target-specific additions to this structure. + * This may be used to cache items from the guest cpu + * page tables for later use by the implementation. + */ +#ifdef TARGET_PAGE_ENTRY_EXTRA + TARGET_PAGE_ENTRY_EXTRA +#endif } CPUTLBEntryFull; /* -- 2.34.1 This bitmap is created and discarded immediately. We gain nothing by its existence. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20220822232338.1727934-2-richard.henderson@linaro.org> --- accel/tcg/translate-all.c | 78 ++------------------------------------- 1 file changed, 4 insertions(+), 74 deletions(-) diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ #define assert_memory_lock() tcg_debug_assert(have_mmap_lock()) #endif -#define SMC_BITMAP_USE_THRESHOLD 10 - typedef struct PageDesc { /* list of TBs intersecting this ram page */ uintptr_t first_tb; -#ifdef CONFIG_SOFTMMU - /* in order to optimize self modifying code, we count the number - of lookups we do to a given page to use a bitmap */ - unsigned long *code_bitmap; - unsigned int code_write_count; -#else +#ifdef CONFIG_USER_ONLY unsigned long flags; void *target_data; #endif -#ifndef CONFIG_USER_ONLY +#ifdef CONFIG_SOFTMMU QemuSpin lock; #endif } PageDesc; @@ -XXX,XX +XXX,XX @@ void tb_htable_init(void) qht_init(&tb_ctx.htable, tb_cmp, CODE_GEN_HTABLE_SIZE, mode); } -/* call with @p->lock held */ -static inline void invalidate_page_bitmap(PageDesc *p) -{ - assert_page_locked(p); -#ifdef CONFIG_SOFTMMU - g_free(p->code_bitmap); - p->code_bitmap = NULL; - p->code_write_count = 0; -#endif -} - /* Set to NULL all the 'first_tb' fields in all PageDescs. */ static void page_flush_tb_1(int level, void **lp) { @@ -XXX,XX +XXX,XX @@ static void page_flush_tb_1(int level, void **lp) for (i = 0; i < V_L2_SIZE; ++i) { page_lock(&pd[i]); pd[i].first_tb = (uintptr_t)NULL; - invalidate_page_bitmap(pd + i); page_unlock(&pd[i]); } } else { @@ -XXX,XX +XXX,XX @@ static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list) if (rm_from_page_list) { p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS); tb_page_remove(p, tb); - invalidate_page_bitmap(p); if (tb->page_addr[1] != -1) { p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS); tb_page_remove(p, tb); - invalidate_page_bitmap(p); } } @@ -XXX,XX +XXX,XX @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) } } -#ifdef CONFIG_SOFTMMU -/* call with @p->lock held */ -static void build_page_bitmap(PageDesc *p) -{ - int n, tb_start, tb_end; - TranslationBlock *tb; - - assert_page_locked(p); - p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE); - - PAGE_FOR_EACH_TB(p, tb, n) { - /* NOTE: this is subtle as a TB may span two physical pages */ - if (n == 0) { - /* NOTE: tb_end may be after the end of the page, but - it is not a problem */ - tb_start = tb->pc & ~TARGET_PAGE_MASK; - tb_end = tb_start + tb->size; - if (tb_end > TARGET_PAGE_SIZE) { - tb_end = TARGET_PAGE_SIZE; - } - } else { - tb_start = 0; - tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); - } - bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start); - } -} -#endif - /* add the tb in the target page and protect it if necessary * * Called with mmap_lock held for user-mode emulation. @@ -XXX,XX +XXX,XX @@ static inline void tb_page_add(PageDesc *p, TranslationBlock *tb, page_already_protected = p->first_tb != (uintptr_t)NULL; #endif p->first_tb = (uintptr_t)tb | n; - invalidate_page_bitmap(p); #if defined(CONFIG_USER_ONLY) /* translator_loop() must have made all TB pages non-writable */ @@ -XXX,XX +XXX,XX @@ tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, /* remove TB from the page(s) if we couldn't insert it */ if (unlikely(existing_tb)) { tb_page_remove(p, tb); - invalidate_page_bitmap(p); if (p2) { tb_page_remove(p2, tb); - invalidate_page_bitmap(p2); } tb = existing_tb; } @@ -XXX,XX +XXX,XX @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages, #if !defined(CONFIG_USER_ONLY) /* if no code remaining, no need to continue to use slow writes */ if (!p->first_tb) { - invalidate_page_bitmap(p); tlb_unprotect_code(start); } #endif @@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_page_fast(struct page_collection *pages, } assert_page_locked(p); - if (!p->code_bitmap && - ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) { - build_page_bitmap(p); - } - if (p->code_bitmap) { - unsigned int nr; - unsigned long b; - - nr = start & ~TARGET_PAGE_MASK; - b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1)); - if (b & ((1 << len) - 1)) { - goto do_invalidate; - } - } else { - do_invalidate: - tb_invalidate_phys_page_range__locked(pages, p, start, start + len, - retaddr); - } + tb_invalidate_phys_page_range__locked(pages, p, start, start + len, + retaddr); } #else /* Called with mmap_lock held. If pc is not 0 then it indicates the -- 2.34.1 Bool is more appropriate type for the alloc parameter. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/translate-all.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ void page_init(void) #endif } -static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc) +static PageDesc *page_find_alloc(tb_page_addr_t index, bool alloc) { PageDesc *pd; void **lp; @@ -XXX,XX +XXX,XX @@ static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc) static inline PageDesc *page_find(tb_page_addr_t index) { - return page_find_alloc(index, 0); + return page_find_alloc(index, false); } static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1, - PageDesc **ret_p2, tb_page_addr_t phys2, int alloc); + PageDesc **ret_p2, tb_page_addr_t phys2, bool alloc); /* In user-mode page locks aren't used; mmap_lock is enough */ #ifdef CONFIG_USER_ONLY @@ -XXX,XX +XXX,XX @@ static inline void page_unlock(PageDesc *pd) /* lock the page(s) of a TB in the correct acquisition order */ static inline void page_lock_tb(const TranslationBlock *tb) { - page_lock_pair(NULL, tb->page_addr[0], NULL, tb->page_addr[1], 0); + page_lock_pair(NULL, tb->page_addr[0], NULL, tb->page_addr[1], false); } static inline void page_unlock_tb(const TranslationBlock *tb) @@ -XXX,XX +XXX,XX @@ void page_collection_unlock(struct page_collection *set) #endif /* !CONFIG_USER_ONLY */ static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1, - PageDesc **ret_p2, tb_page_addr_t phys2, int alloc) + PageDesc **ret_p2, tb_page_addr_t phys2, bool alloc) { PageDesc *p1, *p2; tb_page_addr_t page1; @@ -XXX,XX +XXX,XX @@ tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, * Note that inserting into the hash table first isn't an option, since * we can only insert TBs that are fully initialized. */ - page_lock_pair(&p, phys_pc, &p2, phys_page2, 1); + page_lock_pair(&p, phys_pc, &p2, phys_page2, true); tb_page_add(p, tb, 0, phys_pc & TARGET_PAGE_MASK); if (p2) { tb_page_add(p2, tb, 1, phys_page2); @@ -XXX,XX +XXX,XX @@ void page_set_flags(target_ulong start, target_ulong end, int flags) for (addr = start, len = end - start; len != 0; len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { - PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1); + PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, true); /* If the write protection bit is set, then we invalidate the code inside. */ -- 2.34.1 Use the pc coming from db->pc_first rather than the TB. Use the cached host_addr rather than re-computing for the first page. We still need a separate lookup for the second page because it won't be computed for DisasContextBase until the translator actually performs a read from the page. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/plugin-gen.h | 7 ++++--- accel/tcg/plugin-gen.c | 22 +++++++++++----------- accel/tcg/translator.c | 2 +- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/include/exec/plugin-gen.h b/include/exec/plugin-gen.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/plugin-gen.h +++ b/include/exec/plugin-gen.h @@ -XXX,XX +XXX,XX @@ struct DisasContextBase; #ifdef CONFIG_PLUGIN -bool plugin_gen_tb_start(CPUState *cpu, const TranslationBlock *tb, bool supress); +bool plugin_gen_tb_start(CPUState *cpu, const struct DisasContextBase *db, + bool supress); void plugin_gen_tb_end(CPUState *cpu); void plugin_gen_insn_start(CPUState *cpu, const struct DisasContextBase *db); void plugin_gen_insn_end(void); @@ -XXX,XX +XXX,XX @@ static inline void plugin_insn_append(abi_ptr pc, const void *from, size_t size) #else /* !CONFIG_PLUGIN */ -static inline -bool plugin_gen_tb_start(CPUState *cpu, const TranslationBlock *tb, bool supress) +static inline bool +plugin_gen_tb_start(CPUState *cpu, const struct DisasContextBase *db, bool sup) { return false; } diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -XXX,XX +XXX,XX @@ static void plugin_gen_inject(const struct qemu_plugin_tb *plugin_tb) pr_ops(); } -bool plugin_gen_tb_start(CPUState *cpu, const TranslationBlock *tb, bool mem_only) +bool plugin_gen_tb_start(CPUState *cpu, const DisasContextBase *db, + bool mem_only) { bool ret = false; @@ -XXX,XX +XXX,XX @@ bool plugin_gen_tb_start(CPUState *cpu, const TranslationBlock *tb, bool mem_onl ret = true; - ptb->vaddr = tb->pc; + ptb->vaddr = db->pc_first; ptb->vaddr2 = -1; - get_page_addr_code_hostp(cpu->env_ptr, tb->pc, &ptb->haddr1); + ptb->haddr1 = db->host_addr[0]; ptb->haddr2 = NULL; ptb->mem_only = mem_only; @@ -XXX,XX +XXX,XX @@ void plugin_gen_insn_start(CPUState *cpu, const DisasContextBase *db) * Note that we skip this when haddr1 == NULL, e.g. when we're * fetching instructions from a region not backed by RAM. */ - if (likely(ptb->haddr1 != NULL && ptb->vaddr2 == -1) && - unlikely((db->pc_next & TARGET_PAGE_MASK) != - (db->pc_first & TARGET_PAGE_MASK))) { - get_page_addr_code_hostp(cpu->env_ptr, db->pc_next, - &ptb->haddr2); - ptb->vaddr2 = db->pc_next; - } - if (likely(ptb->vaddr2 == -1)) { + if (ptb->haddr1 == NULL) { + pinsn->haddr = NULL; + } else if (is_same_page(db, db->pc_next)) { pinsn->haddr = ptb->haddr1 + pinsn->vaddr - ptb->vaddr; } else { + if (ptb->vaddr2 == -1) { + ptb->vaddr2 = TARGET_PAGE_ALIGN(db->pc_first); + get_page_addr_code_hostp(cpu->env_ptr, ptb->vaddr2, &ptb->haddr2); + } pinsn->haddr = ptb->haddr2 + pinsn->vaddr - ptb->vaddr2; } } diff --git a/accel/tcg/translator.c b/accel/tcg/translator.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translator.c +++ b/accel/tcg/translator.c @@ -XXX,XX +XXX,XX @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int max_insns, ops->tb_start(db, cpu); tcg_debug_assert(db->is_jmp == DISAS_NEXT); /* no early exit */ - plugin_enabled = plugin_gen_tb_start(cpu, tb, cflags & CF_MEMI_ONLY); + plugin_enabled = plugin_gen_tb_start(cpu, db, cflags & CF_MEMI_ONLY); while (true) { db->num_insns++; -- 2.34.1 Let tb->page_addr[0] contain the address of the first byte of the translated block, rather than the address of the page containing the start of the translated block. We need to recover this value anyway at various points, and it is easier to discard a page offset when it is not needed, which happens naturally via the existing find_page shift. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cpu-exec.c | 16 ++++++++-------- accel/tcg/cputlb.c | 3 ++- accel/tcg/translate-all.c | 9 +++++---- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ struct tb_desc { target_ulong pc; target_ulong cs_base; CPUArchState *env; - tb_page_addr_t phys_page1; + tb_page_addr_t page_addr0; uint32_t flags; uint32_t cflags; uint32_t trace_vcpu_dstate; @@ -XXX,XX +XXX,XX @@ static bool tb_lookup_cmp(const void *p, const void *d) const struct tb_desc *desc = d; if (tb->pc == desc->pc && - tb->page_addr[0] == desc->phys_page1 && + tb->page_addr[0] == desc->page_addr0 && tb->cs_base == desc->cs_base && tb->flags == desc->flags && tb->trace_vcpu_dstate == desc->trace_vcpu_dstate && @@ -XXX,XX +XXX,XX @@ static bool tb_lookup_cmp(const void *p, const void *d) if (tb->page_addr[1] == -1) { return true; } else { - tb_page_addr_t phys_page2; - target_ulong virt_page2; + tb_page_addr_t phys_page1; + target_ulong virt_page1; /* * We know that the first page matched, and an otherwise valid TB @@ -XXX,XX +XXX,XX @@ static bool tb_lookup_cmp(const void *p, const void *d) * is different for the new TB. Therefore any exception raised * here by the faulting lookup is not premature. */ - virt_page2 = TARGET_PAGE_ALIGN(desc->pc); - phys_page2 = get_page_addr_code(desc->env, virt_page2); - if (tb->page_addr[1] == phys_page2) { + virt_page1 = TARGET_PAGE_ALIGN(desc->pc); + phys_page1 = get_page_addr_code(desc->env, virt_page1); + if (tb->page_addr[1] == phys_page1) { return true; } } @@ -XXX,XX +XXX,XX @@ static TranslationBlock *tb_htable_lookup(CPUState *cpu, target_ulong pc, if (phys_pc == -1) { return NULL; } - desc.phys_page1 = phys_pc & TARGET_PAGE_MASK; + desc.page_addr0 = phys_pc; h = tb_hash_func(phys_pc, pc, flags, cflags, *cpu->trace_dstate); return qht_lookup_custom(&tb_ctx.htable, &desc, h, tb_lookup_cmp); } diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu, can be detected */ void tlb_protect_code(ram_addr_t ram_addr) { - cpu_physical_memory_test_and_clear_dirty(ram_addr, TARGET_PAGE_SIZE, + cpu_physical_memory_test_and_clear_dirty(ram_addr & TARGET_PAGE_MASK, + TARGET_PAGE_SIZE, DIRTY_MEMORY_CODE); } diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list) qemu_spin_unlock(&tb->jmp_lock); /* remove the TB from the hash list */ - phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); + phys_pc = tb->page_addr[0]; h = tb_hash_func(phys_pc, tb->pc, tb->flags, orig_cflags, tb->trace_vcpu_dstate); if (!qht_remove(&tb_ctx.htable, tb, h)) { @@ -XXX,XX +XXX,XX @@ tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, * we can only insert TBs that are fully initialized. */ page_lock_pair(&p, phys_pc, &p2, phys_page2, true); - tb_page_add(p, tb, 0, phys_pc & TARGET_PAGE_MASK); + tb_page_add(p, tb, 0, phys_pc); if (p2) { tb_page_add(p2, tb, 1, phys_page2); } else { @@ -XXX,XX +XXX,XX @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages, if (n == 0) { /* NOTE: tb_end may be after the end of the page, but it is not a problem */ - tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); + tb_start = tb->page_addr[0]; tb_end = tb_start + tb->size; } else { tb_start = tb->page_addr[1]; - tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); + tb_end = tb_start + ((tb->page_addr[0] + tb->size) + & ~TARGET_PAGE_MASK); } if (!(tb_end <= start || tb_start >= end)) { #ifdef TARGET_HAS_PRECISE_SMC -- 2.34.1 This function has two users, who use it incompatibly. In tlb_flush_page_by_mmuidx_async_0, when flushing a single page, we need to flush exactly two pages. In tlb_flush_range_by_mmuidx_async_0, when flushing a range of pages, we need to flush N+1 pages. This avoids double-flushing of jmp cache pages in a range. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cputlb.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr) } } -static void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr) -{ - /* Discard jump cache entries for any tb which might potentially - overlap the flushed page. */ - tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE); - tb_jmp_cache_clear_page(cpu, addr); -} - /** * tlb_mmu_resize_locked() - perform TLB resize bookkeeping; resize if necessary * @desc: The CPUTLBDesc portion of the TLB @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_by_mmuidx_async_0(CPUState *cpu, } qemu_spin_unlock(&env_tlb(env)->c.lock); - tb_flush_jmp_cache(cpu, addr); + /* + * Discard jump cache entries for any tb which might potentially + * overlap the flushed page, which includes the previous. + */ + tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE); + tb_jmp_cache_clear_page(cpu, addr); } /** @@ -XXX,XX +XXX,XX @@ static void tlb_flush_range_by_mmuidx_async_0(CPUState *cpu, return; } - for (target_ulong i = 0; i < d.len; i += TARGET_PAGE_SIZE) { - tb_flush_jmp_cache(cpu, d.addr + i); + /* + * Discard jump cache entries for any tb which might potentially + * overlap the flushed pages, which includes the previous. + */ + d.addr -= TARGET_PAGE_SIZE; + for (target_ulong i = 0, n = d.len / TARGET_PAGE_SIZE + 1; i < n; i++) { + tb_jmp_cache_clear_page(cpu, d.addr); + d.addr += TARGET_PAGE_SIZE; } } -- 2.34.1 Wrap the bare TranslationBlock pointer into a structure. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/tb-hash.h | 1 + accel/tcg/tb-jmp-cache.h | 24 ++++++++++++++++++++++++ include/exec/cpu-common.h | 1 + include/hw/core/cpu.h | 15 +-------------- include/qemu/typedefs.h | 1 + accel/stubs/tcg-stub.c | 4 ++++ accel/tcg/cpu-exec.c | 10 +++++++--- accel/tcg/cputlb.c | 9 +++++---- accel/tcg/translate-all.c | 28 +++++++++++++++++++++++++--- hw/core/cpu-common.c | 3 +-- plugins/core.c | 2 +- trace/control-target.c | 2 +- 12 files changed, 72 insertions(+), 28 deletions(-) create mode 100644 accel/tcg/tb-jmp-cache.h diff --git a/accel/tcg/tb-hash.h b/accel/tcg/tb-hash.h index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/tb-hash.h +++ b/accel/tcg/tb-hash.h @@ -XXX,XX +XXX,XX @@ #include "exec/cpu-defs.h" #include "exec/exec-all.h" #include "qemu/xxhash.h" +#include "tb-jmp-cache.h" #ifdef CONFIG_SOFTMMU diff --git a/accel/tcg/tb-jmp-cache.h b/accel/tcg/tb-jmp-cache.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/accel/tcg/tb-jmp-cache.h @@ -XXX,XX +XXX,XX @@ +/* + * The per-CPU TranslationBlock jump cache. + * + * Copyright (c) 2003 Fabrice Bellard + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#ifndef ACCEL_TCG_TB_JMP_CACHE_H +#define ACCEL_TCG_TB_JMP_CACHE_H + +#define TB_JMP_CACHE_BITS 12 +#define TB_JMP_CACHE_SIZE (1 << TB_JMP_CACHE_BITS) + +/* + * Accessed in parallel; all accesses to 'tb' must be atomic. + */ +struct CPUJumpCache { + struct { + TranslationBlock *tb; + } array[TB_JMP_CACHE_SIZE]; +}; + +#endif /* ACCEL_TCG_TB_JMP_CACHE_H */ diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu-common.h +++ b/include/exec/cpu-common.h @@ -XXX,XX +XXX,XX @@ void cpu_list_unlock(void); unsigned int cpu_list_generation_id_get(void); void tcg_flush_softmmu_tlb(CPUState *cs); +void tcg_flush_jmp_cache(CPUState *cs); void tcg_iommu_init_notifier_list(CPUState *cpu); void tcg_iommu_free_notifier_list(CPUState *cpu); diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/core/cpu.h +++ b/include/hw/core/cpu.h @@ -XXX,XX +XXX,XX @@ struct kvm_run; struct hax_vcpu_state; struct hvf_vcpu_state; -#define TB_JMP_CACHE_BITS 12 -#define TB_JMP_CACHE_SIZE (1 << TB_JMP_CACHE_BITS) - /* work queue */ /* The union type allows passing of 64 bit target pointers on 32 bit @@ -XXX,XX +XXX,XX @@ struct CPUState { CPUArchState *env_ptr; IcountDecr *icount_decr_ptr; - /* Accessed in parallel; all accesses must be atomic */ - TranslationBlock *tb_jmp_cache[TB_JMP_CACHE_SIZE]; + CPUJumpCache *tb_jmp_cache; struct GDBRegisterState *gdb_regs; int gdb_num_regs; @@ -XXX,XX +XXX,XX @@ extern CPUTailQ cpus; extern __thread CPUState *current_cpu; -static inline void cpu_tb_jmp_cache_clear(CPUState *cpu) -{ - unsigned int i; - - for (i = 0; i < TB_JMP_CACHE_SIZE; i++) { - qatomic_set(&cpu->tb_jmp_cache[i], NULL); - } -} - /** * qemu_tcg_mttcg_enabled: * Check whether we are running MultiThread TCG or not. diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/typedefs.h +++ b/include/qemu/typedefs.h @@ -XXX,XX +XXX,XX @@ typedef struct CoMutex CoMutex; typedef struct ConfidentialGuestSupport ConfidentialGuestSupport; typedef struct CPUAddressSpace CPUAddressSpace; typedef struct CPUArchState CPUArchState; +typedef struct CPUJumpCache CPUJumpCache; typedef struct CPUState CPUState; typedef struct CPUTLBEntryFull CPUTLBEntryFull; typedef struct DeviceListener DeviceListener; diff --git a/accel/stubs/tcg-stub.c b/accel/stubs/tcg-stub.c index XXXXXXX..XXXXXXX 100644 --- a/accel/stubs/tcg-stub.c +++ b/accel/stubs/tcg-stub.c @@ -XXX,XX +XXX,XX @@ void tlb_set_dirty(CPUState *cpu, target_ulong vaddr) { } +void tcg_flush_jmp_cache(CPUState *cpu) +{ +} + int probe_access_flags(CPUArchState *env, target_ulong addr, MMUAccessType access_type, int mmu_idx, bool nonfault, void **phost, uintptr_t retaddr) diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ #include "sysemu/replay.h" #include "sysemu/tcg.h" #include "exec/helper-proto.h" +#include "tb-jmp-cache.h" #include "tb-hash.h" #include "tb-context.h" #include "internal.h" @@ -XXX,XX +XXX,XX @@ static inline TranslationBlock *tb_lookup(CPUState *cpu, target_ulong pc, tcg_debug_assert(!(cflags & CF_INVALID)); hash = tb_jmp_cache_hash_func(pc); - tb = qatomic_rcu_read(&cpu->tb_jmp_cache[hash]); + tb = qatomic_rcu_read(&cpu->tb_jmp_cache->array[hash].tb); if (likely(tb && tb->pc == pc && @@ -XXX,XX +XXX,XX @@ static inline TranslationBlock *tb_lookup(CPUState *cpu, target_ulong pc, if (tb == NULL) { return NULL; } - qatomic_set(&cpu->tb_jmp_cache[hash], tb); + qatomic_set(&cpu->tb_jmp_cache->array[hash].tb, tb); return tb; } @@ -XXX,XX +XXX,XX @@ int cpu_exec(CPUState *cpu) tb = tb_lookup(cpu, pc, cs_base, flags, cflags); if (tb == NULL) { + uint32_t h; + mmap_lock(); tb = tb_gen_code(cpu, pc, cs_base, flags, cflags); mmap_unlock(); @@ -XXX,XX +XXX,XX @@ int cpu_exec(CPUState *cpu) * We add the TB in the virtual pc hash table * for the fast lookup */ - qatomic_set(&cpu->tb_jmp_cache[tb_jmp_cache_hash_func(pc)], tb); + h = tb_jmp_cache_hash_func(pc); + qatomic_set(&cpu->tb_jmp_cache->array[h].tb, tb); } #ifndef CONFIG_USER_ONLY diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ static void tlb_window_reset(CPUTLBDesc *desc, int64_t ns, static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr) { - unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr); + int i, i0 = tb_jmp_cache_hash_page(page_addr); + CPUJumpCache *jc = cpu->tb_jmp_cache; for (i = 0; i < TB_JMP_PAGE_SIZE; i++) { - qatomic_set(&cpu->tb_jmp_cache[i0 + i], NULL); + qatomic_set(&jc->array[i0 + i].tb, NULL); } } @@ -XXX,XX +XXX,XX @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data) qemu_spin_unlock(&env_tlb(env)->c.lock); - cpu_tb_jmp_cache_clear(cpu); + tcg_flush_jmp_cache(cpu); if (to_clean == ALL_MMUIDX_BITS) { qatomic_set(&env_tlb(env)->c.full_flush_count, @@ -XXX,XX +XXX,XX @@ static void tlb_flush_range_by_mmuidx_async_0(CPUState *cpu, * longer to clear each entry individually than it will to clear it all. */ if (d.len >= (TARGET_PAGE_SIZE * TB_JMP_CACHE_SIZE)) { - cpu_tb_jmp_cache_clear(cpu); + tcg_flush_jmp_cache(cpu); return; } diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ #include "sysemu/tcg.h" #include "qapi/error.h" #include "hw/core/tcg-cpu-ops.h" +#include "tb-jmp-cache.h" #include "tb-hash.h" #include "tb-context.h" #include "internal.h" @@ -XXX,XX +XXX,XX @@ static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) } CPU_FOREACH(cpu) { - cpu_tb_jmp_cache_clear(cpu); + tcg_flush_jmp_cache(cpu); } qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE); @@ -XXX,XX +XXX,XX @@ static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list) /* remove the TB from the hash list */ h = tb_jmp_cache_hash_func(tb->pc); CPU_FOREACH(cpu) { - if (qatomic_read(&cpu->tb_jmp_cache[h]) == tb) { - qatomic_set(&cpu->tb_jmp_cache[h], NULL); + CPUJumpCache *jc = cpu->tb_jmp_cache; + if (qatomic_read(&jc->array[h].tb) == tb) { + qatomic_set(&jc->array[h].tb, NULL); } } @@ -XXX,XX +XXX,XX @@ int page_unprotect(target_ulong address, uintptr_t pc) } #endif /* CONFIG_USER_ONLY */ +/* + * Called by generic code at e.g. cpu reset after cpu creation, + * therefore we must be prepared to allocate the jump cache. + */ +void tcg_flush_jmp_cache(CPUState *cpu) +{ + CPUJumpCache *jc = cpu->tb_jmp_cache; + + if (likely(jc)) { + for (int i = 0; i < TB_JMP_CACHE_SIZE; i++) { + qatomic_set(&jc->array[i].tb, NULL); + } + } else { + /* This should happen once during realize, and thus never race. */ + jc = g_new0(CPUJumpCache, 1); + jc = qatomic_xchg(&cpu->tb_jmp_cache, jc); + assert(jc == NULL); + } +} + /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */ void tcg_flush_softmmu_tlb(CPUState *cs) { diff --git a/hw/core/cpu-common.c b/hw/core/cpu-common.c index XXXXXXX..XXXXXXX 100644 --- a/hw/core/cpu-common.c +++ b/hw/core/cpu-common.c @@ -XXX,XX +XXX,XX @@ static void cpu_common_reset(DeviceState *dev) cpu->cflags_next_tb = -1; if (tcg_enabled()) { - cpu_tb_jmp_cache_clear(cpu); - + tcg_flush_jmp_cache(cpu); tcg_flush_softmmu_tlb(cpu); } } diff --git a/plugins/core.c b/plugins/core.c index XXXXXXX..XXXXXXX 100644 --- a/plugins/core.c +++ b/plugins/core.c @@ -XXX,XX +XXX,XX @@ struct qemu_plugin_ctx *plugin_id_to_ctx_locked(qemu_plugin_id_t id) static void plugin_cpu_update__async(CPUState *cpu, run_on_cpu_data data) { bitmap_copy(cpu->plugin_mask, &data.host_ulong, QEMU_PLUGIN_EV_MAX); - cpu_tb_jmp_cache_clear(cpu); + tcg_flush_jmp_cache(cpu); } static void plugin_cpu_update__locked(gpointer k, gpointer v, gpointer udata) diff --git a/trace/control-target.c b/trace/control-target.c index XXXXXXX..XXXXXXX 100644 --- a/trace/control-target.c +++ b/trace/control-target.c @@ -XXX,XX +XXX,XX @@ static void trace_event_synchronize_vcpu_state_dynamic( { bitmap_copy(vcpu->trace_dstate, vcpu->trace_dstate_delayed, CPU_TRACE_DSTATE_MAX_EVENTS); - cpu_tb_jmp_cache_clear(vcpu); + tcg_flush_jmp_cache(vcpu); } void trace_event_set_vcpu_state_dynamic(CPUState *vcpu, -- 2.34.1 Populate this new method for all targets. Always match the result that would be given by cpu_get_tb_cpu_state, as we will want these values to correspond in the logs. Reviewed-by: Taylor Simpson <tsimpson@quicinc.com> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Mark Cave-Ayland <mark.cave-ayland@ilande.co.uk> (target/sparc) Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- Cc: Eduardo Habkost <eduardo@habkost.net> (supporter:Machine core) Cc: Marcel Apfelbaum <marcel.apfelbaum@gmail.com> (supporter:Machine core) Cc: "Philippe Mathieu-Daudé" <f4bug@amsat.org> (reviewer:Machine core) Cc: Yanan Wang <wangyanan55@huawei.com> (reviewer:Machine core) Cc: Michael Rolnik <mrolnik@gmail.com> (maintainer:AVR TCG CPUs) Cc: "Edgar E. Iglesias" <edgar.iglesias@gmail.com> (maintainer:CRIS TCG CPUs) Cc: Taylor Simpson <tsimpson@quicinc.com> (supporter:Hexagon TCG CPUs) Cc: Song Gao <gaosong@loongson.cn> (maintainer:LoongArch TCG CPUs) Cc: Xiaojuan Yang <yangxiaojuan@loongson.cn> (maintainer:LoongArch TCG CPUs) Cc: Laurent Vivier <laurent@vivier.eu> (maintainer:M68K TCG CPUs) Cc: Jiaxun Yang <jiaxun.yang@flygoat.com> (reviewer:MIPS TCG CPUs) Cc: Aleksandar Rikalo <aleksandar.rikalo@syrmia.com> (reviewer:MIPS TCG CPUs) Cc: Chris Wulff <crwulff@gmail.com> (maintainer:NiosII TCG CPUs) Cc: Marek Vasut <marex@denx.de> (maintainer:NiosII TCG CPUs) Cc: Stafford Horne <shorne@gmail.com> (odd fixer:OpenRISC TCG CPUs) Cc: Yoshinori Sato <ysato@users.sourceforge.jp> (reviewer:RENESAS RX CPUs) Cc: Mark Cave-Ayland <mark.cave-ayland@ilande.co.uk> (maintainer:SPARC TCG CPUs) Cc: Bastian Koppelmann <kbastian@mail.uni-paderborn.de> (maintainer:TriCore TCG CPUs) Cc: Max Filippov <jcmvbkbc@gmail.com> (maintainer:Xtensa TCG CPUs) Cc: qemu-arm@nongnu.org (open list:ARM TCG CPUs) Cc: qemu-ppc@nongnu.org (open list:PowerPC TCG CPUs) Cc: qemu-riscv@nongnu.org (open list:RISC-V TCG CPUs) Cc: qemu-s390x@nongnu.org (open list:S390 TCG CPUs) --- include/hw/core/cpu.h | 3 +++ target/alpha/cpu.c | 9 +++++++++ target/arm/cpu.c | 13 +++++++++++++ target/avr/cpu.c | 8 ++++++++ target/cris/cpu.c | 8 ++++++++ target/hexagon/cpu.c | 8 ++++++++ target/hppa/cpu.c | 8 ++++++++ target/i386/cpu.c | 9 +++++++++ target/loongarch/cpu.c | 9 +++++++++ target/m68k/cpu.c | 8 ++++++++ target/microblaze/cpu.c | 8 ++++++++ target/mips/cpu.c | 8 ++++++++ target/nios2/cpu.c | 9 +++++++++ target/openrisc/cpu.c | 8 ++++++++ target/ppc/cpu_init.c | 8 ++++++++ target/riscv/cpu.c | 13 +++++++++++++ target/rx/cpu.c | 8 ++++++++ target/s390x/cpu.c | 8 ++++++++ target/sh4/cpu.c | 8 ++++++++ target/sparc/cpu.c | 8 ++++++++ target/tricore/cpu.c | 9 +++++++++ target/xtensa/cpu.c | 8 ++++++++ 22 files changed, 186 insertions(+) diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/core/cpu.h +++ b/include/hw/core/cpu.h @@ -XXX,XX +XXX,XX @@ struct SysemuCPUOps; * If the target behaviour here is anything other than "set * the PC register to the value passed in" then the target must * also implement the synchronize_from_tb hook. + * @get_pc: Callback for getting the Program Counter register. + * As above, with the semantics of the target architecture. * @gdb_read_register: Callback for letting GDB read a register. * @gdb_write_register: Callback for letting GDB write a register. * @gdb_adjust_breakpoint: Callback for adjusting the address of a @@ -XXX,XX +XXX,XX @@ struct CPUClass { void (*dump_state)(CPUState *cpu, FILE *, int flags); int64_t (*get_arch_id)(CPUState *cpu); void (*set_pc)(CPUState *cpu, vaddr value); + vaddr (*get_pc)(CPUState *cpu); int (*gdb_read_register)(CPUState *cpu, GByteArray *buf, int reg); int (*gdb_write_register)(CPUState *cpu, uint8_t *buf, int reg); vaddr (*gdb_adjust_breakpoint)(CPUState *cpu, vaddr addr); diff --git a/target/alpha/cpu.c b/target/alpha/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/alpha/cpu.c +++ b/target/alpha/cpu.c @@ -XXX,XX +XXX,XX @@ static void alpha_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.pc = value; } +static vaddr alpha_cpu_get_pc(CPUState *cs) +{ + AlphaCPU *cpu = ALPHA_CPU(cs); + + return cpu->env.pc; +} + + static bool alpha_cpu_has_work(CPUState *cs) { /* Here we are checking to see if the CPU should wake up from HALT. @@ -XXX,XX +XXX,XX @@ static void alpha_cpu_class_init(ObjectClass *oc, void *data) cc->has_work = alpha_cpu_has_work; cc->dump_state = alpha_cpu_dump_state; cc->set_pc = alpha_cpu_set_pc; + cc->get_pc = alpha_cpu_get_pc; cc->gdb_read_register = alpha_cpu_gdb_read_register; cc->gdb_write_register = alpha_cpu_gdb_write_register; #ifndef CONFIG_USER_ONLY diff --git a/target/arm/cpu.c b/target/arm/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -XXX,XX +XXX,XX @@ static void arm_cpu_set_pc(CPUState *cs, vaddr value) } } +static vaddr arm_cpu_get_pc(CPUState *cs) +{ + ARMCPU *cpu = ARM_CPU(cs); + CPUARMState *env = &cpu->env; + + if (is_a64(env)) { + return env->pc; + } else { + return env->regs[15]; + } +} + #ifdef CONFIG_TCG void arm_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) @@ -XXX,XX +XXX,XX @@ static void arm_cpu_class_init(ObjectClass *oc, void *data) cc->has_work = arm_cpu_has_work; cc->dump_state = arm_cpu_dump_state; cc->set_pc = arm_cpu_set_pc; + cc->get_pc = arm_cpu_get_pc; cc->gdb_read_register = arm_cpu_gdb_read_register; cc->gdb_write_register = arm_cpu_gdb_write_register; #ifndef CONFIG_USER_ONLY diff --git a/target/avr/cpu.c b/target/avr/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/cpu.c +++ b/target/avr/cpu.c @@ -XXX,XX +XXX,XX @@ static void avr_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.pc_w = value / 2; /* internally PC points to words */ } +static vaddr avr_cpu_get_pc(CPUState *cs) +{ + AVRCPU *cpu = AVR_CPU(cs); + + return cpu->env.pc_w * 2; +} + static bool avr_cpu_has_work(CPUState *cs) { AVRCPU *cpu = AVR_CPU(cs); @@ -XXX,XX +XXX,XX @@ static void avr_cpu_class_init(ObjectClass *oc, void *data) cc->has_work = avr_cpu_has_work; cc->dump_state = avr_cpu_dump_state; cc->set_pc = avr_cpu_set_pc; + cc->get_pc = avr_cpu_get_pc; dc->vmsd = &vms_avr_cpu; cc->sysemu_ops = &avr_sysemu_ops; cc->disas_set_info = avr_cpu_disas_set_info; diff --git a/target/cris/cpu.c b/target/cris/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/cris/cpu.c +++ b/target/cris/cpu.c @@ -XXX,XX +XXX,XX @@ static void cris_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.pc = value; } +static vaddr cris_cpu_get_pc(CPUState *cs) +{ + CRISCPU *cpu = CRIS_CPU(cs); + + return cpu->env.pc; +} + static bool cris_cpu_has_work(CPUState *cs) { return cs->interrupt_request & (CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI); @@ -XXX,XX +XXX,XX @@ static void cris_cpu_class_init(ObjectClass *oc, void *data) cc->has_work = cris_cpu_has_work; cc->dump_state = cris_cpu_dump_state; cc->set_pc = cris_cpu_set_pc; + cc->get_pc = cris_cpu_get_pc; cc->gdb_read_register = cris_cpu_gdb_read_register; cc->gdb_write_register = cris_cpu_gdb_write_register; #ifndef CONFIG_USER_ONLY diff --git a/target/hexagon/cpu.c b/target/hexagon/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/cpu.c +++ b/target/hexagon/cpu.c @@ -XXX,XX +XXX,XX @@ static void hexagon_cpu_set_pc(CPUState *cs, vaddr value) env->gpr[HEX_REG_PC] = value; } +static vaddr hexagon_cpu_get_pc(CPUState *cs) +{ + HexagonCPU *cpu = HEXAGON_CPU(cs); + CPUHexagonState *env = &cpu->env; + return env->gpr[HEX_REG_PC]; +} + static void hexagon_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -XXX,XX +XXX,XX @@ static void hexagon_cpu_class_init(ObjectClass *c, void *data) cc->has_work = hexagon_cpu_has_work; cc->dump_state = hexagon_dump_state; cc->set_pc = hexagon_cpu_set_pc; + cc->get_pc = hexagon_cpu_get_pc; cc->gdb_read_register = hexagon_gdb_read_register; cc->gdb_write_register = hexagon_gdb_write_register; cc->gdb_num_core_regs = TOTAL_PER_THREAD_REGS + NUM_VREGS + NUM_QREGS; diff --git a/target/hppa/cpu.c b/target/hppa/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hppa/cpu.c +++ b/target/hppa/cpu.c @@ -XXX,XX +XXX,XX @@ static void hppa_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.iaoq_b = value + 4; } +static vaddr hppa_cpu_get_pc(CPUState *cs) +{ + HPPACPU *cpu = HPPA_CPU(cs); + + return cpu->env.iaoq_f; +} + static void hppa_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -XXX,XX +XXX,XX @@ static void hppa_cpu_class_init(ObjectClass *oc, void *data) cc->has_work = hppa_cpu_has_work; cc->dump_state = hppa_cpu_dump_state; cc->set_pc = hppa_cpu_set_pc; + cc->get_pc = hppa_cpu_get_pc; cc->gdb_read_register = hppa_cpu_gdb_read_register; cc->gdb_write_register = hppa_cpu_gdb_write_register; #ifndef CONFIG_USER_ONLY diff --git a/target/i386/cpu.c b/target/i386/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -XXX,XX +XXX,XX @@ static void x86_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.eip = value; } +static vaddr x86_cpu_get_pc(CPUState *cs) +{ + X86CPU *cpu = X86_CPU(cs); + + /* Match cpu_get_tb_cpu_state. */ + return cpu->env.eip + cpu->env.segs[R_CS].base; +} + int x86_cpu_pending_interrupt(CPUState *cs, int interrupt_request) { X86CPU *cpu = X86_CPU(cs); @@ -XXX,XX +XXX,XX @@ static void x86_cpu_common_class_init(ObjectClass *oc, void *data) cc->has_work = x86_cpu_has_work; cc->dump_state = x86_cpu_dump_state; cc->set_pc = x86_cpu_set_pc; + cc->get_pc = x86_cpu_get_pc; cc->gdb_read_register = x86_cpu_gdb_read_register; cc->gdb_write_register = x86_cpu_gdb_write_register; cc->get_arch_id = x86_cpu_get_arch_id; diff --git a/target/loongarch/cpu.c b/target/loongarch/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/loongarch/cpu.c +++ b/target/loongarch/cpu.c @@ -XXX,XX +XXX,XX @@ static void loongarch_cpu_set_pc(CPUState *cs, vaddr value) env->pc = value; } +static vaddr loongarch_cpu_get_pc(CPUState *cs) +{ + LoongArchCPU *cpu = LOONGARCH_CPU(cs); + CPULoongArchState *env = &cpu->env; + + return env->pc; +} + #ifndef CONFIG_USER_ONLY #include "hw/loongarch/virt.h" @@ -XXX,XX +XXX,XX @@ static void loongarch_cpu_class_init(ObjectClass *c, void *data) cc->has_work = loongarch_cpu_has_work; cc->dump_state = loongarch_cpu_dump_state; cc->set_pc = loongarch_cpu_set_pc; + cc->get_pc = loongarch_cpu_get_pc; #ifndef CONFIG_USER_ONLY dc->vmsd = &vmstate_loongarch_cpu; cc->sysemu_ops = &loongarch_sysemu_ops; diff --git a/target/m68k/cpu.c b/target/m68k/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/m68k/cpu.c +++ b/target/m68k/cpu.c @@ -XXX,XX +XXX,XX @@ static void m68k_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.pc = value; } +static vaddr m68k_cpu_get_pc(CPUState *cs) +{ + M68kCPU *cpu = M68K_CPU(cs); + + return cpu->env.pc; +} + static bool m68k_cpu_has_work(CPUState *cs) { return cs->interrupt_request & CPU_INTERRUPT_HARD; @@ -XXX,XX +XXX,XX @@ static void m68k_cpu_class_init(ObjectClass *c, void *data) cc->has_work = m68k_cpu_has_work; cc->dump_state = m68k_cpu_dump_state; cc->set_pc = m68k_cpu_set_pc; + cc->get_pc = m68k_cpu_get_pc; cc->gdb_read_register = m68k_cpu_gdb_read_register; cc->gdb_write_register = m68k_cpu_gdb_write_register; #if defined(CONFIG_SOFTMMU) diff --git a/target/microblaze/cpu.c b/target/microblaze/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/microblaze/cpu.c +++ b/target/microblaze/cpu.c @@ -XXX,XX +XXX,XX @@ static void mb_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.iflags = 0; } +static vaddr mb_cpu_get_pc(CPUState *cs) +{ + MicroBlazeCPU *cpu = MICROBLAZE_CPU(cs); + + return cpu->env.pc; +} + static void mb_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -XXX,XX +XXX,XX @@ static void mb_cpu_class_init(ObjectClass *oc, void *data) cc->dump_state = mb_cpu_dump_state; cc->set_pc = mb_cpu_set_pc; + cc->get_pc = mb_cpu_get_pc; cc->gdb_read_register = mb_cpu_gdb_read_register; cc->gdb_write_register = mb_cpu_gdb_write_register; diff --git a/target/mips/cpu.c b/target/mips/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/mips/cpu.c +++ b/target/mips/cpu.c @@ -XXX,XX +XXX,XX @@ static void mips_cpu_set_pc(CPUState *cs, vaddr value) mips_env_set_pc(&cpu->env, value); } +static vaddr mips_cpu_get_pc(CPUState *cs) +{ + MIPSCPU *cpu = MIPS_CPU(cs); + + return cpu->env.active_tc.PC; +} + static bool mips_cpu_has_work(CPUState *cs) { MIPSCPU *cpu = MIPS_CPU(cs); @@ -XXX,XX +XXX,XX @@ static void mips_cpu_class_init(ObjectClass *c, void *data) cc->has_work = mips_cpu_has_work; cc->dump_state = mips_cpu_dump_state; cc->set_pc = mips_cpu_set_pc; + cc->get_pc = mips_cpu_get_pc; cc->gdb_read_register = mips_cpu_gdb_read_register; cc->gdb_write_register = mips_cpu_gdb_write_register; #ifndef CONFIG_USER_ONLY diff --git a/target/nios2/cpu.c b/target/nios2/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/nios2/cpu.c +++ b/target/nios2/cpu.c @@ -XXX,XX +XXX,XX @@ static void nios2_cpu_set_pc(CPUState *cs, vaddr value) env->pc = value; } +static vaddr nios2_cpu_get_pc(CPUState *cs) +{ + Nios2CPU *cpu = NIOS2_CPU(cs); + CPUNios2State *env = &cpu->env; + + return env->pc; +} + static bool nios2_cpu_has_work(CPUState *cs) { return cs->interrupt_request & CPU_INTERRUPT_HARD; @@ -XXX,XX +XXX,XX @@ static void nios2_cpu_class_init(ObjectClass *oc, void *data) cc->has_work = nios2_cpu_has_work; cc->dump_state = nios2_cpu_dump_state; cc->set_pc = nios2_cpu_set_pc; + cc->get_pc = nios2_cpu_get_pc; cc->disas_set_info = nios2_cpu_disas_set_info; #ifndef CONFIG_USER_ONLY cc->sysemu_ops = &nios2_sysemu_ops; diff --git a/target/openrisc/cpu.c b/target/openrisc/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/openrisc/cpu.c +++ b/target/openrisc/cpu.c @@ -XXX,XX +XXX,XX @@ static void openrisc_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.dflag = 0; } +static vaddr openrisc_cpu_get_pc(CPUState *cs) +{ + OpenRISCCPU *cpu = OPENRISC_CPU(cs); + + return cpu->env.pc; +} + static void openrisc_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -XXX,XX +XXX,XX @@ static void openrisc_cpu_class_init(ObjectClass *oc, void *data) cc->has_work = openrisc_cpu_has_work; cc->dump_state = openrisc_cpu_dump_state; cc->set_pc = openrisc_cpu_set_pc; + cc->get_pc = openrisc_cpu_get_pc; cc->gdb_read_register = openrisc_cpu_gdb_read_register; cc->gdb_write_register = openrisc_cpu_gdb_write_register; #ifndef CONFIG_USER_ONLY diff --git a/target/ppc/cpu_init.c b/target/ppc/cpu_init.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/cpu_init.c +++ b/target/ppc/cpu_init.c @@ -XXX,XX +XXX,XX @@ static void ppc_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.nip = value; } +static vaddr ppc_cpu_get_pc(CPUState *cs) +{ + PowerPCCPU *cpu = POWERPC_CPU(cs); + + return cpu->env.nip; +} + static bool ppc_cpu_has_work(CPUState *cs) { PowerPCCPU *cpu = POWERPC_CPU(cs); @@ -XXX,XX +XXX,XX @@ static void ppc_cpu_class_init(ObjectClass *oc, void *data) cc->has_work = ppc_cpu_has_work; cc->dump_state = ppc_cpu_dump_state; cc->set_pc = ppc_cpu_set_pc; + cc->get_pc = ppc_cpu_get_pc; cc->gdb_read_register = ppc_cpu_gdb_read_register; cc->gdb_write_register = ppc_cpu_gdb_write_register; #ifndef CONFIG_USER_ONLY diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/cpu.c +++ b/target/riscv/cpu.c @@ -XXX,XX +XXX,XX @@ static void riscv_cpu_set_pc(CPUState *cs, vaddr value) } } +static vaddr riscv_cpu_get_pc(CPUState *cs) +{ + RISCVCPU *cpu = RISCV_CPU(cs); + CPURISCVState *env = &cpu->env; + + /* Match cpu_get_tb_cpu_state. */ + if (env->xl == MXL_RV32) { + return env->pc & UINT32_MAX; + } + return env->pc; +} + static void riscv_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -XXX,XX +XXX,XX @@ static void riscv_cpu_class_init(ObjectClass *c, void *data) cc->has_work = riscv_cpu_has_work; cc->dump_state = riscv_cpu_dump_state; cc->set_pc = riscv_cpu_set_pc; + cc->get_pc = riscv_cpu_get_pc; cc->gdb_read_register = riscv_cpu_gdb_read_register; cc->gdb_write_register = riscv_cpu_gdb_write_register; cc->gdb_num_core_regs = 33; diff --git a/target/rx/cpu.c b/target/rx/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/rx/cpu.c +++ b/target/rx/cpu.c @@ -XXX,XX +XXX,XX @@ static void rx_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.pc = value; } +static vaddr rx_cpu_get_pc(CPUState *cs) +{ + RXCPU *cpu = RX_CPU(cs); + + return cpu->env.pc; +} + static void rx_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -XXX,XX +XXX,XX @@ static void rx_cpu_class_init(ObjectClass *klass, void *data) cc->has_work = rx_cpu_has_work; cc->dump_state = rx_cpu_dump_state; cc->set_pc = rx_cpu_set_pc; + cc->get_pc = rx_cpu_get_pc; #ifndef CONFIG_USER_ONLY cc->sysemu_ops = &rx_sysemu_ops; diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/cpu.c +++ b/target/s390x/cpu.c @@ -XXX,XX +XXX,XX @@ static void s390_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.psw.addr = value; } +static vaddr s390_cpu_get_pc(CPUState *cs) +{ + S390CPU *cpu = S390_CPU(cs); + + return cpu->env.psw.addr; +} + static bool s390_cpu_has_work(CPUState *cs) { S390CPU *cpu = S390_CPU(cs); @@ -XXX,XX +XXX,XX @@ static void s390_cpu_class_init(ObjectClass *oc, void *data) cc->has_work = s390_cpu_has_work; cc->dump_state = s390_cpu_dump_state; cc->set_pc = s390_cpu_set_pc; + cc->get_pc = s390_cpu_get_pc; cc->gdb_read_register = s390_cpu_gdb_read_register; cc->gdb_write_register = s390_cpu_gdb_write_register; #ifndef CONFIG_USER_ONLY diff --git a/target/sh4/cpu.c b/target/sh4/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/sh4/cpu.c +++ b/target/sh4/cpu.c @@ -XXX,XX +XXX,XX @@ static void superh_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.pc = value; } +static vaddr superh_cpu_get_pc(CPUState *cs) +{ + SuperHCPU *cpu = SUPERH_CPU(cs); + + return cpu->env.pc; +} + static void superh_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -XXX,XX +XXX,XX @@ static void superh_cpu_class_init(ObjectClass *oc, void *data) cc->has_work = superh_cpu_has_work; cc->dump_state = superh_cpu_dump_state; cc->set_pc = superh_cpu_set_pc; + cc->get_pc = superh_cpu_get_pc; cc->gdb_read_register = superh_cpu_gdb_read_register; cc->gdb_write_register = superh_cpu_gdb_write_register; #ifndef CONFIG_USER_ONLY diff --git a/target/sparc/cpu.c b/target/sparc/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/cpu.c +++ b/target/sparc/cpu.c @@ -XXX,XX +XXX,XX @@ static void sparc_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.npc = value + 4; } +static vaddr sparc_cpu_get_pc(CPUState *cs) +{ + SPARCCPU *cpu = SPARC_CPU(cs); + + return cpu->env.pc; +} + static void sparc_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -XXX,XX +XXX,XX @@ static void sparc_cpu_class_init(ObjectClass *oc, void *data) cc->memory_rw_debug = sparc_cpu_memory_rw_debug; #endif cc->set_pc = sparc_cpu_set_pc; + cc->get_pc = sparc_cpu_get_pc; cc->gdb_read_register = sparc_cpu_gdb_read_register; cc->gdb_write_register = sparc_cpu_gdb_write_register; #ifndef CONFIG_USER_ONLY diff --git a/target/tricore/cpu.c b/target/tricore/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/tricore/cpu.c +++ b/target/tricore/cpu.c @@ -XXX,XX +XXX,XX @@ static void tricore_cpu_set_pc(CPUState *cs, vaddr value) env->PC = value & ~(target_ulong)1; } +static vaddr tricore_cpu_get_pc(CPUState *cs) +{ + TriCoreCPU *cpu = TRICORE_CPU(cs); + CPUTriCoreState *env = &cpu->env; + + return env->PC; +} + static void tricore_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -XXX,XX +XXX,XX @@ static void tricore_cpu_class_init(ObjectClass *c, void *data) cc->dump_state = tricore_cpu_dump_state; cc->set_pc = tricore_cpu_set_pc; + cc->get_pc = tricore_cpu_get_pc; cc->sysemu_ops = &tricore_sysemu_ops; cc->tcg_ops = &tricore_tcg_ops; } diff --git a/target/xtensa/cpu.c b/target/xtensa/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/xtensa/cpu.c +++ b/target/xtensa/cpu.c @@ -XXX,XX +XXX,XX @@ static void xtensa_cpu_set_pc(CPUState *cs, vaddr value) cpu->env.pc = value; } +static vaddr xtensa_cpu_get_pc(CPUState *cs) +{ + XtensaCPU *cpu = XTENSA_CPU(cs); + + return cpu->env.pc; +} + static bool xtensa_cpu_has_work(CPUState *cs) { #ifndef CONFIG_USER_ONLY @@ -XXX,XX +XXX,XX @@ static void xtensa_cpu_class_init(ObjectClass *oc, void *data) cc->has_work = xtensa_cpu_has_work; cc->dump_state = xtensa_cpu_dump_state; cc->set_pc = xtensa_cpu_set_pc; + cc->get_pc = xtensa_cpu_get_pc; cc->gdb_read_register = xtensa_cpu_gdb_read_register; cc->gdb_write_register = xtensa_cpu_gdb_write_register; cc->gdb_stop_before_watchpoint = true; -- 2.34.1 The availability of tb->pc will shortly be conditional. Introduce accessor functions to minimize ifdefs. Pass around a known pc to places like tcg_gen_code, where the caller must already have the value. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/internal.h | 6 ++++ include/exec/exec-all.h | 6 ++++ include/tcg/tcg.h | 2 +- accel/tcg/cpu-exec.c | 46 ++++++++++++++----------- accel/tcg/translate-all.c | 37 +++++++++++--------- target/arm/cpu.c | 4 +-- target/avr/cpu.c | 2 +- target/hexagon/cpu.c | 2 +- target/hppa/cpu.c | 4 +-- target/i386/tcg/tcg-cpu.c | 2 +- target/loongarch/cpu.c | 2 +- target/microblaze/cpu.c | 2 +- target/mips/tcg/exception.c | 2 +- target/mips/tcg/sysemu/special_helper.c | 2 +- target/openrisc/cpu.c | 2 +- target/riscv/cpu.c | 4 +-- target/rx/cpu.c | 2 +- target/sh4/cpu.c | 4 +-- target/sparc/cpu.c | 2 +- target/tricore/cpu.c | 2 +- tcg/tcg.c | 8 ++--- 21 files changed, 82 insertions(+), 61 deletions(-) diff --git a/accel/tcg/internal.h b/accel/tcg/internal.h index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/internal.h +++ b/accel/tcg/internal.h @@ -XXX,XX +XXX,XX @@ G_NORETURN void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr); void page_init(void); void tb_htable_init(void); +/* Return the current PC from CPU, which may be cached in TB. */ +static inline target_ulong log_pc(CPUState *cpu, const TranslationBlock *tb) +{ + return tb_pc(tb); +} + #endif /* ACCEL_TCG_INTERNAL_H */ diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ struct TranslationBlock { uintptr_t jmp_dest[2]; }; +/* Hide the read to avoid ifdefs for TARGET_TB_PCREL. */ +static inline target_ulong tb_pc(const TranslationBlock *tb) +{ + return tb->pc; +} + /* Hide the qatomic_read to make code a little easier on the eyes */ static inline uint32_t tb_cflags(const TranslationBlock *tb) { diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -XXX,XX +XXX,XX @@ void tcg_register_thread(void); void tcg_prologue_init(TCGContext *s); void tcg_func_start(TCGContext *s); -int tcg_gen_code(TCGContext *s, TranslationBlock *tb); +int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start); void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size); diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ static bool tb_lookup_cmp(const void *p, const void *d) const TranslationBlock *tb = p; const struct tb_desc *desc = d; - if (tb->pc == desc->pc && + if (tb_pc(tb) == desc->pc && tb->page_addr[0] == desc->page_addr0 && tb->cs_base == desc->cs_base && tb->flags == desc->flags && @@ -XXX,XX +XXX,XX @@ static inline TranslationBlock *tb_lookup(CPUState *cpu, target_ulong pc, return tb; } -static inline void log_cpu_exec(target_ulong pc, CPUState *cpu, - const TranslationBlock *tb) +static void log_cpu_exec(target_ulong pc, CPUState *cpu, + const TranslationBlock *tb) { - if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_CPU | CPU_LOG_EXEC)) - && qemu_log_in_addr_range(pc)) { - + if (qemu_log_in_addr_range(pc)) { qemu_log_mask(CPU_LOG_EXEC, "Trace %d: %p [" TARGET_FMT_lx "/" TARGET_FMT_lx "/%08x/%08x] %s\n", @@ -XXX,XX +XXX,XX @@ const void *HELPER(lookup_tb_ptr)(CPUArchState *env) return tcg_code_gen_epilogue; } - log_cpu_exec(pc, cpu, tb); + if (qemu_loglevel_mask(CPU_LOG_TB_CPU | CPU_LOG_EXEC)) { + log_cpu_exec(pc, cpu, tb); + } return tb->tc.ptr; } @@ -XXX,XX +XXX,XX @@ cpu_tb_exec(CPUState *cpu, TranslationBlock *itb, int *tb_exit) TranslationBlock *last_tb; const void *tb_ptr = itb->tc.ptr; - log_cpu_exec(itb->pc, cpu, itb); + if (qemu_loglevel_mask(CPU_LOG_TB_CPU | CPU_LOG_EXEC)) { + log_cpu_exec(log_pc(cpu, itb), cpu, itb); + } qemu_thread_jit_execute(); ret = tcg_qemu_tb_exec(env, tb_ptr); @@ -XXX,XX +XXX,XX @@ cpu_tb_exec(CPUState *cpu, TranslationBlock *itb, int *tb_exit) * of the start of the TB. */ CPUClass *cc = CPU_GET_CLASS(cpu); - qemu_log_mask_and_addr(CPU_LOG_EXEC, last_tb->pc, - "Stopped execution of TB chain before %p [" - TARGET_FMT_lx "] %s\n", - last_tb->tc.ptr, last_tb->pc, - lookup_symbol(last_tb->pc)); + if (cc->tcg_ops->synchronize_from_tb) { cc->tcg_ops->synchronize_from_tb(cpu, last_tb); } else { assert(cc->set_pc); - cc->set_pc(cpu, last_tb->pc); + cc->set_pc(cpu, tb_pc(last_tb)); + } + if (qemu_loglevel_mask(CPU_LOG_EXEC)) { + target_ulong pc = log_pc(cpu, last_tb); + if (qemu_log_in_addr_range(pc)) { + qemu_log("Stopped execution of TB chain before %p [" + TARGET_FMT_lx "] %s\n", + last_tb->tc.ptr, pc, lookup_symbol(pc)); + } } } @@ -XXX,XX +XXX,XX @@ static inline void tb_add_jump(TranslationBlock *tb, int n, qemu_spin_unlock(&tb_next->jmp_lock); - qemu_log_mask_and_addr(CPU_LOG_EXEC, tb->pc, - "Linking TBs %p [" TARGET_FMT_lx - "] index %d -> %p [" TARGET_FMT_lx "]\n", - tb->tc.ptr, tb->pc, n, - tb_next->tc.ptr, tb_next->pc); + qemu_log_mask(CPU_LOG_EXEC, "Linking TBs %p index %d -> %p\n", + tb->tc.ptr, n, tb_next->tc.ptr); return; out_unlock_next: @@ -XXX,XX +XXX,XX @@ static inline bool cpu_handle_interrupt(CPUState *cpu, } static inline void cpu_loop_exec_tb(CPUState *cpu, TranslationBlock *tb, + target_ulong pc, TranslationBlock **last_tb, int *tb_exit) { int32_t insns_left; - trace_exec_tb(tb, tb->pc); + trace_exec_tb(tb, pc); tb = cpu_tb_exec(cpu, tb, tb_exit); if (*tb_exit != TB_EXIT_REQUESTED) { *last_tb = tb; @@ -XXX,XX +XXX,XX @@ int cpu_exec(CPUState *cpu) tb_add_jump(last_tb, tb_exit, tb); } - cpu_loop_exec_tb(cpu, tb, &last_tb, &tb_exit); + cpu_loop_exec_tb(cpu, tb, pc, &last_tb, &tb_exit); /* Try to align the host and virtual clocks if the guest is in advance */ diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ static int encode_search(TranslationBlock *tb, uint8_t *block) for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { if (i == 0) { - prev = (j == 0 ? tb->pc : 0); + prev = (j == 0 ? tb_pc(tb) : 0); } else { prev = tcg_ctx->gen_insn_data[i - 1][j]; } @@ -XXX,XX +XXX,XX @@ static int encode_search(TranslationBlock *tb, uint8_t *block) static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, uintptr_t searched_pc, bool reset_icount) { - target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc }; + target_ulong data[TARGET_INSN_START_WORDS] = { tb_pc(tb) }; uintptr_t host_pc = (uintptr_t)tb->tc.ptr; CPUArchState *env = cpu->env_ptr; const uint8_t *p = tb->tc.ptr + tb->tc.size; @@ -XXX,XX +XXX,XX @@ static bool tb_cmp(const void *ap, const void *bp) const TranslationBlock *a = ap; const TranslationBlock *b = bp; - return a->pc == b->pc && + return tb_pc(a) == tb_pc(b) && a->cs_base == b->cs_base && a->flags == b->flags && (tb_cflags(a) & ~CF_INVALID) == (tb_cflags(b) & ~CF_INVALID) && @@ -XXX,XX +XXX,XX @@ static void do_tb_invalidate_check(void *p, uint32_t hash, void *userp) TranslationBlock *tb = p; target_ulong addr = *(target_ulong *)userp; - if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) { + if (!(addr + TARGET_PAGE_SIZE <= tb_pc(tb) || + addr >= tb_pc(tb) + tb->size)) { printf("ERROR invalidate: address=" TARGET_FMT_lx - " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size); + " PC=%08lx size=%04x\n", addr, (long)tb_pc(tb), tb->size); } } @@ -XXX,XX +XXX,XX @@ static void do_tb_page_check(void *p, uint32_t hash, void *userp) TranslationBlock *tb = p; int flags1, flags2; - flags1 = page_get_flags(tb->pc); - flags2 = page_get_flags(tb->pc + tb->size - 1); + flags1 = page_get_flags(tb_pc(tb)); + flags2 = page_get_flags(tb_pc(tb) + tb->size - 1); if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) { printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n", - (long)tb->pc, tb->size, flags1, flags2); + (long)tb_pc(tb), tb->size, flags1, flags2); } } @@ -XXX,XX +XXX,XX @@ static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list) /* remove the TB from the hash list */ phys_pc = tb->page_addr[0]; - h = tb_hash_func(phys_pc, tb->pc, tb->flags, orig_cflags, + h = tb_hash_func(phys_pc, tb_pc(tb), tb->flags, orig_cflags, tb->trace_vcpu_dstate); if (!qht_remove(&tb_ctx.htable, tb, h)) { return; @@ -XXX,XX +XXX,XX @@ tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, } /* add in the hash table */ - h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags, + h = tb_hash_func(phys_pc, tb_pc(tb), tb->flags, tb->cflags, tb->trace_vcpu_dstate); qht_insert(&tb_ctx.htable, tb, h, &existing_tb); @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, tcg_ctx->cpu = NULL; max_insns = tb->icount; - trace_translate_block(tb, tb->pc, tb->tc.ptr); + trace_translate_block(tb, pc, tb->tc.ptr); /* generate machine code */ tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, ti = profile_getclock(); #endif - gen_code_size = tcg_gen_code(tcg_ctx, tb); + gen_code_size = tcg_gen_code(tcg_ctx, tb, pc); if (unlikely(gen_code_size < 0)) { error_return: switch (gen_code_size) { @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, #ifdef DEBUG_DISAS if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && - qemu_log_in_addr_range(tb->pc)) { + qemu_log_in_addr_range(pc)) { FILE *logfile = qemu_log_trylock(); if (logfile) { int code_size, data_size; @@ -XXX,XX +XXX,XX @@ void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) */ cpu->cflags_next_tb = curr_cflags(cpu) | CF_MEMI_ONLY | CF_LAST_IO | n; - qemu_log_mask_and_addr(CPU_LOG_EXEC, tb->pc, - "cpu_io_recompile: rewound execution of TB to " - TARGET_FMT_lx "\n", tb->pc); + if (qemu_loglevel_mask(CPU_LOG_EXEC)) { + target_ulong pc = log_pc(cpu, tb); + if (qemu_log_in_addr_range(pc)) { + qemu_log("cpu_io_recompile: rewound execution of TB to " + TARGET_FMT_lx "\n", pc); + } + } cpu_loop_exit_noexc(cpu); } diff --git a/target/arm/cpu.c b/target/arm/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -XXX,XX +XXX,XX @@ void arm_cpu_synchronize_from_tb(CPUState *cs, * never possible for an AArch64 TB to chain to an AArch32 TB. */ if (is_a64(env)) { - env->pc = tb->pc; + env->pc = tb_pc(tb); } else { - env->regs[15] = tb->pc; + env->regs[15] = tb_pc(tb); } } #endif /* CONFIG_TCG */ diff --git a/target/avr/cpu.c b/target/avr/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/cpu.c +++ b/target/avr/cpu.c @@ -XXX,XX +XXX,XX @@ static void avr_cpu_synchronize_from_tb(CPUState *cs, AVRCPU *cpu = AVR_CPU(cs); CPUAVRState *env = &cpu->env; - env->pc_w = tb->pc / 2; /* internally PC points to words */ + env->pc_w = tb_pc(tb) / 2; /* internally PC points to words */ } static void avr_cpu_reset(DeviceState *ds) diff --git a/target/hexagon/cpu.c b/target/hexagon/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/cpu.c +++ b/target/hexagon/cpu.c @@ -XXX,XX +XXX,XX @@ static void hexagon_cpu_synchronize_from_tb(CPUState *cs, { HexagonCPU *cpu = HEXAGON_CPU(cs); CPUHexagonState *env = &cpu->env; - env->gpr[HEX_REG_PC] = tb->pc; + env->gpr[HEX_REG_PC] = tb_pc(tb); } static bool hexagon_cpu_has_work(CPUState *cs) diff --git a/target/hppa/cpu.c b/target/hppa/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hppa/cpu.c +++ b/target/hppa/cpu.c @@ -XXX,XX +XXX,XX @@ static void hppa_cpu_synchronize_from_tb(CPUState *cs, HPPACPU *cpu = HPPA_CPU(cs); #ifdef CONFIG_USER_ONLY - cpu->env.iaoq_f = tb->pc; + cpu->env.iaoq_f = tb_pc(tb); cpu->env.iaoq_b = tb->cs_base; #else /* Recover the IAOQ values from the GVA + PRIV. */ @@ -XXX,XX +XXX,XX @@ static void hppa_cpu_synchronize_from_tb(CPUState *cs, int32_t diff = cs_base; cpu->env.iasq_f = iasq_f; - cpu->env.iaoq_f = (tb->pc & ~iasq_f) + priv; + cpu->env.iaoq_f = (tb_pc(tb) & ~iasq_f) + priv; if (diff) { cpu->env.iaoq_b = cpu->env.iaoq_f + diff; } diff --git a/target/i386/tcg/tcg-cpu.c b/target/i386/tcg/tcg-cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/tcg/tcg-cpu.c +++ b/target/i386/tcg/tcg-cpu.c @@ -XXX,XX +XXX,XX @@ static void x86_cpu_synchronize_from_tb(CPUState *cs, { X86CPU *cpu = X86_CPU(cs); - cpu->env.eip = tb->pc - tb->cs_base; + cpu->env.eip = tb_pc(tb) - tb->cs_base; } #ifndef CONFIG_USER_ONLY diff --git a/target/loongarch/cpu.c b/target/loongarch/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/loongarch/cpu.c +++ b/target/loongarch/cpu.c @@ -XXX,XX +XXX,XX @@ static void loongarch_cpu_synchronize_from_tb(CPUState *cs, LoongArchCPU *cpu = LOONGARCH_CPU(cs); CPULoongArchState *env = &cpu->env; - env->pc = tb->pc; + env->pc = tb_pc(tb); } #endif /* CONFIG_TCG */ diff --git a/target/microblaze/cpu.c b/target/microblaze/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/microblaze/cpu.c +++ b/target/microblaze/cpu.c @@ -XXX,XX +XXX,XX @@ static void mb_cpu_synchronize_from_tb(CPUState *cs, { MicroBlazeCPU *cpu = MICROBLAZE_CPU(cs); - cpu->env.pc = tb->pc; + cpu->env.pc = tb_pc(tb); cpu->env.iflags = tb->flags & IFLAGS_TB_MASK; } diff --git a/target/mips/tcg/exception.c b/target/mips/tcg/exception.c index XXXXXXX..XXXXXXX 100644 --- a/target/mips/tcg/exception.c +++ b/target/mips/tcg/exception.c @@ -XXX,XX +XXX,XX @@ void mips_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) MIPSCPU *cpu = MIPS_CPU(cs); CPUMIPSState *env = &cpu->env; - env->active_tc.PC = tb->pc; + env->active_tc.PC = tb_pc(tb); env->hflags &= ~MIPS_HFLAG_BMASK; env->hflags |= tb->flags & MIPS_HFLAG_BMASK; } diff --git a/target/mips/tcg/sysemu/special_helper.c b/target/mips/tcg/sysemu/special_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/mips/tcg/sysemu/special_helper.c +++ b/target/mips/tcg/sysemu/special_helper.c @@ -XXX,XX +XXX,XX @@ bool mips_io_recompile_replay_branch(CPUState *cs, const TranslationBlock *tb) CPUMIPSState *env = &cpu->env; if ((env->hflags & MIPS_HFLAG_BMASK) != 0 - && env->active_tc.PC != tb->pc) { + && env->active_tc.PC != tb_pc(tb)) { env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4); env->hflags &= ~MIPS_HFLAG_BMASK; return true; diff --git a/target/openrisc/cpu.c b/target/openrisc/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/openrisc/cpu.c +++ b/target/openrisc/cpu.c @@ -XXX,XX +XXX,XX @@ static void openrisc_cpu_synchronize_from_tb(CPUState *cs, { OpenRISCCPU *cpu = OPENRISC_CPU(cs); - cpu->env.pc = tb->pc; + cpu->env.pc = tb_pc(tb); } diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/cpu.c +++ b/target/riscv/cpu.c @@ -XXX,XX +XXX,XX @@ static void riscv_cpu_synchronize_from_tb(CPUState *cs, RISCVMXL xl = FIELD_EX32(tb->flags, TB_FLAGS, XL); if (xl == MXL_RV32) { - env->pc = (int32_t)tb->pc; + env->pc = (int32_t)tb_pc(tb); } else { - env->pc = tb->pc; + env->pc = tb_pc(tb); } } diff --git a/target/rx/cpu.c b/target/rx/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/rx/cpu.c +++ b/target/rx/cpu.c @@ -XXX,XX +XXX,XX @@ static void rx_cpu_synchronize_from_tb(CPUState *cs, { RXCPU *cpu = RX_CPU(cs); - cpu->env.pc = tb->pc; + cpu->env.pc = tb_pc(tb); } static bool rx_cpu_has_work(CPUState *cs) diff --git a/target/sh4/cpu.c b/target/sh4/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/sh4/cpu.c +++ b/target/sh4/cpu.c @@ -XXX,XX +XXX,XX @@ static void superh_cpu_synchronize_from_tb(CPUState *cs, { SuperHCPU *cpu = SUPERH_CPU(cs); - cpu->env.pc = tb->pc; + cpu->env.pc = tb_pc(tb); cpu->env.flags = tb->flags & TB_FLAG_ENVFLAGS_MASK; } @@ -XXX,XX +XXX,XX @@ static bool superh_io_recompile_replay_branch(CPUState *cs, CPUSH4State *env = &cpu->env; if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0 - && env->pc != tb->pc) { + && env->pc != tb_pc(tb)) { env->pc -= 2; env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL); return true; diff --git a/target/sparc/cpu.c b/target/sparc/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/cpu.c +++ b/target/sparc/cpu.c @@ -XXX,XX +XXX,XX @@ static void sparc_cpu_synchronize_from_tb(CPUState *cs, { SPARCCPU *cpu = SPARC_CPU(cs); - cpu->env.pc = tb->pc; + cpu->env.pc = tb_pc(tb); cpu->env.npc = tb->cs_base; } diff --git a/target/tricore/cpu.c b/target/tricore/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/tricore/cpu.c +++ b/target/tricore/cpu.c @@ -XXX,XX +XXX,XX @@ static void tricore_cpu_synchronize_from_tb(CPUState *cs, TriCoreCPU *cpu = TRICORE_CPU(cs); CPUTriCoreState *env = &cpu->env; - env->PC = tb->pc; + env->PC = tb_pc(tb); } static void tricore_cpu_reset(DeviceState *dev) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ int64_t tcg_cpu_exec_time(void) #endif -int tcg_gen_code(TCGContext *s, TranslationBlock *tb) +int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) { #ifdef CONFIG_PROFILER TCGProfile *prof = &s->prof; @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb) #ifdef DEBUG_DISAS if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) - && qemu_log_in_addr_range(tb->pc))) { + && qemu_log_in_addr_range(pc_start))) { FILE *logfile = qemu_log_trylock(); if (logfile) { fprintf(logfile, "OP:\n"); @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb) if (s->nb_indirects > 0) { #ifdef DEBUG_DISAS if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND) - && qemu_log_in_addr_range(tb->pc))) { + && qemu_log_in_addr_range(pc_start))) { FILE *logfile = qemu_log_trylock(); if (logfile) { fprintf(logfile, "OP before indirect lowering:\n"); @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb) #ifdef DEBUG_DISAS if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) - && qemu_log_in_addr_range(tb->pc))) { + && qemu_log_in_addr_range(pc_start))) { FILE *logfile = qemu_log_trylock(); if (logfile) { fprintf(logfile, "OP after optimization and liveness analysis:\n"); -- 2.34.1 Prepare for targets to be able to produce TBs that can run in more than one virtual context. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/internal.h | 4 +++ accel/tcg/tb-jmp-cache.h | 41 +++++++++++++++++++++++++ include/exec/cpu-defs.h | 3 ++ include/exec/exec-all.h | 32 ++++++++++++++++++-- accel/tcg/cpu-exec.c | 16 ++++++---- accel/tcg/translate-all.c | 64 ++++++++++++++++++++++++++------------- 6 files changed, 131 insertions(+), 29 deletions(-) diff --git a/accel/tcg/internal.h b/accel/tcg/internal.h index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/internal.h +++ b/accel/tcg/internal.h @@ -XXX,XX +XXX,XX @@ void tb_htable_init(void); /* Return the current PC from CPU, which may be cached in TB. */ static inline target_ulong log_pc(CPUState *cpu, const TranslationBlock *tb) { +#if TARGET_TB_PCREL + return cpu->cc->get_pc(cpu); +#else return tb_pc(tb); +#endif } #endif /* ACCEL_TCG_INTERNAL_H */ diff --git a/accel/tcg/tb-jmp-cache.h b/accel/tcg/tb-jmp-cache.h index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/tb-jmp-cache.h +++ b/accel/tcg/tb-jmp-cache.h @@ -XXX,XX +XXX,XX @@ /* * Accessed in parallel; all accesses to 'tb' must be atomic. + * For TARGET_TB_PCREL, accesses to 'pc' must be protected by + * a load_acquire/store_release to 'tb'. */ struct CPUJumpCache { struct { TranslationBlock *tb; +#if TARGET_TB_PCREL + target_ulong pc; +#endif } array[TB_JMP_CACHE_SIZE]; }; +static inline TranslationBlock * +tb_jmp_cache_get_tb(CPUJumpCache *jc, uint32_t hash) +{ +#if TARGET_TB_PCREL + /* Use acquire to ensure current load of pc from jc. */ + return qatomic_load_acquire(&jc->array[hash].tb); +#else + /* Use rcu_read to ensure current load of pc from *tb. */ + return qatomic_rcu_read(&jc->array[hash].tb); +#endif +} + +static inline target_ulong +tb_jmp_cache_get_pc(CPUJumpCache *jc, uint32_t hash, TranslationBlock *tb) +{ +#if TARGET_TB_PCREL + return jc->array[hash].pc; +#else + return tb_pc(tb); +#endif +} + +static inline void +tb_jmp_cache_set(CPUJumpCache *jc, uint32_t hash, + TranslationBlock *tb, target_ulong pc) +{ +#if TARGET_TB_PCREL + jc->array[hash].pc = pc; + /* Use store_release on tb to ensure pc is written first. */ + qatomic_store_release(&jc->array[hash].tb, tb); +#else + /* Use the pc value already stored in tb->pc. */ + qatomic_set(&jc->array[hash].tb, tb); +#endif +} + #endif /* ACCEL_TCG_TB_JMP_CACHE_H */ diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/cpu-defs.h +++ b/include/exec/cpu-defs.h @@ -XXX,XX +XXX,XX @@ # error TARGET_PAGE_BITS must be defined in cpu-param.h # endif #endif +#ifndef TARGET_TB_PCREL +# define TARGET_TB_PCREL 0 +#endif #define TARGET_LONG_SIZE (TARGET_LONG_BITS / 8) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ struct tb_tc { }; struct TranslationBlock { - target_ulong pc; /* simulated PC corresponding to this block (EIP + CS base) */ - target_ulong cs_base; /* CS base for this block */ +#if !TARGET_TB_PCREL + /* + * Guest PC corresponding to this block. This must be the true + * virtual address. Therefore e.g. x86 stores EIP + CS_BASE, and + * targets like Arm, MIPS, HP-PA, which reuse low bits for ISA or + * privilege, must store those bits elsewhere. + * + * If TARGET_TB_PCREL, the opcodes for the TranslationBlock are + * written such that the TB is associated only with the physical + * page and may be run in any virtual address context. In this case, + * PC must always be taken from ENV in a target-specific manner. + * Unwind information is taken as offsets from the page, to be + * deposited into the "current" PC. + */ + target_ulong pc; +#endif + + /* + * Target-specific data associated with the TranslationBlock, e.g.: + * x86: the original user, the Code Segment virtual base, + * arm: an extension of tb->flags, + * s390x: instruction data for EXECUTE, + * sparc: the next pc of the instruction queue (for delay slots). + */ + target_ulong cs_base; + uint32_t flags; /* flags defining in which context the code was generated */ uint32_t cflags; /* compile flags */ @@ -XXX,XX +XXX,XX @@ struct TranslationBlock { /* Hide the read to avoid ifdefs for TARGET_TB_PCREL. */ static inline target_ulong tb_pc(const TranslationBlock *tb) { +#if TARGET_TB_PCREL + qemu_build_not_reached(); +#else return tb->pc; +#endif } /* Hide the qatomic_read to make code a little easier on the eyes */ diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ static bool tb_lookup_cmp(const void *p, const void *d) const TranslationBlock *tb = p; const struct tb_desc *desc = d; - if (tb_pc(tb) == desc->pc && + if ((TARGET_TB_PCREL || tb_pc(tb) == desc->pc) && tb->page_addr[0] == desc->page_addr0 && tb->cs_base == desc->cs_base && tb->flags == desc->flags && @@ -XXX,XX +XXX,XX @@ static TranslationBlock *tb_htable_lookup(CPUState *cpu, target_ulong pc, return NULL; } desc.page_addr0 = phys_pc; - h = tb_hash_func(phys_pc, pc, flags, cflags, *cpu->trace_dstate); + h = tb_hash_func(phys_pc, (TARGET_TB_PCREL ? 0 : pc), + flags, cflags, *cpu->trace_dstate); return qht_lookup_custom(&tb_ctx.htable, &desc, h, tb_lookup_cmp); } @@ -XXX,XX +XXX,XX @@ static inline TranslationBlock *tb_lookup(CPUState *cpu, target_ulong pc, uint32_t flags, uint32_t cflags) { TranslationBlock *tb; + CPUJumpCache *jc; uint32_t hash; /* we should never be trying to look up an INVALID tb */ tcg_debug_assert(!(cflags & CF_INVALID)); hash = tb_jmp_cache_hash_func(pc); - tb = qatomic_rcu_read(&cpu->tb_jmp_cache->array[hash].tb); + jc = cpu->tb_jmp_cache; + tb = tb_jmp_cache_get_tb(jc, hash); if (likely(tb && - tb->pc == pc && + tb_jmp_cache_get_pc(jc, hash, tb) == pc && tb->cs_base == cs_base && tb->flags == flags && tb->trace_vcpu_dstate == *cpu->trace_dstate && @@ -XXX,XX +XXX,XX @@ static inline TranslationBlock *tb_lookup(CPUState *cpu, target_ulong pc, if (tb == NULL) { return NULL; } - qatomic_set(&cpu->tb_jmp_cache->array[hash].tb, tb); + tb_jmp_cache_set(jc, hash, tb, pc); return tb; } @@ -XXX,XX +XXX,XX @@ cpu_tb_exec(CPUState *cpu, TranslationBlock *itb, int *tb_exit) if (cc->tcg_ops->synchronize_from_tb) { cc->tcg_ops->synchronize_from_tb(cpu, last_tb); } else { + assert(!TARGET_TB_PCREL); assert(cc->set_pc); cc->set_pc(cpu, tb_pc(last_tb)); } @@ -XXX,XX +XXX,XX @@ int cpu_exec(CPUState *cpu) * for the fast lookup */ h = tb_jmp_cache_hash_func(pc); - qatomic_set(&cpu->tb_jmp_cache->array[h].tb, tb); + tb_jmp_cache_set(cpu->tb_jmp_cache, h, tb, pc); } #ifndef CONFIG_USER_ONLY diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ static int encode_search(TranslationBlock *tb, uint8_t *block) for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { if (i == 0) { - prev = (j == 0 ? tb_pc(tb) : 0); + prev = (!TARGET_TB_PCREL && j == 0 ? tb_pc(tb) : 0); } else { prev = tcg_ctx->gen_insn_data[i - 1][j]; } @@ -XXX,XX +XXX,XX @@ static int encode_search(TranslationBlock *tb, uint8_t *block) static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, uintptr_t searched_pc, bool reset_icount) { - target_ulong data[TARGET_INSN_START_WORDS] = { tb_pc(tb) }; + target_ulong data[TARGET_INSN_START_WORDS]; uintptr_t host_pc = (uintptr_t)tb->tc.ptr; CPUArchState *env = cpu->env_ptr; const uint8_t *p = tb->tc.ptr + tb->tc.size; @@ -XXX,XX +XXX,XX @@ static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, return -1; } + memset(data, 0, sizeof(data)); + if (!TARGET_TB_PCREL) { + data[0] = tb_pc(tb); + } + /* Reconstruct the stored insn data while looking for the point at which the end of the insn exceeds the searched_pc. */ for (i = 0; i < num_insns; ++i) { @@ -XXX,XX +XXX,XX @@ static bool tb_cmp(const void *ap, const void *bp) const TranslationBlock *a = ap; const TranslationBlock *b = bp; - return tb_pc(a) == tb_pc(b) && - a->cs_base == b->cs_base && - a->flags == b->flags && - (tb_cflags(a) & ~CF_INVALID) == (tb_cflags(b) & ~CF_INVALID) && - a->trace_vcpu_dstate == b->trace_vcpu_dstate && - a->page_addr[0] == b->page_addr[0] && - a->page_addr[1] == b->page_addr[1]; + return ((TARGET_TB_PCREL || tb_pc(a) == tb_pc(b)) && + a->cs_base == b->cs_base && + a->flags == b->flags && + (tb_cflags(a) & ~CF_INVALID) == (tb_cflags(b) & ~CF_INVALID) && + a->trace_vcpu_dstate == b->trace_vcpu_dstate && + a->page_addr[0] == b->page_addr[0] && + a->page_addr[1] == b->page_addr[1]); } void tb_htable_init(void) @@ -XXX,XX +XXX,XX @@ static inline void tb_jmp_unlink(TranslationBlock *dest) qemu_spin_unlock(&dest->jmp_lock); } +static void tb_jmp_cache_inval_tb(TranslationBlock *tb) +{ + CPUState *cpu; + + if (TARGET_TB_PCREL) { + /* A TB may be at any virtual address */ + CPU_FOREACH(cpu) { + tcg_flush_jmp_cache(cpu); + } + } else { + uint32_t h = tb_jmp_cache_hash_func(tb_pc(tb)); + + CPU_FOREACH(cpu) { + CPUJumpCache *jc = cpu->tb_jmp_cache; + + if (qatomic_read(&jc->array[h].tb) == tb) { + qatomic_set(&jc->array[h].tb, NULL); + } + } + } +} + /* * In user-mode, call with mmap_lock held. * In !user-mode, if @rm_from_page_list is set, call with the TB's pages' @@ -XXX,XX +XXX,XX @@ static inline void tb_jmp_unlink(TranslationBlock *dest) */ static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list) { - CPUState *cpu; PageDesc *p; uint32_t h; tb_page_addr_t phys_pc; @@ -XXX,XX +XXX,XX @@ static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list) /* remove the TB from the hash list */ phys_pc = tb->page_addr[0]; - h = tb_hash_func(phys_pc, tb_pc(tb), tb->flags, orig_cflags, - tb->trace_vcpu_dstate); + h = tb_hash_func(phys_pc, (TARGET_TB_PCREL ? 0 : tb_pc(tb)), + tb->flags, orig_cflags, tb->trace_vcpu_dstate); if (!qht_remove(&tb_ctx.htable, tb, h)) { return; } @@ -XXX,XX +XXX,XX @@ static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list) } /* remove the TB from the hash list */ - h = tb_jmp_cache_hash_func(tb->pc); - CPU_FOREACH(cpu) { - CPUJumpCache *jc = cpu->tb_jmp_cache; - if (qatomic_read(&jc->array[h].tb) == tb) { - qatomic_set(&jc->array[h].tb, NULL); - } - } + tb_jmp_cache_inval_tb(tb); /* suppress this TB from the two jump lists */ tb_remove_from_jmp_list(tb, 0); @@ -XXX,XX +XXX,XX @@ tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, } /* add in the hash table */ - h = tb_hash_func(phys_pc, tb_pc(tb), tb->flags, tb->cflags, - tb->trace_vcpu_dstate); + h = tb_hash_func(phys_pc, (TARGET_TB_PCREL ? 0 : tb_pc(tb)), + tb->flags, tb->cflags, tb->trace_vcpu_dstate); qht_insert(&tb_ctx.htable, tb, h, &existing_tb); /* remove TB from the page(s) if we couldn't insert it */ @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, gen_code_buf = tcg_ctx->code_gen_ptr; tb->tc.ptr = tcg_splitwx_to_rx(gen_code_buf); +#if !TARGET_TB_PCREL tb->pc = pc; +#endif tb->cs_base = cs_base; tb->flags = flags; tb->cflags = cflags; -- 2.34.1 From: Leandro Lupori <leandro.lupori@eldorado.org.br> PowerPC64 processors handle direct branches better than indirect ones, resulting in less stalled cycles and branch misses. However, PPC's tb_target_set_jmp_target() was only using direct branches for 16-bit jumps, while PowerPC64's unconditional branch instructions are able to handle displacements of up to 26 bits. To take advantage of this, now jumps whose displacements fit in between 17 and 26 bits are also converted to direct branches. Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Leandro Lupori <leandro.lupori@eldorado.org.br> [rth: Expanded some commentary.] Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/ppc/tcg-target.c.inc | 119 +++++++++++++++++++++++++++++---------- 1 file changed, 88 insertions(+), 31 deletions(-) diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_mb(TCGContext *s, TCGArg a0) tcg_out32(s, insn); } +static inline uint64_t make_pair(tcg_insn_unit i1, tcg_insn_unit i2) +{ + if (HOST_BIG_ENDIAN) { + return (uint64_t)i1 << 32 | i2; + } + return (uint64_t)i2 << 32 | i1; +} + +static inline void ppc64_replace2(uintptr_t rx, uintptr_t rw, + tcg_insn_unit i0, tcg_insn_unit i1) +{ +#if TCG_TARGET_REG_BITS == 64 + qatomic_set((uint64_t *)rw, make_pair(i0, i1)); + flush_idcache_range(rx, rw, 8); +#else + qemu_build_not_reached(); +#endif +} + +static inline void ppc64_replace4(uintptr_t rx, uintptr_t rw, + tcg_insn_unit i0, tcg_insn_unit i1, + tcg_insn_unit i2, tcg_insn_unit i3) +{ + uint64_t p[2]; + + p[!HOST_BIG_ENDIAN] = make_pair(i0, i1); + p[HOST_BIG_ENDIAN] = make_pair(i2, i3); + + /* + * There's no convenient way to get the compiler to allocate a pair + * of registers at an even index, so copy into r6/r7 and clobber. + */ + asm("mr %%r6, %1\n\t" + "mr %%r7, %2\n\t" + "stq %%r6, %0" + : "=Q"(*(__int128 *)rw) : "r"(p[0]), "r"(p[1]) : "r6", "r7"); + flush_idcache_range(rx, rw, 16); +} + void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, uintptr_t jmp_rw, uintptr_t addr) { - if (TCG_TARGET_REG_BITS == 64) { - tcg_insn_unit i1, i2; - intptr_t tb_diff = addr - tc_ptr; - intptr_t br_diff = addr - (jmp_rx + 4); - uint64_t pair; + tcg_insn_unit i0, i1, i2, i3; + intptr_t tb_diff = addr - tc_ptr; + intptr_t br_diff = addr - (jmp_rx + 4); + intptr_t lo, hi; - /* This does not exercise the range of the branch, but we do - still need to be able to load the new value of TCG_REG_TB. - But this does still happen quite often. */ - if (tb_diff == (int16_t)tb_diff) { - i1 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, tb_diff); - i2 = B | (br_diff & 0x3fffffc); - } else { - intptr_t lo = (int16_t)tb_diff; - intptr_t hi = (int32_t)(tb_diff - lo); - assert(tb_diff == hi + lo); - i1 = ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, hi >> 16); - i2 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, lo); - } -#if HOST_BIG_ENDIAN - pair = (uint64_t)i1 << 32 | i2; -#else - pair = (uint64_t)i2 << 32 | i1; -#endif - - /* As per the enclosing if, this is ppc64. Avoid the _Static_assert - within qatomic_set that would fail to build a ppc32 host. */ - qatomic_set__nocheck((uint64_t *)jmp_rw, pair); - flush_idcache_range(jmp_rx, jmp_rw, 8); - } else { + if (TCG_TARGET_REG_BITS == 32) { intptr_t diff = addr - jmp_rx; tcg_debug_assert(in_range_b(diff)); qatomic_set((uint32_t *)jmp_rw, B | (diff & 0x3fffffc)); flush_idcache_range(jmp_rx, jmp_rw, 4); + return; } + + /* + * For 16-bit displacements, we can use a single add + branch. + * This happens quite often. + */ + if (tb_diff == (int16_t)tb_diff) { + i0 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, tb_diff); + i1 = B | (br_diff & 0x3fffffc); + ppc64_replace2(jmp_rx, jmp_rw, i0, i1); + return; + } + + lo = (int16_t)tb_diff; + hi = (int32_t)(tb_diff - lo); + assert(tb_diff == hi + lo); + i0 = ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, hi >> 16); + i1 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, lo); + + /* + * Without stq from 2.07, we can only update two insns, + * and those must be the ones that load the target address. + */ + if (!have_isa_2_07) { + ppc64_replace2(jmp_rx, jmp_rw, i0, i1); + return; + } + + /* + * For 26-bit displacements, we can use a direct branch. + * Otherwise we still need the indirect branch, which we + * must restore after a potential direct branch write. + */ + br_diff -= 4; + if (in_range_b(br_diff)) { + i2 = B | (br_diff & 0x3fffffc); + i3 = NOP; + } else { + i2 = MTSPR | RS(TCG_REG_TB) | CTR; + i3 = BCCTR | BO_ALWAYS; + } + ppc64_replace4(jmp_rx, jmp_rw, i0, i1, i2, i3); } static void tcg_out_call_int(TCGContext *s, int lk, @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, if (s->tb_jmp_insn_offset) { /* Direct jump. */ if (TCG_TARGET_REG_BITS == 64) { - /* Ensure the next insns are 8-byte aligned. */ - if ((uintptr_t)s->code_ptr & 7) { + /* Ensure the next insns are 8 or 16-byte aligned. */ + while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) { tcg_out32(s, NOP); } s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); -- 2.34.1 The value previously chosen overlaps GUSA_MASK. Rename all DELAY_SLOT_* and GUSA_* defines to emphasize that they are included in TB_FLAGs. Add aliases for the FPSCR and SR bits that are included in TB_FLAGS, so that we don't accidentally reassign those bits. Fixes: 4da06fb3062 ("target/sh4: Implement prctl_unalign_sigbus") Resolves: https://gitlab.com/qemu-project/qemu/-/issues/856 Reviewed-by: Yoshinori Sato <ysato@users.sourceforge.jp> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/sh4/cpu.h | 56 +++++++++++++------------ linux-user/sh4/signal.c | 6 +-- target/sh4/cpu.c | 6 +-- target/sh4/helper.c | 6 +-- target/sh4/translate.c | 90 ++++++++++++++++++++++------------------- 5 files changed, 88 insertions(+), 76 deletions(-) diff --git a/target/sh4/cpu.h b/target/sh4/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/sh4/cpu.h +++ b/target/sh4/cpu.h @@ -XXX,XX +XXX,XX @@ #define FPSCR_RM_NEAREST (0 << 0) #define FPSCR_RM_ZERO (1 << 0) -#define DELAY_SLOT_MASK 0x7 -#define DELAY_SLOT (1 << 0) -#define DELAY_SLOT_CONDITIONAL (1 << 1) -#define DELAY_SLOT_RTE (1 << 2) +#define TB_FLAG_DELAY_SLOT (1 << 0) +#define TB_FLAG_DELAY_SLOT_COND (1 << 1) +#define TB_FLAG_DELAY_SLOT_RTE (1 << 2) +#define TB_FLAG_PENDING_MOVCA (1 << 3) +#define TB_FLAG_GUSA_SHIFT 4 /* [11:4] */ +#define TB_FLAG_GUSA_EXCLUSIVE (1 << 12) +#define TB_FLAG_UNALIGN (1 << 13) +#define TB_FLAG_SR_FD (1 << SR_FD) /* 15 */ +#define TB_FLAG_FPSCR_PR FPSCR_PR /* 19 */ +#define TB_FLAG_FPSCR_SZ FPSCR_SZ /* 20 */ +#define TB_FLAG_FPSCR_FR FPSCR_FR /* 21 */ +#define TB_FLAG_SR_RB (1 << SR_RB) /* 29 */ +#define TB_FLAG_SR_MD (1 << SR_MD) /* 30 */ -#define TB_FLAG_PENDING_MOVCA (1 << 3) -#define TB_FLAG_UNALIGN (1 << 4) - -#define GUSA_SHIFT 4 -#ifdef CONFIG_USER_ONLY -#define GUSA_EXCLUSIVE (1 << 12) -#define GUSA_MASK ((0xff << GUSA_SHIFT) | GUSA_EXCLUSIVE) -#else -/* Provide dummy versions of the above to allow tests against tbflags - to be elided while avoiding ifdefs. */ -#define GUSA_EXCLUSIVE 0 -#define GUSA_MASK 0 -#endif - -#define TB_FLAG_ENVFLAGS_MASK (DELAY_SLOT_MASK | GUSA_MASK) +#define TB_FLAG_DELAY_SLOT_MASK (TB_FLAG_DELAY_SLOT | \ + TB_FLAG_DELAY_SLOT_COND | \ + TB_FLAG_DELAY_SLOT_RTE) +#define TB_FLAG_GUSA_MASK ((0xff << TB_FLAG_GUSA_SHIFT) | \ + TB_FLAG_GUSA_EXCLUSIVE) +#define TB_FLAG_FPSCR_MASK (TB_FLAG_FPSCR_PR | \ + TB_FLAG_FPSCR_SZ | \ + TB_FLAG_FPSCR_FR) +#define TB_FLAG_SR_MASK (TB_FLAG_SR_FD | \ + TB_FLAG_SR_RB | \ + TB_FLAG_SR_MD) +#define TB_FLAG_ENVFLAGS_MASK (TB_FLAG_DELAY_SLOT_MASK | \ + TB_FLAG_GUSA_MASK) typedef struct tlb_t { uint32_t vpn; /* virtual page number */ @@ -XXX,XX +XXX,XX @@ static inline int cpu_mmu_index (CPUSH4State *env, bool ifetch) { /* The instruction in a RTE delay slot is fetched in privileged mode, but executed in user mode. */ - if (ifetch && (env->flags & DELAY_SLOT_RTE)) { + if (ifetch && (env->flags & TB_FLAG_DELAY_SLOT_RTE)) { return 0; } else { return (env->sr & (1u << SR_MD)) == 0 ? 1 : 0; @@ -XXX,XX +XXX,XX @@ static inline void cpu_get_tb_cpu_state(CPUSH4State *env, target_ulong *pc, { *pc = env->pc; /* For a gUSA region, notice the end of the region. */ - *cs_base = env->flags & GUSA_MASK ? env->gregs[0] : 0; - *flags = env->flags /* TB_FLAG_ENVFLAGS_MASK: bits 0-2, 4-12 */ - | (env->fpscr & (FPSCR_FR | FPSCR_SZ | FPSCR_PR)) /* Bits 19-21 */ - | (env->sr & ((1u << SR_MD) | (1u << SR_RB))) /* Bits 29-30 */ - | (env->sr & (1u << SR_FD)) /* Bit 15 */ + *cs_base = env->flags & TB_FLAG_GUSA_MASK ? env->gregs[0] : 0; + *flags = env->flags + | (env->fpscr & TB_FLAG_FPSCR_MASK) + | (env->sr & TB_FLAG_SR_MASK) | (env->movcal_backup ? TB_FLAG_PENDING_MOVCA : 0); /* Bit 3 */ #ifdef CONFIG_USER_ONLY *flags |= TB_FLAG_UNALIGN * !env_cpu(env)->prctl_unalign_sigbus; diff --git a/linux-user/sh4/signal.c b/linux-user/sh4/signal.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/sh4/signal.c +++ b/linux-user/sh4/signal.c @@ -XXX,XX +XXX,XX @@ static void restore_sigcontext(CPUSH4State *regs, struct target_sigcontext *sc) __get_user(regs->fpul, &sc->sc_fpul); regs->tra = -1; /* disable syscall checks */ - regs->flags &= ~(DELAY_SLOT_MASK | GUSA_MASK); + regs->flags = 0; } void setup_frame(int sig, struct target_sigaction *ka, @@ -XXX,XX +XXX,XX @@ void setup_frame(int sig, struct target_sigaction *ka, regs->gregs[5] = 0; regs->gregs[6] = frame_addr += offsetof(typeof(*frame), sc); regs->pc = (unsigned long) ka->_sa_handler; - regs->flags &= ~(DELAY_SLOT_MASK | GUSA_MASK); + regs->flags &= ~(TB_FLAG_DELAY_SLOT_MASK | TB_FLAG_GUSA_MASK); unlock_user_struct(frame, frame_addr, 1); return; @@ -XXX,XX +XXX,XX @@ void setup_rt_frame(int sig, struct target_sigaction *ka, regs->gregs[5] = frame_addr + offsetof(typeof(*frame), info); regs->gregs[6] = frame_addr + offsetof(typeof(*frame), uc); regs->pc = (unsigned long) ka->_sa_handler; - regs->flags &= ~(DELAY_SLOT_MASK | GUSA_MASK); + regs->flags &= ~(TB_FLAG_DELAY_SLOT_MASK | TB_FLAG_GUSA_MASK); unlock_user_struct(frame, frame_addr, 1); return; diff --git a/target/sh4/cpu.c b/target/sh4/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/sh4/cpu.c +++ b/target/sh4/cpu.c @@ -XXX,XX +XXX,XX @@ static void superh_cpu_synchronize_from_tb(CPUState *cs, SuperHCPU *cpu = SUPERH_CPU(cs); cpu->env.pc = tb_pc(tb); - cpu->env.flags = tb->flags & TB_FLAG_ENVFLAGS_MASK; + cpu->env.flags = tb->flags; } #ifndef CONFIG_USER_ONLY @@ -XXX,XX +XXX,XX @@ static bool superh_io_recompile_replay_branch(CPUState *cs, SuperHCPU *cpu = SUPERH_CPU(cs); CPUSH4State *env = &cpu->env; - if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0 + if ((env->flags & (TB_FLAG_DELAY_SLOT | TB_FLAG_DELAY_SLOT_COND)) && env->pc != tb_pc(tb)) { env->pc -= 2; - env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL); + env->flags &= ~(TB_FLAG_DELAY_SLOT | TB_FLAG_DELAY_SLOT_COND); return true; } return false; diff --git a/target/sh4/helper.c b/target/sh4/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/sh4/helper.c +++ b/target/sh4/helper.c @@ -XXX,XX +XXX,XX @@ void superh_cpu_do_interrupt(CPUState *cs) env->sr |= (1u << SR_BL) | (1u << SR_MD) | (1u << SR_RB); env->lock_addr = -1; - if (env->flags & DELAY_SLOT_MASK) { + if (env->flags & TB_FLAG_DELAY_SLOT_MASK) { /* Branch instruction should be executed again before delay slot. */ env->spc -= 2; /* Clear flags for exception/interrupt routine. */ - env->flags &= ~DELAY_SLOT_MASK; + env->flags &= ~TB_FLAG_DELAY_SLOT_MASK; } if (do_exp) { @@ -XXX,XX +XXX,XX @@ bool superh_cpu_exec_interrupt(CPUState *cs, int interrupt_request) CPUSH4State *env = &cpu->env; /* Delay slots are indivisible, ignore interrupts */ - if (env->flags & DELAY_SLOT_MASK) { + if (env->flags & TB_FLAG_DELAY_SLOT_MASK) { return false; } else { superh_cpu_do_interrupt(cs); diff --git a/target/sh4/translate.c b/target/sh4/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/sh4/translate.c +++ b/target/sh4/translate.c @@ -XXX,XX +XXX,XX @@ void superh_cpu_dump_state(CPUState *cs, FILE *f, int flags) i, env->gregs[i], i + 1, env->gregs[i + 1], i + 2, env->gregs[i + 2], i + 3, env->gregs[i + 3]); } - if (env->flags & DELAY_SLOT) { + if (env->flags & TB_FLAG_DELAY_SLOT) { qemu_printf("in delay slot (delayed_pc=0x%08x)\n", env->delayed_pc); - } else if (env->flags & DELAY_SLOT_CONDITIONAL) { + } else if (env->flags & TB_FLAG_DELAY_SLOT_COND) { qemu_printf("in conditional delay slot (delayed_pc=0x%08x)\n", env->delayed_pc); - } else if (env->flags & DELAY_SLOT_RTE) { + } else if (env->flags & TB_FLAG_DELAY_SLOT_RTE) { qemu_fprintf(f, "in rte delay slot (delayed_pc=0x%08x)\n", env->delayed_pc); } @@ -XXX,XX +XXX,XX @@ static inline void gen_save_cpu_state(DisasContext *ctx, bool save_pc) static inline bool use_exit_tb(DisasContext *ctx) { - return (ctx->tbflags & GUSA_EXCLUSIVE) != 0; + return (ctx->tbflags & TB_FLAG_GUSA_EXCLUSIVE) != 0; } static bool use_goto_tb(DisasContext *ctx, target_ulong dest) @@ -XXX,XX +XXX,XX @@ static void gen_conditional_jump(DisasContext *ctx, target_ulong dest, TCGLabel *l1 = gen_new_label(); TCGCond cond_not_taken = jump_if_true ? TCG_COND_EQ : TCG_COND_NE; - if (ctx->tbflags & GUSA_EXCLUSIVE) { + if (ctx->tbflags & TB_FLAG_GUSA_EXCLUSIVE) { /* When in an exclusive region, we must continue to the end. Therefore, exit the region on a taken branch, but otherwise fall through to the next instruction. */ tcg_gen_brcondi_i32(cond_not_taken, cpu_sr_t, 0, l1); - tcg_gen_movi_i32(cpu_flags, ctx->envflags & ~GUSA_MASK); + tcg_gen_movi_i32(cpu_flags, ctx->envflags & ~TB_FLAG_GUSA_MASK); /* Note that this won't actually use a goto_tb opcode because we disallow it in use_goto_tb, but it handles exit + singlestep. */ gen_goto_tb(ctx, 0, dest); @@ -XXX,XX +XXX,XX @@ static void gen_delayed_conditional_jump(DisasContext * ctx) tcg_gen_mov_i32(ds, cpu_delayed_cond); tcg_gen_discard_i32(cpu_delayed_cond); - if (ctx->tbflags & GUSA_EXCLUSIVE) { + if (ctx->tbflags & TB_FLAG_GUSA_EXCLUSIVE) { /* When in an exclusive region, we must continue to the end. Therefore, exit the region on a taken branch, but otherwise fall through to the next instruction. */ tcg_gen_brcondi_i32(TCG_COND_EQ, ds, 0, l1); /* Leave the gUSA region. */ - tcg_gen_movi_i32(cpu_flags, ctx->envflags & ~GUSA_MASK); + tcg_gen_movi_i32(cpu_flags, ctx->envflags & ~TB_FLAG_GUSA_MASK); gen_jump(ctx); gen_set_label(l1); @@ -XXX,XX +XXX,XX @@ static inline void gen_store_fpr64(DisasContext *ctx, TCGv_i64 t, int reg) #define XHACK(x) ((((x) & 1 ) << 4) | ((x) & 0xe)) #define CHECK_NOT_DELAY_SLOT \ - if (ctx->envflags & DELAY_SLOT_MASK) { \ - goto do_illegal_slot; \ + if (ctx->envflags & TB_FLAG_DELAY_SLOT_MASK) { \ + goto do_illegal_slot; \ } #define CHECK_PRIVILEGED \ @@ -XXX,XX +XXX,XX @@ static void _decode_opc(DisasContext * ctx) case 0x000b: /* rts */ CHECK_NOT_DELAY_SLOT tcg_gen_mov_i32(cpu_delayed_pc, cpu_pr); - ctx->envflags |= DELAY_SLOT; + ctx->envflags |= TB_FLAG_DELAY_SLOT; ctx->delayed_pc = (uint32_t) - 1; return; case 0x0028: /* clrmac */ @@ -XXX,XX +XXX,XX @@ static void _decode_opc(DisasContext * ctx) CHECK_NOT_DELAY_SLOT gen_write_sr(cpu_ssr); tcg_gen_mov_i32(cpu_delayed_pc, cpu_spc); - ctx->envflags |= DELAY_SLOT_RTE; + ctx->envflags |= TB_FLAG_DELAY_SLOT_RTE; ctx->delayed_pc = (uint32_t) - 1; ctx->base.is_jmp = DISAS_STOP; return; @@ -XXX,XX +XXX,XX @@ static void _decode_opc(DisasContext * ctx) return; case 0xe000: /* mov #imm,Rn */ #ifdef CONFIG_USER_ONLY - /* Detect the start of a gUSA region. If so, update envflags - and end the TB. This will allow us to see the end of the - region (stored in R0) in the next TB. */ + /* + * Detect the start of a gUSA region (mov #-n, r15). + * If so, update envflags and end the TB. This will allow us + * to see the end of the region (stored in R0) in the next TB. + */ if (B11_8 == 15 && B7_0s < 0 && (tb_cflags(ctx->base.tb) & CF_PARALLEL)) { - ctx->envflags = deposit32(ctx->envflags, GUSA_SHIFT, 8, B7_0s); + ctx->envflags = + deposit32(ctx->envflags, TB_FLAG_GUSA_SHIFT, 8, B7_0s); ctx->base.is_jmp = DISAS_STOP; } #endif @@ -XXX,XX +XXX,XX @@ static void _decode_opc(DisasContext * ctx) case 0xa000: /* bra disp */ CHECK_NOT_DELAY_SLOT ctx->delayed_pc = ctx->base.pc_next + 4 + B11_0s * 2; - ctx->envflags |= DELAY_SLOT; + ctx->envflags |= TB_FLAG_DELAY_SLOT; return; case 0xb000: /* bsr disp */ CHECK_NOT_DELAY_SLOT tcg_gen_movi_i32(cpu_pr, ctx->base.pc_next + 4); ctx->delayed_pc = ctx->base.pc_next + 4 + B11_0s * 2; - ctx->envflags |= DELAY_SLOT; + ctx->envflags |= TB_FLAG_DELAY_SLOT; return; } @@ -XXX,XX +XXX,XX @@ static void _decode_opc(DisasContext * ctx) CHECK_NOT_DELAY_SLOT tcg_gen_xori_i32(cpu_delayed_cond, cpu_sr_t, 1); ctx->delayed_pc = ctx->base.pc_next + 4 + B7_0s * 2; - ctx->envflags |= DELAY_SLOT_CONDITIONAL; + ctx->envflags |= TB_FLAG_DELAY_SLOT_COND; return; case 0x8900: /* bt label */ CHECK_NOT_DELAY_SLOT @@ -XXX,XX +XXX,XX @@ static void _decode_opc(DisasContext * ctx) CHECK_NOT_DELAY_SLOT tcg_gen_mov_i32(cpu_delayed_cond, cpu_sr_t); ctx->delayed_pc = ctx->base.pc_next + 4 + B7_0s * 2; - ctx->envflags |= DELAY_SLOT_CONDITIONAL; + ctx->envflags |= TB_FLAG_DELAY_SLOT_COND; return; case 0x8800: /* cmp/eq #imm,R0 */ tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_sr_t, REG(0), B7_0s); @@ -XXX,XX +XXX,XX @@ static void _decode_opc(DisasContext * ctx) case 0x0023: /* braf Rn */ CHECK_NOT_DELAY_SLOT tcg_gen_addi_i32(cpu_delayed_pc, REG(B11_8), ctx->base.pc_next + 4); - ctx->envflags |= DELAY_SLOT; + ctx->envflags |= TB_FLAG_DELAY_SLOT; ctx->delayed_pc = (uint32_t) - 1; return; case 0x0003: /* bsrf Rn */ CHECK_NOT_DELAY_SLOT tcg_gen_movi_i32(cpu_pr, ctx->base.pc_next + 4); tcg_gen_add_i32(cpu_delayed_pc, REG(B11_8), cpu_pr); - ctx->envflags |= DELAY_SLOT; + ctx->envflags |= TB_FLAG_DELAY_SLOT; ctx->delayed_pc = (uint32_t) - 1; return; case 0x4015: /* cmp/pl Rn */ @@ -XXX,XX +XXX,XX @@ static void _decode_opc(DisasContext * ctx) case 0x402b: /* jmp @Rn */ CHECK_NOT_DELAY_SLOT tcg_gen_mov_i32(cpu_delayed_pc, REG(B11_8)); - ctx->envflags |= DELAY_SLOT; + ctx->envflags |= TB_FLAG_DELAY_SLOT; ctx->delayed_pc = (uint32_t) - 1; return; case 0x400b: /* jsr @Rn */ CHECK_NOT_DELAY_SLOT tcg_gen_movi_i32(cpu_pr, ctx->base.pc_next + 4); tcg_gen_mov_i32(cpu_delayed_pc, REG(B11_8)); - ctx->envflags |= DELAY_SLOT; + ctx->envflags |= TB_FLAG_DELAY_SLOT; ctx->delayed_pc = (uint32_t) - 1; return; case 0x400e: /* ldc Rm,SR */ @@ -XXX,XX +XXX,XX @@ static void _decode_opc(DisasContext * ctx) fflush(stderr); #endif do_illegal: - if (ctx->envflags & DELAY_SLOT_MASK) { + if (ctx->envflags & TB_FLAG_DELAY_SLOT_MASK) { do_illegal_slot: gen_save_cpu_state(ctx, true); gen_helper_raise_slot_illegal_instruction(cpu_env); @@ -XXX,XX +XXX,XX @@ static void _decode_opc(DisasContext * ctx) do_fpu_disabled: gen_save_cpu_state(ctx, true); - if (ctx->envflags & DELAY_SLOT_MASK) { + if (ctx->envflags & TB_FLAG_DELAY_SLOT_MASK) { gen_helper_raise_slot_fpu_disable(cpu_env); } else { gen_helper_raise_fpu_disable(cpu_env); @@ -XXX,XX +XXX,XX @@ static void decode_opc(DisasContext * ctx) _decode_opc(ctx); - if (old_flags & DELAY_SLOT_MASK) { + if (old_flags & TB_FLAG_DELAY_SLOT_MASK) { /* go out of the delay slot */ - ctx->envflags &= ~DELAY_SLOT_MASK; + ctx->envflags &= ~TB_FLAG_DELAY_SLOT_MASK; /* When in an exclusive region, we must continue to the end for conditional branches. */ - if (ctx->tbflags & GUSA_EXCLUSIVE - && old_flags & DELAY_SLOT_CONDITIONAL) { + if (ctx->tbflags & TB_FLAG_GUSA_EXCLUSIVE + && old_flags & TB_FLAG_DELAY_SLOT_COND) { gen_delayed_conditional_jump(ctx); return; } /* Otherwise this is probably an invalid gUSA region. Drop the GUSA bits so the next TB doesn't see them. */ - ctx->envflags &= ~GUSA_MASK; + ctx->envflags &= ~TB_FLAG_GUSA_MASK; tcg_gen_movi_i32(cpu_flags, ctx->envflags); - if (old_flags & DELAY_SLOT_CONDITIONAL) { + if (old_flags & TB_FLAG_DELAY_SLOT_COND) { gen_delayed_conditional_jump(ctx); } else { gen_jump(ctx); @@ -XXX,XX +XXX,XX @@ static void decode_gusa(DisasContext *ctx, CPUSH4State *env) } /* The entire region has been translated. */ - ctx->envflags &= ~GUSA_MASK; + ctx->envflags &= ~TB_FLAG_GUSA_MASK; ctx->base.pc_next = pc_end; ctx->base.num_insns += max_insns - 1; return; @@ -XXX,XX +XXX,XX @@ static void decode_gusa(DisasContext *ctx, CPUSH4State *env) /* Restart with the EXCLUSIVE bit set, within a TB run via cpu_exec_step_atomic holding the exclusive lock. */ - ctx->envflags |= GUSA_EXCLUSIVE; + ctx->envflags |= TB_FLAG_GUSA_EXCLUSIVE; gen_save_cpu_state(ctx, false); gen_helper_exclusive(cpu_env); ctx->base.is_jmp = DISAS_NORETURN; @@ -XXX,XX +XXX,XX @@ static void sh4_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs) (tbflags & (1 << SR_RB))) * 0x10; ctx->fbank = tbflags & FPSCR_FR ? 0x10 : 0; - if (tbflags & GUSA_MASK) { +#ifdef CONFIG_USER_ONLY + if (tbflags & TB_FLAG_GUSA_MASK) { + /* In gUSA exclusive region. */ uint32_t pc = ctx->base.pc_next; uint32_t pc_end = ctx->base.tb->cs_base; - int backup = sextract32(ctx->tbflags, GUSA_SHIFT, 8); + int backup = sextract32(ctx->tbflags, TB_FLAG_GUSA_SHIFT, 8); int max_insns = (pc_end - pc) / 2; if (pc != pc_end + backup || max_insns < 2) { /* This is a malformed gUSA region. Don't do anything special, since the interpreter is likely to get confused. */ - ctx->envflags &= ~GUSA_MASK; - } else if (tbflags & GUSA_EXCLUSIVE) { + ctx->envflags &= ~TB_FLAG_GUSA_MASK; + } else if (tbflags & TB_FLAG_GUSA_EXCLUSIVE) { /* Regardless of single-stepping or the end of the page, we must complete execution of the gUSA region while holding the exclusive lock. */ @@ -XXX,XX +XXX,XX @@ static void sh4_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs) return; } } +#endif /* Since the ISA is fixed-width, we can bound by the number of instructions remaining on the page. */ @@ -XXX,XX +XXX,XX @@ static void sh4_tr_translate_insn(DisasContextBase *dcbase, CPUState *cs) DisasContext *ctx = container_of(dcbase, DisasContext, base); #ifdef CONFIG_USER_ONLY - if (unlikely(ctx->envflags & GUSA_MASK) - && !(ctx->envflags & GUSA_EXCLUSIVE)) { + if (unlikely(ctx->envflags & TB_FLAG_GUSA_MASK) + && !(ctx->envflags & TB_FLAG_GUSA_EXCLUSIVE)) { /* We're in an gUSA region, and we have not already fallen back on using an exclusive region. Attempt to parse the region into a single supported atomic operation. Failure @@ -XXX,XX +XXX,XX @@ static void sh4_tr_tb_stop(DisasContextBase *dcbase, CPUState *cs) { DisasContext *ctx = container_of(dcbase, DisasContext, base); - if (ctx->tbflags & GUSA_EXCLUSIVE) { + if (ctx->tbflags & TB_FLAG_GUSA_EXCLUSIVE) { /* Ending the region of exclusivity. Clear the bits. */ - ctx->envflags &= ~GUSA_MASK; + ctx->envflags &= ~TB_FLAG_GUSA_MASK; } switch (ctx->base.is_jmp) { -- 2.34.1 The following changes since commit aa3a285b5bc56a4208b3b57d4a55291e9c260107: Merge tag 'mem-2024-12-21' of https://github.com/davidhildenbrand/qemu into staging (2024-12-22 14:33:27 -0500) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20241224 for you to fetch changes up to e4a8e093dc74be049f4829831dce76e5edab0003: accel/tcg: Move gen_intermediate_code to TCGCPUOps.translate_core (2024-12-24 08:32:15 -0800) ---------------------------------------------------------------- tcg/optimize: Remove in-flight mask data from OptContext fpu: Add float*_muladd_scalbn fpu: Remove float_muladd_halve_result fpu: Add float_round_nearest_even_max fpu: Add float_muladd_suppress_add_product_zero target/hexagon: Use float32_muladd accel/tcg: Move gen_intermediate_code to TCGCPUOps.translate_core ---------------------------------------------------------------- Ilya Leoshkevich (1): tests/tcg: Do not use inttypes.h in multiarch/system/memory.c Pierrick Bouvier (1): plugins: optimize cpu_index code generation Richard Henderson (70): tcg/optimize: Split out finish_bb, finish_ebb tcg/optimize: Split out fold_affected_mask tcg/optimize: Copy mask writeback to fold_masks tcg/optimize: Split out fold_masks_zs tcg/optimize: Augment s_mask from z_mask in fold_masks_zs tcg/optimize: Change representation of s_mask tcg/optimize: Use finish_folding in fold_add, fold_add_vec, fold_addsub2 tcg/optimize: Introduce const value accessors for TempOptInfo tcg/optimize: Use fold_masks_zs in fold_and tcg/optimize: Use fold_masks_zs in fold_andc tcg/optimize: Use fold_masks_zs in fold_bswap tcg/optimize: Use fold_masks_zs in fold_count_zeros tcg/optimize: Use fold_masks_z in fold_ctpop tcg/optimize: Use fold_and and fold_masks_z in fold_deposit tcg/optimize: Compute sign mask in fold_deposit tcg/optimize: Use finish_folding in fold_divide tcg/optimize: Use finish_folding in fold_dup, fold_dup2 tcg/optimize: Use fold_masks_s in fold_eqv tcg/optimize: Use fold_masks_z in fold_extract tcg/optimize: Use finish_folding in fold_extract2 tcg/optimize: Use fold_masks_zs in fold_exts tcg/optimize: Use fold_masks_z in fold_extu tcg/optimize: Use fold_masks_zs in fold_movcond tcg/optimize: Use finish_folding in fold_mul* tcg/optimize: Use fold_masks_s in fold_nand tcg/optimize: Use fold_masks_z in fold_neg_no_const tcg/optimize: Use fold_masks_s in fold_nor tcg/optimize: Use fold_masks_s in fold_not tcg/optimize: Use fold_masks_zs in fold_or tcg/optimize: Use fold_masks_zs in fold_orc tcg/optimize: Use fold_masks_zs in fold_qemu_ld tcg/optimize: Return true from fold_qemu_st, fold_tcg_st tcg/optimize: Use finish_folding in fold_remainder tcg/optimize: Distinguish simplification in fold_setcond_zmask tcg/optimize: Use fold_masks_z in fold_setcond tcg/optimize: Use fold_masks_s in fold_negsetcond tcg/optimize: Use fold_masks_z in fold_setcond2 tcg/optimize: Use finish_folding in fold_cmp_vec tcg/optimize: Use finish_folding in fold_cmpsel_vec tcg/optimize: Use fold_masks_zs in fold_sextract tcg/optimize: Use fold_masks_zs, fold_masks_s in fold_shift tcg/optimize: Simplify sign bit test in fold_shift tcg/optimize: Use finish_folding in fold_sub, fold_sub_vec tcg/optimize: Use fold_masks_zs in fold_tcg_ld tcg/optimize: Use finish_folding in fold_tcg_ld_memcopy tcg/optimize: Use fold_masks_zs in fold_xor tcg/optimize: Use finish_folding in fold_bitsel_vec tcg/optimize: Use finish_folding as default in tcg_optimize tcg/optimize: Remove z_mask, s_mask from OptContext tcg/optimize: Re-enable sign-mask optimizations tcg/optimize: Move fold_bitsel_vec into alphabetic sort tcg/optimize: Move fold_cmp_vec, fold_cmpsel_vec into alphabetic sort softfloat: Add float{16,32,64}_muladd_scalbn target/arm: Use float*_muladd_scalbn target/sparc: Use float*_muladd_scalbn softfloat: Remove float_muladd_halve_result softfloat: Add float_round_nearest_even_max softfloat: Add float_muladd_suppress_add_product_zero target/hexagon: Use float32_mul in helper_sfmpy target/hexagon: Use float32_muladd for helper_sffma target/hexagon: Use float32_muladd for helper_sffms target/hexagon: Use float32_muladd_scalbn for helper_sffma_sc target/hexagon: Use float32_muladd for helper_sffm[as]_lib target/hexagon: Remove internal_fmafx target/hexagon: Expand GEN_XF_ROUND target/hexagon: Remove Float target/hexagon: Remove Double target/hexagon: Use mulu64 for int128_mul_6464 target/hexagon: Simplify internal_mpyhh setup accel/tcg: Move gen_intermediate_code to TCGCPUOps.translate_core include/exec/translator.h | 14 - include/fpu/softfloat-types.h | 2 + include/fpu/softfloat.h | 14 +- include/hw/core/tcg-cpu-ops.h | 13 + target/alpha/cpu.h | 2 + target/arm/internals.h | 2 + target/avr/cpu.h | 2 + target/hexagon/cpu.h | 2 + target/hexagon/fma_emu.h | 3 - target/hppa/cpu.h | 2 + target/i386/tcg/helper-tcg.h | 2 + target/loongarch/internals.h | 2 + target/m68k/cpu.h | 2 + target/microblaze/cpu.h | 2 + target/mips/tcg/tcg-internal.h | 2 + target/openrisc/cpu.h | 2 + target/ppc/cpu.h | 2 + target/riscv/cpu.h | 3 + target/rx/cpu.h | 2 + target/s390x/s390x-internal.h | 2 + target/sh4/cpu.h | 2 + target/sparc/cpu.h | 2 + target/sparc/helper.h | 4 +- target/tricore/cpu.h | 2 + target/xtensa/cpu.h | 2 + accel/tcg/cpu-exec.c | 8 +- accel/tcg/plugin-gen.c | 9 + accel/tcg/translate-all.c | 8 +- fpu/softfloat.c | 63 +-- target/alpha/cpu.c | 1 + target/alpha/translate.c | 4 +- target/arm/cpu.c | 1 + target/arm/tcg/cpu-v7m.c | 1 + target/arm/tcg/helper-a64.c | 6 +- target/arm/tcg/translate.c | 5 +- target/avr/cpu.c | 1 + target/avr/translate.c | 6 +- target/hexagon/cpu.c | 1 + target/hexagon/fma_emu.c | 496 ++++++--------------- target/hexagon/op_helper.c | 125 ++---- target/hexagon/translate.c | 4 +- target/hppa/cpu.c | 1 + target/hppa/translate.c | 4 +- target/i386/tcg/tcg-cpu.c | 1 + target/i386/tcg/translate.c | 5 +- target/loongarch/cpu.c | 1 + target/loongarch/tcg/translate.c | 4 +- target/m68k/cpu.c | 1 + target/m68k/translate.c | 4 +- target/microblaze/cpu.c | 1 + target/microblaze/translate.c | 4 +- target/mips/cpu.c | 1 + target/mips/tcg/translate.c | 4 +- target/openrisc/cpu.c | 1 + target/openrisc/translate.c | 4 +- target/ppc/cpu_init.c | 1 + target/ppc/translate.c | 4 +- target/riscv/tcg/tcg-cpu.c | 1 + target/riscv/translate.c | 4 +- target/rx/cpu.c | 1 + target/rx/translate.c | 4 +- target/s390x/cpu.c | 1 + target/s390x/tcg/translate.c | 4 +- target/sh4/cpu.c | 1 + target/sh4/translate.c | 4 +- target/sparc/cpu.c | 1 + target/sparc/fop_helper.c | 8 +- target/sparc/translate.c | 84 ++-- target/tricore/cpu.c | 1 + target/tricore/translate.c | 5 +- target/xtensa/cpu.c | 1 + target/xtensa/translate.c | 4 +- tcg/optimize.c | 857 +++++++++++++++++++----------------- tests/tcg/multiarch/system/memory.c | 9 +- fpu/softfloat-parts.c.inc | 16 +- 75 files changed, 866 insertions(+), 1009 deletions(-) From: Ilya Leoshkevich <iii@linux.ibm.com> make check-tcg fails on Fedora with the following error message: alpha-linux-gnu-gcc [...] qemu/tests/tcg/multiarch/system/memory.c -o memory [...] qemu/tests/tcg/multiarch/system/memory.c:17:10: fatal error: inttypes.h: No such file or directory 17 | #include <inttypes.h> | ^~~~~~~~~~~~ compilation terminated. The reason is that Fedora has cross-compilers, but no cross-glibc headers. Fix by hardcoding the format specifiers and dropping the include. An alternative fix would be to introduce a configure check for inttypes.h. But this would make it impossible to use Fedora cross-compilers for softmmu tests, which used to work so far. Fixes: ecbcc9ead2f8 ("tests/tcg: add a system test to check memory instrumentation") Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com> Reviewed-by: Paolo Bonzini <pbonzini@redhat.com> Message-ID: <20241010085906.226249-1-iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tests/tcg/multiarch/system/memory.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/tcg/multiarch/system/memory.c b/tests/tcg/multiarch/system/memory.c index XXXXXXX..XXXXXXX 100644 --- a/tests/tcg/multiarch/system/memory.c +++ b/tests/tcg/multiarch/system/memory.c @@ -XXX,XX +XXX,XX @@ #include <stdint.h> #include <stdbool.h> -#include <inttypes.h> #include <minilib.h> #ifndef CHECK_UNALIGNED @@ -XXX,XX +XXX,XX @@ int main(void) int i; bool ok = true; - ml_printf("Test data start: 0x%"PRIxPTR"\n", &test_data[0]); - ml_printf("Test data end: 0x%"PRIxPTR"\n", &test_data[TEST_SIZE]); + ml_printf("Test data start: 0x%lx\n", (unsigned long)&test_data[0]); + ml_printf("Test data end: 0x%lx\n", (unsigned long)&test_data[TEST_SIZE]); /* Run through the unsigned tests first */ for (i = 0; i < ARRAY_SIZE(init_ufns) && ok; i++) { @@ -XXX,XX +XXX,XX @@ int main(void) ok = do_signed_reads(true); } - ml_printf("Test data read: %"PRId32"\n", test_read_count); - ml_printf("Test data write: %"PRId32"\n", test_write_count); + ml_printf("Test data read: %lu\n", (unsigned long)test_read_count); + ml_printf("Test data write: %lu\n", (unsigned long)test_write_count); ml_printf("Test complete: %s\n", ok ? "PASSED" : "FAILED"); return ok ? 0 : -1; } -- 2.43.0 From: Pierrick Bouvier <pierrick.bouvier@linaro.org> When running with a single vcpu, we can return a constant instead of a load when accessing cpu_index. A side effect is that all tcg operations using it are optimized, most notably scoreboard access. When running a simple loop in user-mode, the speedup is around 20%. Signed-off-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-ID: <20241128213843.1023080-1-pierrick.bouvier@linaro.org> --- accel/tcg/plugin-gen.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -XXX,XX +XXX,XX @@ static void gen_disable_mem_helper(void) static TCGv_i32 gen_cpu_index(void) { + /* + * Optimize when we run with a single vcpu. All values using cpu_index, + * including scoreboard index, will be optimized out. + * User-mode calls tb_flush when setting this flag. In system-mode, all + * vcpus are created before generating code. + */ + if (!tcg_cflags_has(current_cpu, CF_PARALLEL)) { + return tcg_constant_i32(current_cpu->cpu_index); + } TCGv_i32 cpu_index = tcg_temp_ebb_new_i32(); tcg_gen_ld_i32(cpu_index, tcg_env, -offsetof(ArchCPU, env) + offsetof(CPUState, cpu_index)); -- 2.43.0 Call them directly from the opcode switch statement in tcg_optimize, rather than in finish_folding based on opcode flags. Adjust folding of conditional branches to match. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 47 +++++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void copy_propagate(OptContext *ctx, TCGOp *op, } } +static void finish_bb(OptContext *ctx) +{ + /* We only optimize memory barriers across basic blocks. */ + ctx->prev_mb = NULL; +} + +static void finish_ebb(OptContext *ctx) +{ + finish_bb(ctx); + /* We only optimize across extended basic blocks. */ + memset(&ctx->temps_used, 0, sizeof(ctx->temps_used)); + remove_mem_copy_all(ctx); +} + static void finish_folding(OptContext *ctx, TCGOp *op) { const TCGOpDef *def = &tcg_op_defs[op->opc]; int i, nb_oargs; - /* - * We only optimize extended basic blocks. If the opcode ends a BB - * and is not a conditional branch, reset all temp data. - */ - if (def->flags & TCG_OPF_BB_END) { - ctx->prev_mb = NULL; - if (!(def->flags & TCG_OPF_COND_BRANCH)) { - memset(&ctx->temps_used, 0, sizeof(ctx->temps_used)); - remove_mem_copy_all(ctx); - } - return; - } - nb_oargs = def->nb_oargs; for (i = 0; i < nb_oargs; i++) { TCGTemp *ts = arg_temp(op->args[i]); @@ -XXX,XX +XXX,XX @@ static bool fold_brcond(OptContext *ctx, TCGOp *op) if (i > 0) { op->opc = INDEX_op_br; op->args[0] = op->args[3]; + finish_ebb(ctx); + } else { + finish_bb(ctx); } - return false; + return true; } static bool fold_brcond2(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) } op->opc = INDEX_op_br; op->args[0] = label; - break; + finish_ebb(ctx); + return true; } - return false; + + finish_bb(ctx); + return true; } static bool fold_bswap(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(xor): done = fold_xor(&ctx, op); break; + case INDEX_op_set_label: + case INDEX_op_br: + case INDEX_op_exit_tb: + case INDEX_op_goto_tb: + case INDEX_op_goto_ptr: + finish_ebb(&ctx); + done = true; + break; default: break; } -- 2.43.0 There are only a few logical operations which can compute an "affected" mask. Split out handling of this optimization to a separate function, only to be called when applicable. Remove the a_mask field from OptContext, as the mask is no longer stored anywhere. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { QSIMPLEQ_HEAD(, MemCopyInfo) mem_free; /* In flight values from optimization. */ - uint64_t a_mask; /* mask bit is 0 iff value identical to first input */ uint64_t z_mask; /* mask bit is 0 iff value bit is 0 */ uint64_t s_mask; /* mask of clrsb(value) bits */ TCGType type; @@ -XXX,XX +XXX,XX @@ static bool fold_const2_commutative(OptContext *ctx, TCGOp *op) static bool fold_masks(OptContext *ctx, TCGOp *op) { - uint64_t a_mask = ctx->a_mask; uint64_t z_mask = ctx->z_mask; uint64_t s_mask = ctx->s_mask; @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) * type changing opcodes. */ if (ctx->type == TCG_TYPE_I32) { - a_mask = (int32_t)a_mask; z_mask = (int32_t)z_mask; s_mask |= MAKE_64BIT_MASK(32, 32); ctx->z_mask = z_mask; @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) if (z_mask == 0) { return tcg_opt_gen_movi(ctx, op, op->args[0], 0); } + return false; +} + +/* + * An "affected" mask bit is 0 if and only if the result is identical + * to the first input. Thus if the entire mask is 0, the operation + * is equivalent to a copy. + */ +static bool fold_affected_mask(OptContext *ctx, TCGOp *op, uint64_t a_mask) +{ + if (ctx->type == TCG_TYPE_I32) { + a_mask = (uint32_t)a_mask; + } if (a_mask == 0) { return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); } @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) * Known-zeros does not imply known-ones. Therefore unless * arg2 is constant, we can't infer affected bits from it. */ - if (arg_is_const(op->args[2])) { - ctx->a_mask = z1 & ~z2; + if (arg_is_const(op->args[2]) && + fold_affected_mask(ctx, op, z1 & ~z2)) { + return true; } return fold_masks(ctx, op); @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) */ if (arg_is_const(op->args[2])) { uint64_t z2 = ~arg_info(op->args[2])->z_mask; - ctx->a_mask = z1 & ~z2; + if (fold_affected_mask(ctx, op, z1 & ~z2)) { + return true; + } z1 &= z2; } ctx->z_mask = z1; @@ -XXX,XX +XXX,XX @@ static bool fold_extract(OptContext *ctx, TCGOp *op) z_mask_old = arg_info(op->args[1])->z_mask; z_mask = extract64(z_mask_old, pos, len); - if (pos == 0) { - ctx->a_mask = z_mask_old ^ z_mask; + if (pos == 0 && fold_affected_mask(ctx, op, z_mask_old ^ z_mask)) { + return true; } ctx->z_mask = z_mask; ctx->s_mask = smask_from_zmask(z_mask); @@ -XXX,XX +XXX,XX @@ static bool fold_exts(OptContext *ctx, TCGOp *op) ctx->z_mask = z_mask; ctx->s_mask = s_mask; - if (!type_change) { - ctx->a_mask = s_mask & ~s_mask_old; + if (!type_change && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { + return true; } return fold_masks(ctx, op); @@ -XXX,XX +XXX,XX @@ static bool fold_extu(OptContext *ctx, TCGOp *op) ctx->z_mask = z_mask; ctx->s_mask = smask_from_zmask(z_mask); - if (!type_change) { - ctx->a_mask = z_mask_old ^ z_mask; + if (!type_change && fold_affected_mask(ctx, op, z_mask_old ^ z_mask)) { + return true; } return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) s_mask |= MAKE_64BIT_MASK(len, 64 - len); ctx->s_mask = s_mask; - if (pos == 0) { - ctx->a_mask = s_mask & ~s_mask_old; + if (pos == 0 && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { + return true; } return fold_masks(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } /* Assume all bits affected, no bits known zero, no sign reps. */ - ctx.a_mask = -1; ctx.z_mask = -1; ctx.s_mask = 0; -- 2.43.0 Use of fold_masks should be restricted to those opcodes that can reliably make use of it -- those with a single output, and from higher-level folders that set up the masks. Prepare for conversion of each folder in turn. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) { uint64_t z_mask = ctx->z_mask; uint64_t s_mask = ctx->s_mask; + const TCGOpDef *def = &tcg_op_defs[op->opc]; + TCGTemp *ts; + TempOptInfo *ti; + + /* Only single-output opcodes are supported here. */ + tcg_debug_assert(def->nb_oargs == 1); /* * 32-bit ops generate 32-bit results, which for the purpose of @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) if (ctx->type == TCG_TYPE_I32) { z_mask = (int32_t)z_mask; s_mask |= MAKE_64BIT_MASK(32, 32); - ctx->z_mask = z_mask; - ctx->s_mask = s_mask; } if (z_mask == 0) { return tcg_opt_gen_movi(ctx, op, op->args[0], 0); } - return false; + + ts = arg_temp(op->args[0]); + reset_ts(ctx, ts); + + ti = ts_info(ts); + ti->z_mask = z_mask; + ti->s_mask = s_mask; + return true; } /* -- 2.43.0 Add a routine to which masks can be passed directly, rather than storing them into OptContext. To be used in upcoming patches. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_const2_commutative(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } -static bool fold_masks(OptContext *ctx, TCGOp *op) +/* + * Record "zero" and "sign" masks for the single output of @op. + * See TempOptInfo definition of z_mask and s_mask. + * If z_mask allows, fold the output to constant zero. + */ +static bool fold_masks_zs(OptContext *ctx, TCGOp *op, + uint64_t z_mask, uint64_t s_mask) { - uint64_t z_mask = ctx->z_mask; - uint64_t s_mask = ctx->s_mask; const TCGOpDef *def = &tcg_op_defs[op->opc]; TCGTemp *ts; TempOptInfo *ti; @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) return true; } +static bool fold_masks(OptContext *ctx, TCGOp *op) +{ + return fold_masks_zs(ctx, op, ctx->z_mask, ctx->s_mask); +} + /* * An "affected" mask bit is 0 if and only if the result is identical * to the first input. Thus if the entire mask is 0, the operation -- 2.43.0 Consider the passed s_mask to be a minimum deduced from either existing s_mask or from a sign-extension operation. We may be able to deduce more from the set of known zeros. Remove identical logic from several opcode folders. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_const2_commutative(OptContext *ctx, TCGOp *op) * Record "zero" and "sign" masks for the single output of @op. * See TempOptInfo definition of z_mask and s_mask. * If z_mask allows, fold the output to constant zero. + * The passed s_mask may be augmented by z_mask. */ static bool fold_masks_zs(OptContext *ctx, TCGOp *op, uint64_t z_mask, uint64_t s_mask) @@ -XXX,XX +XXX,XX @@ static bool fold_masks_zs(OptContext *ctx, TCGOp *op, ti = ts_info(ts); ti->z_mask = z_mask; - ti->s_mask = s_mask; + ti->s_mask = s_mask | smask_from_zmask(z_mask); return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) default: g_assert_not_reached(); } - s_mask = smask_from_zmask(z_mask); + s_mask = 0; switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { case TCG_BSWAP_OZ: break; @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) default: /* The high bits are undefined: force all bits above the sign to 1. */ z_mask |= sign << 1; - s_mask = 0; break; } ctx->z_mask = z_mask; @@ -XXX,XX +XXX,XX @@ static bool fold_count_zeros(OptContext *ctx, TCGOp *op) g_assert_not_reached(); } ctx->z_mask = arg_info(op->args[2])->z_mask | z_mask; - ctx->s_mask = smask_from_zmask(ctx->z_mask); return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_ctpop(OptContext *ctx, TCGOp *op) default: g_assert_not_reached(); } - ctx->s_mask = smask_from_zmask(ctx->z_mask); return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_extract(OptContext *ctx, TCGOp *op) return true; } ctx->z_mask = z_mask; - ctx->s_mask = smask_from_zmask(z_mask); return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_extu(OptContext *ctx, TCGOp *op) } ctx->z_mask = z_mask; - ctx->s_mask = smask_from_zmask(z_mask); if (!type_change && fold_affected_mask(ctx, op, z_mask_old ^ z_mask)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) int width = 8 * memop_size(mop); if (width < 64) { - ctx->s_mask = MAKE_64BIT_MASK(width, 64 - width); - if (!(mop & MO_SIGN)) { + if (mop & MO_SIGN) { + ctx->s_mask = MAKE_64BIT_MASK(width, 64 - width); + } else { ctx->z_mask = MAKE_64BIT_MASK(0, width); - ctx->s_mask <<= 1; } } @@ -XXX,XX +XXX,XX @@ static bool fold_setcond(OptContext *ctx, TCGOp *op) fold_setcond_tst_pow2(ctx, op, false); ctx->z_mask = 1; - ctx->s_mask = smask_from_zmask(1); return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) } ctx->z_mask = 1; - ctx->s_mask = smask_from_zmask(1); return false; do_setcond_const: @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) break; CASE_OP_32_64(ld8u): ctx->z_mask = MAKE_64BIT_MASK(0, 8); - ctx->s_mask = MAKE_64BIT_MASK(9, 55); break; CASE_OP_32_64(ld16s): ctx->s_mask = MAKE_64BIT_MASK(16, 48); break; CASE_OP_32_64(ld16u): ctx->z_mask = MAKE_64BIT_MASK(0, 16); - ctx->s_mask = MAKE_64BIT_MASK(17, 47); break; case INDEX_op_ld32s_i64: ctx->s_mask = MAKE_64BIT_MASK(32, 32); break; case INDEX_op_ld32u_i64: ctx->z_mask = MAKE_64BIT_MASK(0, 32); - ctx->s_mask = MAKE_64BIT_MASK(33, 31); break; default: g_assert_not_reached(); -- 2.43.0 Change the representation from sign bit repetitions to all bits equal to the sign bit, including the sign bit itself. The previous format has a problem in that it is difficult to recreate a valid sign mask after a shift operation: the "repetitions" part of the previous format meant that applying the same shift as for the value lead to an off-by-one value. The new format, including the sign bit itself, means that the sign mask can be manipulated in exactly the same way as the value, canonicalization is easier. Canonicalize the s_mask in fold_masks_zs, rather than requiring callers to do so. Treat 0 as a non-canonical but typeless input for no sign information, which will be reset as appropriate for the data type. We can easily fold in the data from z_mask while canonicalizing. Temporarily disable optimizations using s_mask while each operation is converted to use fold_masks_zs and to the new form. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 64 ++++++++++++-------------------------------------- 1 file changed, 15 insertions(+), 49 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { QSIMPLEQ_HEAD(, MemCopyInfo) mem_copy; uint64_t val; uint64_t z_mask; /* mask bit is 0 if and only if value bit is 0 */ - uint64_t s_mask; /* a left-aligned mask of clrsb(value) bits. */ + uint64_t s_mask; /* mask bit is 1 if value bit matches msb */ } TempOptInfo; typedef struct OptContext { @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { /* In flight values from optimization. */ uint64_t z_mask; /* mask bit is 0 iff value bit is 0 */ - uint64_t s_mask; /* mask of clrsb(value) bits */ + uint64_t s_mask; /* mask bit is 1 if value bit matches msb */ TCGType type; } OptContext; -/* Calculate the smask for a specific value. */ -static uint64_t smask_from_value(uint64_t value) -{ - int rep = clrsb64(value); - return ~(~0ull >> rep); -} - -/* - * Calculate the smask for a given set of known-zeros. - * If there are lots of zeros on the left, we can consider the remainder - * an unsigned field, and thus the corresponding signed field is one bit - * larger. - */ -static uint64_t smask_from_zmask(uint64_t zmask) -{ - /* - * Only the 0 bits are significant for zmask, thus the msb itself - * must be zero, else we have no sign information. - */ - int rep = clz64(zmask); - if (rep == 0) { - return 0; - } - rep -= 1; - return ~(~0ull >> rep); -} - -/* - * Recreate a properly left-aligned smask after manipulation. - * Some bit-shuffling, particularly shifts and rotates, may - * retain sign bits on the left, but may scatter disconnected - * sign bits on the right. Retain only what remains to the left. - */ -static uint64_t smask_from_smask(int64_t smask) -{ - /* Only the 1 bits are significant for smask */ - return smask_from_zmask(~smask); -} - static inline TempOptInfo *ts_info(TCGTemp *ts) { return ts->state_ptr; @@ -XXX,XX +XXX,XX @@ static void init_ts_info(OptContext *ctx, TCGTemp *ts) ti->is_const = true; ti->val = ts->val; ti->z_mask = ts->val; - ti->s_mask = smask_from_value(ts->val); + ti->s_mask = INT64_MIN >> clrsb64(ts->val); } else { ti->is_const = false; ti->z_mask = -1; @@ -XXX,XX +XXX,XX @@ static void finish_folding(OptContext *ctx, TCGOp *op) */ if (i == 0) { ts_info(ts)->z_mask = ctx->z_mask; - ts_info(ts)->s_mask = ctx->s_mask; } } } @@ -XXX,XX +XXX,XX @@ static bool fold_const2_commutative(OptContext *ctx, TCGOp *op) * The passed s_mask may be augmented by z_mask. */ static bool fold_masks_zs(OptContext *ctx, TCGOp *op, - uint64_t z_mask, uint64_t s_mask) + uint64_t z_mask, int64_t s_mask) { const TCGOpDef *def = &tcg_op_defs[op->opc]; TCGTemp *ts; TempOptInfo *ti; + int rep; /* Only single-output opcodes are supported here. */ tcg_debug_assert(def->nb_oargs == 1); @@ -XXX,XX +XXX,XX @@ static bool fold_masks_zs(OptContext *ctx, TCGOp *op, */ if (ctx->type == TCG_TYPE_I32) { z_mask = (int32_t)z_mask; - s_mask |= MAKE_64BIT_MASK(32, 32); + s_mask |= INT32_MIN; } if (z_mask == 0) { @@ -XXX,XX +XXX,XX @@ static bool fold_masks_zs(OptContext *ctx, TCGOp *op, ti = ts_info(ts); ti->z_mask = z_mask; - ti->s_mask = s_mask | smask_from_zmask(z_mask); + + /* Canonicalize s_mask and incorporate data from z_mask. */ + rep = clz64(~s_mask); + rep = MAX(rep, clz64(z_mask)); + rep = MAX(rep - 1, 0); + ti->s_mask = INT64_MIN >> rep; + return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_exts(OptContext *ctx, TCGOp *op) ctx->z_mask = z_mask; ctx->s_mask = s_mask; - if (!type_change && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { + if (0 && !type_change && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) s_mask |= MAKE_64BIT_MASK(len, 64 - len); ctx->s_mask = s_mask; - if (pos == 0 && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { + if (0 && pos == 0 && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_shift(OptContext *ctx, TCGOp *op) ctx->z_mask = do_constant_folding(op->opc, ctx->type, z_mask, sh); s_mask = do_constant_folding(op->opc, ctx->type, s_mask, sh); - ctx->s_mask = smask_from_smask(s_mask); return fold_masks(ctx, op); } -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void finish_ebb(OptContext *ctx) remove_mem_copy_all(ctx); } -static void finish_folding(OptContext *ctx, TCGOp *op) +static bool finish_folding(OptContext *ctx, TCGOp *op) { const TCGOpDef *def = &tcg_op_defs[op->opc]; int i, nb_oargs; @@ -XXX,XX +XXX,XX @@ static void finish_folding(OptContext *ctx, TCGOp *op) ts_info(ts)->z_mask = ctx->z_mask; } } + return true; } /* @@ -XXX,XX +XXX,XX @@ static bool fold_add(OptContext *ctx, TCGOp *op) fold_xi_to_x(ctx, op, 0)) { return true; } - return false; + return finish_folding(ctx, op); } /* We cannot as yet do_constant_folding with vectors. */ @@ -XXX,XX +XXX,XX @@ static bool fold_add_vec(OptContext *ctx, TCGOp *op) fold_xi_to_x(ctx, op, 0)) { return true; } - return false; + return finish_folding(ctx, op); } static bool fold_addsub2(OptContext *ctx, TCGOp *op, bool add) @@ -XXX,XX +XXX,XX @@ static bool fold_addsub2(OptContext *ctx, TCGOp *op, bool add) op->args[4] = arg_new_constant(ctx, bl); op->args[5] = arg_new_constant(ctx, bh); } - return false; + return finish_folding(ctx, op); } static bool fold_add2(OptContext *ctx, TCGOp *op) -- 2.43.0 Introduce ti_is_const, ti_const_val, ti_is_const_val. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static inline TempOptInfo *arg_info(TCGArg arg) return ts_info(arg_temp(arg)); } +static inline bool ti_is_const(TempOptInfo *ti) +{ + return ti->is_const; +} + +static inline uint64_t ti_const_val(TempOptInfo *ti) +{ + return ti->val; +} + +static inline bool ti_is_const_val(TempOptInfo *ti, uint64_t val) +{ + return ti_is_const(ti) && ti_const_val(ti) == val; +} + static inline bool ts_is_const(TCGTemp *ts) { - return ts_info(ts)->is_const; + return ti_is_const(ts_info(ts)); } static inline bool ts_is_const_val(TCGTemp *ts, uint64_t val) { - TempOptInfo *ti = ts_info(ts); - return ti->is_const && ti->val == val; + return ti_is_const_val(ts_info(ts), val); } static inline bool arg_is_const(TCGArg arg) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Sink mask computation below fold_affected_mask early exit. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_add2(OptContext *ctx, TCGOp *op) static bool fold_and(OptContext *ctx, TCGOp *op) { - uint64_t z1, z2; + uint64_t z1, z2, z_mask, s_mask; + TempOptInfo *t1, *t2; if (fold_const2_commutative(ctx, op) || fold_xi_to_i(ctx, op, 0) || @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) return true; } - z1 = arg_info(op->args[1])->z_mask; - z2 = arg_info(op->args[2])->z_mask; - ctx->z_mask = z1 & z2; - - /* - * Sign repetitions are perforce all identical, whether they are 1 or 0. - * Bitwise operations preserve the relative quantity of the repetitions. - */ - ctx->s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; + t1 = arg_info(op->args[1]); + t2 = arg_info(op->args[2]); + z1 = t1->z_mask; + z2 = t2->z_mask; /* * Known-zeros does not imply known-ones. Therefore unless * arg2 is constant, we can't infer affected bits from it. */ - if (arg_is_const(op->args[2]) && - fold_affected_mask(ctx, op, z1 & ~z2)) { + if (ti_is_const(t2) && fold_affected_mask(ctx, op, z1 & ~z2)) { return true; } - return fold_masks(ctx, op); + z_mask = z1 & z2; + + /* + * Sign repetitions are perforce all identical, whether they are 1 or 0. + * Bitwise operations preserve the relative quantity of the repetitions. + */ + s_mask = t1->s_mask & t2->s_mask; + + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_andc(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Avoid double inversion of the value of second const operand. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) static bool fold_andc(OptContext *ctx, TCGOp *op) { - uint64_t z1; + uint64_t z_mask, s_mask; + TempOptInfo *t1, *t2; if (fold_const2(ctx, op) || fold_xx_to_i(ctx, op, 0) || @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) return true; } - z1 = arg_info(op->args[1])->z_mask; + t1 = arg_info(op->args[1]); + t2 = arg_info(op->args[2]); + z_mask = t1->z_mask; /* * Known-zeros does not imply known-ones. Therefore unless * arg2 is constant, we can't infer anything from it. */ - if (arg_is_const(op->args[2])) { - uint64_t z2 = ~arg_info(op->args[2])->z_mask; - if (fold_affected_mask(ctx, op, z1 & ~z2)) { + if (ti_is_const(t2)) { + uint64_t v2 = ti_const_val(t2); + if (fold_affected_mask(ctx, op, z_mask & v2)) { return true; } - z1 &= z2; + z_mask &= ~v2; } - ctx->z_mask = z1; - ctx->s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; - return fold_masks(ctx, op); + s_mask = t1->s_mask & t2->s_mask; + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_brcond(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Always set s_mask along the BSWAP_OS path, since the result is being explicitly sign-extended. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) static bool fold_bswap(OptContext *ctx, TCGOp *op) { uint64_t z_mask, s_mask, sign; + TempOptInfo *t1 = arg_info(op->args[1]); - if (arg_is_const(op->args[1])) { - uint64_t t = arg_info(op->args[1])->val; - - t = do_constant_folding(op->opc, ctx->type, t, op->args[2]); - return tcg_opt_gen_movi(ctx, op, op->args[0], t); + if (ti_is_const(t1)) { + return tcg_opt_gen_movi(ctx, op, op->args[0], + do_constant_folding(op->opc, ctx->type, + ti_const_val(t1), + op->args[2])); } - z_mask = arg_info(op->args[1])->z_mask; - + z_mask = t1->z_mask; switch (op->opc) { case INDEX_op_bswap16_i32: case INDEX_op_bswap16_i64: @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) /* If the sign bit may be 1, force all the bits above to 1. */ if (z_mask & sign) { z_mask |= sign; - s_mask = sign << 1; } + /* The value and therefore s_mask is explicitly sign-extended. */ + s_mask = sign; break; default: /* The high bits are undefined: force all bits above the sign to 1. */ z_mask |= sign << 1; break; } - ctx->z_mask = z_mask; - ctx->s_mask = s_mask; - return fold_masks(ctx, op); + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_call(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Compute s_mask from the union of the maximum count and the op2 fallback for op1 being zero. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_call(OptContext *ctx, TCGOp *op) static bool fold_count_zeros(OptContext *ctx, TCGOp *op) { - uint64_t z_mask; + uint64_t z_mask, s_mask; + TempOptInfo *t1 = arg_info(op->args[1]); + TempOptInfo *t2 = arg_info(op->args[2]); - if (arg_is_const(op->args[1])) { - uint64_t t = arg_info(op->args[1])->val; + if (ti_is_const(t1)) { + uint64_t t = ti_const_val(t1); if (t != 0) { t = do_constant_folding(op->opc, ctx->type, t, 0); @@ -XXX,XX +XXX,XX @@ static bool fold_count_zeros(OptContext *ctx, TCGOp *op) default: g_assert_not_reached(); } - ctx->z_mask = arg_info(op->args[2])->z_mask | z_mask; - return false; + s_mask = ~z_mask; + z_mask |= t2->z_mask; + s_mask &= t2->s_mask; + + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_ctpop(OptContext *ctx, TCGOp *op) -- 2.43.0 Add fold_masks_z as a trivial wrapper around fold_masks_zs. Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_masks_zs(OptContext *ctx, TCGOp *op, return true; } +static bool fold_masks_z(OptContext *ctx, TCGOp *op, uint64_t z_mask) +{ + return fold_masks_zs(ctx, op, z_mask, 0); +} + static bool fold_masks(OptContext *ctx, TCGOp *op) { return fold_masks_zs(ctx, op, ctx->z_mask, ctx->s_mask); @@ -XXX,XX +XXX,XX @@ static bool fold_count_zeros(OptContext *ctx, TCGOp *op) static bool fold_ctpop(OptContext *ctx, TCGOp *op) { + uint64_t z_mask; + if (fold_const1(ctx, op)) { return true; } switch (ctx->type) { case TCG_TYPE_I32: - ctx->z_mask = 32 | 31; + z_mask = 32 | 31; break; case TCG_TYPE_I64: - ctx->z_mask = 64 | 63; + z_mask = 64 | 63; break; default: g_assert_not_reached(); } - return false; + return fold_masks_z(ctx, op, z_mask); } static bool fold_deposit(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. When we fold to and, use fold_and. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_ctpop(OptContext *ctx, TCGOp *op) static bool fold_deposit(OptContext *ctx, TCGOp *op) { + TempOptInfo *t1 = arg_info(op->args[1]); + TempOptInfo *t2 = arg_info(op->args[2]); + int ofs = op->args[3]; + int len = op->args[4]; TCGOpcode and_opc; + uint64_t z_mask; - if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - uint64_t t1 = arg_info(op->args[1])->val; - uint64_t t2 = arg_info(op->args[2])->val; - - t1 = deposit64(t1, op->args[3], op->args[4], t2); - return tcg_opt_gen_movi(ctx, op, op->args[0], t1); + if (ti_is_const(t1) && ti_is_const(t2)) { + return tcg_opt_gen_movi(ctx, op, op->args[0], + deposit64(ti_const_val(t1), ofs, len, + ti_const_val(t2))); } switch (ctx->type) { @@ -XXX,XX +XXX,XX @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) } /* Inserting a value into zero at offset 0. */ - if (arg_is_const_val(op->args[1], 0) && op->args[3] == 0) { - uint64_t mask = MAKE_64BIT_MASK(0, op->args[4]); + if (ti_is_const_val(t1, 0) && ofs == 0) { + uint64_t mask = MAKE_64BIT_MASK(0, len); op->opc = and_opc; op->args[1] = op->args[2]; op->args[2] = arg_new_constant(ctx, mask); - ctx->z_mask = mask & arg_info(op->args[1])->z_mask; - return false; + return fold_and(ctx, op); } /* Inserting zero into a value. */ - if (arg_is_const_val(op->args[2], 0)) { - uint64_t mask = deposit64(-1, op->args[3], op->args[4], 0); + if (ti_is_const_val(t2, 0)) { + uint64_t mask = deposit64(-1, ofs, len, 0); op->opc = and_opc; op->args[2] = arg_new_constant(ctx, mask); - ctx->z_mask = mask & arg_info(op->args[1])->z_mask; - return false; + return fold_and(ctx, op); } - ctx->z_mask = deposit64(arg_info(op->args[1])->z_mask, - op->args[3], op->args[4], - arg_info(op->args[2])->z_mask); - return false; + z_mask = deposit64(t1->z_mask, ofs, len, t2->z_mask); + return fold_masks_z(ctx, op, z_mask); } static bool fold_divide(OptContext *ctx, TCGOp *op) -- 2.43.0 The input which overlaps the sign bit of the output can have its input s_mask propagated to the output s_mask. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) TempOptInfo *t2 = arg_info(op->args[2]); int ofs = op->args[3]; int len = op->args[4]; + int width; TCGOpcode and_opc; - uint64_t z_mask; + uint64_t z_mask, s_mask; if (ti_is_const(t1) && ti_is_const(t2)) { return tcg_opt_gen_movi(ctx, op, op->args[0], @@ -XXX,XX +XXX,XX @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) switch (ctx->type) { case TCG_TYPE_I32: and_opc = INDEX_op_and_i32; + width = 32; break; case TCG_TYPE_I64: and_opc = INDEX_op_and_i64; + width = 64; break; default: g_assert_not_reached(); @@ -XXX,XX +XXX,XX @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) return fold_and(ctx, op); } + /* The s_mask from the top portion of the deposit is still valid. */ + if (ofs + len == width) { + s_mask = t2->s_mask << ofs; + } else { + s_mask = t1->s_mask & ~MAKE_64BIT_MASK(0, ofs + len); + } + z_mask = deposit64(t1->z_mask, ofs, len, t2->z_mask); - return fold_masks_z(ctx, op, z_mask); + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_divide(OptContext *ctx, TCGOp *op) -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_divide(OptContext *ctx, TCGOp *op) fold_xi_to_x(ctx, op, 1)) { return true; } - return false; + return finish_folding(ctx, op); } static bool fold_dup(OptContext *ctx, TCGOp *op) -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_dup(OptContext *ctx, TCGOp *op) t = dup_const(TCGOP_VECE(op), t); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } - return false; + return finish_folding(ctx, op); } static bool fold_dup2(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_dup2(OptContext *ctx, TCGOp *op) op->opc = INDEX_op_dup_vec; TCGOP_VECE(op) = MO_32; } - return false; + return finish_folding(ctx, op); } static bool fold_eqv(OptContext *ctx, TCGOp *op) -- 2.43.0 Add fold_masks_s as a trivial wrapper around fold_masks_zs. Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_masks_z(OptContext *ctx, TCGOp *op, uint64_t z_mask) return fold_masks_zs(ctx, op, z_mask, 0); } +static bool fold_masks_s(OptContext *ctx, TCGOp *op, uint64_t s_mask) +{ + return fold_masks_zs(ctx, op, -1, s_mask); +} + static bool fold_masks(OptContext *ctx, TCGOp *op) { return fold_masks_zs(ctx, op, ctx->z_mask, ctx->s_mask); @@ -XXX,XX +XXX,XX @@ static bool fold_dup2(OptContext *ctx, TCGOp *op) static bool fold_eqv(OptContext *ctx, TCGOp *op) { + uint64_t s_mask; + if (fold_const2_commutative(ctx, op) || fold_xi_to_x(ctx, op, -1) || fold_xi_to_not(ctx, op, 0)) { return true; } - ctx->s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; - return false; + s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; + return fold_masks_s(ctx, op, s_mask); } static bool fold_extract(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) static bool fold_extract(OptContext *ctx, TCGOp *op) { uint64_t z_mask_old, z_mask; + TempOptInfo *t1 = arg_info(op->args[1]); int pos = op->args[2]; int len = op->args[3]; - if (arg_is_const(op->args[1])) { - uint64_t t; - - t = arg_info(op->args[1])->val; - t = extract64(t, pos, len); - return tcg_opt_gen_movi(ctx, op, op->args[0], t); + if (ti_is_const(t1)) { + return tcg_opt_gen_movi(ctx, op, op->args[0], + extract64(ti_const_val(t1), pos, len)); } - z_mask_old = arg_info(op->args[1])->z_mask; + z_mask_old = t1->z_mask; z_mask = extract64(z_mask_old, pos, len); if (pos == 0 && fold_affected_mask(ctx, op, z_mask_old ^ z_mask)) { return true; } - ctx->z_mask = z_mask; - return fold_masks(ctx, op); + return fold_masks_z(ctx, op, z_mask); } static bool fold_extract2(OptContext *ctx, TCGOp *op) -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_extract2(OptContext *ctx, TCGOp *op) } return tcg_opt_gen_movi(ctx, op, op->args[0], v1 | v2); } - return false; + return finish_folding(ctx, op); } static bool fold_exts(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Explicitly sign-extend z_mask instead of doing that manually. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_extract2(OptContext *ctx, TCGOp *op) static bool fold_exts(OptContext *ctx, TCGOp *op) { - uint64_t s_mask_old, s_mask, z_mask, sign; + uint64_t s_mask_old, s_mask, z_mask; bool type_change = false; + TempOptInfo *t1; if (fold_const1(ctx, op)) { return true; } - z_mask = arg_info(op->args[1])->z_mask; - s_mask = arg_info(op->args[1])->s_mask; + t1 = arg_info(op->args[1]); + z_mask = t1->z_mask; + s_mask = t1->s_mask; s_mask_old = s_mask; switch (op->opc) { CASE_OP_32_64(ext8s): - sign = INT8_MIN; - z_mask = (uint8_t)z_mask; + s_mask |= INT8_MIN; + z_mask = (int8_t)z_mask; break; CASE_OP_32_64(ext16s): - sign = INT16_MIN; - z_mask = (uint16_t)z_mask; + s_mask |= INT16_MIN; + z_mask = (int16_t)z_mask; break; case INDEX_op_ext_i32_i64: type_change = true; QEMU_FALLTHROUGH; case INDEX_op_ext32s_i64: - sign = INT32_MIN; - z_mask = (uint32_t)z_mask; + s_mask |= INT32_MIN; + z_mask = (int32_t)z_mask; break; default: g_assert_not_reached(); } - if (z_mask & sign) { - z_mask |= sign; - } - s_mask |= sign << 1; - - ctx->z_mask = z_mask; - ctx->s_mask = s_mask; if (0 && !type_change && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { return true; } - return fold_masks(ctx, op); + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_extu(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_extu(OptContext *ctx, TCGOp *op) g_assert_not_reached(); } - ctx->z_mask = z_mask; if (!type_change && fold_affected_mask(ctx, op, z_mask_old ^ z_mask)) { return true; } - return fold_masks(ctx, op); + + return fold_masks_z(ctx, op, z_mask); } static bool fold_mb(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_mov(OptContext *ctx, TCGOp *op) static bool fold_movcond(OptContext *ctx, TCGOp *op) { + uint64_t z_mask, s_mask; + TempOptInfo *tt, *ft; int i; /* If true and false values are the same, eliminate the cmp. */ @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]); } - ctx->z_mask = arg_info(op->args[3])->z_mask - | arg_info(op->args[4])->z_mask; - ctx->s_mask = arg_info(op->args[3])->s_mask - & arg_info(op->args[4])->s_mask; + tt = arg_info(op->args[3]); + ft = arg_info(op->args[4]); + z_mask = tt->z_mask | ft->z_mask; + s_mask = tt->s_mask & ft->s_mask; - if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { - uint64_t tv = arg_info(op->args[3])->val; - uint64_t fv = arg_info(op->args[4])->val; + if (ti_is_const(tt) && ti_is_const(ft)) { + uint64_t tv = ti_const_val(tt); + uint64_t fv = ti_const_val(ft); TCGOpcode opc, negopc = 0; TCGCond cond = op->args[5]; @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) } } } - return false; + + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_mul(OptContext *ctx, TCGOp *op) -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_mul(OptContext *ctx, TCGOp *op) fold_xi_to_x(ctx, op, 1)) { return true; } - return false; + return finish_folding(ctx, op); } static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) fold_xi_to_i(ctx, op, 0)) { return true; } - return false; + return finish_folding(ctx, op); } static bool fold_multiply2(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_multiply2(OptContext *ctx, TCGOp *op) tcg_opt_gen_movi(ctx, op2, rh, h); return true; } - return false; + return finish_folding(ctx, op); } static bool fold_nand(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_multiply2(OptContext *ctx, TCGOp *op) static bool fold_nand(OptContext *ctx, TCGOp *op) { + uint64_t s_mask; + if (fold_const2_commutative(ctx, op) || fold_xi_to_not(ctx, op, -1)) { return true; } - ctx->s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; - return false; + s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; + return fold_masks_s(ctx, op, s_mask); } static bool fold_neg_no_const(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_neg_no_const(OptContext *ctx, TCGOp *op) { /* Set to 1 all bits to the left of the rightmost. */ uint64_t z_mask = arg_info(op->args[1])->z_mask; - ctx->z_mask = -(z_mask & -z_mask); + z_mask = -(z_mask & -z_mask); - /* - * Because of fold_sub_to_neg, we want to always return true, - * via finish_folding. - */ - finish_folding(ctx, op); - return true; + return fold_masks_z(ctx, op, z_mask); } static bool fold_neg(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_neg(OptContext *ctx, TCGOp *op) static bool fold_nor(OptContext *ctx, TCGOp *op) { + uint64_t s_mask; + if (fold_const2_commutative(ctx, op) || fold_xi_to_not(ctx, op, 0)) { return true; } - ctx->s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; - return false; + s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; + return fold_masks_s(ctx, op, s_mask); } static bool fold_not(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_not(OptContext *ctx, TCGOp *op) if (fold_const1(ctx, op)) { return true; } - - ctx->s_mask = arg_info(op->args[1])->s_mask; - - /* Because of fold_to_not, we want to always return true, via finish. */ - finish_folding(ctx, op); - return true; + return fold_masks_s(ctx, op, arg_info(op->args[1])->s_mask); } static bool fold_or(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_not(OptContext *ctx, TCGOp *op) static bool fold_or(OptContext *ctx, TCGOp *op) { + uint64_t z_mask, s_mask; + TempOptInfo *t1, *t2; + if (fold_const2_commutative(ctx, op) || fold_xi_to_x(ctx, op, 0) || fold_xx_to_x(ctx, op)) { return true; } - ctx->z_mask = arg_info(op->args[1])->z_mask - | arg_info(op->args[2])->z_mask; - ctx->s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; - return fold_masks(ctx, op); + t1 = arg_info(op->args[1]); + t2 = arg_info(op->args[2]); + z_mask = t1->z_mask | t2->z_mask; + s_mask = t1->s_mask & t2->s_mask; + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_orc(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_or(OptContext *ctx, TCGOp *op) static bool fold_orc(OptContext *ctx, TCGOp *op) { + uint64_t s_mask; + if (fold_const2(ctx, op) || fold_xx_to_i(ctx, op, -1) || fold_xi_to_x(ctx, op, -1) || @@ -XXX,XX +XXX,XX @@ static bool fold_orc(OptContext *ctx, TCGOp *op) return true; } - ctx->s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; - return false; + s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; + return fold_masks_s(ctx, op, s_mask); } static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Be careful not to call fold_masks_zs when the memory operation is wide enough to require multiple outputs, so split into two functions: fold_qemu_ld_1reg and fold_qemu_ld_2reg. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_orc(OptContext *ctx, TCGOp *op) return fold_masks_s(ctx, op, s_mask); } -static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) +static bool fold_qemu_ld_1reg(OptContext *ctx, TCGOp *op) { const TCGOpDef *def = &tcg_op_defs[op->opc]; MemOpIdx oi = op->args[def->nb_oargs + def->nb_iargs]; MemOp mop = get_memop(oi); int width = 8 * memop_size(mop); + uint64_t z_mask = -1, s_mask = 0; if (width < 64) { if (mop & MO_SIGN) { - ctx->s_mask = MAKE_64BIT_MASK(width, 64 - width); + s_mask = MAKE_64BIT_MASK(width - 1, 64 - (width - 1)); } else { - ctx->z_mask = MAKE_64BIT_MASK(0, width); + z_mask = MAKE_64BIT_MASK(0, width); } } /* Opcodes that touch guest memory stop the mb optimization. */ ctx->prev_mb = NULL; - return false; + + return fold_masks_zs(ctx, op, z_mask, s_mask); +} + +static bool fold_qemu_ld_2reg(OptContext *ctx, TCGOp *op) +{ + /* Opcodes that touch guest memory stop the mb optimization. */ + ctx->prev_mb = NULL; + return finish_folding(ctx, op); } static bool fold_qemu_st(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; case INDEX_op_qemu_ld_a32_i32: case INDEX_op_qemu_ld_a64_i32: + done = fold_qemu_ld_1reg(&ctx, op); + break; case INDEX_op_qemu_ld_a32_i64: case INDEX_op_qemu_ld_a64_i64: + if (TCG_TARGET_REG_BITS == 64) { + done = fold_qemu_ld_1reg(&ctx, op); + break; + } + QEMU_FALLTHROUGH; case INDEX_op_qemu_ld_a32_i128: case INDEX_op_qemu_ld_a64_i128: - done = fold_qemu_ld(&ctx, op); + done = fold_qemu_ld_2reg(&ctx, op); break; case INDEX_op_qemu_st8_a32_i32: case INDEX_op_qemu_st8_a64_i32: -- 2.43.0 Stores have no output operands, and so need no further work. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_qemu_st(OptContext *ctx, TCGOp *op) { /* Opcodes that touch guest memory stop the mb optimization. */ ctx->prev_mb = NULL; - return false; + return true; } static bool fold_remainder(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_st(OptContext *ctx, TCGOp *op) if (op->args[1] != tcgv_ptr_arg(tcg_env)) { remove_mem_copy_all(ctx); - return false; + return true; } switch (op->opc) { @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_st(OptContext *ctx, TCGOp *op) g_assert_not_reached(); } remove_mem_copy_in(ctx, ofs, ofs + lm1); - return false; + return true; } static bool fold_tcg_st_memcopy(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_st_memcopy(OptContext *ctx, TCGOp *op) TCGType type; if (op->args[1] != tcgv_ptr_arg(tcg_env)) { - fold_tcg_st(ctx, op); - return false; + return fold_tcg_st(ctx, op); } src = arg_temp(op->args[0]); @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_st_memcopy(OptContext *ctx, TCGOp *op) last = ofs + tcg_type_size(type) - 1; remove_mem_copy_in(ctx, ofs, last); record_mem_copy(ctx, type, src, ofs, last); - return false; + return true; } static bool fold_xor(OptContext *ctx, TCGOp *op) -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_remainder(OptContext *ctx, TCGOp *op) fold_xx_to_i(ctx, op, 0)) { return true; } - return false; + return finish_folding(ctx, op); } static bool fold_setcond_zmask(OptContext *ctx, TCGOp *op, bool neg) -- 2.43.0 Change return from bool to int; distinguish between complete folding, simplification, and no change. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_remainder(OptContext *ctx, TCGOp *op) return finish_folding(ctx, op); } -static bool fold_setcond_zmask(OptContext *ctx, TCGOp *op, bool neg) +/* Return 1 if finished, -1 if simplified, 0 if unchanged. */ +static int fold_setcond_zmask(OptContext *ctx, TCGOp *op, bool neg) { uint64_t a_zmask, b_val; TCGCond cond; @@ -XXX,XX +XXX,XX @@ static bool fold_setcond_zmask(OptContext *ctx, TCGOp *op, bool neg) op->opc = xor_opc; op->args[2] = arg_new_constant(ctx, 1); } - return false; + return -1; } } - - return false; + return 0; } static void fold_setcond_tst_pow2(OptContext *ctx, TCGOp *op, bool neg) @@ -XXX,XX +XXX,XX @@ static bool fold_setcond(OptContext *ctx, TCGOp *op) return tcg_opt_gen_movi(ctx, op, op->args[0], i); } - if (fold_setcond_zmask(ctx, op, false)) { + i = fold_setcond_zmask(ctx, op, false); + if (i > 0) { return true; } - fold_setcond_tst_pow2(ctx, op, false); + if (i == 0) { + fold_setcond_tst_pow2(ctx, op, false); + } ctx->z_mask = 1; return false; @@ -XXX,XX +XXX,XX @@ static bool fold_negsetcond(OptContext *ctx, TCGOp *op) return tcg_opt_gen_movi(ctx, op, op->args[0], -i); } - if (fold_setcond_zmask(ctx, op, true)) { + i = fold_setcond_zmask(ctx, op, true); + if (i > 0) { return true; } - fold_setcond_tst_pow2(ctx, op, true); + if (i == 0) { + fold_setcond_tst_pow2(ctx, op, true); + } /* Value is {0,-1} so all bits are repetitions of the sign. */ ctx->s_mask = -1; -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_setcond(OptContext *ctx, TCGOp *op) fold_setcond_tst_pow2(ctx, op, false); } - ctx->z_mask = 1; - return false; + return fold_masks_z(ctx, op, 1); } static bool fold_negsetcond(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_negsetcond(OptContext *ctx, TCGOp *op) } /* Value is {0,-1} so all bits are repetitions of the sign. */ - ctx->s_mask = -1; - return false; + return fold_masks_s(ctx, op, -1); } static bool fold_setcond2(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) return fold_setcond(ctx, op); } - ctx->z_mask = 1; - return false; + return fold_masks_z(ctx, op, 1); do_setcond_const: return tcg_opt_gen_movi(ctx, op, op->args[0], i); -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_cmp_vec(OptContext *ctx, TCGOp *op) if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) { op->args[3] = tcg_swap_cond(op->args[3]); } - return false; + return finish_folding(ctx, op); } static bool fold_cmpsel_vec(OptContext *ctx, TCGOp *op) -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_cmpsel_vec(OptContext *ctx, TCGOp *op) if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) { op->args[5] = tcg_invert_cond(op->args[5]); } - return false; + return finish_folding(ctx, op); } static bool fold_sextract(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_cmpsel_vec(OptContext *ctx, TCGOp *op) static bool fold_sextract(OptContext *ctx, TCGOp *op) { uint64_t z_mask, s_mask, s_mask_old; + TempOptInfo *t1 = arg_info(op->args[1]); int pos = op->args[2]; int len = op->args[3]; - if (arg_is_const(op->args[1])) { - uint64_t t; - - t = arg_info(op->args[1])->val; - t = sextract64(t, pos, len); - return tcg_opt_gen_movi(ctx, op, op->args[0], t); + if (ti_is_const(t1)) { + return tcg_opt_gen_movi(ctx, op, op->args[0], + sextract64(ti_const_val(t1), pos, len)); } - z_mask = arg_info(op->args[1])->z_mask; - z_mask = sextract64(z_mask, pos, len); - ctx->z_mask = z_mask; - - s_mask_old = arg_info(op->args[1])->s_mask; - s_mask = sextract64(s_mask_old, pos, len); - s_mask |= MAKE_64BIT_MASK(len, 64 - len); - ctx->s_mask = s_mask; + s_mask_old = t1->s_mask; + s_mask = s_mask_old >> pos; + s_mask |= -1ull << (len - 1); if (0 && pos == 0 && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { return true; } - return fold_masks(ctx, op); + z_mask = sextract64(t1->z_mask, pos, len); + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_shift(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) static bool fold_shift(OptContext *ctx, TCGOp *op) { uint64_t s_mask, z_mask, sign; + TempOptInfo *t1, *t2; if (fold_const2(ctx, op) || fold_ix_to_i(ctx, op, 0) || @@ -XXX,XX +XXX,XX @@ static bool fold_shift(OptContext *ctx, TCGOp *op) return true; } - s_mask = arg_info(op->args[1])->s_mask; - z_mask = arg_info(op->args[1])->z_mask; + t1 = arg_info(op->args[1]); + t2 = arg_info(op->args[2]); + s_mask = t1->s_mask; + z_mask = t1->z_mask; - if (arg_is_const(op->args[2])) { - int sh = arg_info(op->args[2])->val; - - ctx->z_mask = do_constant_folding(op->opc, ctx->type, z_mask, sh); + if (ti_is_const(t2)) { + int sh = ti_const_val(t2); + z_mask = do_constant_folding(op->opc, ctx->type, z_mask, sh); s_mask = do_constant_folding(op->opc, ctx->type, s_mask, sh); - return fold_masks(ctx, op); + return fold_masks_zs(ctx, op, z_mask, s_mask); } switch (op->opc) { @@ -XXX,XX +XXX,XX @@ static bool fold_shift(OptContext *ctx, TCGOp *op) * Arithmetic right shift will not reduce the number of * input sign repetitions. */ - ctx->s_mask = s_mask; - break; + return fold_masks_s(ctx, op, s_mask); CASE_OP_32_64(shr): /* * If the sign bit is known zero, then logical right shift - * will not reduced the number of input sign repetitions. + * will not reduce the number of input sign repetitions. */ - sign = (s_mask & -s_mask) >> 1; + sign = -s_mask; if (sign && !(z_mask & sign)) { - ctx->s_mask = s_mask; + return fold_masks_s(ctx, op, s_mask); } break; default: break; } - return false; + return finish_folding(ctx, op); } static bool fold_sub_to_neg(OptContext *ctx, TCGOp *op) -- 2.43.0 Merge the two conditions, sign != 0 && !(z_mask & sign), by testing ~z_mask & sign. If sign == 0, the logical and will produce false. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) static bool fold_shift(OptContext *ctx, TCGOp *op) { - uint64_t s_mask, z_mask, sign; + uint64_t s_mask, z_mask; TempOptInfo *t1, *t2; if (fold_const2(ctx, op) || @@ -XXX,XX +XXX,XX @@ static bool fold_shift(OptContext *ctx, TCGOp *op) * If the sign bit is known zero, then logical right shift * will not reduce the number of input sign repetitions. */ - sign = -s_mask; - if (sign && !(z_mask & sign)) { + if (~z_mask & -s_mask) { return fold_masks_s(ctx, op, s_mask); } break; -- 2.43.0 Duplicate fold_sub_vec into fold_sub instead of calling it, now that fold_sub_vec always returns true. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_sub_vec(OptContext *ctx, TCGOp *op) fold_sub_to_neg(ctx, op)) { return true; } - return false; + return finish_folding(ctx, op); } static bool fold_sub(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || fold_sub_vec(ctx, op)) { + if (fold_const2(ctx, op) || + fold_xx_to_i(ctx, op, 0) || + fold_xi_to_x(ctx, op, 0) || + fold_sub_to_neg(ctx, op)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_sub(OptContext *ctx, TCGOp *op) ? INDEX_op_add_i32 : INDEX_op_add_i64); op->args[2] = arg_new_constant(ctx, -val); } - return false; + return finish_folding(ctx, op); } static bool fold_sub2(OptContext *ctx, TCGOp *op) -- 2.43.0 Avoid the use of the OptContext slots. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_sub2(OptContext *ctx, TCGOp *op) static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) { + uint64_t z_mask = -1, s_mask = 0; + /* We can't do any folding with a load, but we can record bits. */ switch (op->opc) { CASE_OP_32_64(ld8s): - ctx->s_mask = MAKE_64BIT_MASK(8, 56); + s_mask = INT8_MIN; break; CASE_OP_32_64(ld8u): - ctx->z_mask = MAKE_64BIT_MASK(0, 8); + z_mask = MAKE_64BIT_MASK(0, 8); break; CASE_OP_32_64(ld16s): - ctx->s_mask = MAKE_64BIT_MASK(16, 48); + s_mask = INT16_MIN; break; CASE_OP_32_64(ld16u): - ctx->z_mask = MAKE_64BIT_MASK(0, 16); + z_mask = MAKE_64BIT_MASK(0, 16); break; case INDEX_op_ld32s_i64: - ctx->s_mask = MAKE_64BIT_MASK(32, 32); + s_mask = INT32_MIN; break; case INDEX_op_ld32u_i64: - ctx->z_mask = MAKE_64BIT_MASK(0, 32); + z_mask = MAKE_64BIT_MASK(0, 32); break; default: g_assert_not_reached(); } - return false; + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_tcg_ld_memcopy(OptContext *ctx, TCGOp *op) -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_ld_memcopy(OptContext *ctx, TCGOp *op) TCGType type; if (op->args[1] != tcgv_ptr_arg(tcg_env)) { - return false; + return finish_folding(ctx, op); } type = ctx->type; -- 2.43.0 Avoid the use of the OptContext slots. Find TempOptInfo once. Remove fold_masks as the function becomes unused. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_masks_s(OptContext *ctx, TCGOp *op, uint64_t s_mask) return fold_masks_zs(ctx, op, -1, s_mask); } -static bool fold_masks(OptContext *ctx, TCGOp *op) -{ - return fold_masks_zs(ctx, op, ctx->z_mask, ctx->s_mask); -} - /* * An "affected" mask bit is 0 if and only if the result is identical * to the first input. Thus if the entire mask is 0, the operation @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_st_memcopy(OptContext *ctx, TCGOp *op) static bool fold_xor(OptContext *ctx, TCGOp *op) { + uint64_t z_mask, s_mask; + TempOptInfo *t1, *t2; + if (fold_const2_commutative(ctx, op) || fold_xx_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, 0) || @@ -XXX,XX +XXX,XX @@ static bool fold_xor(OptContext *ctx, TCGOp *op) return true; } - ctx->z_mask = arg_info(op->args[1])->z_mask - | arg_info(op->args[2])->z_mask; - ctx->s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; - return fold_masks(ctx, op); + t1 = arg_info(op->args[1]); + t2 = arg_info(op->args[2]); + z_mask = t1->z_mask | t2->z_mask; + s_mask = t1->s_mask & t2->s_mask; + return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_bitsel_vec(OptContext *ctx, TCGOp *op) -- 2.43.0 Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_bitsel_vec(OptContext *ctx, TCGOp *op) return fold_orc(ctx, op); } } - return false; + return finish_folding(ctx, op); } /* Propagate constants and copies, fold constant expressions. */ -- 2.43.0 All non-default cases now finish folding within each function. Do the same with the default case and assert it is done after. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) done = true; break; default: + done = finish_folding(&ctx, op); break; } - - if (!done) { - finish_folding(&ctx, op); - } + tcg_debug_assert(done); } } -- 2.43.0 All mask setting is now done with parameters via fold_masks_*. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { QSIMPLEQ_HEAD(, MemCopyInfo) mem_free; /* In flight values from optimization. */ - uint64_t z_mask; /* mask bit is 0 iff value bit is 0 */ - uint64_t s_mask; /* mask bit is 1 if value bit matches msb */ TCGType type; } OptContext; @@ -XXX,XX +XXX,XX @@ static bool finish_folding(OptContext *ctx, TCGOp *op) for (i = 0; i < nb_oargs; i++) { TCGTemp *ts = arg_temp(op->args[i]); reset_ts(ctx, ts); - /* - * Save the corresponding known-zero/sign bits mask for the - * first output argument (only one supported so far). - */ - if (i == 0) { - ts_info(ts)->z_mask = ctx->z_mask; - } } return true; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) ctx.type = TCG_TYPE_I32; } - /* Assume all bits affected, no bits known zero, no sign reps. */ - ctx.z_mask = -1; - ctx.s_mask = 0; - /* * Process each opcode. * Sorted alphabetically by opcode as much as possible. -- 2.43.0 All instances of s_mask have been converted to the new representation. We can now re-enable usage. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_exts(OptContext *ctx, TCGOp *op) g_assert_not_reached(); } - if (0 && !type_change && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { + if (!type_change && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) s_mask = s_mask_old >> pos; s_mask |= -1ull << (len - 1); - if (0 && pos == 0 && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { + if (pos == 0 && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { return true; } -- 2.43.0 The big comment just above says functions should be sorted. Add forward declarations as needed. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 114 +++++++++++++++++++++++++------------------------ 1 file changed, 59 insertions(+), 55 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_xx_to_x(OptContext *ctx, TCGOp *op) * 3) those that produce information about the result value. */ +static bool fold_or(OptContext *ctx, TCGOp *op); +static bool fold_orc(OptContext *ctx, TCGOp *op); +static bool fold_xor(OptContext *ctx, TCGOp *op); + static bool fold_add(OptContext *ctx, TCGOp *op) { if (fold_const2_commutative(ctx, op) || @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) return fold_masks_zs(ctx, op, z_mask, s_mask); } +static bool fold_bitsel_vec(OptContext *ctx, TCGOp *op) +{ + /* If true and false values are the same, eliminate the cmp. */ + if (args_are_copies(op->args[2], op->args[3])) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]); + } + + if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { + uint64_t tv = arg_info(op->args[2])->val; + uint64_t fv = arg_info(op->args[3])->val; + + if (tv == -1 && fv == 0) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); + } + if (tv == 0 && fv == -1) { + if (TCG_TARGET_HAS_not_vec) { + op->opc = INDEX_op_not_vec; + return fold_not(ctx, op); + } else { + op->opc = INDEX_op_xor_vec; + op->args[2] = arg_new_constant(ctx, -1); + return fold_xor(ctx, op); + } + } + } + if (arg_is_const(op->args[2])) { + uint64_t tv = arg_info(op->args[2])->val; + if (tv == -1) { + op->opc = INDEX_op_or_vec; + op->args[2] = op->args[3]; + return fold_or(ctx, op); + } + if (tv == 0 && TCG_TARGET_HAS_andc_vec) { + op->opc = INDEX_op_andc_vec; + op->args[2] = op->args[1]; + op->args[1] = op->args[3]; + return fold_andc(ctx, op); + } + } + if (arg_is_const(op->args[3])) { + uint64_t fv = arg_info(op->args[3])->val; + if (fv == 0) { + op->opc = INDEX_op_and_vec; + return fold_and(ctx, op); + } + if (fv == -1 && TCG_TARGET_HAS_orc_vec) { + op->opc = INDEX_op_orc_vec; + op->args[2] = op->args[1]; + op->args[1] = op->args[3]; + return fold_orc(ctx, op); + } + } + return finish_folding(ctx, op); +} + static bool fold_brcond(OptContext *ctx, TCGOp *op) { int i = do_constant_folding_cond1(ctx, op, NO_DEST, &op->args[0], @@ -XXX,XX +XXX,XX @@ static bool fold_xor(OptContext *ctx, TCGOp *op) return fold_masks_zs(ctx, op, z_mask, s_mask); } -static bool fold_bitsel_vec(OptContext *ctx, TCGOp *op) -{ - /* If true and false values are the same, eliminate the cmp. */ - if (args_are_copies(op->args[2], op->args[3])) { - return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]); - } - - if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { - uint64_t tv = arg_info(op->args[2])->val; - uint64_t fv = arg_info(op->args[3])->val; - - if (tv == -1 && fv == 0) { - return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); - } - if (tv == 0 && fv == -1) { - if (TCG_TARGET_HAS_not_vec) { - op->opc = INDEX_op_not_vec; - return fold_not(ctx, op); - } else { - op->opc = INDEX_op_xor_vec; - op->args[2] = arg_new_constant(ctx, -1); - return fold_xor(ctx, op); - } - } - } - if (arg_is_const(op->args[2])) { - uint64_t tv = arg_info(op->args[2])->val; - if (tv == -1) { - op->opc = INDEX_op_or_vec; - op->args[2] = op->args[3]; - return fold_or(ctx, op); - } - if (tv == 0 && TCG_TARGET_HAS_andc_vec) { - op->opc = INDEX_op_andc_vec; - op->args[2] = op->args[1]; - op->args[1] = op->args[3]; - return fold_andc(ctx, op); - } - } - if (arg_is_const(op->args[3])) { - uint64_t fv = arg_info(op->args[3])->val; - if (fv == 0) { - op->opc = INDEX_op_and_vec; - return fold_and(ctx, op); - } - if (fv == -1 && TCG_TARGET_HAS_orc_vec) { - op->opc = INDEX_op_orc_vec; - op->args[2] = op->args[1]; - op->args[1] = op->args[3]; - return fold_orc(ctx, op); - } - } - return finish_folding(ctx, op); -} - /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { -- 2.43.0 The big comment just above says functions should be sorted. Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 60 +++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_call(OptContext *ctx, TCGOp *op) return true; } +static bool fold_cmp_vec(OptContext *ctx, TCGOp *op) +{ + /* Canonicalize the comparison to put immediate second. */ + if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) { + op->args[3] = tcg_swap_cond(op->args[3]); + } + return finish_folding(ctx, op); +} + +static bool fold_cmpsel_vec(OptContext *ctx, TCGOp *op) +{ + /* If true and false values are the same, eliminate the cmp. */ + if (args_are_copies(op->args[3], op->args[4])) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[3]); + } + + /* Canonicalize the comparison to put immediate second. */ + if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) { + op->args[5] = tcg_swap_cond(op->args[5]); + } + /* + * Canonicalize the "false" input reg to match the destination, + * so that the tcg backend can implement "move if true". + */ + if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) { + op->args[5] = tcg_invert_cond(op->args[5]); + } + return finish_folding(ctx, op); +} + static bool fold_count_zeros(OptContext *ctx, TCGOp *op) { uint64_t z_mask, s_mask; @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) return tcg_opt_gen_movi(ctx, op, op->args[0], i); } -static bool fold_cmp_vec(OptContext *ctx, TCGOp *op) -{ - /* Canonicalize the comparison to put immediate second. */ - if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) { - op->args[3] = tcg_swap_cond(op->args[3]); - } - return finish_folding(ctx, op); -} - -static bool fold_cmpsel_vec(OptContext *ctx, TCGOp *op) -{ - /* If true and false values are the same, eliminate the cmp. */ - if (args_are_copies(op->args[3], op->args[4])) { - return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[3]); - } - - /* Canonicalize the comparison to put immediate second. */ - if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) { - op->args[5] = tcg_swap_cond(op->args[5]); - } - /* - * Canonicalize the "false" input reg to match the destination, - * so that the tcg backend can implement "move if true". - */ - if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) { - op->args[5] = tcg_invert_cond(op->args[5]); - } - return finish_folding(ctx, op); -} - static bool fold_sextract(OptContext *ctx, TCGOp *op) { uint64_t z_mask, s_mask, s_mask_old; -- 2.43.0 We currently have a flag, float_muladd_halve_result, to scale the result by 2**-1. Extend this to handle arbitrary scaling. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat.h | 6 ++++ fpu/softfloat.c | 58 ++++++++++++++++++++++----------------- fpu/softfloat-parts.c.inc | 7 +++-- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/include/fpu/softfloat.h b/include/fpu/softfloat.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat.h +++ b/include/fpu/softfloat.h @@ -XXX,XX +XXX,XX @@ float16 float16_add(float16, float16, float_status *status); float16 float16_sub(float16, float16, float_status *status); float16 float16_mul(float16, float16, float_status *status); float16 float16_muladd(float16, float16, float16, int, float_status *status); +float16 float16_muladd_scalbn(float16, float16, float16, + int, int, float_status *status); float16 float16_div(float16, float16, float_status *status); float16 float16_scalbn(float16, int, float_status *status); float16 float16_min(float16, float16, float_status *status); @@ -XXX,XX +XXX,XX @@ float32 float32_mul(float32, float32, float_status *status); float32 float32_div(float32, float32, float_status *status); float32 float32_rem(float32, float32, float_status *status); float32 float32_muladd(float32, float32, float32, int, float_status *status); +float32 float32_muladd_scalbn(float32, float32, float32, + int, int, float_status *status); float32 float32_sqrt(float32, float_status *status); float32 float32_exp2(float32, float_status *status); float32 float32_log2(float32, float_status *status); @@ -XXX,XX +XXX,XX @@ float64 float64_mul(float64, float64, float_status *status); float64 float64_div(float64, float64, float_status *status); float64 float64_rem(float64, float64, float_status *status); float64 float64_muladd(float64, float64, float64, int, float_status *status); +float64 float64_muladd_scalbn(float64, float64, float64, + int, int, float_status *status); float64 float64_sqrt(float64, float_status *status); float64 float64_log2(float64, float_status *status); FloatRelation float64_compare(float64, float64, float_status *status); diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static FloatParts128 *parts128_mul(FloatParts128 *a, FloatParts128 *b, #define parts_mul(A, B, S) \ PARTS_GENERIC_64_128(mul, A)(A, B, S) -static FloatParts64 *parts64_muladd(FloatParts64 *a, FloatParts64 *b, - FloatParts64 *c, int flags, - float_status *s); -static FloatParts128 *parts128_muladd(FloatParts128 *a, FloatParts128 *b, - FloatParts128 *c, int flags, - float_status *s); +static FloatParts64 *parts64_muladd_scalbn(FloatParts64 *a, FloatParts64 *b, + FloatParts64 *c, int scale, + int flags, float_status *s); +static FloatParts128 *parts128_muladd_scalbn(FloatParts128 *a, FloatParts128 *b, + FloatParts128 *c, int scale, + int flags, float_status *s); -#define parts_muladd(A, B, C, Z, S) \ - PARTS_GENERIC_64_128(muladd, A)(A, B, C, Z, S) +#define parts_muladd_scalbn(A, B, C, Z, Y, S) \ + PARTS_GENERIC_64_128(muladd_scalbn, A)(A, B, C, Z, Y, S) static FloatParts64 *parts64_div(FloatParts64 *a, FloatParts64 *b, float_status *s); @@ -XXX,XX +XXX,XX @@ floatx80_mul(floatx80 a, floatx80 b, float_status *status) * Fused multiply-add */ -float16 QEMU_FLATTEN float16_muladd(float16 a, float16 b, float16 c, - int flags, float_status *status) +float16 QEMU_FLATTEN +float16_muladd_scalbn(float16 a, float16 b, float16 c, + int scale, int flags, float_status *status) { FloatParts64 pa, pb, pc, *pr; float16_unpack_canonical(&pa, a, status); float16_unpack_canonical(&pb, b, status); float16_unpack_canonical(&pc, c, status); - pr = parts_muladd(&pa, &pb, &pc, flags, status); + pr = parts_muladd_scalbn(&pa, &pb, &pc, scale, flags, status); return float16_round_pack_canonical(pr, status); } -static float32 QEMU_SOFTFLOAT_ATTR -soft_f32_muladd(float32 a, float32 b, float32 c, int flags, - float_status *status) +float16 float16_muladd(float16 a, float16 b, float16 c, + int flags, float_status *status) +{ + return float16_muladd_scalbn(a, b, c, 0, flags, status); +} + +float32 QEMU_SOFTFLOAT_ATTR +float32_muladd_scalbn(float32 a, float32 b, float32 c, + int scale, int flags, float_status *status) { FloatParts64 pa, pb, pc, *pr; float32_unpack_canonical(&pa, a, status); float32_unpack_canonical(&pb, b, status); float32_unpack_canonical(&pc, c, status); - pr = parts_muladd(&pa, &pb, &pc, flags, status); + pr = parts_muladd_scalbn(&pa, &pb, &pc, scale, flags, status); return float32_round_pack_canonical(pr, status); } -static float64 QEMU_SOFTFLOAT_ATTR -soft_f64_muladd(float64 a, float64 b, float64 c, int flags, - float_status *status) +float64 QEMU_SOFTFLOAT_ATTR +float64_muladd_scalbn(float64 a, float64 b, float64 c, + int scale, int flags, float_status *status) { FloatParts64 pa, pb, pc, *pr; float64_unpack_canonical(&pa, a, status); float64_unpack_canonical(&pb, b, status); float64_unpack_canonical(&pc, c, status); - pr = parts_muladd(&pa, &pb, &pc, flags, status); + pr = parts_muladd_scalbn(&pa, &pb, &pc, scale, flags, status); return float64_round_pack_canonical(pr, status); } @@ -XXX,XX +XXX,XX @@ float32_muladd(float32 xa, float32 xb, float32 xc, int flags, float_status *s) return ur.s; soft: - return soft_f32_muladd(ua.s, ub.s, uc.s, flags, s); + return float32_muladd_scalbn(ua.s, ub.s, uc.s, 0, flags, s); } float64 QEMU_FLATTEN @@ -XXX,XX +XXX,XX @@ float64_muladd(float64 xa, float64 xb, float64 xc, int flags, float_status *s) return ur.s; soft: - return soft_f64_muladd(ua.s, ub.s, uc.s, flags, s); + return float64_muladd_scalbn(ua.s, ub.s, uc.s, 0, flags, s); } float64 float64r32_muladd(float64 a, float64 b, float64 c, @@ -XXX,XX +XXX,XX @@ float64 float64r32_muladd(float64 a, float64 b, float64 c, float64_unpack_canonical(&pa, a, status); float64_unpack_canonical(&pb, b, status); float64_unpack_canonical(&pc, c, status); - pr = parts_muladd(&pa, &pb, &pc, flags, status); + pr = parts_muladd_scalbn(&pa, &pb, &pc, 0, flags, status); return float64r32_round_pack_canonical(pr, status); } @@ -XXX,XX +XXX,XX @@ bfloat16 QEMU_FLATTEN bfloat16_muladd(bfloat16 a, bfloat16 b, bfloat16 c, bfloat16_unpack_canonical(&pa, a, status); bfloat16_unpack_canonical(&pb, b, status); bfloat16_unpack_canonical(&pc, c, status); - pr = parts_muladd(&pa, &pb, &pc, flags, status); + pr = parts_muladd_scalbn(&pa, &pb, &pc, 0, flags, status); return bfloat16_round_pack_canonical(pr, status); } @@ -XXX,XX +XXX,XX @@ float128 QEMU_FLATTEN float128_muladd(float128 a, float128 b, float128 c, float128_unpack_canonical(&pa, a, status); float128_unpack_canonical(&pb, b, status); float128_unpack_canonical(&pc, c, status); - pr = parts_muladd(&pa, &pb, &pc, flags, status); + pr = parts_muladd_scalbn(&pa, &pb, &pc, 0, flags, status); return float128_round_pack_canonical(pr, status); } @@ -XXX,XX +XXX,XX @@ float32 float32_exp2(float32 a, float_status *status) float64_unpack_canonical(&rp, float64_one, status); for (i = 0 ; i < 15 ; i++) { + float64_unpack_canonical(&tp, float32_exp2_coefficients[i], status); - rp = *parts_muladd(&tp, &xnp, &rp, 0, status); + rp = *parts_muladd_scalbn(&tp, &xnp, &rp, 0, 0, status); xnp = *parts_mul(&xnp, &xp, status); } diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(mul)(FloatPartsN *a, FloatPartsN *b, * Requires A and C extracted into a double-sized structure to provide the * extra space for the widening multiply. */ -static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, - FloatPartsN *c, int flags, float_status *s) +static FloatPartsN *partsN(muladd_scalbn)(FloatPartsN *a, FloatPartsN *b, + FloatPartsN *c, int scale, + int flags, float_status *s) { int ab_mask, abc_mask; FloatPartsW p_widen, c_widen; @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, a->exp = p_widen.exp; return_normal: + /* TODO: Replace all use of float_muladd_halve_result with scale. */ if (flags & float_muladd_halve_result) { a->exp -= 1; } + a->exp += scale; finish_sign: if (flags & float_muladd_negate_result) { a->sign ^= 1; -- 2.43.0 Use the scalbn interface instead of float_muladd_halve_result. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/arm/tcg/helper-a64.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/target/arm/tcg/helper-a64.c b/target/arm/tcg/helper-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/tcg/helper-a64.c +++ b/target/arm/tcg/helper-a64.c @@ -XXX,XX +XXX,XX @@ uint32_t HELPER(rsqrtsf_f16)(uint32_t a, uint32_t b, float_status *fpst) (float16_is_infinity(b) && float16_is_zero(a))) { return float16_one_point_five; } - return float16_muladd(a, b, float16_three, float_muladd_halve_result, fpst); + return float16_muladd_scalbn(a, b, float16_three, -1, 0, fpst); } float32 HELPER(rsqrtsf_f32)(float32 a, float32 b, float_status *fpst) @@ -XXX,XX +XXX,XX @@ float32 HELPER(rsqrtsf_f32)(float32 a, float32 b, float_status *fpst) (float32_is_infinity(b) && float32_is_zero(a))) { return float32_one_point_five; } - return float32_muladd(a, b, float32_three, float_muladd_halve_result, fpst); + return float32_muladd_scalbn(a, b, float32_three, -1, 0, fpst); } float64 HELPER(rsqrtsf_f64)(float64 a, float64 b, float_status *fpst) @@ -XXX,XX +XXX,XX @@ float64 HELPER(rsqrtsf_f64)(float64 a, float64 b, float_status *fpst) (float64_is_infinity(b) && float64_is_zero(a))) { return float64_one_point_five; } - return float64_muladd(a, b, float64_three, float_muladd_halve_result, fpst); + return float64_muladd_scalbn(a, b, float64_three, -1, 0, fpst); } /* Floating-point reciprocal exponent - see FPRecpX in ARM ARM */ -- 2.43.0 Use the scalbn interface instead of float_muladd_halve_result. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/sparc/helper.h | 4 +- target/sparc/fop_helper.c | 8 ++-- target/sparc/translate.c | 80 +++++++++++++++++++++++---------------- 3 files changed, 54 insertions(+), 38 deletions(-) diff --git a/target/sparc/helper.h b/target/sparc/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/helper.h +++ b/target/sparc/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_3(faddd, TCG_CALL_NO_WG, f64, env, f64, f64) DEF_HELPER_FLAGS_3(fsubd, TCG_CALL_NO_WG, f64, env, f64, f64) DEF_HELPER_FLAGS_3(fmuld, TCG_CALL_NO_WG, f64, env, f64, f64) DEF_HELPER_FLAGS_3(fdivd, TCG_CALL_NO_WG, f64, env, f64, f64) -DEF_HELPER_FLAGS_5(fmaddd, TCG_CALL_NO_WG, f64, env, f64, f64, f64, i32) +DEF_HELPER_FLAGS_6(fmaddd, TCG_CALL_NO_WG, f64, env, f64, f64, f64, s32, i32) DEF_HELPER_FLAGS_3(fnaddd, TCG_CALL_NO_WG, f64, env, f64, f64) DEF_HELPER_FLAGS_3(fnmuld, TCG_CALL_NO_WG, f64, env, f64, f64) @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_3(fadds, TCG_CALL_NO_WG, f32, env, f32, f32) DEF_HELPER_FLAGS_3(fsubs, TCG_CALL_NO_WG, f32, env, f32, f32) DEF_HELPER_FLAGS_3(fmuls, TCG_CALL_NO_WG, f32, env, f32, f32) DEF_HELPER_FLAGS_3(fdivs, TCG_CALL_NO_WG, f32, env, f32, f32) -DEF_HELPER_FLAGS_5(fmadds, TCG_CALL_NO_WG, f32, env, f32, f32, f32, i32) +DEF_HELPER_FLAGS_6(fmadds, TCG_CALL_NO_WG, f32, env, f32, f32, f32, s32, i32) DEF_HELPER_FLAGS_3(fnadds, TCG_CALL_NO_WG, f32, env, f32, f32) DEF_HELPER_FLAGS_3(fnmuls, TCG_CALL_NO_WG, f32, env, f32, f32) diff --git a/target/sparc/fop_helper.c b/target/sparc/fop_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/fop_helper.c +++ b/target/sparc/fop_helper.c @@ -XXX,XX +XXX,XX @@ Int128 helper_fsqrtq(CPUSPARCState *env, Int128 src) } float32 helper_fmadds(CPUSPARCState *env, float32 s1, - float32 s2, float32 s3, uint32_t op) + float32 s2, float32 s3, int32_t sc, uint32_t op) { - float32 ret = float32_muladd(s1, s2, s3, op, &env->fp_status); + float32 ret = float32_muladd_scalbn(s1, s2, s3, sc, op, &env->fp_status); check_ieee_exceptions(env, GETPC()); return ret; } float64 helper_fmaddd(CPUSPARCState *env, float64 s1, - float64 s2, float64 s3, uint32_t op) + float64 s2, float64 s3, int32_t sc, uint32_t op) { - float64 ret = float64_muladd(s1, s2, s3, op, &env->fp_status); + float64 ret = float64_muladd_scalbn(s1, s2, s3, sc, op, &env->fp_status); check_ieee_exceptions(env, GETPC()); return ret; } diff --git a/target/sparc/translate.c b/target/sparc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/translate.c +++ b/target/sparc/translate.c @@ -XXX,XX +XXX,XX @@ static void gen_op_fabsq(TCGv_i128 dst, TCGv_i128 src) static void gen_op_fmadds(TCGv_i32 d, TCGv_i32 s1, TCGv_i32 s2, TCGv_i32 s3) { - gen_helper_fmadds(d, tcg_env, s1, s2, s3, tcg_constant_i32(0)); + TCGv_i32 z = tcg_constant_i32(0); + gen_helper_fmadds(d, tcg_env, s1, s2, s3, z, z); } static void gen_op_fmaddd(TCGv_i64 d, TCGv_i64 s1, TCGv_i64 s2, TCGv_i64 s3) { - gen_helper_fmaddd(d, tcg_env, s1, s2, s3, tcg_constant_i32(0)); + TCGv_i32 z = tcg_constant_i32(0); + gen_helper_fmaddd(d, tcg_env, s1, s2, s3, z, z); } static void gen_op_fmsubs(TCGv_i32 d, TCGv_i32 s1, TCGv_i32 s2, TCGv_i32 s3) { - int op = float_muladd_negate_c; - gen_helper_fmadds(d, tcg_env, s1, s2, s3, tcg_constant_i32(op)); + TCGv_i32 z = tcg_constant_i32(0); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_c); + gen_helper_fmadds(d, tcg_env, s1, s2, s3, z, op); } static void gen_op_fmsubd(TCGv_i64 d, TCGv_i64 s1, TCGv_i64 s2, TCGv_i64 s3) { - int op = float_muladd_negate_c; - gen_helper_fmaddd(d, tcg_env, s1, s2, s3, tcg_constant_i32(op)); + TCGv_i32 z = tcg_constant_i32(0); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_c); + gen_helper_fmaddd(d, tcg_env, s1, s2, s3, z, op); } static void gen_op_fnmsubs(TCGv_i32 d, TCGv_i32 s1, TCGv_i32 s2, TCGv_i32 s3) { - int op = float_muladd_negate_c | float_muladd_negate_result; - gen_helper_fmadds(d, tcg_env, s1, s2, s3, tcg_constant_i32(op)); + TCGv_i32 z = tcg_constant_i32(0); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_c | + float_muladd_negate_result); + gen_helper_fmadds(d, tcg_env, s1, s2, s3, z, op); } static void gen_op_fnmsubd(TCGv_i64 d, TCGv_i64 s1, TCGv_i64 s2, TCGv_i64 s3) { - int op = float_muladd_negate_c | float_muladd_negate_result; - gen_helper_fmaddd(d, tcg_env, s1, s2, s3, tcg_constant_i32(op)); + TCGv_i32 z = tcg_constant_i32(0); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_c | + float_muladd_negate_result); + gen_helper_fmaddd(d, tcg_env, s1, s2, s3, z, op); } static void gen_op_fnmadds(TCGv_i32 d, TCGv_i32 s1, TCGv_i32 s2, TCGv_i32 s3) { - int op = float_muladd_negate_result; - gen_helper_fmadds(d, tcg_env, s1, s2, s3, tcg_constant_i32(op)); + TCGv_i32 z = tcg_constant_i32(0); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_result); + gen_helper_fmadds(d, tcg_env, s1, s2, s3, z, op); } static void gen_op_fnmaddd(TCGv_i64 d, TCGv_i64 s1, TCGv_i64 s2, TCGv_i64 s3) { - int op = float_muladd_negate_result; - gen_helper_fmaddd(d, tcg_env, s1, s2, s3, tcg_constant_i32(op)); + TCGv_i32 z = tcg_constant_i32(0); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_result); + gen_helper_fmaddd(d, tcg_env, s1, s2, s3, z, op); } /* Use muladd to compute (1 * src1) + src2 / 2 with one rounding. */ static void gen_op_fhadds(TCGv_i32 d, TCGv_i32 s1, TCGv_i32 s2) { - TCGv_i32 one = tcg_constant_i32(float32_one); - int op = float_muladd_halve_result; - gen_helper_fmadds(d, tcg_env, one, s1, s2, tcg_constant_i32(op)); + TCGv_i32 fone = tcg_constant_i32(float32_one); + TCGv_i32 mone = tcg_constant_i32(-1); + TCGv_i32 op = tcg_constant_i32(0); + gen_helper_fmadds(d, tcg_env, fone, s1, s2, mone, op); } static void gen_op_fhaddd(TCGv_i64 d, TCGv_i64 s1, TCGv_i64 s2) { - TCGv_i64 one = tcg_constant_i64(float64_one); - int op = float_muladd_halve_result; - gen_helper_fmaddd(d, tcg_env, one, s1, s2, tcg_constant_i32(op)); + TCGv_i64 fone = tcg_constant_i64(float64_one); + TCGv_i32 mone = tcg_constant_i32(-1); + TCGv_i32 op = tcg_constant_i32(0); + gen_helper_fmaddd(d, tcg_env, fone, s1, s2, mone, op); } /* Use muladd to compute (1 * src1) - src2 / 2 with one rounding. */ static void gen_op_fhsubs(TCGv_i32 d, TCGv_i32 s1, TCGv_i32 s2) { - TCGv_i32 one = tcg_constant_i32(float32_one); - int op = float_muladd_negate_c | float_muladd_halve_result; - gen_helper_fmadds(d, tcg_env, one, s1, s2, tcg_constant_i32(op)); + TCGv_i32 fone = tcg_constant_i32(float32_one); + TCGv_i32 mone = tcg_constant_i32(-1); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_c); + gen_helper_fmadds(d, tcg_env, fone, s1, s2, mone, op); } static void gen_op_fhsubd(TCGv_i64 d, TCGv_i64 s1, TCGv_i64 s2) { - TCGv_i64 one = tcg_constant_i64(float64_one); - int op = float_muladd_negate_c | float_muladd_halve_result; - gen_helper_fmaddd(d, tcg_env, one, s1, s2, tcg_constant_i32(op)); + TCGv_i64 fone = tcg_constant_i64(float64_one); + TCGv_i32 mone = tcg_constant_i32(-1); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_c); + gen_helper_fmaddd(d, tcg_env, fone, s1, s2, mone, op); } /* Use muladd to compute -((1 * src1) + src2 / 2) with one rounding. */ static void gen_op_fnhadds(TCGv_i32 d, TCGv_i32 s1, TCGv_i32 s2) { - TCGv_i32 one = tcg_constant_i32(float32_one); - int op = float_muladd_negate_result | float_muladd_halve_result; - gen_helper_fmadds(d, tcg_env, one, s1, s2, tcg_constant_i32(op)); + TCGv_i32 fone = tcg_constant_i32(float32_one); + TCGv_i32 mone = tcg_constant_i32(-1); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_result); + gen_helper_fmadds(d, tcg_env, fone, s1, s2, mone, op); } static void gen_op_fnhaddd(TCGv_i64 d, TCGv_i64 s1, TCGv_i64 s2) { - TCGv_i64 one = tcg_constant_i64(float64_one); - int op = float_muladd_negate_result | float_muladd_halve_result; - gen_helper_fmaddd(d, tcg_env, one, s1, s2, tcg_constant_i32(op)); + TCGv_i64 fone = tcg_constant_i64(float64_one); + TCGv_i32 mone = tcg_constant_i32(-1); + TCGv_i32 op = tcg_constant_i32(float_muladd_negate_result); + gen_helper_fmaddd(d, tcg_env, fone, s1, s2, mone, op); } static void gen_op_fpexception_im(DisasContext *dc, int ftt) -- 2.43.0 All uses have been convered to float*_muladd_scalbn. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat.h | 3 --- fpu/softfloat.c | 6 ------ fpu/softfloat-parts.c.inc | 4 ---- 3 files changed, 13 deletions(-) diff --git a/include/fpu/softfloat.h b/include/fpu/softfloat.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat.h +++ b/include/fpu/softfloat.h @@ -XXX,XX +XXX,XX @@ bfloat16 bfloat16_squash_input_denormal(bfloat16 a, float_status *status); | Using these differs from negating an input or output before calling | the muladd function in that this means that a NaN doesn't have its | sign bit inverted before it is propagated. -| We also support halving the result before rounding, as a special -| case to support the ARM fused-sqrt-step instruction FRSQRTS. *----------------------------------------------------------------------------*/ enum { float_muladd_negate_c = 1, float_muladd_negate_product = 2, float_muladd_negate_result = 4, - float_muladd_halve_result = 8, }; /*---------------------------------------------------------------------------- diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ float32_muladd(float32 xa, float32 xb, float32 xc, int flags, float_status *s) if (unlikely(!can_use_fpu(s))) { goto soft; } - if (unlikely(flags & float_muladd_halve_result)) { - goto soft; - } float32_input_flush3(&ua.s, &ub.s, &uc.s, s); if (unlikely(!f32_is_zon3(ua, ub, uc))) { @@ -XXX,XX +XXX,XX @@ float64_muladd(float64 xa, float64 xb, float64 xc, int flags, float_status *s) if (unlikely(!can_use_fpu(s))) { goto soft; } - if (unlikely(flags & float_muladd_halve_result)) { - goto soft; - } float64_input_flush3(&ua.s, &ub.s, &uc.s, s); if (unlikely(!f64_is_zon3(ua, ub, uc))) { diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd_scalbn)(FloatPartsN *a, FloatPartsN *b, a->exp = p_widen.exp; return_normal: - /* TODO: Replace all use of float_muladd_halve_result with scale. */ - if (flags & float_muladd_halve_result) { - a->exp -= 1; - } a->exp += scale; finish_sign: if (flags & float_muladd_negate_result) { -- 2.43.0 This rounding mode is used by Hexagon. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-types.h | 2 ++ fpu/softfloat-parts.c.inc | 3 +++ 2 files changed, 5 insertions(+) diff --git a/include/fpu/softfloat-types.h b/include/fpu/softfloat-types.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-types.h +++ b/include/fpu/softfloat-types.h @@ -XXX,XX +XXX,XX @@ typedef enum __attribute__((__packed__)) { float_round_to_odd = 5, /* Not an IEEE rounding mode: round to closest odd, overflow to inf */ float_round_to_odd_inf = 6, + /* Not an IEEE rounding mode: round to nearest even, overflow to max */ + float_round_nearest_even_max = 7, } FloatRoundMode; /* diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static void partsN(uncanon_normal)(FloatPartsN *p, float_status *s, int exp, flags = 0; switch (s->float_rounding_mode) { + case float_round_nearest_even_max: + overflow_norm = true; + /* fall through */ case float_round_nearest_even: if (N > 64 && frac_lsb == 0) { inc = ((p->frac_hi & 1) || (p->frac_lo & round_mask) != frac_lsbm1 -- 2.43.0 Certain Hexagon instructions suppress changes to the result when the product of fma() is a true zero. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat.h | 5 +++++ fpu/softfloat.c | 3 +++ fpu/softfloat-parts.c.inc | 4 +++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/include/fpu/softfloat.h b/include/fpu/softfloat.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat.h +++ b/include/fpu/softfloat.h @@ -XXX,XX +XXX,XX @@ bfloat16 bfloat16_squash_input_denormal(bfloat16 a, float_status *status); | Using these differs from negating an input or output before calling | the muladd function in that this means that a NaN doesn't have its | sign bit inverted before it is propagated. +| +| With float_muladd_suppress_add_product_zero, if A or B is zero +| such that the product is a true zero, then return C without addition. +| This preserves the sign of C when C is +/- 0. Used for Hexagon. *----------------------------------------------------------------------------*/ enum { float_muladd_negate_c = 1, float_muladd_negate_product = 2, float_muladd_negate_result = 4, + float_muladd_suppress_add_product_zero = 8, }; /*---------------------------------------------------------------------------- diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ float32_muladd(float32 xa, float32 xb, float32 xc, int flags, float_status *s) if (unlikely(!can_use_fpu(s))) { goto soft; } + if (unlikely(flags & float_muladd_suppress_add_product_zero)) { + goto soft; + } float32_input_flush3(&ua.s, &ub.s, &uc.s, s); if (unlikely(!f32_is_zon3(ua, ub, uc))) { diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd_scalbn)(FloatPartsN *a, FloatPartsN *b, goto return_normal; } if (c->cls == float_class_zero) { - if (a->sign != c->sign) { + if (flags & float_muladd_suppress_add_product_zero) { + a->sign = c->sign; + } else if (a->sign != c->sign) { goto return_sub_zero; } goto return_zero; -- 2.43.0 There are no special cases for this instruction. Remove internal_mpyf as unused. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/fma_emu.h | 1 - target/hexagon/fma_emu.c | 8 -------- target/hexagon/op_helper.c | 2 +- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/target/hexagon/fma_emu.h b/target/hexagon/fma_emu.h index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.h +++ b/target/hexagon/fma_emu.h @@ -XXX,XX +XXX,XX @@ int32_t float32_getexp(float32 f32); float32 infinite_float32(uint8_t sign); float32 internal_fmafx(float32 a, float32 b, float32 c, int scale, float_status *fp_status); -float32 internal_mpyf(float32 a, float32 b, float_status *fp_status); float64 internal_mpyhh(float64 a, float64 b, unsigned long long int accumulated, float_status *fp_status); diff --git a/target/hexagon/fma_emu.c b/target/hexagon/fma_emu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.c +++ b/target/hexagon/fma_emu.c @@ -XXX,XX +XXX,XX @@ float32 internal_fmafx(float32 a, float32 b, float32 c, int scale, return accum_round_float32(result, fp_status); } -float32 internal_mpyf(float32 a, float32 b, float_status *fp_status) -{ - if (float32_is_zero(a) || float32_is_zero(b)) { - return float32_mul(a, b, fp_status); - } - return internal_fmafx(a, b, float32_zero, 0, fp_status); -} - float64 internal_mpyhh(float64 a, float64 b, unsigned long long int accumulated, float_status *fp_status) diff --git a/target/hexagon/op_helper.c b/target/hexagon/op_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/op_helper.c +++ b/target/hexagon/op_helper.c @@ -XXX,XX +XXX,XX @@ float32 HELPER(sfmpy)(CPUHexagonState *env, float32 RsV, float32 RtV) { float32 RdV; arch_fpop_start(env); - RdV = internal_mpyf(RsV, RtV, &env->fp_status); + RdV = float32_mul(RsV, RtV, &env->fp_status); arch_fpop_end(env); return RdV; } -- 2.43.0 There are no special cases for this instruction. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/op_helper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/hexagon/op_helper.c b/target/hexagon/op_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/op_helper.c +++ b/target/hexagon/op_helper.c @@ -XXX,XX +XXX,XX @@ float32 HELPER(sffma)(CPUHexagonState *env, float32 RxV, float32 RsV, float32 RtV) { arch_fpop_start(env); - RxV = internal_fmafx(RsV, RtV, RxV, 0, &env->fp_status); + RxV = float32_muladd(RsV, RtV, RxV, 0, &env->fp_status); arch_fpop_end(env); return RxV; } -- 2.43.0 There are no special cases for this instruction. Since hexagon always uses default-nan mode, explicitly negating the first input is unnecessary. Use float_muladd_negate_product instead. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/op_helper.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/target/hexagon/op_helper.c b/target/hexagon/op_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/op_helper.c +++ b/target/hexagon/op_helper.c @@ -XXX,XX +XXX,XX @@ float32 HELPER(sffma_sc)(CPUHexagonState *env, float32 RxV, float32 HELPER(sffms)(CPUHexagonState *env, float32 RxV, float32 RsV, float32 RtV) { - float32 neg_RsV; arch_fpop_start(env); - neg_RsV = float32_set_sign(RsV, float32_is_neg(RsV) ? 0 : 1); - RxV = internal_fmafx(neg_RsV, RtV, RxV, 0, &env->fp_status); + RxV = float32_muladd(RsV, RtV, RxV, float_muladd_negate_product, + &env->fp_status); arch_fpop_end(env); return RxV; } -- 2.43.0 This instruction has a special case that 0 * x + c returns c without the normal sign folding that comes with 0 + -0. Use the new float_muladd_suppress_add_product_zero to describe this. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/op_helper.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/target/hexagon/op_helper.c b/target/hexagon/op_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/op_helper.c +++ b/target/hexagon/op_helper.c @@ -XXX,XX +XXX,XX @@ static float32 check_nan(float32 dst, float32 x, float_status *fp_status) float32 HELPER(sffma_sc)(CPUHexagonState *env, float32 RxV, float32 RsV, float32 RtV, float32 PuV) { - size4s_t tmp; arch_fpop_start(env); - RxV = check_nan(RxV, RxV, &env->fp_status); - RxV = check_nan(RxV, RsV, &env->fp_status); - RxV = check_nan(RxV, RtV, &env->fp_status); - tmp = internal_fmafx(RsV, RtV, RxV, fSXTN(8, 64, PuV), &env->fp_status); - if (!(float32_is_zero(RxV) && is_zero_prod(RsV, RtV))) { - RxV = tmp; - } + RxV = float32_muladd_scalbn(RsV, RtV, RxV, fSXTN(8, 64, PuV), + float_muladd_suppress_add_product_zero, + &env->fp_status); arch_fpop_end(env); return RxV; } -- 2.43.0 There are multiple special cases for this instruction. (1) The saturate to normal maximum instead of overflow to infinity is handled by the new float_round_nearest_even_max rounding mode. (2) The 0 * n + c special case is handled by the new float_muladd_suppress_add_product_zero flag. (3) The Inf - Inf -> 0 special case can be detected after the fact by examining float_flag_invalid_isi. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/op_helper.c | 105 +++++++++---------------------------- 1 file changed, 26 insertions(+), 79 deletions(-) diff --git a/target/hexagon/op_helper.c b/target/hexagon/op_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/op_helper.c +++ b/target/hexagon/op_helper.c @@ -XXX,XX +XXX,XX @@ float32 HELPER(sffma)(CPUHexagonState *env, float32 RxV, return RxV; } -static bool is_zero_prod(float32 a, float32 b) -{ - return ((float32_is_zero(a) && is_finite(b)) || - (float32_is_zero(b) && is_finite(a))); -} - -static float32 check_nan(float32 dst, float32 x, float_status *fp_status) -{ - float32 ret = dst; - if (float32_is_any_nan(x)) { - if (extract32(x, 22, 1) == 0) { - float_raise(float_flag_invalid, fp_status); - } - ret = make_float32(0xffffffff); /* nan */ - } - return ret; -} - float32 HELPER(sffma_sc)(CPUHexagonState *env, float32 RxV, float32 RsV, float32 RtV, float32 PuV) { @@ -XXX,XX +XXX,XX @@ float32 HELPER(sffms)(CPUHexagonState *env, float32 RxV, return RxV; } -static bool is_inf_prod(int32_t a, int32_t b) +static float32 do_sffma_lib(CPUHexagonState *env, float32 RxV, + float32 RsV, float32 RtV, int negate) { - return (float32_is_infinity(a) && float32_is_infinity(b)) || - (float32_is_infinity(a) && is_finite(b) && !float32_is_zero(b)) || - (float32_is_infinity(b) && is_finite(a) && !float32_is_zero(a)); + int flags; + + arch_fpop_start(env); + + set_float_rounding_mode(float_round_nearest_even_max, &env->fp_status); + RxV = float32_muladd(RsV, RtV, RxV, + negate | float_muladd_suppress_add_product_zero, + &env->fp_status); + + flags = get_float_exception_flags(&env->fp_status); + if (flags) { + /* Flags are suppressed by this instruction. */ + set_float_exception_flags(0, &env->fp_status); + + /* Return 0 for Inf - Inf. */ + if (flags & float_flag_invalid_isi) { + RxV = 0; + } + } + + arch_fpop_end(env); + return RxV; } float32 HELPER(sffma_lib)(CPUHexagonState *env, float32 RxV, float32 RsV, float32 RtV) { - bool infinp; - bool infminusinf; - float32 tmp; - - arch_fpop_start(env); - set_float_rounding_mode(float_round_nearest_even, &env->fp_status); - infminusinf = float32_is_infinity(RxV) && - is_inf_prod(RsV, RtV) && - (fGETBIT(31, RsV ^ RxV ^ RtV) != 0); - infinp = float32_is_infinity(RxV) || - float32_is_infinity(RtV) || - float32_is_infinity(RsV); - RxV = check_nan(RxV, RxV, &env->fp_status); - RxV = check_nan(RxV, RsV, &env->fp_status); - RxV = check_nan(RxV, RtV, &env->fp_status); - tmp = internal_fmafx(RsV, RtV, RxV, 0, &env->fp_status); - if (!(float32_is_zero(RxV) && is_zero_prod(RsV, RtV))) { - RxV = tmp; - } - set_float_exception_flags(0, &env->fp_status); - if (float32_is_infinity(RxV) && !infinp) { - RxV = RxV - 1; - } - if (infminusinf) { - RxV = 0; - } - arch_fpop_end(env); - return RxV; + return do_sffma_lib(env, RxV, RsV, RtV, 0); } float32 HELPER(sffms_lib)(CPUHexagonState *env, float32 RxV, float32 RsV, float32 RtV) { - bool infinp; - bool infminusinf; - float32 tmp; - - arch_fpop_start(env); - set_float_rounding_mode(float_round_nearest_even, &env->fp_status); - infminusinf = float32_is_infinity(RxV) && - is_inf_prod(RsV, RtV) && - (fGETBIT(31, RsV ^ RxV ^ RtV) == 0); - infinp = float32_is_infinity(RxV) || - float32_is_infinity(RtV) || - float32_is_infinity(RsV); - RxV = check_nan(RxV, RxV, &env->fp_status); - RxV = check_nan(RxV, RsV, &env->fp_status); - RxV = check_nan(RxV, RtV, &env->fp_status); - float32 minus_RsV = float32_sub(float32_zero, RsV, &env->fp_status); - tmp = internal_fmafx(minus_RsV, RtV, RxV, 0, &env->fp_status); - if (!(float32_is_zero(RxV) && is_zero_prod(RsV, RtV))) { - RxV = tmp; - } - set_float_exception_flags(0, &env->fp_status); - if (float32_is_infinity(RxV) && !infinp) { - RxV = RxV - 1; - } - if (infminusinf) { - RxV = 0; - } - arch_fpop_end(env); - return RxV; + return do_sffma_lib(env, RxV, RsV, RtV, float_muladd_negate_product); } float64 HELPER(dfmpyfix)(CPUHexagonState *env, float64 RssV, float64 RttV) -- 2.43.0 The function is now unused. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/fma_emu.h | 2 - target/hexagon/fma_emu.c | 171 --------------------------------------- 2 files changed, 173 deletions(-) diff --git a/target/hexagon/fma_emu.h b/target/hexagon/fma_emu.h index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.h +++ b/target/hexagon/fma_emu.h @@ -XXX,XX +XXX,XX @@ static inline uint32_t float32_getexp_raw(float32 f32) } int32_t float32_getexp(float32 f32); float32 infinite_float32(uint8_t sign); -float32 internal_fmafx(float32 a, float32 b, float32 c, - int scale, float_status *fp_status); float64 internal_mpyhh(float64 a, float64 b, unsigned long long int accumulated, float_status *fp_status); diff --git a/target/hexagon/fma_emu.c b/target/hexagon/fma_emu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.c +++ b/target/hexagon/fma_emu.c @@ -XXX,XX +XXX,XX @@ int32_t float64_getexp(float64 f64) return -1; } -static uint64_t float32_getmant(float32 f32) -{ - Float a = { .i = f32 }; - if (float32_is_normal(f32)) { - return a.mant | 1ULL << 23; - } - if (float32_is_zero(f32)) { - return 0; - } - if (float32_is_denormal(f32)) { - return a.mant; - } - return ~0ULL; -} - int32_t float32_getexp(float32 f32) { Float a = { .i = f32 }; @@ -XXX,XX +XXX,XX @@ float32 infinite_float32(uint8_t sign) } /* Return a maximum finite value with the requested sign */ -static float32 maxfinite_float32(uint8_t sign) -{ - if (sign) { - return make_float32(SF_MINUS_MAXF); - } else { - return make_float32(SF_MAXF); - } -} - -/* Return a zero value with requested sign */ -static float32 zero_float32(uint8_t sign) -{ - if (sign) { - return make_float32(0x80000000); - } else { - return float32_zero; - } -} - #define GEN_XF_ROUND(SUFFIX, MANTBITS, INF_EXP, INTERNAL_TYPE) \ static SUFFIX accum_round_##SUFFIX(Accum a, float_status * fp_status) \ { \ @@ -XXX,XX +XXX,XX @@ static SUFFIX accum_round_##SUFFIX(Accum a, float_status * fp_status) \ } GEN_XF_ROUND(float64, DF_MANTBITS, DF_INF_EXP, Double) -GEN_XF_ROUND(float32, SF_MANTBITS, SF_INF_EXP, Float) - -static bool is_inf_prod(float64 a, float64 b) -{ - return ((float64_is_infinity(a) && float64_is_infinity(b)) || - (float64_is_infinity(a) && is_finite(b) && (!float64_is_zero(b))) || - (float64_is_infinity(b) && is_finite(a) && (!float64_is_zero(a)))); -} - -static float64 special_fma(float64 a, float64 b, float64 c, - float_status *fp_status) -{ - float64 ret = make_float64(0); - - /* - * If A multiplied by B is an exact infinity and C is also an infinity - * but with the opposite sign, FMA returns NaN and raises invalid. - */ - uint8_t a_sign = float64_is_neg(a); - uint8_t b_sign = float64_is_neg(b); - uint8_t c_sign = float64_is_neg(c); - if (is_inf_prod(a, b) && float64_is_infinity(c)) { - if ((a_sign ^ b_sign) != c_sign) { - ret = make_float64(DF_NAN); - float_raise(float_flag_invalid, fp_status); - return ret; - } - } - if ((float64_is_infinity(a) && float64_is_zero(b)) || - (float64_is_zero(a) && float64_is_infinity(b))) { - ret = make_float64(DF_NAN); - float_raise(float_flag_invalid, fp_status); - return ret; - } - /* - * If none of the above checks are true and C is a NaN, - * a NaN shall be returned - * If A or B are NaN, a NAN shall be returned. - */ - if (float64_is_any_nan(a) || - float64_is_any_nan(b) || - float64_is_any_nan(c)) { - if (float64_is_any_nan(a) && (fGETBIT(51, a) == 0)) { - float_raise(float_flag_invalid, fp_status); - } - if (float64_is_any_nan(b) && (fGETBIT(51, b) == 0)) { - float_raise(float_flag_invalid, fp_status); - } - if (float64_is_any_nan(c) && (fGETBIT(51, c) == 0)) { - float_raise(float_flag_invalid, fp_status); - } - ret = make_float64(DF_NAN); - return ret; - } - /* - * We have checked for adding opposite-signed infinities. - * Other infinities return infinity with the correct sign - */ - if (float64_is_infinity(c)) { - ret = infinite_float64(c_sign); - return ret; - } - if (float64_is_infinity(a) || float64_is_infinity(b)) { - ret = infinite_float64(a_sign ^ b_sign); - return ret; - } - g_assert_not_reached(); -} - -static float32 special_fmaf(float32 a, float32 b, float32 c, - float_status *fp_status) -{ - float64 aa, bb, cc; - aa = float32_to_float64(a, fp_status); - bb = float32_to_float64(b, fp_status); - cc = float32_to_float64(c, fp_status); - return float64_to_float32(special_fma(aa, bb, cc, fp_status), fp_status); -} - -float32 internal_fmafx(float32 a, float32 b, float32 c, int scale, - float_status *fp_status) -{ - Accum prod; - Accum acc; - Accum result; - accum_init(&prod); - accum_init(&acc); - accum_init(&result); - - uint8_t a_sign = float32_is_neg(a); - uint8_t b_sign = float32_is_neg(b); - uint8_t c_sign = float32_is_neg(c); - if (float32_is_infinity(a) || - float32_is_infinity(b) || - float32_is_infinity(c)) { - return special_fmaf(a, b, c, fp_status); - } - if (float32_is_any_nan(a) || - float32_is_any_nan(b) || - float32_is_any_nan(c)) { - return special_fmaf(a, b, c, fp_status); - } - if ((scale == 0) && (float32_is_zero(a) || float32_is_zero(b))) { - float32 tmp = float32_mul(a, b, fp_status); - tmp = float32_add(tmp, c, fp_status); - return tmp; - } - - /* (a * 2**b) * (c * 2**d) == a*c * 2**(b+d) */ - prod.mant = int128_mul_6464(float32_getmant(a), float32_getmant(b)); - - /* - * Note: extracting the mantissa into an int is multiplying by - * 2**23, so adjust here - */ - prod.exp = float32_getexp(a) + float32_getexp(b) - SF_BIAS - 23; - prod.sign = a_sign ^ b_sign; - if (float32_is_zero(a) || float32_is_zero(b)) { - prod.exp = -2 * WAY_BIG_EXP; - } - if ((scale > 0) && float32_is_denormal(c)) { - acc.mant = int128_mul_6464(0, 0); - acc.exp = -WAY_BIG_EXP; - acc.sign = c_sign; - acc.sticky = 1; - result = accum_add(prod, acc); - } else if (!float32_is_zero(c)) { - acc.mant = int128_mul_6464(float32_getmant(c), 1); - acc.exp = float32_getexp(c); - acc.sign = c_sign; - result = accum_add(prod, acc); - } else { - result = prod; - } - result.exp += scale; - return accum_round_float32(result, fp_status); -} float64 internal_mpyhh(float64 a, float64 b, unsigned long long int accumulated, -- 2.43.0 This massive macro is now only used once. Expand it for use only by float64. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/fma_emu.c | 255 +++++++++++++++++++-------------------- 1 file changed, 127 insertions(+), 128 deletions(-) diff --git a/target/hexagon/fma_emu.c b/target/hexagon/fma_emu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.c +++ b/target/hexagon/fma_emu.c @@ -XXX,XX +XXX,XX @@ float32 infinite_float32(uint8_t sign) } /* Return a maximum finite value with the requested sign */ -#define GEN_XF_ROUND(SUFFIX, MANTBITS, INF_EXP, INTERNAL_TYPE) \ -static SUFFIX accum_round_##SUFFIX(Accum a, float_status * fp_status) \ -{ \ - if ((int128_gethi(a.mant) == 0) && (int128_getlo(a.mant) == 0) \ - && ((a.guard | a.round | a.sticky) == 0)) { \ - /* result zero */ \ - switch (fp_status->float_rounding_mode) { \ - case float_round_down: \ - return zero_##SUFFIX(1); \ - default: \ - return zero_##SUFFIX(0); \ - } \ - } \ - /* Normalize right */ \ - /* We want MANTBITS bits of mantissa plus the leading one. */ \ - /* That means that we want MANTBITS+1 bits, or 0x000000000000FF_FFFF */ \ - /* So we need to normalize right while the high word is non-zero and \ - * while the low word is nonzero when masked with 0xffe0_0000_0000_0000 */ \ - while ((int128_gethi(a.mant) != 0) || \ - ((int128_getlo(a.mant) >> (MANTBITS + 1)) != 0)) { \ - a = accum_norm_right(a, 1); \ - } \ - /* \ - * OK, now normalize left \ - * We want to normalize left until we have a leading one in bit 24 \ - * Theoretically, we only need to shift a maximum of one to the left if we \ - * shifted out lots of bits from B, or if we had no shift / 1 shift sticky \ - * should be 0 \ - */ \ - while ((int128_getlo(a.mant) & (1ULL << MANTBITS)) == 0) { \ - a = accum_norm_left(a); \ - } \ - /* \ - * OK, now we might need to denormalize because of potential underflow. \ - * We need to do this before rounding, and rounding might make us normal \ - * again \ - */ \ - while (a.exp <= 0) { \ - a = accum_norm_right(a, 1 - a.exp); \ - /* \ - * Do we have underflow? \ - * That's when we get an inexact answer because we ran out of bits \ - * in a denormal. \ - */ \ - if (a.guard || a.round || a.sticky) { \ - float_raise(float_flag_underflow, fp_status); \ - } \ - } \ - /* OK, we're relatively canonical... now we need to round */ \ - if (a.guard || a.round || a.sticky) { \ - float_raise(float_flag_inexact, fp_status); \ - switch (fp_status->float_rounding_mode) { \ - case float_round_to_zero: \ - /* Chop and we're done */ \ - break; \ - case float_round_up: \ - if (a.sign == 0) { \ - a.mant = int128_add(a.mant, int128_one()); \ - } \ - break; \ - case float_round_down: \ - if (a.sign != 0) { \ - a.mant = int128_add(a.mant, int128_one()); \ - } \ - break; \ - default: \ - if (a.round || a.sticky) { \ - /* round up if guard is 1, down if guard is zero */ \ - a.mant = int128_add(a.mant, int128_make64(a.guard)); \ - } else if (a.guard) { \ - /* exactly .5, round up if odd */ \ - a.mant = int128_add(a.mant, int128_and(a.mant, int128_one())); \ - } \ - break; \ - } \ - } \ - /* \ - * OK, now we might have carried all the way up. \ - * So we might need to shr once \ - * at least we know that the lsb should be zero if we rounded and \ - * got a carry out... \ - */ \ - if ((int128_getlo(a.mant) >> (MANTBITS + 1)) != 0) { \ - a = accum_norm_right(a, 1); \ - } \ - /* Overflow? */ \ - if (a.exp >= INF_EXP) { \ - /* Yep, inf result */ \ - float_raise(float_flag_overflow, fp_status); \ - float_raise(float_flag_inexact, fp_status); \ - switch (fp_status->float_rounding_mode) { \ - case float_round_to_zero: \ - return maxfinite_##SUFFIX(a.sign); \ - case float_round_up: \ - if (a.sign == 0) { \ - return infinite_##SUFFIX(a.sign); \ - } else { \ - return maxfinite_##SUFFIX(a.sign); \ - } \ - case float_round_down: \ - if (a.sign != 0) { \ - return infinite_##SUFFIX(a.sign); \ - } else { \ - return maxfinite_##SUFFIX(a.sign); \ - } \ - default: \ - return infinite_##SUFFIX(a.sign); \ - } \ - } \ - /* Underflow? */ \ - if (int128_getlo(a.mant) & (1ULL << MANTBITS)) { \ - /* Leading one means: No, we're normal. So, we should be done... */ \ - INTERNAL_TYPE ret; \ - ret.i = 0; \ - ret.sign = a.sign; \ - ret.exp = a.exp; \ - ret.mant = int128_getlo(a.mant); \ - return ret.i; \ - } \ - assert(a.exp == 1); \ - INTERNAL_TYPE ret; \ - ret.i = 0; \ - ret.sign = a.sign; \ - ret.exp = 0; \ - ret.mant = int128_getlo(a.mant); \ - return ret.i; \ +static float64 accum_round_float64(Accum a, float_status *fp_status) +{ + if ((int128_gethi(a.mant) == 0) && (int128_getlo(a.mant) == 0) + && ((a.guard | a.round | a.sticky) == 0)) { + /* result zero */ + switch (fp_status->float_rounding_mode) { + case float_round_down: + return zero_float64(1); + default: + return zero_float64(0); + } + } + /* + * Normalize right + * We want DF_MANTBITS bits of mantissa plus the leading one. + * That means that we want DF_MANTBITS+1 bits, or 0x000000000000FF_FFFF + * So we need to normalize right while the high word is non-zero and + * while the low word is nonzero when masked with 0xffe0_0000_0000_0000 + */ + while ((int128_gethi(a.mant) != 0) || + ((int128_getlo(a.mant) >> (DF_MANTBITS + 1)) != 0)) { + a = accum_norm_right(a, 1); + } + /* + * OK, now normalize left + * We want to normalize left until we have a leading one in bit 24 + * Theoretically, we only need to shift a maximum of one to the left if we + * shifted out lots of bits from B, or if we had no shift / 1 shift sticky + * should be 0 + */ + while ((int128_getlo(a.mant) & (1ULL << DF_MANTBITS)) == 0) { + a = accum_norm_left(a); + } + /* + * OK, now we might need to denormalize because of potential underflow. + * We need to do this before rounding, and rounding might make us normal + * again + */ + while (a.exp <= 0) { + a = accum_norm_right(a, 1 - a.exp); + /* + * Do we have underflow? + * That's when we get an inexact answer because we ran out of bits + * in a denormal. + */ + if (a.guard || a.round || a.sticky) { + float_raise(float_flag_underflow, fp_status); + } + } + /* OK, we're relatively canonical... now we need to round */ + if (a.guard || a.round || a.sticky) { + float_raise(float_flag_inexact, fp_status); + switch (fp_status->float_rounding_mode) { + case float_round_to_zero: + /* Chop and we're done */ + break; + case float_round_up: + if (a.sign == 0) { + a.mant = int128_add(a.mant, int128_one()); + } + break; + case float_round_down: + if (a.sign != 0) { + a.mant = int128_add(a.mant, int128_one()); + } + break; + default: + if (a.round || a.sticky) { + /* round up if guard is 1, down if guard is zero */ + a.mant = int128_add(a.mant, int128_make64(a.guard)); + } else if (a.guard) { + /* exactly .5, round up if odd */ + a.mant = int128_add(a.mant, int128_and(a.mant, int128_one())); + } + break; + } + } + /* + * OK, now we might have carried all the way up. + * So we might need to shr once + * at least we know that the lsb should be zero if we rounded and + * got a carry out... + */ + if ((int128_getlo(a.mant) >> (DF_MANTBITS + 1)) != 0) { + a = accum_norm_right(a, 1); + } + /* Overflow? */ + if (a.exp >= DF_INF_EXP) { + /* Yep, inf result */ + float_raise(float_flag_overflow, fp_status); + float_raise(float_flag_inexact, fp_status); + switch (fp_status->float_rounding_mode) { + case float_round_to_zero: + return maxfinite_float64(a.sign); + case float_round_up: + if (a.sign == 0) { + return infinite_float64(a.sign); + } else { + return maxfinite_float64(a.sign); + } + case float_round_down: + if (a.sign != 0) { + return infinite_float64(a.sign); + } else { + return maxfinite_float64(a.sign); + } + default: + return infinite_float64(a.sign); + } + } + /* Underflow? */ + if (int128_getlo(a.mant) & (1ULL << DF_MANTBITS)) { + /* Leading one means: No, we're normal. So, we should be done... */ + Double ret; + ret.i = 0; + ret.sign = a.sign; + ret.exp = a.exp; + ret.mant = int128_getlo(a.mant); + return ret.i; + } + assert(a.exp == 1); + Double ret; + ret.i = 0; + ret.sign = a.sign; + ret.exp = 0; + ret.mant = int128_getlo(a.mant); + return ret.i; } -GEN_XF_ROUND(float64, DF_MANTBITS, DF_INF_EXP, Double) - float64 internal_mpyhh(float64 a, float64 b, unsigned long long int accumulated, float_status *fp_status) -- 2.43.0 This structure, with bitfields, is incorrect for big-endian. Use the existing float32_getexp_raw which uses extract32. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/fma_emu.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/target/hexagon/fma_emu.c b/target/hexagon/fma_emu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.c +++ b/target/hexagon/fma_emu.c @@ -XXX,XX +XXX,XX @@ typedef union { }; } Double; -typedef union { - float f; - uint32_t i; - struct { - uint32_t mant:23; - uint32_t exp:8; - uint32_t sign:1; - }; -} Float; - static uint64_t float64_getmant(float64 f64) { Double a = { .i = f64 }; @@ -XXX,XX +XXX,XX @@ int32_t float64_getexp(float64 f64) int32_t float32_getexp(float32 f32) { - Float a = { .i = f32 }; + int exp = float32_getexp_raw(f32); if (float32_is_normal(f32)) { - return a.exp; + return exp; } if (float32_is_denormal(f32)) { - return a.exp + 1; + return exp + 1; } return -1; } -- 2.43.0 This structure, with bitfields, is incorrect for big-endian. Use extract64 and deposit64 instead. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/fma_emu.c | 46 ++++++++++++++-------------------------- 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/target/hexagon/fma_emu.c b/target/hexagon/fma_emu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.c +++ b/target/hexagon/fma_emu.c @@ -XXX,XX +XXX,XX @@ #define WAY_BIG_EXP 4096 -typedef union { - double f; - uint64_t i; - struct { - uint64_t mant:52; - uint64_t exp:11; - uint64_t sign:1; - }; -} Double; - static uint64_t float64_getmant(float64 f64) { - Double a = { .i = f64 }; + uint64_t mant = extract64(f64, 0, 52); if (float64_is_normal(f64)) { - return a.mant | 1ULL << 52; + return mant | 1ULL << 52; } if (float64_is_zero(f64)) { return 0; } if (float64_is_denormal(f64)) { - return a.mant; + return mant; } return ~0ULL; } int32_t float64_getexp(float64 f64) { - Double a = { .i = f64 }; + int exp = extract64(f64, 52, 11); if (float64_is_normal(f64)) { - return a.exp; + return exp; } if (float64_is_denormal(f64)) { - return a.exp + 1; + return exp + 1; } return -1; } @@ -XXX,XX +XXX,XX @@ float32 infinite_float32(uint8_t sign) /* Return a maximum finite value with the requested sign */ static float64 accum_round_float64(Accum a, float_status *fp_status) { + uint64_t ret; + if ((int128_gethi(a.mant) == 0) && (int128_getlo(a.mant) == 0) && ((a.guard | a.round | a.sticky) == 0)) { /* result zero */ @@ -XXX,XX +XXX,XX @@ static float64 accum_round_float64(Accum a, float_status *fp_status) } } /* Underflow? */ - if (int128_getlo(a.mant) & (1ULL << DF_MANTBITS)) { + ret = int128_getlo(a.mant); + if (ret & (1ULL << DF_MANTBITS)) { /* Leading one means: No, we're normal. So, we should be done... */ - Double ret; - ret.i = 0; - ret.sign = a.sign; - ret.exp = a.exp; - ret.mant = int128_getlo(a.mant); - return ret.i; + ret = deposit64(ret, 52, 11, a.exp); + } else { + assert(a.exp == 1); + ret = deposit64(ret, 52, 11, 0); } - assert(a.exp == 1); - Double ret; - ret.i = 0; - ret.sign = a.sign; - ret.exp = 0; - ret.mant = int128_getlo(a.mant); - return ret.i; + ret = deposit64(ret, 63, 1, a.sign); + return ret; } float64 internal_mpyhh(float64 a, float64 b, -- 2.43.0 No need to open-code 64x64->128-bit multiplication. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/fma_emu.c | 32 +++----------------------------- 1 file changed, 3 insertions(+), 29 deletions(-) diff --git a/target/hexagon/fma_emu.c b/target/hexagon/fma_emu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.c +++ b/target/hexagon/fma_emu.c @@ -XXX,XX +XXX,XX @@ int32_t float32_getexp(float32 f32) return -1; } -static uint32_t int128_getw0(Int128 x) -{ - return int128_getlo(x); -} - -static uint32_t int128_getw1(Int128 x) -{ - return int128_getlo(x) >> 32; -} - static Int128 int128_mul_6464(uint64_t ai, uint64_t bi) { - Int128 a, b; - uint64_t pp0, pp1a, pp1b, pp1s, pp2; + uint64_t l, h; - a = int128_make64(ai); - b = int128_make64(bi); - pp0 = (uint64_t)int128_getw0(a) * (uint64_t)int128_getw0(b); - pp1a = (uint64_t)int128_getw1(a) * (uint64_t)int128_getw0(b); - pp1b = (uint64_t)int128_getw1(b) * (uint64_t)int128_getw0(a); - pp2 = (uint64_t)int128_getw1(a) * (uint64_t)int128_getw1(b); - - pp1s = pp1a + pp1b; - if ((pp1s < pp1a) || (pp1s < pp1b)) { - pp2 += (1ULL << 32); - } - uint64_t ret_low = pp0 + (pp1s << 32); - if ((ret_low < pp0) || (ret_low < (pp1s << 32))) { - pp2 += 1; - } - - return int128_make128(ret_low, pp2 + (pp1s >> 32)); + mulu64(&l, &h, ai, bi); + return int128_make128(l, h); } static Int128 int128_sub_borrow(Int128 a, Int128 b, int borrow) -- 2.43.0 Initialize x with accumulated via direct assignment, rather than multiplying by 1. Reviewed-by: Brian Cain <brian.cain@oss.qualcomm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hexagon/fma_emu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/hexagon/fma_emu.c b/target/hexagon/fma_emu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/fma_emu.c +++ b/target/hexagon/fma_emu.c @@ -XXX,XX +XXX,XX @@ float64 internal_mpyhh(float64 a, float64 b, float64_is_infinity(b)) { return float64_mul(a, b, fp_status); } - x.mant = int128_mul_6464(accumulated, 1); + x.mant = int128_make64(accumulated); x.sticky = sticky; prod = fGETUWORD(1, float64_getmant(a)) * fGETUWORD(1, float64_getmant(b)); x.mant = int128_add(x.mant, int128_mul_6464(prod, 0x100000000ULL)); -- 2.43.0 Convert all targets simultaneously, as the gen_intermediate_code function disappears from the target. While there are possible workarounds, they're larger than simply performing the conversion. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/translator.h | 14 -------------- include/hw/core/tcg-cpu-ops.h | 13 +++++++++++++ target/alpha/cpu.h | 2 ++ target/arm/internals.h | 2 ++ target/avr/cpu.h | 2 ++ target/hexagon/cpu.h | 2 ++ target/hppa/cpu.h | 2 ++ target/i386/tcg/helper-tcg.h | 2 ++ target/loongarch/internals.h | 2 ++ target/m68k/cpu.h | 2 ++ target/microblaze/cpu.h | 2 ++ target/mips/tcg/tcg-internal.h | 2 ++ target/openrisc/cpu.h | 2 ++ target/ppc/cpu.h | 2 ++ target/riscv/cpu.h | 3 +++ target/rx/cpu.h | 2 ++ target/s390x/s390x-internal.h | 2 ++ target/sh4/cpu.h | 2 ++ target/sparc/cpu.h | 2 ++ target/tricore/cpu.h | 2 ++ target/xtensa/cpu.h | 2 ++ accel/tcg/cpu-exec.c | 8 +++++--- accel/tcg/translate-all.c | 8 +++++--- target/alpha/cpu.c | 1 + target/alpha/translate.c | 4 ++-- target/arm/cpu.c | 1 + target/arm/tcg/cpu-v7m.c | 1 + target/arm/tcg/translate.c | 5 ++--- target/avr/cpu.c | 1 + target/avr/translate.c | 6 +++--- target/hexagon/cpu.c | 1 + target/hexagon/translate.c | 4 ++-- target/hppa/cpu.c | 1 + target/hppa/translate.c | 4 ++-- target/i386/tcg/tcg-cpu.c | 1 + target/i386/tcg/translate.c | 5 ++--- target/loongarch/cpu.c | 1 + target/loongarch/tcg/translate.c | 4 ++-- target/m68k/cpu.c | 1 + target/m68k/translate.c | 4 ++-- target/microblaze/cpu.c | 1 + target/microblaze/translate.c | 4 ++-- target/mips/cpu.c | 1 + target/mips/tcg/translate.c | 4 ++-- target/openrisc/cpu.c | 1 + target/openrisc/translate.c | 4 ++-- target/ppc/cpu_init.c | 1 + target/ppc/translate.c | 4 ++-- target/riscv/tcg/tcg-cpu.c | 1 + target/riscv/translate.c | 4 ++-- target/rx/cpu.c | 1 + target/rx/translate.c | 4 ++-- target/s390x/cpu.c | 1 + target/s390x/tcg/translate.c | 4 ++-- target/sh4/cpu.c | 1 + target/sh4/translate.c | 4 ++-- target/sparc/cpu.c | 1 + target/sparc/translate.c | 4 ++-- target/tricore/cpu.c | 1 + target/tricore/translate.c | 5 ++--- target/xtensa/cpu.c | 1 + target/xtensa/translate.c | 4 ++-- 62 files changed, 121 insertions(+), 62 deletions(-) diff --git a/include/exec/translator.h b/include/exec/translator.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/translator.h +++ b/include/exec/translator.h @@ -XXX,XX +XXX,XX @@ #include "qemu/bswap.h" #include "exec/vaddr.h" -/** - * gen_intermediate_code - * @cpu: cpu context - * @tb: translation block - * @max_insns: max number of instructions to translate - * @pc: guest virtual program counter address - * @host_pc: host physical program counter address - * - * This function must be provided by the target, which should create - * the target-specific DisasContext, and then invoke translator_loop. - */ -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc); - /** * DisasJumpType: * @DISAS_NEXT: Next instruction in program order. diff --git a/include/hw/core/tcg-cpu-ops.h b/include/hw/core/tcg-cpu-ops.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/core/tcg-cpu-ops.h +++ b/include/hw/core/tcg-cpu-ops.h @@ -XXX,XX +XXX,XX @@ struct TCGCPUOps { * Called when the first CPU is realized. */ void (*initialize)(void); + /** + * @translate_code: Translate guest instructions to TCGOps + * @cpu: cpu context + * @tb: translation block + * @max_insns: max number of instructions to translate + * @pc: guest virtual program counter address + * @host_pc: host physical program counter address + * + * This function must be provided by the target, which should create + * the target-specific DisasContext, and then invoke translator_loop. + */ + void (*translate_code)(CPUState *cpu, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); /** * @synchronize_from_tb: Synchronize state from a TCG #TranslationBlock * diff --git a/target/alpha/cpu.h b/target/alpha/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/alpha/cpu.h +++ b/target/alpha/cpu.h @@ -XXX,XX +XXX,XX @@ enum { }; void alpha_translate_init(void); +void alpha_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); #define CPU_RESOLVING_TYPE TYPE_ALPHA_CPU diff --git a/target/arm/internals.h b/target/arm/internals.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -XXX,XX +XXX,XX @@ void init_cpreg_list(ARMCPU *cpu); void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu); void arm_translate_init(void); +void arm_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); void arm_cpu_register_gdb_commands(ARMCPU *cpu); void aarch64_cpu_register_gdb_commands(ARMCPU *cpu, GString *, diff --git a/target/avr/cpu.h b/target/avr/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/avr/cpu.h +++ b/target/avr/cpu.h @@ -XXX,XX +XXX,XX @@ static inline void set_avr_feature(CPUAVRState *env, int feature) } void avr_cpu_tcg_init(void); +void avr_cpu_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); int cpu_avr_exec(CPUState *cpu); diff --git a/target/hexagon/cpu.h b/target/hexagon/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/cpu.h +++ b/target/hexagon/cpu.h @@ -XXX,XX +XXX,XX @@ static inline void cpu_get_tb_cpu_state(CPUHexagonState *env, vaddr *pc, typedef HexagonCPU ArchCPU; void hexagon_translate_init(void); +void hexagon_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); #include "exec/cpu-all.h" diff --git a/target/hppa/cpu.h b/target/hppa/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/hppa/cpu.h +++ b/target/hppa/cpu.h @@ -XXX,XX +XXX,XX @@ static inline int HPPA_BTLB_ENTRIES(CPUHPPAState *env) } void hppa_translate_init(void); +void hppa_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); #define CPU_RESOLVING_TYPE TYPE_HPPA_CPU diff --git a/target/i386/tcg/helper-tcg.h b/target/i386/tcg/helper-tcg.h index XXXXXXX..XXXXXXX 100644 --- a/target/i386/tcg/helper-tcg.h +++ b/target/i386/tcg/helper-tcg.h @@ -XXX,XX +XXX,XX @@ static inline target_long lshift(target_long x, int n) /* translate.c */ void tcg_x86_init(void); +void x86_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); /* excp_helper.c */ G_NORETURN void raise_exception(CPUX86State *env, int exception_index); diff --git a/target/loongarch/internals.h b/target/loongarch/internals.h index XXXXXXX..XXXXXXX 100644 --- a/target/loongarch/internals.h +++ b/target/loongarch/internals.h @@ -XXX,XX +XXX,XX @@ #define TARGET_VIRT_MASK MAKE_64BIT_MASK(0, TARGET_VIRT_ADDR_SPACE_BITS) void loongarch_translate_init(void); +void loongarch_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); void G_NORETURN do_raise_exception(CPULoongArchState *env, uint32_t exception, diff --git a/target/m68k/cpu.h b/target/m68k/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/m68k/cpu.h +++ b/target/m68k/cpu.h @@ -XXX,XX +XXX,XX @@ int m68k_cpu_gdb_read_register(CPUState *cpu, GByteArray *buf, int reg); int m68k_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg); void m68k_tcg_init(void); +void m68k_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); void m68k_cpu_init_gdb(M68kCPU *cpu); uint32_t cpu_m68k_get_ccr(CPUM68KState *env); void cpu_m68k_set_ccr(CPUM68KState *env, uint32_t); diff --git a/target/microblaze/cpu.h b/target/microblaze/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/microblaze/cpu.h +++ b/target/microblaze/cpu.h @@ -XXX,XX +XXX,XX @@ static inline void mb_cpu_write_msr(CPUMBState *env, uint32_t val) } void mb_tcg_init(void); +void mb_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); #define CPU_RESOLVING_TYPE TYPE_MICROBLAZE_CPU diff --git a/target/mips/tcg/tcg-internal.h b/target/mips/tcg/tcg-internal.h index XXXXXXX..XXXXXXX 100644 --- a/target/mips/tcg/tcg-internal.h +++ b/target/mips/tcg/tcg-internal.h @@ -XXX,XX +XXX,XX @@ #include "cpu.h" void mips_tcg_init(void); +void mips_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); void mips_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb); G_NORETURN void mips_cpu_do_unaligned_access(CPUState *cpu, vaddr addr, diff --git a/target/openrisc/cpu.h b/target/openrisc/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/openrisc/cpu.h +++ b/target/openrisc/cpu.h @@ -XXX,XX +XXX,XX @@ void openrisc_cpu_dump_state(CPUState *cpu, FILE *f, int flags); int openrisc_cpu_gdb_read_register(CPUState *cpu, GByteArray *buf, int reg); int openrisc_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg); void openrisc_translate_init(void); +void openrisc_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); int print_insn_or1k(bfd_vma addr, disassemble_info *info); #ifndef CONFIG_USER_ONLY diff --git a/target/ppc/cpu.h b/target/ppc/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/cpu.h +++ b/target/ppc/cpu.h @@ -XXX,XX +XXX,XX @@ extern const VMStateDescription vmstate_ppc_cpu; /*****************************************************************************/ void ppc_translate_init(void); +void ppc_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); #if !defined(CONFIG_USER_ONLY) void ppc_store_sdr1(CPUPPCState *env, target_ulong value); diff --git a/target/riscv/cpu.h b/target/riscv/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/cpu.h +++ b/target/riscv/cpu.h @@ -XXX,XX +XXX,XX @@ RISCVException smstateen_acc_ok(CPURISCVState *env, int index, uint64_t bit); void riscv_cpu_set_mode(CPURISCVState *env, target_ulong newpriv, bool virt_en); void riscv_translate_init(void); +void riscv_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); + G_NORETURN void riscv_raise_exception(CPURISCVState *env, uint32_t exception, uintptr_t pc); diff --git a/target/rx/cpu.h b/target/rx/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/rx/cpu.h +++ b/target/rx/cpu.h @@ -XXX,XX +XXX,XX @@ int rx_cpu_gdb_read_register(CPUState *cpu, GByteArray *buf, int reg); int rx_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg); void rx_translate_init(void); +void rx_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); void rx_cpu_unpack_psw(CPURXState *env, uint32_t psw, int rte); #include "exec/cpu-all.h" diff --git a/target/s390x/s390x-internal.h b/target/s390x/s390x-internal.h index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/s390x-internal.h +++ b/target/s390x/s390x-internal.h @@ -XXX,XX +XXX,XX @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, /* translate.c */ void s390x_translate_init(void); +void s390x_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); void s390x_restore_state_to_opc(CPUState *cs, const TranslationBlock *tb, const uint64_t *data); diff --git a/target/sh4/cpu.h b/target/sh4/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/sh4/cpu.h +++ b/target/sh4/cpu.h @@ -XXX,XX +XXX,XX @@ G_NORETURN void superh_cpu_do_unaligned_access(CPUState *cpu, vaddr addr, uintptr_t retaddr); void sh4_translate_init(void); +void sh4_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); #if !defined(CONFIG_USER_ONLY) hwaddr superh_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr); diff --git a/target/sparc/cpu.h b/target/sparc/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/cpu.h +++ b/target/sparc/cpu.h @@ -XXX,XX +XXX,XX @@ int sparc_cpu_memory_rw_debug(CPUState *cpu, vaddr addr, /* translate.c */ void sparc_tcg_init(void); +void sparc_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); /* fop_helper.c */ target_ulong cpu_get_fsr(CPUSPARCState *); diff --git a/target/tricore/cpu.h b/target/tricore/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/tricore/cpu.h +++ b/target/tricore/cpu.h @@ -XXX,XX +XXX,XX @@ FIELD(TB_FLAGS, PRIV, 0, 2) void cpu_state_reset(CPUTriCoreState *s); void tricore_tcg_init(void); +void tricore_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); static inline void cpu_get_tb_cpu_state(CPUTriCoreState *env, vaddr *pc, uint64_t *cs_base, uint32_t *flags) diff --git a/target/xtensa/cpu.h b/target/xtensa/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/xtensa/cpu.h +++ b/target/xtensa/cpu.h @@ -XXX,XX +XXX,XX @@ G_NORETURN void xtensa_cpu_do_unaligned_access(CPUState *cpu, vaddr addr, void xtensa_collect_sr_names(const XtensaConfig *config); void xtensa_translate_init(void); +void xtensa_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc); void **xtensa_get_regfile_by_name(const char *name, int entries, int bits); void xtensa_breakpoint_handler(CPUState *cs); void xtensa_register_core(XtensaConfigList *node); diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ bool tcg_exec_realizefn(CPUState *cpu, Error **errp) if (!tcg_target_initialized) { /* Check mandatory TCGCPUOps handlers */ + const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops; #ifndef CONFIG_USER_ONLY - assert(cpu->cc->tcg_ops->cpu_exec_halt); - assert(cpu->cc->tcg_ops->cpu_exec_interrupt); + assert(tcg_ops->cpu_exec_halt); + assert(tcg_ops->cpu_exec_interrupt); #endif /* !CONFIG_USER_ONLY */ - cpu->cc->tcg_ops->initialize(); + assert(tcg_ops->translate_code); + tcg_ops->initialize(); tcg_target_initialized = true; } diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ static int setjmp_gen_code(CPUArchState *env, TranslationBlock *tb, tcg_func_start(tcg_ctx); - tcg_ctx->cpu = env_cpu(env); - gen_intermediate_code(env_cpu(env), tb, max_insns, pc, host_pc); + CPUState *cs = env_cpu(env); + tcg_ctx->cpu = cs; + cs->cc->tcg_ops->translate_code(cs, tb, max_insns, pc, host_pc); + assert(tb->size != 0); tcg_ctx->cpu = NULL; *max_insns = tb->icount; @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, /* * Overflow of code_gen_buffer, or the current slice of it. * - * TODO: We don't need to re-do gen_intermediate_code, nor + * TODO: We don't need to re-do tcg_ops->translate_code, nor * should we re-do the tcg optimization currently hidden * inside tcg_gen_code. All that should be required is to * flush the TBs, allocate a new TB, re-initialize it per diff --git a/target/alpha/cpu.c b/target/alpha/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/alpha/cpu.c +++ b/target/alpha/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps alpha_sysemu_ops = { static const TCGCPUOps alpha_tcg_ops = { .initialize = alpha_translate_init, + .translate_code = alpha_translate_code, .synchronize_from_tb = alpha_cpu_synchronize_from_tb, .restore_state_to_opc = alpha_restore_state_to_opc, diff --git a/target/alpha/translate.c b/target/alpha/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/alpha/translate.c +++ b/target/alpha/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps alpha_tr_ops = { .tb_stop = alpha_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void alpha_translate_code(CPUState *cpu, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc; translator_loop(cpu, tb, max_insns, pc, host_pc, &alpha_tr_ops, &dc.base); diff --git a/target/arm/cpu.c b/target/arm/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps arm_sysemu_ops = { #ifdef CONFIG_TCG static const TCGCPUOps arm_tcg_ops = { .initialize = arm_translate_init, + .translate_code = arm_translate_code, .synchronize_from_tb = arm_cpu_synchronize_from_tb, .debug_excp_handler = arm_debug_excp_handler, .restore_state_to_opc = arm_restore_state_to_opc, diff --git a/target/arm/tcg/cpu-v7m.c b/target/arm/tcg/cpu-v7m.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/tcg/cpu-v7m.c +++ b/target/arm/tcg/cpu-v7m.c @@ -XXX,XX +XXX,XX @@ static void cortex_m55_initfn(Object *obj) static const TCGCPUOps arm_v7m_tcg_ops = { .initialize = arm_translate_init, + .translate_code = arm_translate_code, .synchronize_from_tb = arm_cpu_synchronize_from_tb, .debug_excp_handler = arm_debug_excp_handler, .restore_state_to_opc = arm_restore_state_to_opc, diff --git a/target/arm/tcg/translate.c b/target/arm/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/tcg/translate.c +++ b/target/arm/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps thumb_translator_ops = { .tb_stop = arm_tr_tb_stop, }; -/* generate intermediate code for basic block 'tb'. */ -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void arm_translate_code(CPUState *cpu, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc = { }; const TranslatorOps *ops = &arm_translator_ops; diff --git a/target/avr/cpu.c b/target/avr/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/cpu.c +++ b/target/avr/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps avr_sysemu_ops = { static const TCGCPUOps avr_tcg_ops = { .initialize = avr_cpu_tcg_init, + .translate_code = avr_cpu_translate_code, .synchronize_from_tb = avr_cpu_synchronize_from_tb, .restore_state_to_opc = avr_restore_state_to_opc, .cpu_exec_interrupt = avr_cpu_exec_interrupt, diff --git a/target/avr/translate.c b/target/avr/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/translate.c +++ b/target/avr/translate.c @@ -XXX,XX +XXX,XX @@ static bool trans_WDR(DisasContext *ctx, arg_WDR *a) * * - translate() * - canonicalize_skip() - * - gen_intermediate_code() + * - translate_code() * - restore_state_to_opc() * */ @@ -XXX,XX +XXX,XX @@ static const TranslatorOps avr_tr_ops = { .tb_stop = avr_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void avr_cpu_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc = { }; translator_loop(cs, tb, max_insns, pc, host_pc, &avr_tr_ops, &dc.base); diff --git a/target/hexagon/cpu.c b/target/hexagon/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/cpu.c +++ b/target/hexagon/cpu.c @@ -XXX,XX +XXX,XX @@ static void hexagon_cpu_init(Object *obj) static const TCGCPUOps hexagon_tcg_ops = { .initialize = hexagon_translate_init, + .translate_code = hexagon_translate_code, .synchronize_from_tb = hexagon_cpu_synchronize_from_tb, .restore_state_to_opc = hexagon_restore_state_to_opc, }; diff --git a/target/hexagon/translate.c b/target/hexagon/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/translate.c +++ b/target/hexagon/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps hexagon_tr_ops = { .tb_stop = hexagon_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void hexagon_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx; diff --git a/target/hppa/cpu.c b/target/hppa/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/hppa/cpu.c +++ b/target/hppa/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps hppa_sysemu_ops = { static const TCGCPUOps hppa_tcg_ops = { .initialize = hppa_translate_init, + .translate_code = hppa_translate_code, .synchronize_from_tb = hppa_cpu_synchronize_from_tb, .restore_state_to_opc = hppa_restore_state_to_opc, diff --git a/target/hppa/translate.c b/target/hppa/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/hppa/translate.c +++ b/target/hppa/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps hppa_tr_ops = { #endif }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void hppa_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx = { }; translator_loop(cs, tb, max_insns, pc, host_pc, &hppa_tr_ops, &ctx.base); diff --git a/target/i386/tcg/tcg-cpu.c b/target/i386/tcg/tcg-cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/tcg/tcg-cpu.c +++ b/target/i386/tcg/tcg-cpu.c @@ -XXX,XX +XXX,XX @@ static bool x86_debug_check_breakpoint(CPUState *cs) static const TCGCPUOps x86_tcg_ops = { .initialize = tcg_x86_init, + .translate_code = x86_translate_code, .synchronize_from_tb = x86_cpu_synchronize_from_tb, .restore_state_to_opc = x86_restore_state_to_opc, .cpu_exec_enter = x86_cpu_exec_enter, diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps i386_tr_ops = { .tb_stop = i386_tr_tb_stop, }; -/* generate intermediate code for basic block 'tb'. */ -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void x86_translate_code(CPUState *cpu, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc; diff --git a/target/loongarch/cpu.c b/target/loongarch/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/loongarch/cpu.c +++ b/target/loongarch/cpu.c @@ -XXX,XX +XXX,XX @@ static void loongarch_cpu_dump_state(CPUState *cs, FILE *f, int flags) static const TCGCPUOps loongarch_tcg_ops = { .initialize = loongarch_translate_init, + .translate_code = loongarch_translate_code, .synchronize_from_tb = loongarch_cpu_synchronize_from_tb, .restore_state_to_opc = loongarch_restore_state_to_opc, diff --git a/target/loongarch/tcg/translate.c b/target/loongarch/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/loongarch/tcg/translate.c +++ b/target/loongarch/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps loongarch_tr_ops = { .tb_stop = loongarch_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void loongarch_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx; diff --git a/target/m68k/cpu.c b/target/m68k/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/m68k/cpu.c +++ b/target/m68k/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps m68k_sysemu_ops = { static const TCGCPUOps m68k_tcg_ops = { .initialize = m68k_tcg_init, + .translate_code = m68k_translate_code, .restore_state_to_opc = m68k_restore_state_to_opc, #ifndef CONFIG_USER_ONLY diff --git a/target/m68k/translate.c b/target/m68k/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/m68k/translate.c +++ b/target/m68k/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps m68k_tr_ops = { .tb_stop = m68k_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void m68k_translate_code(CPUState *cpu, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc; translator_loop(cpu, tb, max_insns, pc, host_pc, &m68k_tr_ops, &dc.base); diff --git a/target/microblaze/cpu.c b/target/microblaze/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/microblaze/cpu.c +++ b/target/microblaze/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps mb_sysemu_ops = { static const TCGCPUOps mb_tcg_ops = { .initialize = mb_tcg_init, + .translate_code = mb_translate_code, .synchronize_from_tb = mb_cpu_synchronize_from_tb, .restore_state_to_opc = mb_restore_state_to_opc, diff --git a/target/microblaze/translate.c b/target/microblaze/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/microblaze/translate.c +++ b/target/microblaze/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps mb_tr_ops = { .tb_stop = mb_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void mb_translate_code(CPUState *cpu, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc; translator_loop(cpu, tb, max_insns, pc, host_pc, &mb_tr_ops, &dc.base); diff --git a/target/mips/cpu.c b/target/mips/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/mips/cpu.c +++ b/target/mips/cpu.c @@ -XXX,XX +XXX,XX @@ static const Property mips_cpu_properties[] = { #include "hw/core/tcg-cpu-ops.h" static const TCGCPUOps mips_tcg_ops = { .initialize = mips_tcg_init, + .translate_code = mips_translate_code, .synchronize_from_tb = mips_cpu_synchronize_from_tb, .restore_state_to_opc = mips_restore_state_to_opc, diff --git a/target/mips/tcg/translate.c b/target/mips/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/mips/tcg/translate.c +++ b/target/mips/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps mips_tr_ops = { .tb_stop = mips_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void mips_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx; diff --git a/target/openrisc/cpu.c b/target/openrisc/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/openrisc/cpu.c +++ b/target/openrisc/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps openrisc_sysemu_ops = { static const TCGCPUOps openrisc_tcg_ops = { .initialize = openrisc_translate_init, + .translate_code = openrisc_translate_code, .synchronize_from_tb = openrisc_cpu_synchronize_from_tb, .restore_state_to_opc = openrisc_restore_state_to_opc, diff --git a/target/openrisc/translate.c b/target/openrisc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/openrisc/translate.c +++ b/target/openrisc/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps openrisc_tr_ops = { .tb_stop = openrisc_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void openrisc_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx; diff --git a/target/ppc/cpu_init.c b/target/ppc/cpu_init.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/cpu_init.c +++ b/target/ppc/cpu_init.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps ppc_sysemu_ops = { static const TCGCPUOps ppc_tcg_ops = { .initialize = ppc_translate_init, + .translate_code = ppc_translate_code, .restore_state_to_opc = ppc_restore_state_to_opc, #ifdef CONFIG_USER_ONLY diff --git a/target/ppc/translate.c b/target/ppc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/translate.c +++ b/target/ppc/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps ppc_tr_ops = { .tb_stop = ppc_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void ppc_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx; diff --git a/target/riscv/tcg/tcg-cpu.c b/target/riscv/tcg/tcg-cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/tcg/tcg-cpu.c +++ b/target/riscv/tcg/tcg-cpu.c @@ -XXX,XX +XXX,XX @@ static void riscv_restore_state_to_opc(CPUState *cs, static const TCGCPUOps riscv_tcg_ops = { .initialize = riscv_translate_init, + .translate_code = riscv_translate_code, .synchronize_from_tb = riscv_cpu_synchronize_from_tb, .restore_state_to_opc = riscv_restore_state_to_opc, diff --git a/target/riscv/translate.c b/target/riscv/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/riscv/translate.c +++ b/target/riscv/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps riscv_tr_ops = { .tb_stop = riscv_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void riscv_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx; diff --git a/target/rx/cpu.c b/target/rx/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/rx/cpu.c +++ b/target/rx/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps rx_sysemu_ops = { static const TCGCPUOps rx_tcg_ops = { .initialize = rx_translate_init, + .translate_code = rx_translate_code, .synchronize_from_tb = rx_cpu_synchronize_from_tb, .restore_state_to_opc = rx_restore_state_to_opc, .tlb_fill = rx_cpu_tlb_fill, diff --git a/target/rx/translate.c b/target/rx/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/rx/translate.c +++ b/target/rx/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps rx_tr_ops = { .tb_stop = rx_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void rx_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc; diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/cpu.c +++ b/target/s390x/cpu.c @@ -XXX,XX +XXX,XX @@ void cpu_get_tb_cpu_state(CPUS390XState *env, vaddr *pc, static const TCGCPUOps s390_tcg_ops = { .initialize = s390x_translate_init, + .translate_code = s390x_translate_code, .restore_state_to_opc = s390x_restore_state_to_opc, #ifdef CONFIG_USER_ONLY diff --git a/target/s390x/tcg/translate.c b/target/s390x/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/tcg/translate.c +++ b/target/s390x/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps s390x_tr_ops = { .disas_log = s390x_tr_disas_log, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void s390x_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc; diff --git a/target/sh4/cpu.c b/target/sh4/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/sh4/cpu.c +++ b/target/sh4/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps sh4_sysemu_ops = { static const TCGCPUOps superh_tcg_ops = { .initialize = sh4_translate_init, + .translate_code = sh4_translate_code, .synchronize_from_tb = superh_cpu_synchronize_from_tb, .restore_state_to_opc = superh_restore_state_to_opc, diff --git a/target/sh4/translate.c b/target/sh4/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/sh4/translate.c +++ b/target/sh4/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps sh4_tr_ops = { .tb_stop = sh4_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void sh4_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx; diff --git a/target/sparc/cpu.c b/target/sparc/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/cpu.c +++ b/target/sparc/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps sparc_sysemu_ops = { static const TCGCPUOps sparc_tcg_ops = { .initialize = sparc_tcg_init, + .translate_code = sparc_translate_code, .synchronize_from_tb = sparc_cpu_synchronize_from_tb, .restore_state_to_opc = sparc_restore_state_to_opc, diff --git a/target/sparc/translate.c b/target/sparc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/translate.c +++ b/target/sparc/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps sparc_tr_ops = { .tb_stop = sparc_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void sparc_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc = {}; diff --git a/target/tricore/cpu.c b/target/tricore/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/tricore/cpu.c +++ b/target/tricore/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps tricore_sysemu_ops = { static const TCGCPUOps tricore_tcg_ops = { .initialize = tricore_tcg_init, + .translate_code = tricore_translate_code, .synchronize_from_tb = tricore_cpu_synchronize_from_tb, .restore_state_to_opc = tricore_restore_state_to_opc, .tlb_fill = tricore_cpu_tlb_fill, diff --git a/target/tricore/translate.c b/target/tricore/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/tricore/translate.c +++ b/target/tricore/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps tricore_tr_ops = { .tb_stop = tricore_tr_tb_stop, }; - -void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void tricore_translate_code(CPUState *cs, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext ctx; translator_loop(cs, tb, max_insns, pc, host_pc, diff --git a/target/xtensa/cpu.c b/target/xtensa/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/xtensa/cpu.c +++ b/target/xtensa/cpu.c @@ -XXX,XX +XXX,XX @@ static const struct SysemuCPUOps xtensa_sysemu_ops = { static const TCGCPUOps xtensa_tcg_ops = { .initialize = xtensa_translate_init, + .translate_code = xtensa_translate_code, .debug_excp_handler = xtensa_breakpoint_handler, .restore_state_to_opc = xtensa_restore_state_to_opc, diff --git a/target/xtensa/translate.c b/target/xtensa/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/xtensa/translate.c +++ b/target/xtensa/translate.c @@ -XXX,XX +XXX,XX @@ static const TranslatorOps xtensa_translator_ops = { .tb_stop = xtensa_tr_tb_stop, }; -void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int *max_insns, - vaddr pc, void *host_pc) +void xtensa_translate_code(CPUState *cpu, TranslationBlock *tb, + int *max_insns, vaddr pc, void *host_pc) { DisasContext dc = {}; translator_loop(cpu, tb, max_insns, pc, host_pc, -- 2.43.0