Changes since v1:

* Added QEMU_ERROR to wrap __attribute__((error)) -- patch 12.

r~

The following changes since commit 77f7c747193662edfadeeb3118d63eed0eac51a6:

Merge remote-tracking branch 'remotes/huth-gitlab/tags/pull-request-2018-10-17' into staging (2018-10-18 13:40:19 +0100)

https://github.com/rth7680/qemu.git tags/pull-tcg-20181018

for you to fetch changes up to 403f290c0603f35f2d09c982bf5549b6d0803ec1:

cputlb: read CPUTLBEntry.addr_write atomically (2018-10-18 19:46:53 -0700)

Queued tcg patches.

Emilio G. Cota (10):

tcg: access cpu->icount_decr.u16.high with atomics

tcg: fix use of uninitialized variable under CONFIG_PROFILER

tcg: plug holes in struct TCGProfile

tcg: distribute tcg_time into TCG contexts

target/alpha: remove tlb_flush from alpha_cpu_initfn

target/unicore32: remove tlb_flush from uc32_init_fn

exec: introduce tlb_init

cputlb: fix assert_cpu_is_self macro

cputlb: serialize tlb updates with env->tlb_lock

cputlb: read CPUTLBEntry.addr_write atomically

Richard Henderson (11):

tcg: Implement CPU_LOG_TB_NOCHAIN during expansion

tcg: Add tlb_index and tlb_entry helpers

tcg: Split CONFIG_ATOMIC128

target/i386: Convert to HAVE_CMPXCHG128

target/arm: Convert to HAVE_CMPXCHG128

target/arm: Check HAVE_CMPXCHG128 at translate time

target/ppc: Convert to HAVE_CMPXCHG128 and HAVE_ATOMIC128

target/s390x: Convert to HAVE_CMPXCHG128 and HAVE_ATOMIC128

target/s390x: Split do_cdsg, do_lpq, do_stpq

target/s390x: Skip wout, cout helpers if op helper does not return

target/s390x: Check HAVE_ATOMIC128 and HAVE_CMPXCHG128 at translate

accel/tcg/atomic_template.h | 20 +++-

accel/tcg/softmmu_template.h | 64 +++++-----

include/exec/cpu-defs.h | 3 +

include/exec/cpu_ldst.h | 30 ++++-

include/exec/cpu_ldst_template.h | 25 ++--

include/exec/exec-all.h | 8 ++

include/qemu/atomic128.h | 153 ++++++++++++++++++++++++

include/qemu/compiler.h | 11 ++

include/qemu/timer.h | 1 -

target/ppc/helper.h | 2 +-

tcg/tcg.h | 20 ++--

accel/tcg/cpu-exec.c | 2 +-

accel/tcg/cputlb.c | 235 +++++++++++++++++++-----------------

accel/tcg/tcg-all.c | 2 +-

accel/tcg/translate-all.c | 2 +-

accel/tcg/user-exec.c | 5 +-

cpus.c | 3 +-

exec.c | 1 +

monitor.c | 13 +-

qom/cpu.c | 2 +-

target/alpha/cpu.c | 1 -

target/arm/helper-a64.c | 251 +++++++++++++++++++--------------------

target/arm/translate-a64.c | 38 +++---

target/i386/mem_helper.c | 9 +-

target/ppc/mem_helper.c | 33 ++++-

target/ppc/translate.c | 115 +++++++++---------

target/s390x/mem_helper.c | 202 +++++++++++++++----------------

target/s390x/translate.c | 45 +++++--

target/unicore32/cpu.c | 2 -

tcg/tcg-op.c | 9 +-

tcg/tcg.c | 25 +++-

configure | 19 +++

32 files changed, 839 insertions(+), 512 deletions(-)

create mode 100644 include/qemu/atomic128.h

Rather than test NOCHAIN before linking, do not emit the

goto_tb opcode at all. We already do this for goto_ptr.

accel/tcg/cpu-exec.c | 2 +-

tcg/tcg-op.c | 9 ++++++++-

2 files changed, 9 insertions(+), 2 deletions(-)

@@ -XXX,XX +XXX,XX @@ void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx)

seen this numbered exit before, via tcg_gen_goto_tb. */

tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));

#endif

+ /* When not chaining, exit without indicating a link. */

+ if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {

+ val = 0;

+ }

} else {

/* This is an exit via the exitreq label. */

tcg_debug_assert(idx == TB_EXIT_REQUESTED);

@@ -XXX,XX +XXX,XX @@ void tcg_gen_goto_tb(unsigned idx)

tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);

tcg_ctx->goto_tb_issue_mask |= 1 << idx;

#endif

- tcg_gen_op1i(INDEX_op_goto_tb, idx);

+ /* When not chaining, we simply fall through to the "fallback" exit. */

+ if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {

+ tcg_gen_op1i(INDEX_op_goto_tb, idx);

}

void tcg_gen_lookup_and_goto_ptr(void)

2.17.2

From: "Emilio G. Cota" <cota@braap.org>

This plugs two 4-byte holes in 64-bit.

Signed-off-by: Emilio G. Cota <cota@braap.org>

Message-Id: <20181010144853.13005-4-cota@braap.org>

tcg/tcg.h | 2 +-

diff --git a/tcg/tcg.h b/tcg/tcg.h

--- a/tcg/tcg.h

+++ b/tcg/tcg.h

@@ -XXX,XX +XXX,XX @@ typedef struct TCGProfile {

int64_t tb_count;

int64_t op_count; /* total insn count */

int op_count_max; /* max insn per TB */

- int64_t temp_count;

int temp_count_max;

+ int64_t temp_count;

int64_t del_op_count;

int64_t code_in_len;

int64_t code_out_len;

2.17.2

From: "Emilio G. Cota" <cota@braap.org>

As far as I can tell tlb_flush does not need to be called

this early. tlb_flush is eventually called after the CPU

has been realized.

This change paves the way to the introduction of tlb_init,

which will be called from cpu_exec_realizefn.

Cc: Guan Xuetao <gxt@mprc.pku.edu.cn>

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Reviewed-by: Richard Henderson <richard.henderson@linaro.org>

Signed-off-by: Emilio G. Cota <cota@braap.org>

Message-Id: <20181009174557.16125-3-cota@braap.org>

target/unicore32/cpu.c | 2 --

1 file changed, 2 deletions(-)

diff --git a/target/unicore32/cpu.c b/target/unicore32/cpu.c

--- a/target/unicore32/cpu.c

+++ b/target/unicore32/cpu.c

@@ -XXX,XX +XXX,XX @@ static void uc32_cpu_initfn(Object *obj)

env->uncached_asr = ASR_MODE_PRIV;

env->regs[31] = 0x03000000;

#endif

-

- tlb_flush(cs);

}

static const VMStateDescription vmstate_uc32_cpu = {

2.17.2

From: "Emilio G. Cota" <cota@braap.org>

Paves the way for the addition of a per-TLB lock.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Reviewed-by: Richard Henderson <richard.henderson@linaro.org>

Signed-off-by: Emilio G. Cota <cota@braap.org>

Message-Id: <20181009174557.16125-4-cota@braap.org>

include/exec/exec-all.h | 8 ++++++++

accel/tcg/cputlb.c | 4 ++++

exec.c | 1 +

3 files changed, 13 insertions(+)

diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h

--- a/include/exec/exec-all.h

+++ b/include/exec/exec-all.h

@@ -XXX,XX +XXX,XX @@ void cpu_address_space_init(CPUState *cpu, int asidx,

#if !defined(CONFIG_USER_ONLY) && defined(CONFIG_TCG)

/* cputlb.c */

+/**

+ * tlb_init - initialize a CPU's TLB

+ * @cpu: CPU whose TLB should be initialized

+ */

+void tlb_init(CPUState *cpu);

/**

* tlb_flush_page:

* @cpu: CPU whose TLB should be flushed

@@ -XXX,XX +XXX,XX @@ void tlb_set_page(CPUState *cpu, target_ulong vaddr,

void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx,

uintptr_t retaddr);

#else

+static inline void tlb_init(CPUState *cpu)

+{

+}

static inline void tlb_flush_page(CPUState *cpu, target_ulong addr)

{

}

diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c

index XXXXXXX..XXXXXXX 100644

--- a/accel/tcg/cputlb.c

+++ b/accel/tcg/cputlb.c

@@ -XXX,XX +XXX,XX @@ QEMU_BUILD_BUG_ON(sizeof(target_ulong) > sizeof(run_on_cpu_data));

QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16);

#define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)

+void tlb_init(CPUState *cpu)

+{

+}

+

/* flush_all_helper: run fn across all cpus

*

* If the wait flag is set then the src cpu's helper will be queued as

diff --git a/exec.c b/exec.c

index XXXXXXX..XXXXXXX 100644

--- a/exec.c

+++ b/exec.c

@@ -XXX,XX +XXX,XX @@ void cpu_exec_realizefn(CPUState *cpu, Error **errp)

tcg_target_initialized = true;

cc->tcg_initialize();

+ tlb_init(cpu);

#ifndef CONFIG_USER_ONLY

if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {

2.17.2

From: "Emilio G. Cota" <cota@braap.org>

Reviewed-by: Richard Henderson <richard.henderson@linaro.org>

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Signed-off-by: Emilio G. Cota <cota@braap.org>

Message-Id: <20181009174557.16125-5-cota@braap.org>

accel/tcg/cputlb.c | 4 ++--

1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c

--- a/accel/tcg/cputlb.c

+++ b/accel/tcg/cputlb.c

@@ -XXX,XX +XXX,XX @@

} \

} while (0)

-#define assert_cpu_is_self(this_cpu) do { \

+#define assert_cpu_is_self(cpu) do { \

if (DEBUG_TLB_GATE) { \

- g_assert(!cpu->created || qemu_cpu_is_self(cpu)); \

+ g_assert(!(cpu)->created || qemu_cpu_is_self(cpu)); \

} \

} while (0)

2.17.2

From: "Emilio G. Cota" <cota@braap.org>

Currently we rely on atomic operations for cross-CPU invalidations.

There are two cases that these atomics miss: cross-CPU invalidations

can race with either (1) vCPU threads flushing their TLB, which

happens via memset, or (2) vCPUs calling tlb_reset_dirty on their TLB,

which updates .addr_write with a regular store. This results in

undefined behaviour, since we're mixing regular and atomic ops

on concurrent accesses.

Fix it by using tlb_lock, a per-vCPU lock. All updaters of tlb_table

and the corresponding victim cache now hold the lock.

The readers that do not hold tlb_lock must use atomic reads when

reading .addr_write, since this field can be updated by other threads;

the conversion to atomic reads is done in the next patch.

Note that an alternative fix would be to expand the use of atomic ops.

However, in the case of TLB flushes this would have a huge performance

impact, since (1) TLB flushes can happen very frequently and (2) we

currently use a full memory barrier to flush each TLB entry, and a TLB

has many entries. Instead, acquiring the lock is barely slower than a

full memory barrier since it is uncontended, and with a single lock

acquisition we can flush the entire TLB.

Tested-by: Alex Bennée <alex.bennee@linaro.org>

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Signed-off-by: Emilio G. Cota <cota@braap.org>

Message-Id: <20181009174557.16125-6-cota@braap.org>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

include/exec/cpu-defs.h | 3 +

accel/tcg/cputlb.c | 155 ++++++++++++++++++++++------------------

2 files changed, 87 insertions(+), 71 deletions(-)

diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h

index XXXXXXX..XXXXXXX 100644

--- a/include/exec/cpu-defs.h

+++ b/include/exec/cpu-defs.h

@@ -XXX,XX +XXX,XX @@

#endif

#include "qemu/host-utils.h"

+#include "qemu/thread.h"

#include "qemu/queue.h"

#ifdef CONFIG_TCG

#include "tcg-target.h"

@@ -XXX,XX +XXX,XX @@ typedef struct CPUIOTLBEntry {

#define CPU_COMMON_TLB \

/* The meaning of the MMU modes is defined in the target code. */ \

+ /* tlb_lock serializes updates to tlb_table and tlb_v_table */ \

+ QemuSpin tlb_lock; \

CPUTLBEntry tlb_table[NB_MMU_MODES][CPU_TLB_SIZE]; \

CPUTLBEntry tlb_v_table[NB_MMU_MODES][CPU_VTLB_SIZE]; \

CPUIOTLBEntry iotlb[NB_MMU_MODES][CPU_TLB_SIZE]; \

diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c

index XXXXXXX..XXXXXXX 100644

--- a/accel/tcg/cputlb.c

+++ b/accel/tcg/cputlb.c

@@ -XXX,XX +XXX,XX @@ QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16);

void tlb_init(CPUState *cpu)

{

+ CPUArchState *env = cpu->env_ptr;

+

+ qemu_spin_init(&env->tlb_lock);

}

/* flush_all_helper: run fn across all cpus

@@ -XXX,XX +XXX,XX @@ static void tlb_flush_nocheck(CPUState *cpu)

atomic_set(&env->tlb_flush_count, env->tlb_flush_count + 1);

tlb_debug("(count: %zu)\n", tlb_flush_count());

+ /*

+ * tlb_table/tlb_v_table updates from any thread must hold tlb_lock.

+ * However, updates from the owner thread (as is the case here; see the

+ * above assert_cpu_is_self) do not need atomic_set because all reads

+ * that do not hold the lock are performed by the same owner thread.

+ */

+ qemu_spin_lock(&env->tlb_lock);

memset(env->tlb_table, -1, sizeof(env->tlb_table));

memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table));

+ qemu_spin_unlock(&env->tlb_lock);

+

cpu_tb_jmp_cache_clear(cpu);

env->vtlb_index = 0;

@@ -XXX,XX +XXX,XX @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)

tlb_debug("start: mmu_idx:0x%04lx\n", mmu_idx_bitmask);

+ qemu_spin_lock(&env->tlb_lock);

for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {

if (test_bit(mmu_idx, &mmu_idx_bitmask)) {

@@ -XXX,XX +XXX,XX @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)

memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0]));

}

}

+ qemu_spin_unlock(&env->tlb_lock);

cpu_tb_jmp_cache_clear(cpu);

@@ -XXX,XX +XXX,XX @@ static inline bool tlb_hit_page_anyprot(CPUTLBEntry *tlb_entry,

tlb_hit_page(tlb_entry->addr_code, page);

}

-static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong page)

+/* Called with tlb_lock held */

+static inline void tlb_flush_entry_locked(CPUTLBEntry *tlb_entry,

+ target_ulong page)

{

if (tlb_hit_page_anyprot(tlb_entry, page)) {

memset(tlb_entry, -1, sizeof(*tlb_entry));

}

}

-static inline void tlb_flush_vtlb_page(CPUArchState *env, int mmu_idx,

- target_ulong page)

+/* Called with tlb_lock held */

+static inline void tlb_flush_vtlb_page_locked(CPUArchState *env, int mmu_idx,

+ target_ulong page)

{

int k;

+

+ assert_cpu_is_self(ENV_GET_CPU(env));

for (k = 0; k < CPU_VTLB_SIZE; k++) {

- tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], page);

+ tlb_flush_entry_locked(&env->tlb_v_table[mmu_idx][k], page);

}

}

@@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data)

addr &= TARGET_PAGE_MASK;

i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

+ qemu_spin_lock(&env->tlb_lock);

for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {

- tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);

- tlb_flush_vtlb_page(env, mmu_idx, addr);

+ tlb_flush_entry_locked(&env->tlb_table[mmu_idx][i], addr);

+ tlb_flush_vtlb_page_locked(env, mmu_idx, addr);

}

+ qemu_spin_unlock(&env->tlb_lock);

tb_flush_jmp_cache(cpu, addr);

}

@@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_by_mmuidx_async_work(CPUState *cpu,

tlb_debug("page:%d addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n",

page, addr, mmu_idx_bitmap);

+ qemu_spin_lock(&env->tlb_lock);

for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {

if (test_bit(mmu_idx, &mmu_idx_bitmap)) {

- tlb_flush_entry(&env->tlb_table[mmu_idx][page], addr);

- tlb_flush_vtlb_page(env, mmu_idx, addr);

+ tlb_flush_entry_locked(&env->tlb_table[mmu_idx][page], addr);

+ tlb_flush_vtlb_page_locked(env, mmu_idx, addr);

}

}

+ qemu_spin_unlock(&env->tlb_lock);

tb_flush_jmp_cache(cpu, addr);

}

@@ -XXX,XX +XXX,XX @@ void tlb_unprotect_code(ram_addr_t ram_addr)

* most usual is detecting writes to code regions which may invalidate

* generated code.

*

- * Because we want other vCPUs to respond to changes straight away we

- * update the te->addr_write field atomically. If the TLB entry has

- * been changed by the vCPU in the mean time we skip the update.

+ * Other vCPUs might be reading their TLBs during guest execution, so we update

+ * te->addr_write with atomic_set. We don't need to worry about this for

+ * oversized guests as MTTCG is disabled for them.

*

- * As this function uses atomic accesses we also need to ensure

- * updates to tlb_entries follow the same access rules. We don't need

- * to worry about this for oversized guests as MTTCG is disabled for

- * them.

+ * Called with tlb_lock held.

*/

-

-static void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,

- uintptr_t length)

+static void tlb_reset_dirty_range_locked(CPUTLBEntry *tlb_entry,

+ uintptr_t start, uintptr_t length)

{

-#if TCG_OVERSIZED_GUEST

uintptr_t addr = tlb_entry->addr_write;

if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) {

addr &= TARGET_PAGE_MASK;

addr += tlb_entry->addend;

if ((addr - start) < length) {

+#if TCG_OVERSIZED_GUEST

tlb_entry->addr_write |= TLB_NOTDIRTY;

- }

- }

#else

- /* paired with atomic_mb_set in tlb_set_page_with_attrs */

- uintptr_t orig_addr = atomic_mb_read(&tlb_entry->addr_write);

- uintptr_t addr = orig_addr;

-

- if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) {

- addr &= TARGET_PAGE_MASK;

- addr += atomic_read(&tlb_entry->addend);

- if ((addr - start) < length) {

- uintptr_t notdirty_addr = orig_addr | TLB_NOTDIRTY;

- atomic_cmpxchg(&tlb_entry->addr_write, orig_addr, notdirty_addr);

+ atomic_set(&tlb_entry->addr_write,

+ tlb_entry->addr_write | TLB_NOTDIRTY);

+#endif

}

}

-#endif

}

-/* For atomic correctness when running MTTCG we need to use the right

- * primitives when copying entries */

-static inline void copy_tlb_helper(CPUTLBEntry *d, CPUTLBEntry *s,

- bool atomic_set)

+/*

+ * Called with tlb_lock held.

+ * Called only from the vCPU context, i.e. the TLB's owner thread.

+ */

+static inline void copy_tlb_helper_locked(CPUTLBEntry *d, const CPUTLBEntry *s)

{

-#if TCG_OVERSIZED_GUEST

*d = *s;

-#else

- if (atomic_set) {

- d->addr_read = s->addr_read;

- d->addr_code = s->addr_code;

- atomic_set(&d->addend, atomic_read(&s->addend));

- /* Pairs with flag setting in tlb_reset_dirty_range */

- atomic_mb_set(&d->addr_write, atomic_read(&s->addr_write));

- } else {

- d->addr_read = s->addr_read;

- d->addr_write = atomic_read(&s->addr_write);

- d->addr_code = s->addr_code;

- d->addend = atomic_read(&s->addend);

- }

-#endif

}

/* This is a cross vCPU call (i.e. another vCPU resetting the flags of

- * the target vCPU). As such care needs to be taken that we don't

- * dangerously race with another vCPU update. The only thing actually

- * updated is the target TLB entry ->addr_write flags.

+ * the target vCPU).

+ * We must take tlb_lock to avoid racing with another vCPU update. The only

+ * thing actually updated is the target TLB entry ->addr_write flags.

*/

void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)

{

@@ -XXX,XX +XXX,XX @@ void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)

int mmu_idx;

env = cpu->env_ptr;

+ qemu_spin_lock(&env->tlb_lock);

for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {

unsigned int i;

for (i = 0; i < CPU_TLB_SIZE; i++) {

- tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],

- start1, length);

+ tlb_reset_dirty_range_locked(&env->tlb_table[mmu_idx][i], start1,

+ length);

}

for (i = 0; i < CPU_VTLB_SIZE; i++) {

- tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],

- start1, length);

+ tlb_reset_dirty_range_locked(&env->tlb_v_table[mmu_idx][i], start1,

+ length);

}

}

+ qemu_spin_unlock(&env->tlb_lock);

}

-static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)

+/* Called with tlb_lock held */

+static inline void tlb_set_dirty1_locked(CPUTLBEntry *tlb_entry,

+ target_ulong vaddr)

{

if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY)) {

tlb_entry->addr_write = vaddr;

@@ -XXX,XX +XXX,XX @@ void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)

vaddr &= TARGET_PAGE_MASK;

i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

+ qemu_spin_lock(&env->tlb_lock);

for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {

- tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);

+ tlb_set_dirty1_locked(&env->tlb_table[mmu_idx][i], vaddr);

}

for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {

int k;

for (k = 0; k < CPU_VTLB_SIZE; k++) {

- tlb_set_dirty1(&env->tlb_v_table[mmu_idx][k], vaddr);

+ tlb_set_dirty1_locked(&env->tlb_v_table[mmu_idx][k], vaddr);

}

}

+ qemu_spin_unlock(&env->tlb_lock);

}

/* Our TLB does not support large pages, so remember the area covered by

@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,

addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat;

}

- /* Make sure there's no cached translation for the new page. */

- tlb_flush_vtlb_page(env, mmu_idx, vaddr_page);

-

code_address = address;

iotlb = memory_region_section_get_iotlb(cpu, section, vaddr_page,

paddr_page, xlat, prot, &address);

@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,

index = (vaddr_page >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

te = &env->tlb_table[mmu_idx][index];

+ /*

+ * Hold the TLB lock for the rest of the function. We could acquire/release

+ * the lock several times in the function, but it is faster to amortize the

+ * acquisition cost by acquiring it just once. Note that this leads to

+ * a longer critical section, but this is not a concern since the TLB lock

+ * is unlikely to be contended.

+ */

+ qemu_spin_lock(&env->tlb_lock);

+

+ /* Make sure there's no cached translation for the new page. */

+ tlb_flush_vtlb_page_locked(env, mmu_idx, vaddr_page);

+

/*

* Only evict the old entry to the victim tlb if it's for a

* different page; otherwise just overwrite the stale data.

@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,

CPUTLBEntry *tv = &env->tlb_v_table[mmu_idx][vidx];

/* Evict the old entry into the victim tlb. */

- copy_tlb_helper(tv, te, true);

+ copy_tlb_helper_locked(tv, te);

env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index];

}

@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,

}

}

- /* Pairs with flag setting in tlb_reset_dirty_range */

- copy_tlb_helper(te, &tn, true);

- /* atomic_mb_set(&te->addr_write, write_address); */

+ copy_tlb_helper_locked(te, &tn);

+ qemu_spin_unlock(&env->tlb_lock);

}

/* Add a new TLB entry, but without specifying the memory

@@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,

size_t elt_ofs, target_ulong page)

{

size_t vidx;

+

+ assert_cpu_is_self(ENV_GET_CPU(env));

for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) {

CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx];

target_ulong cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs);

@@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,

/* Found entry in victim tlb, swap tlb and iotlb. */

CPUTLBEntry tmptlb, *tlb = &env->tlb_table[mmu_idx][index];

- copy_tlb_helper(&tmptlb, tlb, false);

- copy_tlb_helper(tlb, vtlb, true);

- copy_tlb_helper(vtlb, &tmptlb, true);

+ qemu_spin_lock(&env->tlb_lock);

+ copy_tlb_helper_locked(&tmptlb, tlb);

+ copy_tlb_helper_locked(tlb, vtlb);

+ copy_tlb_helper_locked(vtlb, &tmptlb);

+ qemu_spin_unlock(&env->tlb_lock);

CPUIOTLBEntry tmpio, *io = &env->iotlb[mmu_idx][index];

CPUIOTLBEntry *vio = &env->iotlb_v[mmu_idx][vidx];

--

2.17.2

Isolate the computation of an index from an address into a

helper before we change that function.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

[ cota: convert tlb_vaddr_to_host; use atomic_read on addr_write ]

Signed-off-by: Emilio G. Cota <cota@braap.org>

Message-Id: <20181009175129.17888-2-cota@braap.org>

---

accel/tcg/softmmu_template.h | 64 +++++++++++++++++---------------

include/exec/cpu_ldst.h | 19 ++++++++--

include/exec/cpu_ldst_template.h | 25 +++++++------

accel/tcg/cputlb.c | 60 ++++++++++++++----------------

4 files changed, 90 insertions(+), 78 deletions(-)

diff --git a/accel/tcg/softmmu_template.h b/accel/tcg/softmmu_template.h

--- a/accel/tcg/softmmu_template.h

+++ b/accel/tcg/softmmu_template.h

@@ -XXX,XX +XXX,XX @@ static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,

WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,

TCGMemOpIdx oi, uintptr_t retaddr)

{

- unsigned mmu_idx = get_mmuidx(oi);

- int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

- target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;

+ uintptr_t mmu_idx = get_mmuidx(oi);

+ uintptr_t index = tlb_index(env, mmu_idx, addr);

+ CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);

+ target_ulong tlb_addr = entry->ADDR_READ;

unsigned a_bits = get_alignment_bits(get_memop(oi));

uintptr_t haddr;

DATA_TYPE res;

@@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,

tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, READ_ACCESS_TYPE,

mmu_idx, retaddr);

}

- tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;

+ tlb_addr = entry->ADDR_READ;

}

/* Handle an IO access. */

@@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,

return res;

}

- haddr = addr + env->tlb_table[mmu_idx][index].addend;

+ haddr = addr + entry->addend;

#if DATA_SIZE == 1

res = glue(glue(ld, LSUFFIX), _p)((uint8_t *)haddr);

#else

@@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,

WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,

TCGMemOpIdx oi, uintptr_t retaddr)

{

- unsigned mmu_idx = get_mmuidx(oi);

- int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

- target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;

+ uintptr_t mmu_idx = get_mmuidx(oi);

+ uintptr_t index = tlb_index(env, mmu_idx, addr);

+ CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);

+ target_ulong tlb_addr = entry->ADDR_READ;

unsigned a_bits = get_alignment_bits(get_memop(oi));

uintptr_t haddr;

DATA_TYPE res;

@@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,

tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, READ_ACCESS_TYPE,

mmu_idx, retaddr);

}

- tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;

+ tlb_addr = entry->ADDR_READ;

}

/* Handle an IO access. */

@@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,

return res;

}

- haddr = addr + env->tlb_table[mmu_idx][index].addend;

+ haddr = addr + entry->addend;

res = glue(glue(ld, LSUFFIX), _be_p)((uint8_t *)haddr);

return res;

}

@@ -XXX,XX +XXX,XX @@ static inline void glue(io_write, SUFFIX)(CPUArchState *env,

void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,

TCGMemOpIdx oi, uintptr_t retaddr)

{

- unsigned mmu_idx = get_mmuidx(oi);

- int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

- target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;

+ uintptr_t mmu_idx = get_mmuidx(oi);

+ uintptr_t index = tlb_index(env, mmu_idx, addr);

+ CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);

+ target_ulong tlb_addr = entry->addr_write;

unsigned a_bits = get_alignment_bits(get_memop(oi));

uintptr_t haddr;

@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,

tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE,

mmu_idx, retaddr);

}

- tlb_addr = env->tlb_table[mmu_idx][index].addr_write & ~TLB_INVALID_MASK;

+ tlb_addr = entry->addr_write & ~TLB_INVALID_MASK;

}

/* Handle an IO access. */

@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,

if (DATA_SIZE > 1

&& unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1

>= TARGET_PAGE_SIZE)) {

- int i, index2;

- target_ulong page2, tlb_addr2;

+ int i;

+ target_ulong page2;

+ CPUTLBEntry *entry2;

do_unaligned_access:

/* Ensure the second page is in the TLB. Note that the first page

is already guaranteed to be filled, and that the second page

cannot evict the first. */

page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;

- index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

- tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write;

- if (!tlb_hit_page(tlb_addr2, page2)

+ entry2 = tlb_entry(env, mmu_idx, page2);

+ if (!tlb_hit_page(entry2->addr_write, page2)

&& !VICTIM_TLB_HIT(addr_write, page2)) {

tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE,

mmu_idx, retaddr);

@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,

return;

}

- haddr = addr + env->tlb_table[mmu_idx][index].addend;

+ haddr = addr + entry->addend;

#if DATA_SIZE == 1

glue(glue(st, SUFFIX), _p)((uint8_t *)haddr, val);

#else

@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,

void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,

TCGMemOpIdx oi, uintptr_t retaddr)

{

- unsigned mmu_idx = get_mmuidx(oi);

- int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

- target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;

+ uintptr_t mmu_idx = get_mmuidx(oi);

+ uintptr_t index = tlb_index(env, mmu_idx, addr);

+ CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);

+ target_ulong tlb_addr = entry->addr_write;

unsigned a_bits = get_alignment_bits(get_memop(oi));

uintptr_t haddr;

@@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,

tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE,

mmu_idx, retaddr);

}

- tlb_addr = env->tlb_table[mmu_idx][index].addr_write & ~TLB_INVALID_MASK;

+ tlb_addr = entry->addr_write & ~TLB_INVALID_MASK;

}

/* Handle an IO access. */

@@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,

if (DATA_SIZE > 1

&& unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1

>= TARGET_PAGE_SIZE)) {

- int i, index2;

- target_ulong page2, tlb_addr2;

+ int i;

+ target_ulong page2;

+ CPUTLBEntry *entry2;

do_unaligned_access:

/* Ensure the second page is in the TLB. Note that the first page

is already guaranteed to be filled, and that the second page

cannot evict the first. */

page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;

- index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

- tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write;

- if (!tlb_hit_page(tlb_addr2, page2)

+ entry2 = tlb_entry(env, mmu_idx, page2);

+ if (!tlb_hit_page(entry2->addr_write, page2)

&& !VICTIM_TLB_HIT(addr_write, page2)) {

tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE,

mmu_idx, retaddr);

@@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,

return;

}

- haddr = addr + env->tlb_table[mmu_idx][index].addend;

+ haddr = addr + entry->addend;

glue(glue(st, SUFFIX), _be_p)((uint8_t *)haddr, val);

}

#endif /* DATA_SIZE > 1 */

@@ -XXX,XX +XXX,XX @@ extern __thread uintptr_t helper_retaddr;

/* The memory helpers for tcg-generated code need tcg_target_long etc. */

#include "tcg.h"