These bits do not need to vary with the actual page size

used by the guest.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Reviewed-by: David Hildenbrand <david@redhat.com>

Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

include/exec/cpu-all.h | 16 ++++++++++------

1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h

index XXXXXXX..XXXXXXX 100644

--- a/include/exec/cpu-all.h

+++ b/include/exec/cpu-all.h

@@ -XXX,XX +XXX,XX @@ CPUArchState *cpu_copy(CPUArchState *env);

#if !defined(CONFIG_USER_ONLY)

-/* Flags stored in the low bits of the TLB virtual address. These are

- * defined so that fast path ram access is all zeros.

+/*

+ * Flags stored in the low bits of the TLB virtual address.

+ * These are defined so that fast path ram access is all zeros.

* The flags all must be between TARGET_PAGE_BITS and

* maximum address alignment bit.

+ *

+ * Use TARGET_PAGE_BITS_MIN so that these bits are constant

+ * when TARGET_PAGE_BITS_VARY is in effect.

*/

/* Zero if TLB entry is valid. */

-#define TLB_INVALID_MASK (1 << (TARGET_PAGE_BITS - 1))

+#define TLB_INVALID_MASK (1 << (TARGET_PAGE_BITS_MIN - 1))

/* Set if TLB entry references a clean RAM page. The iotlb entry will

contain the page physical address. */

-#define TLB_NOTDIRTY (1 << (TARGET_PAGE_BITS - 2))

+#define TLB_NOTDIRTY (1 << (TARGET_PAGE_BITS_MIN - 2))

/* Set if TLB entry is an IO callback. */

-#define TLB_MMIO (1 << (TARGET_PAGE_BITS - 3))

+#define TLB_MMIO (1 << (TARGET_PAGE_BITS_MIN - 3))

/* Set if TLB entry contains a watchpoint. */

-#define TLB_WATCHPOINT (1 << (TARGET_PAGE_BITS - 4))

+#define TLB_WATCHPOINT (1 << (TARGET_PAGE_BITS_MIN - 4))

/* Use this mask to check interception with an alignment mask

* in a TCG backend.

--

2.17.1

This forced inlining can result in missing symbols,

which makes a debugging build harder to follow.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>

Reviewed-by: David Hildenbrand <david@redhat.com>

Reported-by: Peter Maydell <peter.maydell@linaro.org>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

include/qemu/compiler.h | 11 +++++++++++

accel/tcg/cputlb.c | 4 ++--

2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/include/qemu/compiler.h b/include/qemu/compiler.h

index XXXXXXX..XXXXXXX 100644

--- a/include/qemu/compiler.h

+++ b/include/qemu/compiler.h

@@ -XXX,XX +XXX,XX @@

# define QEMU_NONSTRING

#endif

+/*

+ * Forced inlining may be desired to encourage constant propagation

+ * of function parameters. However, it can also make debugging harder,

+ * so disable it for a non-optimizing build.

+ */

+#if defined(__OPTIMIZE__)

+#define QEMU_ALWAYS_INLINE __attribute__((always_inline))

+#else

+#define QEMU_ALWAYS_INLINE

+#endif

+

/* Implement C11 _Generic via GCC builtins. Example:

*

* QEMU_GENERIC(x, (float, sinf), (long double, sinl), sin) (x)

diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c

index XXXXXXX..XXXXXXX 100644

--- a/accel/tcg/cputlb.c

+++ b/accel/tcg/cputlb.c

@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,

typedef uint64_t FullLoadHelper(CPUArchState *env, target_ulong addr,

TCGMemOpIdx oi, uintptr_t retaddr);

-static inline uint64_t __attribute__((always_inline))

+static inline uint64_t QEMU_ALWAYS_INLINE

load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,

uintptr_t retaddr, MemOp op, bool code_read,

FullLoadHelper *full_load)

@@ -XXX,XX +XXX,XX @@ tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr,

* Store Helpers

*/

-static inline void __attribute__((always_inline))

+static inline void QEMU_ALWAYS_INLINE

store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

TCGMemOpIdx oi, uintptr_t retaddr, MemOp op)

{

--

2.17.1

Use this as a compile-time assert that a particular

code path is not reachable.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

include/qemu/compiler.h | 15 +++++++++++++++

1 file changed, 15 insertions(+)

diff --git a/include/qemu/compiler.h b/include/qemu/compiler.h

index XXXXXXX..XXXXXXX 100644

--- a/include/qemu/compiler.h

+++ b/include/qemu/compiler.h

@@ -XXX,XX +XXX,XX @@

#define QEMU_GENERIC9(x, a0, ...) QEMU_GENERIC_IF(x, a0, QEMU_GENERIC8(x, __VA_ARGS__))

#define QEMU_GENERIC10(x, a0, ...) QEMU_GENERIC_IF(x, a0, QEMU_GENERIC9(x, __VA_ARGS__))

+/**

+ * qemu_build_not_reached()

+ *

+ * The compiler, during optimization, is expected to prove that a call

+ * to this function cannot be reached and remove it. If the compiler

+ * supports QEMU_ERROR, this will be reported at compile time; otherwise

+ * this will be reported at link time due to the missing symbol.

+ */

+#ifdef __OPTIMIZE__

+extern void QEMU_NORETURN QEMU_ERROR("code path is reachable")

+ qemu_build_not_reached(void);

+#else

+#define qemu_build_not_reached() g_assert_not_reached()

+#endif

+

#endif /* COMPILER_H */

--

2.17.1

We will shortly be using these more than once.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Reviewed-by: David Hildenbrand <david@redhat.com>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

accel/tcg/cputlb.c | 107 +++++++++++++++++++++++----------------------

1 file changed, 55 insertions(+), 52 deletions(-)

diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c

index XXXXXXX..XXXXXXX 100644

--- a/accel/tcg/cputlb.c

+++ b/accel/tcg/cputlb.c

@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,

typedef uint64_t FullLoadHelper(CPUArchState *env, target_ulong addr,

TCGMemOpIdx oi, uintptr_t retaddr);

+static inline uint64_t QEMU_ALWAYS_INLINE

+load_memop(const void *haddr, MemOp op)

+{

+ switch (op) {

+ case MO_UB:

+ return ldub_p(haddr);

+ case MO_BEUW:

+ return lduw_be_p(haddr);

+ case MO_LEUW:

+ return lduw_le_p(haddr);

+ case MO_BEUL:

+ return (uint32_t)ldl_be_p(haddr);

+ case MO_LEUL:

+ return (uint32_t)ldl_le_p(haddr);

+ case MO_BEQ:

+ return ldq_be_p(haddr);

+ case MO_LEQ:

+ return ldq_le_p(haddr);

+ default:

+ qemu_build_not_reached();

+ }

+}

+

static inline uint64_t QEMU_ALWAYS_INLINE

load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,

uintptr_t retaddr, MemOp op, bool code_read,

@@ -XXX,XX +XXX,XX @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,

do_aligned_access:

haddr = (void *)((uintptr_t)addr + entry->addend);

- switch (op) {

- case MO_UB:

- res = ldub_p(haddr);

- break;

- case MO_BEUW:

- res = lduw_be_p(haddr);

- break;

- case MO_LEUW:

- res = lduw_le_p(haddr);

- break;

- case MO_BEUL:

- res = (uint32_t)ldl_be_p(haddr);

- break;

- case MO_LEUL:

- res = (uint32_t)ldl_le_p(haddr);

- break;

- case MO_BEQ:

- res = ldq_be_p(haddr);

- break;

- case MO_LEQ:

- res = ldq_le_p(haddr);

- break;

- default:

- qemu_build_not_reached();

- }

-

- return res;

+ return load_memop(haddr, op);

}

/*

@@ -XXX,XX +XXX,XX @@ tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr,

* Store Helpers

*/

+static inline void QEMU_ALWAYS_INLINE

+store_memop(void *haddr, uint64_t val, MemOp op)

+{

+ switch (op) {

+ case MO_UB:

+ stb_p(haddr, val);

+ break;

+ case MO_BEUW:

+ stw_be_p(haddr, val);

+ break;

+ case MO_LEUW:

+ stw_le_p(haddr, val);

+ break;

+ case MO_BEUL:

+ stl_be_p(haddr, val);

+ break;

+ case MO_LEUL:

+ stl_le_p(haddr, val);

+ break;

+ case MO_BEQ:

+ stq_be_p(haddr, val);

+ break;

+ case MO_LEQ:

+ stq_le_p(haddr, val);

+ break;

+ default:

+ qemu_build_not_reached();

+ }

+}

+

static inline void QEMU_ALWAYS_INLINE

store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

TCGMemOpIdx oi, uintptr_t retaddr, MemOp op)

@@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

do_aligned_access:

haddr = (void *)((uintptr_t)addr + entry->addend);

- switch (op) {

- case MO_UB:

- stb_p(haddr, val);

- break;

- case MO_BEUW:

- stw_be_p(haddr, val);

- break;

- case MO_LEUW:

- stw_le_p(haddr, val);

- break;

- case MO_BEUL:

- stl_be_p(haddr, val);

- break;

- case MO_LEUL:

- stl_le_p(haddr, val);

- break;

- case MO_BEQ:

- stq_be_p(haddr, val);

- break;

- case MO_LEQ:

- stq_le_p(haddr, val);

- break;

- default:

- qemu_build_not_reached();

- }

+ store_memop(haddr, val, op);

}

void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,

--

2.17.1

The memory_region_tb_read tracepoint is unreachable, since notdirty

is supposed to apply only to writes. The memory_region_tb_write

tracepoint is mis-named, because notdirty is not only used for TB

invalidation. It is also used for e.g. VGA RAM updates and migration.

Replace memory_region_tb_write with memory_notdirty_write_access,

and place it in memory_notdirty_write_prepare where it can catch

all of the instances. Add memory_notdirty_set_dirty to log when

we no longer intercept writes to a page.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>

Reviewed-by: David Hildenbrand <david@redhat.com>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

exec.c | 3 +++

memory.c | 4 ----

trace-events | 4 ++--

3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/exec.c b/exec.c

index XXXXXXX..XXXXXXX 100644

--- a/exec.c

+++ b/exec.c

@@ -XXX,XX +XXX,XX @@ void memory_notdirty_write_prepare(NotDirtyInfo *ndi,

ndi->size = size;

ndi->pages = NULL;

+ trace_memory_notdirty_write_access(mem_vaddr, ram_addr, size);

+

assert(tcg_enabled());

if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {

ndi->pages = page_collection_lock(ram_addr, ram_addr + size);

@@ -XXX,XX +XXX,XX @@ void memory_notdirty_write_complete(NotDirtyInfo *ndi)

/* we remove the notdirty callback only if the code has been

flushed */

if (!cpu_physical_memory_is_clean(ndi->ram_addr)) {

+ trace_memory_notdirty_set_dirty(ndi->mem_vaddr);

tlb_set_dirty(ndi->cpu, ndi->mem_vaddr);

}

}

diff --git a/memory.c b/memory.c

index XXXXXXX..XXXXXXX 100644

--- a/memory.c

+++ b/memory.c

@@ -XXX,XX +XXX,XX @@ static MemTxResult memory_region_read_accessor(MemoryRegion *mr,

/* Accesses to code which has previously been translated into a TB show

* up in the MMIO path, as accesses to the io_mem_notdirty

* MemoryRegion. */

- trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);

} else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {

hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);

trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);

@@ -XXX,XX +XXX,XX @@ static MemTxResult memory_region_read_with_attrs_accessor(MemoryRegion *mr,

/* Accesses to code which has previously been translated into a TB show

* up in the MMIO path, as accesses to the io_mem_notdirty

* MemoryRegion. */

- trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);

} else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {

hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);

trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);

@@ -XXX,XX +XXX,XX @@ static MemTxResult memory_region_write_accessor(MemoryRegion *mr,

/* Accesses to code which has previously been translated into a TB show

* up in the MMIO path, as accesses to the io_mem_notdirty

* MemoryRegion. */

- trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);

} else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {

hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);

trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);

@@ -XXX,XX +XXX,XX @@ static MemTxResult memory_region_write_with_attrs_accessor(MemoryRegion *mr,

/* Accesses to code which has previously been translated into a TB show

* up in the MMIO path, as accesses to the io_mem_notdirty

* MemoryRegion. */

- trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);

} else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {

hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);

trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);

diff --git a/trace-events b/trace-events

index XXXXXXX..XXXXXXX 100644

--- a/trace-events

+++ b/trace-events

@@ -XXX,XX +XXX,XX @@ dma_map_wait(void *dbs) "dbs=%p"

find_ram_offset(uint64_t size, uint64_t offset) "size: 0x%" PRIx64 " @ 0x%" PRIx64

find_ram_offset_loop(uint64_t size, uint64_t candidate, uint64_t offset, uint64_t next, uint64_t mingap) "trying size: 0x%" PRIx64 " @ 0x%" PRIx64 ", offset: 0x%" PRIx64" next: 0x%" PRIx64 " mingap: 0x%" PRIx64

ram_block_discard_range(const char *rbname, void *hva, size_t length, bool need_madvise, bool need_fallocate, int ret) "%s@%p + 0x%zx: madvise: %d fallocate: %d ret: %d"

+memory_notdirty_write_access(uint64_t vaddr, uint64_t ram_addr, unsigned size) "0x%" PRIx64 " ram_addr 0x%" PRIx64 " size %u"

+memory_notdirty_set_dirty(uint64_t vaddr) "0x%" PRIx64

# memory.c

memory_region_ops_read(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

memory_region_ops_write(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

memory_region_subpage_read(int cpu_index, void *mr, uint64_t offset, uint64_t value, unsigned size) "cpu %d mr %p offset 0x%"PRIx64" value 0x%"PRIx64" size %u"

memory_region_subpage_write(int cpu_index, void *mr, uint64_t offset, uint64_t value, unsigned size) "cpu %d mr %p offset 0x%"PRIx64" value 0x%"PRIx64" size %u"

-memory_region_tb_read(int cpu_index, uint64_t addr, uint64_t value, unsigned size) "cpu %d addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

-memory_region_tb_write(int cpu_index, uint64_t addr, uint64_t value, unsigned size) "cpu %d addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

memory_region_ram_device_read(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

memory_region_ram_device_write(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

flatview_new(void *view, void *root) "%p (root %p)"

--

2.17.1

It does not require going through the whole I/O path

in order to discard a write.

Reviewed-by: David Hildenbrand <david@redhat.com>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

include/exec/cpu-all.h | 5 ++++-

include/exec/cpu-common.h | 1 -

accel/tcg/cputlb.c | 36 ++++++++++++++++++++--------------

exec.c | 41 +--------------------------------------

4 files changed, 26 insertions(+), 57 deletions(-)

diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h

index XXXXXXX..XXXXXXX 100644

--- a/include/exec/cpu-all.h

+++ b/include/exec/cpu-all.h

@@ -XXX,XX +XXX,XX @@ CPUArchState *cpu_copy(CPUArchState *env);

#define TLB_WATCHPOINT (1 << (TARGET_PAGE_BITS_MIN - 4))

/* Set if TLB entry requires byte swap. */

#define TLB_BSWAP (1 << (TARGET_PAGE_BITS_MIN - 5))

+/* Set if TLB entry writes ignored. */

+#define TLB_DISCARD_WRITE (1 << (TARGET_PAGE_BITS_MIN - 6))

/* Use this mask to check interception with an alignment mask

* in a TCG backend.

*/

#define TLB_FLAGS_MASK \

- (TLB_INVALID_MASK | TLB_NOTDIRTY | TLB_MMIO | TLB_WATCHPOINT | TLB_BSWAP)

+ (TLB_INVALID_MASK | TLB_NOTDIRTY | TLB_MMIO \

+ | TLB_WATCHPOINT | TLB_BSWAP | TLB_DISCARD_WRITE)

/**

* tlb_hit_page: return true if page aligned @addr is a hit against the

diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h

index XXXXXXX..XXXXXXX 100644

--- a/include/exec/cpu-common.h

+++ b/include/exec/cpu-common.h

@@ -XXX,XX +XXX,XX @@ void qemu_flush_coalesced_mmio_buffer(void);

void cpu_flush_icache_range(hwaddr start, hwaddr len);

-extern struct MemoryRegion io_mem_rom;

extern struct MemoryRegion io_mem_notdirty;

typedef int (RAMBlockIterFunc)(RAMBlock *rb, void *opaque);

diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c

index XXXXXXX..XXXXXXX 100644

--- a/accel/tcg/cputlb.c

+++ b/accel/tcg/cputlb.c

@@ -XXX,XX +XXX,XX @@ static void tlb_reset_dirty_range_locked(CPUTLBEntry *tlb_entry,

{

uintptr_t addr = tlb_entry->addr_write;

- if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) {

+ if ((addr & (TLB_INVALID_MASK | TLB_MMIO |

+ TLB_DISCARD_WRITE | TLB_NOTDIRTY)) == 0) {

addr &= TARGET_PAGE_MASK;

addr += tlb_entry->addend;

if ((addr - start) < length) {

@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,

address |= TLB_MMIO;

addend = 0;

} else {

- /* TLB_MMIO for rom/romd handled below */

addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat;

}

@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,

tn.addr_write = -1;

if (prot & PAGE_WRITE) {

- if ((memory_region_is_ram(section->mr) && section->readonly)

- || memory_region_is_romd(section->mr)) {

- /* Write access calls the I/O callback. */

- tn.addr_write = address | TLB_MMIO;

- } else if (memory_region_is_ram(section->mr)

- && cpu_physical_memory_is_clean(

- memory_region_get_ram_addr(section->mr) + xlat)) {

- tn.addr_write = address | TLB_NOTDIRTY;

- } else {

- tn.addr_write = address;

+ tn.addr_write = address;

+ if (memory_region_is_romd(section->mr)) {

+ /* Use the MMIO path so that the device can switch states. */

+ tn.addr_write |= TLB_MMIO;

+ } else if (memory_region_is_ram(section->mr)) {

+ if (section->readonly) {

+ tn.addr_write |= TLB_DISCARD_WRITE;

+ } else if (cpu_physical_memory_is_clean(

+ memory_region_get_ram_addr(section->mr) + xlat)) {

+ tn.addr_write |= TLB_NOTDIRTY;

+ }

}

if (prot & PAGE_WRITE_INV) {

tn.addr_write |= TLB_INVALID_MASK;

@@ -XXX,XX +XXX,XX @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry,

mr = section->mr;

mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;

cpu->mem_io_pc = retaddr;

- if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {

+ if (mr != &io_mem_notdirty && !cpu->can_do_io) {

cpu_io_recompile(cpu, retaddr);

}

@@ -XXX,XX +XXX,XX @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry,

section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs);

mr = section->mr;

mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;

- if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {

+ if (mr != &io_mem_notdirty && !cpu->can_do_io) {

cpu_io_recompile(cpu, retaddr);

}

cpu->mem_io_vaddr = addr;

@@ -XXX,XX +XXX,XX @@ void *probe_access(CPUArchState *env, target_ulong addr, int size,

}

/* Reject I/O access, or other required slow-path. */

- if (tlb_addr & (TLB_NOTDIRTY | TLB_MMIO | TLB_BSWAP)) {

+ if (tlb_addr & (TLB_NOTDIRTY | TLB_MMIO | TLB_BSWAP | TLB_DISCARD_WRITE)) {

return NULL;

}

@@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

return;

}

+ /* Ignore writes to ROM. */

+ if (unlikely(tlb_addr & TLB_DISCARD_WRITE)) {

+ return;

+ }

+

haddr = (void *)((uintptr_t)addr + entry->addend);

/*

diff --git a/exec.c b/exec.c

index XXXXXXX..XXXXXXX 100644

--- a/exec.c

+++ b/exec.c

@@ -XXX,XX +XXX,XX @@ static MemoryRegion *system_io;

AddressSpace address_space_io;

AddressSpace address_space_memory;

-MemoryRegion io_mem_rom, io_mem_notdirty;

+MemoryRegion io_mem_notdirty;

static MemoryRegion io_mem_unassigned;

#endif

@@ -XXX,XX +XXX,XX @@ typedef struct subpage_t {

#define PHYS_SECTION_UNASSIGNED 0

#define PHYS_SECTION_NOTDIRTY 1

-#define PHYS_SECTION_ROM 2

static void io_mem_init(void);

static void memory_map_init(void);

@@ -XXX,XX +XXX,XX @@ hwaddr memory_region_section_get_iotlb(CPUState *cpu,

iotlb = memory_region_get_ram_addr(section->mr) + xlat;

if (!section->readonly) {

iotlb |= PHYS_SECTION_NOTDIRTY;

- } else {

- iotlb |= PHYS_SECTION_ROM;

}

} else {

AddressSpaceDispatch *d;

@@ -XXX,XX +XXX,XX @@ static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr)

return phys_section_add(map, &section);

}

-static void readonly_mem_write(void *opaque, hwaddr addr,

- uint64_t val, unsigned size)

-{

- /* Ignore any write to ROM. */

-}

-

-static bool readonly_mem_accepts(void *opaque, hwaddr addr,

- unsigned size, bool is_write,

- MemTxAttrs attrs)

-{

- return is_write;

-}

-

-/* This will only be used for writes, because reads are special cased

- * to directly access the underlying host ram.

- */

-static const MemoryRegionOps readonly_mem_ops = {

- .write = readonly_mem_write,

- .valid.accepts = readonly_mem_accepts,

- .endianness = DEVICE_NATIVE_ENDIAN,

- .valid = {

- .min_access_size = 1,

- .max_access_size = 8,

- .unaligned = false,

- },

- .impl = {

- .min_access_size = 1,

- .max_access_size = 8,

- .unaligned = false,

- },

-};

-

MemoryRegionSection *iotlb_to_section(CPUState *cpu,

hwaddr index, MemTxAttrs attrs)

{

@@ -XXX,XX +XXX,XX @@ MemoryRegionSection *iotlb_to_section(CPUState *cpu,

static void io_mem_init(void)

{

- memory_region_init_io(&io_mem_rom, NULL, &readonly_mem_ops,

- NULL, NULL, UINT64_MAX);

memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,

NULL, UINT64_MAX);

@@ -XXX,XX +XXX,XX @@ AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv)

assert(n == PHYS_SECTION_UNASSIGNED);

n = dummy_section(&d->map, fv, &io_mem_notdirty);

assert(n == PHYS_SECTION_NOTDIRTY);

- n = dummy_section(&d->map, fv, &io_mem_rom);

- assert(n == PHYS_SECTION_ROM);

d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };

--

2.17.1

Pages that we want to track for NOTDIRTY are RAM. We do not

really need to go through the I/O path to handle them.

Acked-by: David Hildenbrand <david@redhat.com>

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

include/exec/cpu-common.h | 2 --

accel/tcg/cputlb.c | 26 +++++++++++++++++---

exec.c | 50 ---------------------------------------

memory.c | 16 -------------

4 files changed, 23 insertions(+), 71 deletions(-)

diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h

index XXXXXXX..XXXXXXX 100644

--- a/include/exec/cpu-common.h

+++ b/include/exec/cpu-common.h

@@ -XXX,XX +XXX,XX @@ void qemu_flush_coalesced_mmio_buffer(void);

void cpu_flush_icache_range(hwaddr start, hwaddr len);

-extern struct MemoryRegion io_mem_notdirty;

-

typedef int (RAMBlockIterFunc)(RAMBlock *rb, void *opaque);

int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque);

diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c

index XXXXXXX..XXXXXXX 100644

--- a/accel/tcg/cputlb.c

+++ b/accel/tcg/cputlb.c

@@ -XXX,XX +XXX,XX @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry,

mr = section->mr;

mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;

cpu->mem_io_pc = retaddr;

- if (mr != &io_mem_notdirty && !cpu->can_do_io) {

+ if (!cpu->can_do_io) {

cpu_io_recompile(cpu, retaddr);

}

@@ -XXX,XX +XXX,XX @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry,

section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs);

mr = section->mr;

mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;

- if (mr != &io_mem_notdirty && !cpu->can_do_io) {

+ if (!cpu->can_do_io) {

cpu_io_recompile(cpu, retaddr);

}

cpu->mem_io_vaddr = addr;

@@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

need_swap = size > 1 && (tlb_addr & TLB_BSWAP);

/* Handle I/O access. */

- if (likely(tlb_addr & (TLB_MMIO | TLB_NOTDIRTY))) {

+ if (tlb_addr & TLB_MMIO) {

io_writex(env, iotlbentry, mmu_idx, val, addr, retaddr,

op ^ (need_swap * MO_BSWAP));

return;

@@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

haddr = (void *)((uintptr_t)addr + entry->addend);

+ /* Handle clean RAM pages. */

+ if (tlb_addr & TLB_NOTDIRTY) {

+ NotDirtyInfo ndi;

+

+ /* We require mem_io_pc in tb_invalidate_phys_page_range. */

+ env_cpu(env)->mem_io_pc = retaddr;

+

+ memory_notdirty_write_prepare(&ndi, env_cpu(env), addr,

+ addr + iotlbentry->addr, size);

+

+ if (unlikely(need_swap)) {

+ store_memop(haddr, val, op ^ MO_BSWAP);

+ } else {

+ store_memop(haddr, val, op);

+ }

+

+ memory_notdirty_write_complete(&ndi);

+ return;

+ }

+

/*

* Keep these two store_memop separate to ensure that the compiler

* is able to fold the entire function to a single instruction.

diff --git a/exec.c b/exec.c

index XXXXXXX..XXXXXXX 100644

--- a/exec.c

+++ b/exec.c

@@ -XXX,XX +XXX,XX @@ static MemoryRegion *system_io;

AddressSpace address_space_io;

AddressSpace address_space_memory;

-MemoryRegion io_mem_notdirty;

static MemoryRegion io_mem_unassigned;

#endif

@@ -XXX,XX +XXX,XX @@ typedef struct subpage_t {

} subpage_t;

#define PHYS_SECTION_UNASSIGNED 0

-#define PHYS_SECTION_NOTDIRTY 1

static void io_mem_init(void);

static void memory_map_init(void);

@@ -XXX,XX +XXX,XX @@ hwaddr memory_region_section_get_iotlb(CPUState *cpu,

if (memory_region_is_ram(section->mr)) {

/* Normal RAM. */

iotlb = memory_region_get_ram_addr(section->mr) + xlat;

- if (!section->readonly) {

- iotlb |= PHYS_SECTION_NOTDIRTY;

- }

} else {

AddressSpaceDispatch *d;

@@ -XXX,XX +XXX,XX @@ void memory_notdirty_write_complete(NotDirtyInfo *ndi)

}

}

-/* Called within RCU critical section. */

-static void notdirty_mem_write(void *opaque, hwaddr ram_addr,

- uint64_t val, unsigned size)

-{

- NotDirtyInfo ndi;

-

- memory_notdirty_write_prepare(&ndi, current_cpu, current_cpu->mem_io_vaddr,

- ram_addr, size);

-

- stn_p(qemu_map_ram_ptr(NULL, ram_addr), size, val);

- memory_notdirty_write_complete(&ndi);

-}

-

-static bool notdirty_mem_accepts(void *opaque, hwaddr addr,

- unsigned size, bool is_write,

- MemTxAttrs attrs)

-{

- return is_write;

-}

-

-static const MemoryRegionOps notdirty_mem_ops = {

- .write = notdirty_mem_write,

- .valid.accepts = notdirty_mem_accepts,

- .endianness = DEVICE_NATIVE_ENDIAN,

- .valid = {

- .min_access_size = 1,

- .max_access_size = 8,

- .unaligned = false,

- },

- .impl = {

- .min_access_size = 1,

- .max_access_size = 8,

- .unaligned = false,

- },

-};

-

/* Generate a debug exception if a watchpoint has been hit. */

void cpu_check_watchpoint(CPUState *cpu, vaddr addr, vaddr len,

MemTxAttrs attrs, int flags, uintptr_t ra)

@@ -XXX,XX +XXX,XX @@ static void io_mem_init(void)

{

memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,

NULL, UINT64_MAX);

-

- /* io_mem_notdirty calls tb_invalidate_phys_page_fast,

- * which can be called without the iothread mutex.

- */

- memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,

- NULL, UINT64_MAX);

- memory_region_clear_global_locking(&io_mem_notdirty);

}

AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv)

@@ -XXX,XX +XXX,XX @@ AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv)

n = dummy_section(&d->map, fv, &io_mem_unassigned);

assert(n == PHYS_SECTION_UNASSIGNED);

- n = dummy_section(&d->map, fv, &io_mem_notdirty);

- assert(n == PHYS_SECTION_NOTDIRTY);

d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };

diff --git a/memory.c b/memory.c

index XXXXXXX..XXXXXXX 100644

--- a/memory.c

+++ b/memory.c

@@ -XXX,XX +XXX,XX @@ static MemTxResult memory_region_read_accessor(MemoryRegion *mr,

tmp = mr->ops->read(mr->opaque, addr, size);

if (mr->subpage) {

trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);

- } else if (mr == &io_mem_notdirty) {

- /* Accesses to code which has previously been translated into a TB show

- * up in the MMIO path, as accesses to the io_mem_notdirty

- * MemoryRegion. */

} else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {

hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);

trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);

@@ -XXX,XX +XXX,XX @@ static MemTxResult memory_region_read_with_attrs_accessor(MemoryRegion *mr,

r = mr->ops->read_with_attrs(mr->opaque, addr, &tmp, size, attrs);

if (mr->subpage) {

trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);

- } else if (mr == &io_mem_notdirty) {

- /* Accesses to code which has previously been translated into a TB show

- * up in the MMIO path, as accesses to the io_mem_notdirty

- * MemoryRegion. */

} else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {

hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);

trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);

@@ -XXX,XX +XXX,XX @@ static MemTxResult memory_region_write_accessor(MemoryRegion *mr,

if (mr->subpage) {

trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);

- } else if (mr == &io_mem_notdirty) {

- /* Accesses to code which has previously been translated into a TB show

- * up in the MMIO path, as accesses to the io_mem_notdirty

- * MemoryRegion. */

} else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {

hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);

trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);

@@ -XXX,XX +XXX,XX @@ static MemTxResult memory_region_write_with_attrs_accessor(MemoryRegion *mr,

if (mr->subpage) {

trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);

- } else if (mr == &io_mem_notdirty) {

- /* Accesses to code which has previously been translated into a TB show

- * up in the MMIO path, as accesses to the io_mem_notdirty

- * MemoryRegion. */

} else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {

hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);

trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);

--

2.17.1

There is only one caller, tlb_set_page_with_attrs. We cannot

inline the entire function because the AddressSpaceDispatch

structure is private to exec.c, and cannot easily be moved to

include/exec/memory-internal.h.

Compute is_ram and is_romd once within tlb_set_page_with_attrs.

Fold the number of tests against these predicates. Compute

cpu_physical_memory_is_clean outside of the tlb lock region.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

include/exec/exec-all.h | 6 +---

accel/tcg/cputlb.c | 68 ++++++++++++++++++++++++++---------------

exec.c | 22 ++-----------

3 files changed, 47 insertions(+), 49 deletions(-)

diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h

index XXXXXXX..XXXXXXX 100644

--- a/include/exec/exec-all.h

+++ b/include/exec/exec-all.h

@@ -XXX,XX +XXX,XX @@ address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,

hwaddr *xlat, hwaddr *plen,

MemTxAttrs attrs, int *prot);

hwaddr memory_region_section_get_iotlb(CPUState *cpu,

- MemoryRegionSection *section,

- target_ulong vaddr,

- hwaddr paddr, hwaddr xlat,

- int prot,

- target_ulong *address);

+ MemoryRegionSection *section);

#endif

/* vl.c */

diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c

index XXXXXXX..XXXXXXX 100644

--- a/accel/tcg/cputlb.c

+++ b/accel/tcg/cputlb.c

@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,

MemoryRegionSection *section;

unsigned int index;

target_ulong address;

- target_ulong code_address;

+ target_ulong write_address;

uintptr_t addend;

CPUTLBEntry *te, tn;

hwaddr iotlb, xlat, sz, paddr_page;

target_ulong vaddr_page;

int asidx = cpu_asidx_from_attrs(cpu, attrs);

int wp_flags;

+ bool is_ram, is_romd;

assert_cpu_is_self(cpu);

@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,

if (attrs.byte_swap) {

address |= TLB_BSWAP;

}

- if (!memory_region_is_ram(section->mr) &&

- !memory_region_is_romd(section->mr)) {

- /* IO memory case */

- address |= TLB_MMIO;

- addend = 0;

- } else {

+

+ is_ram = memory_region_is_ram(section->mr);

+ is_romd = memory_region_is_romd(section->mr);

+

+ if (is_ram || is_romd) {

+ /* RAM and ROMD both have associated host memory. */

addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat;

+ } else {

+ /* I/O does not; force the host address to NULL. */

+ addend = 0;

+ }

+

+ write_address = address;

+ if (is_ram) {

+ iotlb = memory_region_get_ram_addr(section->mr) + xlat;

+ /*

+ * Computing is_clean is expensive; avoid all that unless

+ * the page is actually writable.

+ */

+ if (prot & PAGE_WRITE) {

+ if (section->readonly) {

+ write_address |= TLB_DISCARD_WRITE;

+ } else if (cpu_physical_memory_is_clean(iotlb)) {

+ write_address |= TLB_NOTDIRTY;

+ }

+ }

+ } else {

+ /* I/O or ROMD */

+ iotlb = memory_region_section_get_iotlb(cpu, section) + xlat;

+ /*

+ * Writes to romd devices must go through MMIO to enable write.

+ * Reads to romd devices go through the ram_ptr found above,

+ * but of course reads to I/O must go through MMIO.