These bits do not need to vary with the actual page size

used by the guest.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Reviewed-by: David Hildenbrand <david@redhat.com>

Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

include/exec/cpu-all.h | 16 ++++++++++------

1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h

index XXXXXXX..XXXXXXX 100644

--- a/include/exec/cpu-all.h

+++ b/include/exec/cpu-all.h

@@ -XXX,XX +XXX,XX @@ CPUArchState *cpu_copy(CPUArchState *env);

#if !defined(CONFIG_USER_ONLY)

-/* Flags stored in the low bits of the TLB virtual address. These are

- * defined so that fast path ram access is all zeros.

+/*

+ * Flags stored in the low bits of the TLB virtual address.

+ * These are defined so that fast path ram access is all zeros.

* The flags all must be between TARGET_PAGE_BITS and

* maximum address alignment bit.

+ *

+ * Use TARGET_PAGE_BITS_MIN so that these bits are constant

+ * when TARGET_PAGE_BITS_VARY is in effect.

*/

/* Zero if TLB entry is valid. */

-#define TLB_INVALID_MASK (1 << (TARGET_PAGE_BITS - 1))

+#define TLB_INVALID_MASK (1 << (TARGET_PAGE_BITS_MIN - 1))

/* Set if TLB entry references a clean RAM page. The iotlb entry will

contain the page physical address. */

-#define TLB_NOTDIRTY (1 << (TARGET_PAGE_BITS - 2))

+#define TLB_NOTDIRTY (1 << (TARGET_PAGE_BITS_MIN - 2))

/* Set if TLB entry is an IO callback. */

-#define TLB_MMIO (1 << (TARGET_PAGE_BITS - 3))

+#define TLB_MMIO (1 << (TARGET_PAGE_BITS_MIN - 3))

/* Set if TLB entry contains a watchpoint. */

-#define TLB_WATCHPOINT (1 << (TARGET_PAGE_BITS - 4))

+#define TLB_WATCHPOINT (1 << (TARGET_PAGE_BITS_MIN - 4))

/* Use this mask to check interception with an alignment mask

* in a TCG backend.

--

2.17.1

This forced inlining can result in missing symbols,

which makes a debugging build harder to follow.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>

Reviewed-by: David Hildenbrand <david@redhat.com>

Reported-by: Peter Maydell <peter.maydell@linaro.org>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

include/qemu/compiler.h | 11 +++++++++++

accel/tcg/cputlb.c | 4 ++--

2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/include/qemu/compiler.h b/include/qemu/compiler.h

index XXXXXXX..XXXXXXX 100644

--- a/include/qemu/compiler.h

+++ b/include/qemu/compiler.h

@@ -XXX,XX +XXX,XX @@

# define QEMU_NONSTRING

#endif

+/*

+ * Forced inlining may be desired to encourage constant propagation

+ * of function parameters. However, it can also make debugging harder,

+ * so disable it for a non-optimizing build.

+ */

+#if defined(__OPTIMIZE__)

+#define QEMU_ALWAYS_INLINE __attribute__((always_inline))

+#else

+#define QEMU_ALWAYS_INLINE

+#endif

+

/* Implement C11 _Generic via GCC builtins. Example:

*

* QEMU_GENERIC(x, (float, sinf), (long double, sinl), sin) (x)

diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c

index XXXXXXX..XXXXXXX 100644

--- a/accel/tcg/cputlb.c

+++ b/accel/tcg/cputlb.c

@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,

typedef uint64_t FullLoadHelper(CPUArchState *env, target_ulong addr,

TCGMemOpIdx oi, uintptr_t retaddr);

-static inline uint64_t __attribute__((always_inline))

+static inline uint64_t QEMU_ALWAYS_INLINE

load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,

uintptr_t retaddr, MemOp op, bool code_read,

FullLoadHelper *full_load)

@@ -XXX,XX +XXX,XX @@ tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr,

* Store Helpers

*/

-static inline void __attribute__((always_inline))

+static inline void QEMU_ALWAYS_INLINE

store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

TCGMemOpIdx oi, uintptr_t retaddr, MemOp op)

{

--

2.17.1

Use this as a compile-time assert that a particular

code path is not reachable.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

include/qemu/compiler.h | 15 +++++++++++++++

1 file changed, 15 insertions(+)

diff --git a/include/qemu/compiler.h b/include/qemu/compiler.h

index XXXXXXX..XXXXXXX 100644

--- a/include/qemu/compiler.h

+++ b/include/qemu/compiler.h

@@ -XXX,XX +XXX,XX @@

#define QEMU_GENERIC9(x, a0, ...) QEMU_GENERIC_IF(x, a0, QEMU_GENERIC8(x, __VA_ARGS__))

#define QEMU_GENERIC10(x, a0, ...) QEMU_GENERIC_IF(x, a0, QEMU_GENERIC9(x, __VA_ARGS__))

+/**

+ * qemu_build_not_reached()

+ *

+ * The compiler, during optimization, is expected to prove that a call

+ * to this function cannot be reached and remove it. If the compiler

+ * supports QEMU_ERROR, this will be reported at compile time; otherwise

+ * this will be reported at link time due to the missing symbol.

+ */

+#ifdef __OPTIMIZE__

+extern void QEMU_NORETURN QEMU_ERROR("code path is reachable")

+ qemu_build_not_reached(void);

+#else

+#define qemu_build_not_reached() g_assert_not_reached()

+#endif

+

#endif /* COMPILER_H */

--

2.17.1

Handle bswap on ram directly in load/store_helper. This fixes a

bug with the previous implementation in that one cannot use the

I/O path for RAM.

Fixes: a26fc6f5152b47f1

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Reviewed-by: David Hildenbrand <david@redhat.com>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

include/exec/cpu-all.h | 4 ++-

accel/tcg/cputlb.c | 72 +++++++++++++++++++++++++-----------------

2 files changed, 46 insertions(+), 30 deletions(-)

diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h

index XXXXXXX..XXXXXXX 100644

--- a/include/exec/cpu-all.h

+++ b/include/exec/cpu-all.h

@@ -XXX,XX +XXX,XX @@ CPUArchState *cpu_copy(CPUArchState *env);

#define TLB_MMIO (1 << (TARGET_PAGE_BITS_MIN - 3))

/* Set if TLB entry contains a watchpoint. */

#define TLB_WATCHPOINT (1 << (TARGET_PAGE_BITS_MIN - 4))

+/* Set if TLB entry requires byte swap. */

+#define TLB_BSWAP (1 << (TARGET_PAGE_BITS_MIN - 5))

/* Use this mask to check interception with an alignment mask

* in a TCG backend.

*/

#define TLB_FLAGS_MASK \

- (TLB_INVALID_MASK | TLB_NOTDIRTY | TLB_MMIO | TLB_WATCHPOINT)

+ (TLB_INVALID_MASK | TLB_NOTDIRTY | TLB_MMIO | TLB_WATCHPOINT | TLB_BSWAP)

/**

* tlb_hit_page: return true if page aligned @addr is a hit against the

diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c

index XXXXXXX..XXXXXXX 100644

--- a/accel/tcg/cputlb.c

+++ b/accel/tcg/cputlb.c

@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,

address |= TLB_INVALID_MASK;

}

if (attrs.byte_swap) {

- /* Force the access through the I/O slow path. */

- address |= TLB_MMIO;

+ address |= TLB_BSWAP;

}

if (!memory_region_is_ram(section->mr) &&

!memory_region_is_romd(section->mr)) {

@@ -XXX,XX +XXX,XX @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry,

bool locked = false;

MemTxResult r;

- if (iotlbentry->attrs.byte_swap) {

- op ^= MO_BSWAP;

- }

-

section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs);

mr = section->mr;

mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;

@@ -XXX,XX +XXX,XX @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry,

bool locked = false;

MemTxResult r;

- if (iotlbentry->attrs.byte_swap) {

- op ^= MO_BSWAP;

- }

-

section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs);

mr = section->mr;

mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;

@@ -XXX,XX +XXX,XX @@ void *probe_access(CPUArchState *env, target_ulong addr, int size,

wp_access, retaddr);

}

- if (tlb_addr & (TLB_NOTDIRTY | TLB_MMIO)) {

- /* I/O access */

+ /* Reject I/O access, or other required slow-path. */

+ if (tlb_addr & (TLB_NOTDIRTY | TLB_MMIO | TLB_BSWAP)) {

return NULL;

}

@@ -XXX,XX +XXX,XX @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,

/* Handle anything that isn't just a straight memory access. */

if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {

CPUIOTLBEntry *iotlbentry;

+ bool need_swap;

/* For anything that is unaligned, recurse through full_load. */

if ((addr & (size - 1)) != 0) {

@@ -XXX,XX +XXX,XX @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,

/* On watchpoint hit, this will longjmp out. */

cpu_check_watchpoint(env_cpu(env), addr, size,

iotlbentry->attrs, BP_MEM_READ, retaddr);

-

- /* The backing page may or may not require I/O. */

- tlb_addr &= ~TLB_WATCHPOINT;

- if ((tlb_addr & ~TARGET_PAGE_MASK) == 0) {

- goto do_aligned_access;

- }

}

+ need_swap = size > 1 && (tlb_addr & TLB_BSWAP);

+

/* Handle I/O access. */

- return io_readx(env, iotlbentry, mmu_idx, addr,

- retaddr, access_type, op);

+ if (likely(tlb_addr & TLB_MMIO)) {

+ return io_readx(env, iotlbentry, mmu_idx, addr, retaddr,

+ access_type, op ^ (need_swap * MO_BSWAP));

+ }

+

+ haddr = (void *)((uintptr_t)addr + entry->addend);

+

+ /*

+ * Keep these two load_memop separate to ensure that the compiler

+ * is able to fold the entire function to a single instruction.

+ * There is a build-time assert inside to remind you of this. ;-)

+ */

+ if (unlikely(need_swap)) {

+ return load_memop(haddr, op ^ MO_BSWAP);

+ }

+ return load_memop(haddr, op);

}

/* Handle slow unaligned access (it spans two pages or IO). */

@@ -XXX,XX +XXX,XX @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,

return res & MAKE_64BIT_MASK(0, size * 8);

}

- do_aligned_access:

haddr = (void *)((uintptr_t)addr + entry->addend);

return load_memop(haddr, op);

}

@@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

/* Handle anything that isn't just a straight memory access. */

if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {

CPUIOTLBEntry *iotlbentry;

+ bool need_swap;

/* For anything that is unaligned, recurse through byte stores. */

if ((addr & (size - 1)) != 0) {

@@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

/* On watchpoint hit, this will longjmp out. */

cpu_check_watchpoint(env_cpu(env), addr, size,

iotlbentry->attrs, BP_MEM_WRITE, retaddr);

-

- /* The backing page may or may not require I/O. */

- tlb_addr &= ~TLB_WATCHPOINT;

- if ((tlb_addr & ~TARGET_PAGE_MASK) == 0) {

- goto do_aligned_access;

- }

}

+ need_swap = size > 1 && (tlb_addr & TLB_BSWAP);

+

/* Handle I/O access. */

- io_writex(env, iotlbentry, mmu_idx, val, addr, retaddr, op);

+ if (likely(tlb_addr & (TLB_MMIO | TLB_NOTDIRTY))) {

+ io_writex(env, iotlbentry, mmu_idx, val, addr, retaddr,

+ op ^ (need_swap * MO_BSWAP));

+ return;

+ }

+

+ haddr = (void *)((uintptr_t)addr + entry->addend);

+

+ /*

+ * Keep these two store_memop separate to ensure that the compiler

+ * is able to fold the entire function to a single instruction.

+ * There is a build-time assert inside to remind you of this. ;-)

+ */

+ if (unlikely(need_swap)) {

+ store_memop(haddr, val, op ^ MO_BSWAP);

+ } else {

+ store_memop(haddr, val, op);

+ }

return;

}

@@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

return;

}

- do_aligned_access:

haddr = (void *)((uintptr_t)addr + entry->addend);

store_memop(haddr, val, op);

}

--

2.17.1

The memory_region_tb_read tracepoint is unreachable, since notdirty

is supposed to apply only to writes. The memory_region_tb_write

tracepoint is mis-named, because notdirty is not only used for TB

invalidation. It is also used for e.g. VGA RAM updates and migration.

Replace memory_region_tb_write with memory_notdirty_write_access,

and place it in memory_notdirty_write_prepare where it can catch

all of the instances. Add memory_notdirty_set_dirty to log when

we no longer intercept writes to a page.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>

Reviewed-by: David Hildenbrand <david@redhat.com>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

exec.c | 3 +++

memory.c | 4 ----

trace-events | 4 ++--

3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/exec.c b/exec.c

index XXXXXXX..XXXXXXX 100644

--- a/exec.c

+++ b/exec.c

@@ -XXX,XX +XXX,XX @@ void memory_notdirty_write_prepare(NotDirtyInfo *ndi,

ndi->size = size;

ndi->pages = NULL;

+ trace_memory_notdirty_write_access(mem_vaddr, ram_addr, size);

+

assert(tcg_enabled());

if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {

ndi->pages = page_collection_lock(ram_addr, ram_addr + size);

@@ -XXX,XX +XXX,XX @@ void memory_notdirty_write_complete(NotDirtyInfo *ndi)

/* we remove the notdirty callback only if the code has been

flushed */

if (!cpu_physical_memory_is_clean(ndi->ram_addr)) {

+ trace_memory_notdirty_set_dirty(ndi->mem_vaddr);

tlb_set_dirty(ndi->cpu, ndi->mem_vaddr);

}

}

diff --git a/memory.c b/memory.c

index XXXXXXX..XXXXXXX 100644

--- a/memory.c

+++ b/memory.c

@@ -XXX,XX +XXX,XX @@ static MemTxResult memory_region_read_accessor(MemoryRegion *mr,

/* Accesses to code which has previously been translated into a TB show

* up in the MMIO path, as accesses to the io_mem_notdirty

* MemoryRegion. */

- trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);

} else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {

hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);

trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);

@@ -XXX,XX +XXX,XX @@ static MemTxResult memory_region_read_with_attrs_accessor(MemoryRegion *mr,

/* Accesses to code which has previously been translated into a TB show

* up in the MMIO path, as accesses to the io_mem_notdirty

* MemoryRegion. */

- trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);

} else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {

hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);

trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);

@@ -XXX,XX +XXX,XX @@ static MemTxResult memory_region_write_accessor(MemoryRegion *mr,

/* Accesses to code which has previously been translated into a TB show

* up in the MMIO path, as accesses to the io_mem_notdirty

* MemoryRegion. */

- trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);

} else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {

hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);

trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);

@@ -XXX,XX +XXX,XX @@ static MemTxResult memory_region_write_with_attrs_accessor(MemoryRegion *mr,

/* Accesses to code which has previously been translated into a TB show

* up in the MMIO path, as accesses to the io_mem_notdirty

* MemoryRegion. */

- trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);

} else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {

hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);

trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);

diff --git a/trace-events b/trace-events

index XXXXXXX..XXXXXXX 100644

--- a/trace-events

+++ b/trace-events

@@ -XXX,XX +XXX,XX @@ dma_map_wait(void *dbs) "dbs=%p"

find_ram_offset(uint64_t size, uint64_t offset) "size: 0x%" PRIx64 " @ 0x%" PRIx64

find_ram_offset_loop(uint64_t size, uint64_t candidate, uint64_t offset, uint64_t next, uint64_t mingap) "trying size: 0x%" PRIx64 " @ 0x%" PRIx64 ", offset: 0x%" PRIx64" next: 0x%" PRIx64 " mingap: 0x%" PRIx64

ram_block_discard_range(const char *rbname, void *hva, size_t length, bool need_madvise, bool need_fallocate, int ret) "%s@%p + 0x%zx: madvise: %d fallocate: %d ret: %d"

+memory_notdirty_write_access(uint64_t vaddr, uint64_t ram_addr, unsigned size) "0x%" PRIx64 " ram_addr 0x%" PRIx64 " size %u"

+memory_notdirty_set_dirty(uint64_t vaddr) "0x%" PRIx64

# memory.c

memory_region_ops_read(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

memory_region_ops_write(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

memory_region_subpage_read(int cpu_index, void *mr, uint64_t offset, uint64_t value, unsigned size) "cpu %d mr %p offset 0x%"PRIx64" value 0x%"PRIx64" size %u"

memory_region_subpage_write(int cpu_index, void *mr, uint64_t offset, uint64_t value, unsigned size) "cpu %d mr %p offset 0x%"PRIx64" value 0x%"PRIx64" size %u"

-memory_region_tb_read(int cpu_index, uint64_t addr, uint64_t value, unsigned size) "cpu %d addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

-memory_region_tb_write(int cpu_index, uint64_t addr, uint64_t value, unsigned size) "cpu %d addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

memory_region_ram_device_read(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

memory_region_ram_device_write(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

flatview_new(void *view, void *root) "%p (root %p)"

--

2.17.1

It does not require going through the whole I/O path

in order to discard a write.

Reviewed-by: David Hildenbrand <david@redhat.com>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

include/exec/cpu-all.h | 5 ++++-

include/exec/cpu-common.h | 1 -

accel/tcg/cputlb.c | 36 ++++++++++++++++++++--------------

exec.c | 41 +--------------------------------------

4 files changed, 26 insertions(+), 57 deletions(-)

diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h

index XXXXXXX..XXXXXXX 100644

--- a/include/exec/cpu-all.h

+++ b/include/exec/cpu-all.h

@@ -XXX,XX +XXX,XX @@ CPUArchState *cpu_copy(CPUArchState *env);

#define TLB_WATCHPOINT (1 << (TARGET_PAGE_BITS_MIN - 4))

/* Set if TLB entry requires byte swap. */

#define TLB_BSWAP (1 << (TARGET_PAGE_BITS_MIN - 5))

+/* Set if TLB entry writes ignored. */

+#define TLB_DISCARD_WRITE (1 << (TARGET_PAGE_BITS_MIN - 6))

/* Use this mask to check interception with an alignment mask

* in a TCG backend.

*/

#define TLB_FLAGS_MASK \

- (TLB_INVALID_MASK | TLB_NOTDIRTY | TLB_MMIO | TLB_WATCHPOINT | TLB_BSWAP)

+ (TLB_INVALID_MASK | TLB_NOTDIRTY | TLB_MMIO \

+ | TLB_WATCHPOINT | TLB_BSWAP | TLB_DISCARD_WRITE)

/**

* tlb_hit_page: return true if page aligned @addr is a hit against the

diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h

index XXXXXXX..XXXXXXX 100644

--- a/include/exec/cpu-common.h

+++ b/include/exec/cpu-common.h

@@ -XXX,XX +XXX,XX @@ void qemu_flush_coalesced_mmio_buffer(void);

void cpu_flush_icache_range(hwaddr start, hwaddr len);

-extern struct MemoryRegion io_mem_rom;

extern struct MemoryRegion io_mem_notdirty;

typedef int (RAMBlockIterFunc)(RAMBlock *rb, void *opaque);

diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c

index XXXXXXX..XXXXXXX 100644

--- a/accel/tcg/cputlb.c

+++ b/accel/tcg/cputlb.c

@@ -XXX,XX +XXX,XX @@ static void tlb_reset_dirty_range_locked(CPUTLBEntry *tlb_entry,

{

uintptr_t addr = tlb_entry->addr_write;

- if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) {

+ if ((addr & (TLB_INVALID_MASK | TLB_MMIO |

+ TLB_DISCARD_WRITE | TLB_NOTDIRTY)) == 0) {

addr &= TARGET_PAGE_MASK;

addr += tlb_entry->addend;

if ((addr - start) < length) {

@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,

address |= TLB_MMIO;

addend = 0;

} else {

- /* TLB_MMIO for rom/romd handled below */

addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat;

}

@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,

tn.addr_write = -1;

if (prot & PAGE_WRITE) {

- if ((memory_region_is_ram(section->mr) && section->readonly)

- || memory_region_is_romd(section->mr)) {

- /* Write access calls the I/O callback. */

- tn.addr_write = address | TLB_MMIO;

- } else if (memory_region_is_ram(section->mr)

- && cpu_physical_memory_is_clean(

- memory_region_get_ram_addr(section->mr) + xlat)) {

- tn.addr_write = address | TLB_NOTDIRTY;

- } else {

- tn.addr_write = address;

+ tn.addr_write = address;

+ if (memory_region_is_romd(section->mr)) {

+ /* Use the MMIO path so that the device can switch states. */

+ tn.addr_write |= TLB_MMIO;

+ } else if (memory_region_is_ram(section->mr)) {

+ if (section->readonly) {

+ tn.addr_write |= TLB_DISCARD_WRITE;

+ } else if (cpu_physical_memory_is_clean(

+ memory_region_get_ram_addr(section->mr) + xlat)) {

+ tn.addr_write |= TLB_NOTDIRTY;

+ }

}

if (prot & PAGE_WRITE_INV) {

tn.addr_write |= TLB_INVALID_MASK;

@@ -XXX,XX +XXX,XX @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry,

mr = section->mr;

mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;

cpu->mem_io_pc = retaddr;

- if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {

+ if (mr != &io_mem_notdirty && !cpu->can_do_io) {

cpu_io_recompile(cpu, retaddr);

}

@@ -XXX,XX +XXX,XX @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry,

section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs);

mr = section->mr;

mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;

- if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {

+ if (mr != &io_mem_notdirty && !cpu->can_do_io) {

cpu_io_recompile(cpu, retaddr);

}

cpu->mem_io_vaddr = addr;

@@ -XXX,XX +XXX,XX @@ void *probe_access(CPUArchState *env, target_ulong addr, int size,

}

/* Reject I/O access, or other required slow-path. */

- if (tlb_addr & (TLB_NOTDIRTY | TLB_MMIO | TLB_BSWAP)) {

+ if (tlb_addr & (TLB_NOTDIRTY | TLB_MMIO | TLB_BSWAP | TLB_DISCARD_WRITE)) {

return NULL;

}

@@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

return;

}

+ /* Ignore writes to ROM. */

+ if (unlikely(tlb_addr & TLB_DISCARD_WRITE)) {

+ return;

+ }

+

haddr = (void *)((uintptr_t)addr + entry->addend);

/*

diff --git a/exec.c b/exec.c

index XXXXXXX..XXXXXXX 100644

--- a/exec.c

+++ b/exec.c

@@ -XXX,XX +XXX,XX @@ static MemoryRegion *system_io;

AddressSpace address_space_io;

AddressSpace address_space_memory;

-MemoryRegion io_mem_rom, io_mem_notdirty;

+MemoryRegion io_mem_notdirty;

static MemoryRegion io_mem_unassigned;

#endif

@@ -XXX,XX +XXX,XX @@ typedef struct subpage_t {

#define PHYS_SECTION_UNASSIGNED 0

#define PHYS_SECTION_NOTDIRTY 1

-#define PHYS_SECTION_ROM 2

static void io_mem_init(void);

static void memory_map_init(void);

@@ -XXX,XX +XXX,XX @@ hwaddr memory_region_section_get_iotlb(CPUState *cpu,

iotlb = memory_region_get_ram_addr(section->mr) + xlat;

if (!section->readonly) {

iotlb |= PHYS_SECTION_NOTDIRTY;

- } else {

- iotlb |= PHYS_SECTION_ROM;

}

} else {

AddressSpaceDispatch *d;

@@ -XXX,XX +XXX,XX @@ static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr)

return phys_section_add(map, &section);

}

-static void readonly_mem_write(void *opaque, hwaddr addr,

- uint64_t val, unsigned size)

-{

- /* Ignore any write to ROM. */

-}

-

-static bool readonly_mem_accepts(void *opaque, hwaddr addr,

- unsigned size, bool is_write,

- MemTxAttrs attrs)

-{

- return is_write;

-}

-

-/* This will only be used for writes, because reads are special cased

- * to directly access the underlying host ram.

- */

-static const MemoryRegionOps readonly_mem_ops = {

- .write = readonly_mem_write,

- .valid.accepts = readonly_mem_accepts,

- .endianness = DEVICE_NATIVE_ENDIAN,

- .valid = {

- .min_access_size = 1,

- .max_access_size = 8,

- .unaligned = false,

- },

- .impl = {

- .min_access_size = 1,

- .max_access_size = 8,

- .unaligned = false,

- },

-};

-

MemoryRegionSection *iotlb_to_section(CPUState *cpu,

hwaddr index, MemTxAttrs attrs)

{

@@ -XXX,XX +XXX,XX @@ MemoryRegionSection *iotlb_to_section(CPUState *cpu,

static void io_mem_init(void)

{

- memory_region_init_io(&io_mem_rom, NULL, &readonly_mem_ops,

- NULL, NULL, UINT64_MAX);

memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,

NULL, UINT64_MAX);

@@ -XXX,XX +XXX,XX @@ AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv)

assert(n == PHYS_SECTION_UNASSIGNED);

n = dummy_section(&d->map, fv, &io_mem_notdirty);

assert(n == PHYS_SECTION_NOTDIRTY);

- n = dummy_section(&d->map, fv, &io_mem_rom);

- assert(n == PHYS_SECTION_ROM);

d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };

--

2.17.1

Pages that we want to track for NOTDIRTY are RAM. We do not

really need to go through the I/O path to handle them.

Acked-by: David Hildenbrand <david@redhat.com>

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

include/exec/cpu-common.h | 2 --

accel/tcg/cputlb.c | 26 +++++++++++++++++---

exec.c | 50 ---------------------------------------

memory.c | 16 -------------

4 files changed, 23 insertions(+), 71 deletions(-)

diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h

index XXXXXXX..XXXXXXX 100644

--- a/include/exec/cpu-common.h

+++ b/include/exec/cpu-common.h

@@ -XXX,XX +XXX,XX @@ void qemu_flush_coalesced_mmio_buffer(void);

void cpu_flush_icache_range(hwaddr start, hwaddr len);

-extern struct MemoryRegion io_mem_notdirty;

-

typedef int (RAMBlockIterFunc)(RAMBlock *rb, void *opaque);

int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque);

diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c

index XXXXXXX..XXXXXXX 100644

--- a/accel/tcg/cputlb.c

+++ b/accel/tcg/cputlb.c

@@ -XXX,XX +XXX,XX @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry,

mr = section->mr;

mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;

cpu->mem_io_pc = retaddr;

- if (mr != &io_mem_notdirty && !cpu->can_do_io) {

+ if (!cpu->can_do_io) {

cpu_io_recompile(cpu, retaddr);

}

@@ -XXX,XX +XXX,XX @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry,

section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs);

mr = section->mr;

mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;

- if (mr != &io_mem_notdirty && !cpu->can_do_io) {

+ if (!cpu->can_do_io) {

cpu_io_recompile(cpu, retaddr);

}

cpu->mem_io_vaddr = addr;

@@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

need_swap = size > 1 && (tlb_addr & TLB_BSWAP);

/* Handle I/O access. */

- if (likely(tlb_addr & (TLB_MMIO | TLB_NOTDIRTY))) {

+ if (tlb_addr & TLB_MMIO) {

io_writex(env, iotlbentry, mmu_idx, val, addr, retaddr,

op ^ (need_swap * MO_BSWAP));

return;

@@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

haddr = (void *)((uintptr_t)addr + entry->addend);

+ /* Handle clean RAM pages. */

+ if (tlb_addr & TLB_NOTDIRTY) {

+ NotDirtyInfo ndi;

+

+ /* We require mem_io_pc in tb_invalidate_phys_page_range. */

+ env_cpu(env)->mem_io_pc = retaddr;

+

+ memory_notdirty_write_prepare(&ndi, env_cpu(env), addr,