This is v4 of my notdirty + rom patch set with two suggested name

changes (qemu_build_not_reached, TLB_DISCARD_WRITE) from David and Alex.

r~

The following changes since commit 240ab11fb72049d6373cbbec8d788f8e411a00bc:

Merge remote-tracking branch 'remotes/aperard/tags/pull-xen-20190924' into staging (2019-09-24 15:36:31 +0100)

https://github.com/rth7680/qemu.git tags/pull-tcg-20190925

for you to fetch changes up to ae57db63acf5a0399232f852acc5c1d83ef63400:

cputlb: Pass retaddr to tb_check_watchpoint (2019-09-25 10:56:28 -0700)

Fixes for TLB_BSWAP

Coversion of NOTDIRTY and ROM handling to cputlb

Followup cleanups to cputlb

Richard Henderson (16):

exec: Use TARGET_PAGE_BITS_MIN for TLB flags

cputlb: Disable __always_inline__ without optimization

qemu/compiler.h: Add qemu_build_not_reached

cputlb: Use qemu_build_not_reached in load/store_helpers

cputlb: Split out load/store_memop

cputlb: Introduce TLB_BSWAP

exec: Adjust notdirty tracing

cputlb: Move ROM handling from I/O path to TLB path

cputlb: Move NOTDIRTY handling from I/O path to TLB path

cputlb: Partially inline memory_region_section_get_iotlb

cputlb: Merge and move memory_notdirty_write_{prepare,complete}

cputlb: Handle TLB_NOTDIRTY in probe_access

cputlb: Remove cpu->mem_io_vaddr

cputlb: Remove tb_invalidate_phys_page_range is_cpu_write_access

cputlb: Pass retaddr to tb_invalidate_phys_page_fast

cputlb: Pass retaddr to tb_check_watchpoint

accel/tcg/translate-all.h | 8 +-

include/exec/cpu-all.h | 23 ++-

include/exec/cpu-common.h | 3 -

include/exec/exec-all.h | 6 +-

include/exec/memory-internal.h | 65 --------

include/hw/core/cpu.h | 2 -

include/qemu/compiler.h | 26 +++

accel/tcg/cputlb.c | 348 +++++++++++++++++++++++++----------------

accel/tcg/translate-all.c | 51 +++---

exec.c | 158 +------------------

hw/core/cpu.c | 1 -

memory.c | 20 ---

trace-events | 4 +-

13 files changed, 288 insertions(+), 427 deletions(-)

We can use notdirty_write for the write and return a valid host

pointer for this case.

Reviewed-by: David Hildenbrand <david@redhat.com>

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

accel/tcg/cputlb.c | 26 +++++++++++++++++---------

1 file changed, 17 insertions(+), 9 deletions(-)

@@ -XXX,XX +XXX,XX @@ void *probe_access(CPUArchState *env, target_ulong addr, int size,

return NULL;

- /* Handle watchpoints. */

- if (tlb_addr & TLB_WATCHPOINT) {

- cpu_check_watchpoint(env_cpu(env), addr, size,

- env_tlb(env)->d[mmu_idx].iotlb[index].attrs,

- wp_access, retaddr);

- }

+ if (unlikely(tlb_addr & TLB_FLAGS_MASK)) {

+ CPUIOTLBEntry *iotlbentry = &env_tlb(env)->d[mmu_idx].iotlb[index];

- /* Reject I/O access, or other required slow-path. */

- if (tlb_addr & (TLB_NOTDIRTY | TLB_MMIO | TLB_BSWAP | TLB_DISCARD_WRITE)) {

- return NULL;

+ /* Reject I/O access, or other required slow-path. */

+ if (tlb_addr & (TLB_MMIO | TLB_BSWAP | TLB_DISCARD_WRITE)) {

+ return NULL;

+

+ /* Handle watchpoints. */

+ if (tlb_addr & TLB_WATCHPOINT) {

+ cpu_check_watchpoint(env_cpu(env), addr, size,

+ iotlbentry->attrs, wp_access, retaddr);

+ /* Handle clean RAM pages. */

+ if (tlb_addr & TLB_NOTDIRTY) {

+ notdirty_write(env_cpu(env), addr, size, iotlbentry, retaddr);

+ }

}

return (void *)((uintptr_t)addr + entry->addend);

2.17.1

Fixes the previous TLB_WATCHPOINT patches because we are currently

failing to set cpu->mem_io_pc with the call to cpu_check_watchpoint.

Pass down the retaddr directly because it's readily available.

Fixes: 50b107c5d61

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Reviewed-by: David Hildenbrand <david@redhat.com>

accel/tcg/translate-all.h | 2 +-

accel/tcg/translate-all.c | 6 +++---

exec.c | 2 +-

3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/accel/tcg/translate-all.h b/accel/tcg/translate-all.h

--- a/accel/tcg/translate-all.h

+++ b/accel/tcg/translate-all.h

@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_page_fast(struct page_collection *pages,

tb_page_addr_t start, int len,

uintptr_t retaddr);

void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end);

-void tb_check_watchpoint(CPUState *cpu);

+void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr);

#ifdef CONFIG_USER_ONLY

int page_unprotect(target_ulong address, uintptr_t pc);

diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c

index XXXXXXX..XXXXXXX 100644

--- a/accel/tcg/translate-all.c

+++ b/accel/tcg/translate-all.c

@@ -XXX,XX +XXX,XX @@ static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc)

#endif

/* user-mode: call with mmap_lock held */

-void tb_check_watchpoint(CPUState *cpu)

+void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr)

TranslationBlock *tb;

assert_memory_lock();

- tb = tcg_tb_lookup(cpu->mem_io_pc);

+ tb = tcg_tb_lookup(retaddr);

if (tb) {

/* We can use retranslation to find the PC. */

- cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc, true);

+ cpu_restore_state_from_tb(cpu, tb, retaddr, true);

tb_phys_invalidate(tb, -1);

} else {

/* The exception probably happened in a helper. The CPU state should

diff --git a/exec.c b/exec.c

index XXXXXXX..XXXXXXX 100644

--- a/exec.c

+++ b/exec.c

@@ -XXX,XX +XXX,XX @@ void cpu_check_watchpoint(CPUState *cpu, vaddr addr, vaddr len,

cpu->watchpoint_hit = wp;

mmap_lock();

- tb_check_watchpoint(cpu);

+ tb_check_watchpoint(cpu, ra);

if (wp->flags & BP_STOP_BEFORE_ACCESS) {

cpu->exception_index = EXCP_DEBUG;

mmap_unlock();

2.17.1

All callers pass false to this argument. Remove it and pass the

constant on to tb_invalidate_phys_page_range__locked.

Reviewed-by: David Hildenbrand <david@redhat.com>

accel/tcg/translate-all.h | 3 +--

accel/tcg/translate-all.c | 6 ++----

exec.c | 4 ++--

3 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/accel/tcg/translate-all.h b/accel/tcg/translate-all.h

--- a/accel/tcg/translate-all.h

+++ b/accel/tcg/translate-all.h

@@ -XXX,XX +XXX,XX @@ struct page_collection *page_collection_lock(tb_page_addr_t start,

void page_collection_unlock(struct page_collection *set);

void tb_invalidate_phys_page_fast(struct page_collection *pages,

tb_page_addr_t start, int len);

-void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,

- int is_cpu_write_access);

+void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end);

void tb_check_watchpoint(CPUState *cpu);

#ifdef CONFIG_USER_ONLY

diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c

index XXXXXXX..XXXXXXX 100644

--- a/accel/tcg/translate-all.c

+++ b/accel/tcg/translate-all.c

@@ -XXX,XX +XXX,XX @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages,

*

* Called with mmap_lock held for user-mode emulation

*/

-void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,

- int is_cpu_write_access)

+void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end)

struct page_collection *pages;

PageDesc *p;

@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,

return;

}

pages = page_collection_lock(start, end);

- tb_invalidate_phys_page_range__locked(pages, p, start, end,

- is_cpu_write_access);

+ tb_invalidate_phys_page_range__locked(pages, p, start, end, 0);

page_collection_unlock(pages);

}

diff --git a/exec.c b/exec.c

index XXXXXXX..XXXXXXX 100644

--- a/exec.c

+++ b/exec.c

@@ -XXX,XX +XXX,XX @@ const char *parse_cpu_option(const char *cpu_option)

void tb_invalidate_phys_addr(target_ulong addr)

{

mmap_lock();

- tb_invalidate_phys_page_range(addr, addr + 1, 0);

+ tb_invalidate_phys_page_range(addr, addr + 1);

mmap_unlock();

}

@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs)

return;

}

ram_addr = memory_region_get_ram_addr(mr) + addr;

- tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);

+ tb_invalidate_phys_page_range(ram_addr, ram_addr + 1);

rcu_read_unlock();

}

2.17.1

Handle bswap on ram directly in load/store_helper. This fixes a

bug with the previous implementation in that one cannot use the

I/O path for RAM.

Fixes: a26fc6f5152b47f1

Reviewed-by: David Hildenbrand <david@redhat.com>

include/exec/cpu-all.h | 4 ++-

accel/tcg/cputlb.c | 72 +++++++++++++++++++++++++-----------------

2 files changed, 46 insertions(+), 30 deletions(-)

diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h

--- a/include/exec/cpu-all.h

+++ b/include/exec/cpu-all.h

@@ -XXX,XX +XXX,XX @@ CPUArchState *cpu_copy(CPUArchState *env);

#define TLB_MMIO (1 << (TARGET_PAGE_BITS_MIN - 3))

/* Set if TLB entry contains a watchpoint. */

#define TLB_WATCHPOINT (1 << (TARGET_PAGE_BITS_MIN - 4))

+/* Set if TLB entry requires byte swap. */

+#define TLB_BSWAP (1 << (TARGET_PAGE_BITS_MIN - 5))

/* Use this mask to check interception with an alignment mask

* in a TCG backend.

*/

#define TLB_FLAGS_MASK \

- (TLB_INVALID_MASK | TLB_NOTDIRTY | TLB_MMIO | TLB_WATCHPOINT)

+ (TLB_INVALID_MASK | TLB_NOTDIRTY | TLB_MMIO | TLB_WATCHPOINT | TLB_BSWAP)

/**

* tlb_hit_page: return true if page aligned @addr is a hit against the

diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c

index XXXXXXX..XXXXXXX 100644

--- a/accel/tcg/cputlb.c

+++ b/accel/tcg/cputlb.c

@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,

address |= TLB_INVALID_MASK;

}

if (attrs.byte_swap) {

- /* Force the access through the I/O slow path. */

- address |= TLB_MMIO;

+ address |= TLB_BSWAP;

}

if (!memory_region_is_ram(section->mr) &&

!memory_region_is_romd(section->mr)) {

@@ -XXX,XX +XXX,XX @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry,

bool locked = false;

MemTxResult r;

- if (iotlbentry->attrs.byte_swap) {

- op ^= MO_BSWAP;

- }

-

section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs);

mr = section->mr;

mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;

@@ -XXX,XX +XXX,XX @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry,

bool locked = false;

MemTxResult r;

- if (iotlbentry->attrs.byte_swap) {

- op ^= MO_BSWAP;

- }

-

section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs);

mr = section->mr;

mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;

@@ -XXX,XX +XXX,XX @@ void *probe_access(CPUArchState *env, target_ulong addr, int size,

wp_access, retaddr);

}

- if (tlb_addr & (TLB_NOTDIRTY | TLB_MMIO)) {

- /* I/O access */

+ /* Reject I/O access, or other required slow-path. */

+ if (tlb_addr & (TLB_NOTDIRTY | TLB_MMIO | TLB_BSWAP)) {

return NULL;

}

@@ -XXX,XX +XXX,XX @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,

/* Handle anything that isn't just a straight memory access. */

if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {

CPUIOTLBEntry *iotlbentry;

+ bool need_swap;

/* For anything that is unaligned, recurse through full_load. */

if ((addr & (size - 1)) != 0) {

@@ -XXX,XX +XXX,XX @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,

/* On watchpoint hit, this will longjmp out. */

cpu_check_watchpoint(env_cpu(env), addr, size,

iotlbentry->attrs, BP_MEM_READ, retaddr);

-

- /* The backing page may or may not require I/O. */

- tlb_addr &= ~TLB_WATCHPOINT;

- if ((tlb_addr & ~TARGET_PAGE_MASK) == 0) {

- goto do_aligned_access;

- }

}

+ need_swap = size > 1 && (tlb_addr & TLB_BSWAP);

/* Handle I/O access. */

- return io_readx(env, iotlbentry, mmu_idx, addr,

- retaddr, access_type, op);

+ if (likely(tlb_addr & TLB_MMIO)) {

+ return io_readx(env, iotlbentry, mmu_idx, addr, retaddr,

+ access_type, op ^ (need_swap * MO_BSWAP));

+ }

+

+ haddr = (void *)((uintptr_t)addr + entry->addend);

+

+ /*

+ * Keep these two load_memop separate to ensure that the compiler

+ * is able to fold the entire function to a single instruction.

+ * There is a build-time assert inside to remind you of this. ;-)

+ */

+ if (unlikely(need_swap)) {

+ return load_memop(haddr, op ^ MO_BSWAP);

+ }

+ return load_memop(haddr, op);

}

/* Handle slow unaligned access (it spans two pages or IO). */

@@ -XXX,XX +XXX,XX @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,

return res & MAKE_64BIT_MASK(0, size * 8);

}

- do_aligned_access:

haddr = (void *)((uintptr_t)addr + entry->addend);

return load_memop(haddr, op);

}

@@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

/* Handle anything that isn't just a straight memory access. */

if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {

CPUIOTLBEntry *iotlbentry;

+ bool need_swap;

/* For anything that is unaligned, recurse through byte stores. */

if ((addr & (size - 1)) != 0) {

@@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

/* On watchpoint hit, this will longjmp out. */

cpu_check_watchpoint(env_cpu(env), addr, size,

iotlbentry->attrs, BP_MEM_WRITE, retaddr);

-

- /* The backing page may or may not require I/O. */

- tlb_addr &= ~TLB_WATCHPOINT;

- if ((tlb_addr & ~TARGET_PAGE_MASK) == 0) {

- goto do_aligned_access;

- }

}

+ need_swap = size > 1 && (tlb_addr & TLB_BSWAP);

+

/* Handle I/O access. */

- io_writex(env, iotlbentry, mmu_idx, val, addr, retaddr, op);

+ if (likely(tlb_addr & (TLB_MMIO | TLB_NOTDIRTY))) {

+ io_writex(env, iotlbentry, mmu_idx, val, addr, retaddr,

+ op ^ (need_swap * MO_BSWAP));

+ return;

+ }

+

+ haddr = (void *)((uintptr_t)addr + entry->addend);

+

+ /*

+ * Keep these two store_memop separate to ensure that the compiler

+ * is able to fold the entire function to a single instruction.

+ * There is a build-time assert inside to remind you of this. ;-)

+ */

+ if (unlikely(need_swap)) {

+ store_memop(haddr, val, op ^ MO_BSWAP);

+ } else {

+ store_memop(haddr, val, op);

+ }

return;

}

@@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

return;

}

- do_aligned_access:

haddr = (void *)((uintptr_t)addr + entry->addend);

store_memop(haddr, val, op);

}

2.17.1

The memory_region_tb_read tracepoint is unreachable, since notdirty

is supposed to apply only to writes. The memory_region_tb_write

tracepoint is mis-named, because notdirty is not only used for TB

invalidation. It is also used for e.g. VGA RAM updates and migration.

Replace memory_region_tb_write with memory_notdirty_write_access,

and place it in memory_notdirty_write_prepare where it can catch

all of the instances. Add memory_notdirty_set_dirty to log when

we no longer intercept writes to a page.

Reviewed-by: David Hildenbrand <david@redhat.com>

exec.c | 3 +++

memory.c | 4 ----

trace-events | 4 ++--

3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/exec.c b/exec.c

--- a/exec.c

+++ b/exec.c

@@ -XXX,XX +XXX,XX @@ void memory_notdirty_write_prepare(NotDirtyInfo *ndi,

ndi->size = size;

ndi->pages = NULL;

+ trace_memory_notdirty_write_access(mem_vaddr, ram_addr, size);

+

assert(tcg_enabled());

if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {

ndi->pages = page_collection_lock(ram_addr, ram_addr + size);

@@ -XXX,XX +XXX,XX @@ void memory_notdirty_write_complete(NotDirtyInfo *ndi)

/* we remove the notdirty callback only if the code has been

flushed */

if (!cpu_physical_memory_is_clean(ndi->ram_addr)) {

+ trace_memory_notdirty_set_dirty(ndi->mem_vaddr);

tlb_set_dirty(ndi->cpu, ndi->mem_vaddr);

}

}

diff --git a/memory.c b/memory.c

index XXXXXXX..XXXXXXX 100644

--- a/memory.c

+++ b/memory.c

@@ -XXX,XX +XXX,XX @@ static MemTxResult memory_region_read_accessor(MemoryRegion *mr,

/* Accesses to code which has previously been translated into a TB show

* up in the MMIO path, as accesses to the io_mem_notdirty

* MemoryRegion. */

- trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);

} else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {

hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);

trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);

@@ -XXX,XX +XXX,XX @@ static MemTxResult memory_region_read_with_attrs_accessor(MemoryRegion *mr,

/* Accesses to code which has previously been translated into a TB show

* up in the MMIO path, as accesses to the io_mem_notdirty

* MemoryRegion. */

- trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);

} else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {

hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);

trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);

@@ -XXX,XX +XXX,XX @@ static MemTxResult memory_region_write_accessor(MemoryRegion *mr,

/* Accesses to code which has previously been translated into a TB show

* up in the MMIO path, as accesses to the io_mem_notdirty

* MemoryRegion. */

- trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);

} else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {

hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);

trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);

@@ -XXX,XX +XXX,XX @@ static MemTxResult memory_region_write_with_attrs_accessor(MemoryRegion *mr,

/* Accesses to code which has previously been translated into a TB show

* up in the MMIO path, as accesses to the io_mem_notdirty

* MemoryRegion. */

- trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);

} else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {

hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);

trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);

diff --git a/trace-events b/trace-events

index XXXXXXX..XXXXXXX 100644

--- a/trace-events

+++ b/trace-events

@@ -XXX,XX +XXX,XX @@ dma_map_wait(void *dbs) "dbs=%p"

find_ram_offset(uint64_t size, uint64_t offset) "size: 0x%" PRIx64 " @ 0x%" PRIx64

find_ram_offset_loop(uint64_t size, uint64_t candidate, uint64_t offset, uint64_t next, uint64_t mingap) "trying size: 0x%" PRIx64 " @ 0x%" PRIx64 ", offset: 0x%" PRIx64" next: 0x%" PRIx64 " mingap: 0x%" PRIx64

ram_block_discard_range(const char *rbname, void *hva, size_t length, bool need_madvise, bool need_fallocate, int ret) "%s@%p + 0x%zx: madvise: %d fallocate: %d ret: %d"

+memory_notdirty_write_access(uint64_t vaddr, uint64_t ram_addr, unsigned size) "0x%" PRIx64 " ram_addr 0x%" PRIx64 " size %u"

+memory_notdirty_set_dirty(uint64_t vaddr) "0x%" PRIx64

# memory.c

memory_region_ops_read(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

memory_region_ops_write(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

memory_region_subpage_read(int cpu_index, void *mr, uint64_t offset, uint64_t value, unsigned size) "cpu %d mr %p offset 0x%"PRIx64" value 0x%"PRIx64" size %u"

memory_region_subpage_write(int cpu_index, void *mr, uint64_t offset, uint64_t value, unsigned size) "cpu %d mr %p offset 0x%"PRIx64" value 0x%"PRIx64" size %u"

-memory_region_tb_read(int cpu_index, uint64_t addr, uint64_t value, unsigned size) "cpu %d addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

-memory_region_tb_write(int cpu_index, uint64_t addr, uint64_t value, unsigned size) "cpu %d addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

memory_region_ram_device_read(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

memory_region_ram_device_write(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u"

flatview_new(void *view, void *root) "%p (root %p)"

2.17.1

Use this as a compile-time assert that a particular

code path is not reachable.

include/qemu/compiler.h | 15 +++++++++++++++

1 file changed, 15 insertions(+)

diff --git a/include/qemu/compiler.h b/include/qemu/compiler.h

--- a/include/qemu/compiler.h

+++ b/include/qemu/compiler.h

@@ -XXX,XX +XXX,XX @@

#define QEMU_GENERIC9(x, a0, ...) QEMU_GENERIC_IF(x, a0, QEMU_GENERIC8(x, __VA_ARGS__))

#define QEMU_GENERIC10(x, a0, ...) QEMU_GENERIC_IF(x, a0, QEMU_GENERIC9(x, __VA_ARGS__))

+/**

+ * qemu_build_not_reached()

+ *

+ * The compiler, during optimization, is expected to prove that a call

+ * to this function cannot be reached and remove it. If the compiler

+ * supports QEMU_ERROR, this will be reported at compile time; otherwise

+ * this will be reported at link time due to the missing symbol.

+ */

+#ifdef __OPTIMIZE__

+extern void QEMU_NORETURN QEMU_ERROR("code path is reachable")

+ qemu_build_not_reached(void);

+#else

+#define qemu_build_not_reached() g_assert_not_reached()

+#endif

#endif /* COMPILER_H */

2.17.1

Since 9458a9a1df1a, all readers of the dirty bitmaps wait

for the rcu lock, which means that they wait until the end

of any executing TranslationBlock.

include/exec/memory-internal.h | 65 -----------------------------

accel/tcg/cputlb.c | 76 +++++++++++++++++++---------------

exec.c | 44 --------------------

3 files changed, 42 insertions(+), 143 deletions(-)

@@ -XXX,XX +XXX,XX @@

#include "exec/helper-proto.h"

#include "qemu/atomic.h"

#include "qemu/atomic128.h"

+#include "translate-all.h"

/* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */

/* #define DEBUG_TLB */

@@ -XXX,XX +XXX,XX @@ tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr)

return qemu_ram_addr_from_host_nofail(p);

+static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size,

+ CPUIOTLBEntry *iotlbentry, uintptr_t retaddr)

+{

+ ram_addr_t ram_addr = mem_vaddr + iotlbentry->addr;

+

+ trace_memory_notdirty_write_access(mem_vaddr, ram_addr, size);

+

+ if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {

+ struct page_collection *pages

+ = page_collection_lock(ram_addr, ram_addr + size);

+

+ /* We require mem_io_pc in tb_invalidate_phys_page_range. */

+ cpu->mem_io_pc = retaddr;

+

+ tb_invalidate_phys_page_fast(pages, ram_addr, size);

+ page_collection_unlock(pages);

+ }

+

+ /*

+ * Set both VGA and migration bits for simplicity and to remove

+ * the notdirty callback faster.

+ */

+ cpu_physical_memory_set_dirty_range(ram_addr, size, DIRTY_CLIENTS_NOCODE);

+

+ /* We remove the notdirty callback only if the code has been flushed. */

+ if (!cpu_physical_memory_is_clean(ram_addr)) {

+ trace_memory_notdirty_set_dirty(mem_vaddr);

+ tlb_set_dirty(cpu, mem_vaddr);

+ }

+}

+

/*

* Probe for whether the specified guest access is permitted. If it is not

* permitted then an exception will be taken in the same way as if this

@@ -XXX,XX +XXX,XX @@ void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr,

/* Probe for a read-modify-write atomic operation. Do not allow unaligned

* operations, or io operations to proceed. Return the host address. */

static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,

- TCGMemOpIdx oi, uintptr_t retaddr,

- NotDirtyInfo *ndi)

+ TCGMemOpIdx oi, uintptr_t retaddr)

size_t mmu_idx = get_mmuidx(oi);

uintptr_t index = tlb_index(env, mmu_idx, addr);

@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,

hostaddr = (void *)((uintptr_t)addr + tlbe->addend);

- ndi->active = false;

if (unlikely(tlb_addr & TLB_NOTDIRTY)) {

- ndi->active = true;

- memory_notdirty_write_prepare(ndi, env_cpu(env), addr,

- qemu_ram_addr_from_host_nofail(hostaddr),

- 1 << s_bits);

+ notdirty_write(env_cpu(env), addr, 1 << s_bits,

+ &env_tlb(env)->d[mmu_idx].iotlb[index], retaddr);

}

return hostaddr;

@@ -XXX,XX +XXX,XX @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

return;

}

- haddr = (void *)((uintptr_t)addr + entry->addend);

-

/* Handle clean RAM pages. */

if (tlb_addr & TLB_NOTDIRTY) {

- NotDirtyInfo ndi;

-

- /* We require mem_io_pc in tb_invalidate_phys_page_range. */

- env_cpu(env)->mem_io_pc = retaddr;

-

- memory_notdirty_write_prepare(&ndi, env_cpu(env), addr,

- addr + iotlbentry->addr, size);

-

- if (unlikely(need_swap)) {

- store_memop(haddr, val, op ^ MO_BSWAP);

- } else {

- store_memop(haddr, val, op);

- }

-

- memory_notdirty_write_complete(&ndi);

- return;

+ notdirty_write(env_cpu(env), addr, size, iotlbentry, retaddr);

}

+ haddr = (void *)((uintptr_t)addr + entry->addend);

+

/*

* Keep these two store_memop separate to ensure that the compiler

* is able to fold the entire function to a single instruction.

@@ -XXX,XX +XXX,XX @@ void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,

#define EXTRA_ARGS , TCGMemOpIdx oi, uintptr_t retaddr

#define ATOMIC_NAME(X) \

HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu))

-#define ATOMIC_MMU_DECLS NotDirtyInfo ndi

-#define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, retaddr, &ndi)

-#define ATOMIC_MMU_CLEANUP \

- do { \

- if (unlikely(ndi.active)) { \

- memory_notdirty_write_complete(&ndi); \

- } \

- } while (0)

+#define ATOMIC_MMU_DECLS

+#define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, retaddr)

+#define ATOMIC_MMU_CLEANUP

#define DATA_SIZE 1

#include "atomic_template.h"

@@ -XXX,XX +XXX,XX @@ void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,

#undef ATOMIC_MMU_LOOKUP

#define EXTRA_ARGS , TCGMemOpIdx oi

#define ATOMIC_NAME(X) HELPER(glue(glue(atomic_ ## X, SUFFIX), END))

-#define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, GETPC(), &ndi)

+#define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, GETPC())

#define DATA_SIZE 1

#include "atomic_template.h"

diff --git a/exec.c b/exec.c

index XXXXXXX..XXXXXXX 100644

--- a/exec.c

+++ b/exec.c

@@ -XXX,XX +XXX,XX @@ ram_addr_t qemu_ram_addr_from_host(void *ptr)

return block->offset + offset;

-/* Called within RCU critical section. */

-void memory_notdirty_write_prepare(NotDirtyInfo *ndi,

- CPUState *cpu,

- vaddr mem_vaddr,

- ram_addr_t ram_addr,

- unsigned size)

- ndi->cpu = cpu;

- ndi->ram_addr = ram_addr;

- ndi->mem_vaddr = mem_vaddr;

- ndi->size = size;

- ndi->pages = NULL;

-

- trace_memory_notdirty_write_access(mem_vaddr, ram_addr, size);

-

- assert(tcg_enabled());

- if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {

- ndi->pages = page_collection_lock(ram_addr, ram_addr + size);

- tb_invalidate_phys_page_fast(ndi->pages, ram_addr, size);

- }

-/* Called within RCU critical section. */

-void memory_notdirty_write_complete(NotDirtyInfo *ndi)

-{

- if (ndi->pages) {

- assert(tcg_enabled());

- page_collection_unlock(ndi->pages);

- ndi->pages = NULL;

- }

-

- /* Set both VGA and migration bits for simplicity and to remove

- * the notdirty callback faster.

- */

- cpu_physical_memory_set_dirty_range(ndi->ram_addr, ndi->size,

- DIRTY_CLIENTS_NOCODE);

- /* we remove the notdirty callback only if the code has been

- flushed */

- if (!cpu_physical_memory_is_clean(ndi->ram_addr)) {

- trace_memory_notdirty_set_dirty(ndi->mem_vaddr);

- tlb_set_dirty(ndi->cpu, ndi->mem_vaddr);

- }

-}

-

/* Generate a debug exception if a watchpoint has been hit. */

void cpu_check_watchpoint(CPUState *cpu, vaddr addr, vaddr len,

MemTxAttrs attrs, int flags, uintptr_t ra)

2.17.1

This forced inlining can result in missing symbols,

which makes a debugging build harder to follow.

include/qemu/compiler.h | 11 +++++++++++

accel/tcg/cputlb.c | 4 ++--

2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/include/qemu/compiler.h b/include/qemu/compiler.h

--- a/include/qemu/compiler.h

+++ b/include/qemu/compiler.h

@@ -XXX,XX +XXX,XX @@

# define QEMU_NONSTRING

#endif

+/*

+ * Forced inlining may be desired to encourage constant propagation

+ * of function parameters. However, it can also make debugging harder,

+ * so disable it for a non-optimizing build.

+ */

+#if defined(__OPTIMIZE__)

+#define QEMU_ALWAYS_INLINE __attribute__((always_inline))

+#else

+#define QEMU_ALWAYS_INLINE

+#endif

+

/* Implement C11 _Generic via GCC builtins. Example:

*

* QEMU_GENERIC(x, (float, sinf), (long double, sinl), sin) (x)

@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,

typedef uint64_t FullLoadHelper(CPUArchState *env, target_ulong addr,

TCGMemOpIdx oi, uintptr_t retaddr);

-static inline uint64_t __attribute__((always_inline))

+static inline uint64_t QEMU_ALWAYS_INLINE

load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,

uintptr_t retaddr, MemOp op, bool code_read,

FullLoadHelper *full_load)

@@ -XXX,XX +XXX,XX @@ tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr,

* Store Helpers

*/

-static inline void __attribute__((always_inline))

+static inline void QEMU_ALWAYS_INLINE

store_helper(CPUArchState *env, target_ulong addr, uint64_t val,

TCGMemOpIdx oi, uintptr_t retaddr, MemOp op)

2.17.1

We will shortly be using these more than once.

Reviewed-by: David Hildenbrand <david@redhat.com>

accel/tcg/cputlb.c | 107 +++++++++++++++++++++++----------------------

1 file changed, 55 insertions(+), 52 deletions(-)

@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,

typedef uint64_t FullLoadHelper(CPUArchState *env, target_ulong addr,

TCGMemOpIdx oi, uintptr_t retaddr);

+static inline uint64_t QEMU_ALWAYS_INLINE

+load_memop(const void *haddr, MemOp op)

+{

+ switch (op) {

+ case MO_UB:

+ return ldub_p(haddr);

+ case MO_BEUW:

+ return lduw_be_p(haddr);

+ case MO_LEUW:

+ return lduw_le_p(haddr);

+ case MO_BEUL:

+ return (uint32_t)ldl_be_p(haddr);

+ case MO_LEUL:

+ return (uint32_t)ldl_le_p(haddr);

+ case MO_BEQ:

+ return ldq_be_p(haddr);

+ case MO_LEQ:

+ return ldq_le_p(haddr);

+ default:

+ qemu_build_not_reached();

+ }

+}

+

static inline uint64_t QEMU_ALWAYS_INLINE

load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,

uintptr_t retaddr, MemOp op, bool code_read,

@@ -XXX,XX +XXX,XX @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,

do_aligned_access:

haddr = (void *)((uintptr_t)addr + entry->addend);

- switch (op) {

- case MO_UB:

- res = ldub_p(haddr);

- break;

- case MO_BEUW: