Series comparison

-[PULL 00/22] tcg patch queue
+[PATCH for-8.0 00/11] tcg patch queue
-Second pull for this week, since this set is large enough by itself.
+Posting pre-PR because I had to adjust Emilio's QTree patch [1],
+and added a new patch to avoid an assert that can be generated
 with incorrect -R reserved_va values vs the ARM commpage.
 r~
+[1] https://gitlab.com/rth7680/qemu/-/jobs/3975817279#L92
-The following changes since commit 7c9236d6d61f30583d5d860097d88dbf0fe487bf:
+Emilio Cota (2):
   util: import GTree as QTree
   tcg: use QTree instead of GTree
-  Merge tag 'pull-tcg-20230116' of https://gitlab.com/rth7680/qemu into staging (2023-01-17 10:24:16 +0000)
+Richard Henderson (9):
   linux-user: Diagnose misaligned -R size
   include/exec: Change reserved_va semantics to last byte
   accel/tcg: Pass last not end to page_set_flags
   accel/tcg: Pass last not end to page_reset_target_data
   accel/tcg: Pass last not end to PAGE_FOR_EACH_TB
   accel/tcg: Pass last not end to page_collection_lock
   accel/tcg: Pass last not end to tb_invalidate_phys_page_range__locked
   accel/tcg: Pass last not end to tb_invalidate_phys_range
   linux-user/arm: Take more care allocating commpage
-are available in the Git repository at:
+ configure                   |   15 +
  meson.build                 |    4 +
  include/exec/cpu-all.h      |   15 +-
  include/exec/exec-all.h     |    2 +-
  include/qemu/qtree.h        |  201 +++++
  linux-user/arm/target_cpu.h |    2 +-
  accel/tcg/tb-maint.c        |  112 +--
  accel/tcg/translate-all.c   |    2 +-
  accel/tcg/user-exec.c       |   25 +-
  bsd-user/main.c             |   10 +-
  bsd-user/mmap.c             |   10 +-
  linux-user/elfload.c        |   67 +-
  linux-user/main.c           |   31 +-
  linux-user/mmap.c           |   22 +-
  linux-user/syscall.c        |    4 +-
  softmmu/physmem.c           |    2 +-
  tcg/region.c                |   19 +-
  tests/bench/qtree-bench.c   |  286 +++++++
  tests/unit/test-qtree.c     |  333 +++++++++
  util/qtree.c                | 1390 +++++++++++++++++++++++++++++++++++
  tests/bench/meson.build     |    4 +
  tests/unit/meson.build      |    1 +
  util/meson.build            |    1 +
 files changed, 2412 insertions(+), 146 deletions(-)
  create mode 100644 include/qemu/qtree.h
  create mode 100644 tests/bench/qtree-bench.c
  create mode 100644 tests/unit/test-qtree.c
  create mode 100644 util/qtree.c
-  https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20230117
+--
+.34.1
 for you to fetch changes up to 493c9b19a7fb7f387c4fcf57d3836504d5242bf5:
   tcg/riscv: Implement direct branch for goto_tb (2023-01-17 22:36:17 +0000)
 ----------------------------------------------------------------
 tcg: Fix race conditions in (most) goto_tb implementations
 ----------------------------------------------------------------
 Richard Henderson (22):
       tcg: Split out tcg_out_exit_tb
       tcg/i386: Remove unused goto_tb code for indirect jump
       tcg/ppc: Remove unused goto_tb code for indirect jump
       tcg/sparc64: Remove unused goto_tb code for indirect jump
       tcg: Replace asserts on tcg_jmp_insn_offset
       tcg: Introduce set_jmp_insn_offset
       tcg: Introduce get_jmp_target_addr
       tcg: Split out tcg_out_goto_tb
       tcg: Rename TB_JMP_RESET_OFFSET_INVALID to TB_JMP_OFFSET_INVALID
       tcg: Add gen_tb to TCGContext
       tcg: Add TranslationBlock.jmp_insn_offset
       tcg: Change tb_target_set_jmp_target arguments
       tcg: Move tb_target_set_jmp_target declaration to tcg.h
       tcg: Always define tb_target_set_jmp_target
       tcg: Remove TCG_TARGET_HAS_direct_jump
       tcg/aarch64: Reorg goto_tb implementation
       tcg/ppc: Reorg goto_tb implementation
       tcg/sparc64: Remove USE_REG_TB
       tcg/sparc64: Reorg goto_tb implementation
       tcg/arm: Implement direct branch for goto_tb
       tcg/riscv: Introduce OPC_NOP
       tcg/riscv: Implement direct branch for goto_tb
  include/exec/exec-all.h          |   5 +-
  include/tcg/tcg.h                |  14 ++-
  tcg/aarch64/tcg-target.h         |   6 +-
  tcg/arm/tcg-target.h             |   5 -
  tcg/i386/tcg-target.h            |   9 --
  tcg/loongarch64/tcg-target.h     |   3 -
  tcg/mips/tcg-target.h            |   5 -
  tcg/ppc/tcg-target.h             |   7 +-
  tcg/riscv/tcg-target.h           |   4 -
  tcg/s390x/tcg-target.h           |  11 ---
  tcg/sparc64/tcg-target.h         |   4 -
  tcg/tci/tcg-target.h             |   4 -
  accel/tcg/cpu-exec.c             |  21 ++--
  accel/tcg/translate-all.c        |  10 +-
  tcg/tcg-op.c                     |  14 +--
  tcg/tcg.c                        |  42 +++++---
  tcg/aarch64/tcg-target.c.inc     | 106 ++++++++++-----------
  tcg/arm/tcg-target.c.inc         |  89 +++++++++++------
  tcg/i386/tcg-target.c.inc        |  68 +++++++------
  tcg/loongarch64/tcg-target.c.inc |  66 +++++++------
  tcg/mips/tcg-target.c.inc        |  59 +++++++-----
  tcg/ppc/tcg-target.c.inc         | 193 ++++++++++++-------------------------
  tcg/riscv/tcg-target.c.inc       |  65 +++++++++----
  tcg/s390x/tcg-target.c.inc       |  67 ++++++++-----
  tcg/sparc64/tcg-target.c.inc     | 201 +++++++++++++++------------------------
  tcg/tci/tcg-target.c.inc         |  31 +++---
 files changed, 528 insertions(+), 581 deletions(-)

-[PULL 22/22] tcg/riscv: Implement direct branch for goto_tb
+[PATCH for-8.0 01/11] util: import GTree as QTree
-Now that tcg can handle direct and indirect goto_tb simultaneously,
+From: Emilio Cota <cota@braap.org>
 we can optimistically leave space for a direct branch and fall back
 to loading the pointer from the TB for an indirect branch.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
+The only reason to add this implementation is to control the memory allocator
 used. Some users (e.g. TCG) cannot work reliably in multi-threaded
 environments (e.g. forking in user-mode) with GTree's allocator, GSlice.
 See https://gitlab.com/qemu-project/qemu/-/issues/285 for details.
 Importing GTree is a temporary workaround until GTree migrates away
 from GSlice.
 This implementation is identical to that in glib v2.75.0, except that
 we don't import recent additions to the API nor deprecated API calls,
 none of which are used in QEMU.
 I've imported tests from glib and added a benchmark just to
 make sure that performance is similar. Note: it cannot be identical
 because (1) we are not using GSlice, (2) we use different compilation flags
 (e.g. -fPIC) and (3) we're linking statically.
 $ cat /proc/cpuinfo| grep 'model name' | head -1
 model name      : AMD Ryzen 7 PRO 5850U with Radeon Graphics
 $ echo '0' | sudo tee /sys/devices/system/cpu/cpufreq/boost
 $ tests/bench/qtree-bench
  Tree         Op      32            1024            4096          131072         1048576
 ------------------------------------------------------------------------------------------------
 GTree     Lookup   83.23           43.08           25.31           19.40           16.22
 QTree     Lookup  113.42 (1.36x)   53.83 (1.25x)   28.38 (1.12x)   17.64 (0.91x)   13.04 (0.80x)
 GTree     Insert   44.23           29.37           25.83           19.49           17.03
 QTree     Insert   46.87 (1.06x)   25.62 (0.87x)   24.29 (0.94x)   16.83 (0.86x)   12.97 (0.76x)
 GTree     Remove   53.27           35.15           31.43           24.64           16.70
 QTree     Remove   57.32 (1.08x)   41.76 (1.19x)   38.37 (1.22x)   29.30 (1.19x)   15.07 (0.90x)
 GTree  RemoveAll  135.44          127.52          126.72          120.11           64.34
 QTree  RemoveAll  127.15 (0.94x)  110.37 (0.87x)  107.97 (0.85x)   97.13 (0.81x)   55.10 (0.86x)
 GTree   Traverse  277.71          276.09          272.78          246.72           98.47
 QTree   Traverse  370.33 (1.33x)  411.97 (1.49x)  400.23 (1.47x)  262.82 (1.07x)   78.52 (0.80x)
 ------------------------------------------------------------------------------------------------
 As a sanity check, the same benchmark when Glib's version
 is >= $glib_dropped_gslice_version (i.e. QTree == GTree):
  Tree         Op      32            1024            4096          131072         1048576
 ------------------------------------------------------------------------------------------------
 GTree     Lookup   82.72           43.09           24.18           19.73           16.09
 QTree     Lookup   81.82 (0.99x)   43.10 (1.00x)   24.20 (1.00x)   19.76 (1.00x)   16.26 (1.01x)
 GTree     Insert   45.07           29.62           26.34           19.90           17.18
 QTree     Insert   45.72 (1.01x)   29.60 (1.00x)   26.38 (1.00x)   19.71 (0.99x)   17.20 (1.00x)
 GTree     Remove   54.48           35.36           31.77           24.97           16.95
 QTree     Remove   54.46 (1.00x)   35.32 (1.00x)   31.77 (1.00x)   24.91 (1.00x)   17.15 (1.01x)
 GTree  RemoveAll  140.68          127.36          125.43          121.45           68.20
 QTree  RemoveAll  140.65 (1.00x)  127.64 (1.00x)  125.01 (1.00x)  121.73 (1.00x)   67.06 (0.98x)
 GTree   Traverse  278.68          276.05          266.75          251.65          104.93
 QTree   Traverse  278.31 (1.00x)  275.78 (1.00x)  266.42 (1.00x)  247.89 (0.99x)  104.58 (1.00x)
 ------------------------------------------------------------------------------------------------
 Signed-off-by: Emilio Cota <cota@braap.org>
 Message-Id: <20230205163758.416992-2-cota@braap.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- tcg/riscv/tcg-target.c.inc | 19 +++++++++++++++++--
+ configure                 |   15 +
-file changed, 17 insertions(+), 2 deletions(-)
+ meson.build               |    4 +
  include/qemu/qtree.h      |  201 ++++++
  tests/bench/qtree-bench.c |  286 ++++++++
  tests/unit/test-qtree.c   |  333 +++++++++
  util/qtree.c              | 1390 +++++++++++++++++++++++++++++++++++++
  tests/bench/meson.build   |    4 +
  tests/unit/meson.build    |    1 +
  util/meson.build          |    1 +
 files changed, 2235 insertions(+)
  create mode 100644 include/qemu/qtree.h
  create mode 100644 tests/bench/qtree-bench.c
  create mode 100644 tests/unit/test-qtree.c
  create mode 100644 util/qtree.c
-diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
+diff --git a/configure b/configure
 index XXXXXXX..XXXXXXX 100755
 --- a/configure
 +++ b/configure
@@ -XXX,XX +XXX,XX @@ safe_stack=""
  use_containers="yes"
  gdb_bin=$(command -v "gdb-multiarch" || command -v "gdb")
  gdb_arches=""
 +glib_has_gslice="no"
  if test -e "$source_path/.git"
  then
@@ -XXX,XX +XXX,XX @@ for i in $glib_modules; do
      fi
  done
 +# Check whether glib has gslice, which we have to avoid for correctness.
 +# TODO: remove this check and the corresponding workaround (qtree) when
 +# the minimum supported glib is >= $glib_dropped_gslice_version.
 +glib_dropped_gslice_version=2.75.3
 +for i in $glib_modules; do
 +    if ! $pkg_config --atleast-version=$glib_dropped_gslice_version $i; then
 +        glib_has_gslice="yes"
 +    break
 +    fi
 +done
 +
  glib_bindir="$($pkg_config --variable=bindir glib-2.0)"
  if test -z "$glib_bindir" ; then
      glib_bindir="$($pkg_config --variable=prefix glib-2.0)"/bin
@@ -XXX,XX +XXX,XX @@ echo "GLIB_CFLAGS=$glib_cflags" >> $config_host_mak
  echo "GLIB_LIBS=$glib_libs" >> $config_host_mak
  echo "GLIB_BINDIR=$glib_bindir" >> $config_host_mak
  echo "GLIB_VERSION=$($pkg_config --modversion glib-2.0)" >> $config_host_mak
 +if test "$glib_has_gslice" = "yes" ; then
 +    echo "HAVE_GLIB_WITH_SLICE_ALLOCATOR=y" >> $config_host_mak
 +fi
  echo "QEMU_LDFLAGS=$QEMU_LDFLAGS" >> $config_host_mak
  echo "EXESUF=$EXESUF" >> $config_host_mak
 diff --git a/meson.build b/meson.build
 index XXXXXXX..XXXXXXX 100644
---- a/tcg/riscv/tcg-target.c.inc
+--- a/meson.build
-+++ b/tcg/riscv/tcg-target.c.inc
++++ b/meson.build
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
+@@ -XXX,XX +XXX,XX @@ glib = declare_dependency(compile_args: config_host['GLIB_CFLAGS'].split(),
+                           })
- static void tcg_out_goto_tb(TCGContext *s, int which)
+ # override glib dep with the configure results (for subprojects)
- {
+ meson.override_dependency('glib-2.0', glib)
--    /* indirect jump method */
++# pass down whether Glib has the slice allocator
-+    /* Direct branch will be patched by tb_target_set_jmp_target. */
++if config_host.has_key('HAVE_GLIB_WITH_SLICE_ALLOCATOR')
-+    set_jmp_insn_offset(s, which);
++  config_host_data.set('HAVE_GLIB_WITH_SLICE_ALLOCATOR', true)
-+    tcg_out32(s, OPC_JAL);
++endif
-+
-+    /* When branch is out of range, fall through to indirect. */
+ gio = not_found
-     tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO,
+ gdbus_codegen = not_found
-                get_jmp_target_addr(s, which));
+diff --git a/include/qemu/qtree.h b/include/qemu/qtree.h
-     tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0);
+new file mode 100644
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
+index XXXXXXX..XXXXXXX
- void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+--- /dev/null
-                               uintptr_t jmp_rx, uintptr_t jmp_rw)
++++ b/include/qemu/qtree.h
- {
+@@ -XXX,XX +XXX,XX @@
--    /* Always indirect, nothing to do */
++/*
-+    uintptr_t addr = tb->jmp_target_addr[n];
++ * GLIB - Library of useful routines for C programming
-+    ptrdiff_t offset = addr - jmp_rx;
++ * Copyright (C) 1995-1997  Peter Mattis, Spencer Kimball and Josh MacDonald
-+    tcg_insn_unit insn;
++ *
-+
++ * SPDX-License-Identifier: LGPL-2.1-or-later
-+    /* Either directly branch, or fall through to indirect branch. */
++ *
-+    if (offset == sextreg(offset, 0, 20)) {
++ * This library is free software; you can redistribute it and/or
-+        insn = encode_uj(OPC_JAL, TCG_REG_ZERO, offset);
++ * modify it under the terms of the GNU Lesser General Public
 + * License as published by the Free Software Foundation; either
 + * version 2.1 of the License, or (at your option) any later version.
 + *
 + * This library is distributed in the hope that it will be useful,
 + * but WITHOUT ANY WARRANTY; without even the implied warranty of
 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 + * Lesser General Public License for more details.
 + *
 + * You should have received a copy of the GNU Lesser General Public
 + * License along with this library; if not, see <http://www.gnu.org/licenses/>.
 + */
 +
 +/*
 + * Modified by the GLib Team and others 1997-2000.  See the AUTHORS
 + * file for a list of people on the GLib Team.  See the ChangeLog
 + * files for a list of changes.  These files are distributed with
 + * GLib at ftp://ftp.gtk.org/pub/gtk/.
 + */
 +
 +/*
 + * QTree is a partial import of Glib's GTree. The parts excluded correspond
 + * to API calls either deprecated (e.g. g_tree_traverse) or recently added
 + * (e.g. g_tree_search_node, added in 2.68); neither have callers in QEMU.
 + *
 + * The reason for this import is to allow us to control the memory allocator
 + * used by the tree implementation. Until Glib 2.75.3, GTree uses Glib's
 + * slice allocator, which causes problems when forking in user-mode;
 + * see https://gitlab.com/qemu-project/qemu/-/issues/285 and glib's
 + * "45b5a6c1e gslice: Remove slice allocator and use malloc() instead".
 + *
 + * TODO: remove QTree when QEMU's minimum Glib version is >= 2.75.3.
 + */
 +
 +#ifndef QEMU_QTREE_H
 +#define QEMU_QTREE_H
 +
 +#include "qemu/osdep.h"
 +
 +#ifdef HAVE_GLIB_WITH_SLICE_ALLOCATOR
 +
 +typedef struct _QTree  QTree;
 +
 +typedef struct _QTreeNode QTreeNode;
 +
 +typedef gboolean (*QTraverseNodeFunc)(QTreeNode *node,
 +                                      gpointer user_data);
 +
 +/*
 + * Balanced binary trees
 + */
 +QTree *q_tree_new(GCompareFunc key_compare_func);
 +QTree *q_tree_new_with_data(GCompareDataFunc key_compare_func,
 +                            gpointer key_compare_data);
 +QTree *q_tree_new_full(GCompareDataFunc key_compare_func,
 +                       gpointer key_compare_data,
 +                       GDestroyNotify key_destroy_func,
 +                       GDestroyNotify value_destroy_func);
 +QTree *q_tree_ref(QTree *tree);
 +void q_tree_unref(QTree *tree);
 +void q_tree_destroy(QTree *tree);
 +void q_tree_insert(QTree *tree,
 +                   gpointer key,
 +                   gpointer value);
 +void q_tree_replace(QTree *tree,
 +                    gpointer key,
 +                    gpointer value);
 +gboolean q_tree_remove(QTree *tree,
 +                       gconstpointer key);
 +gboolean q_tree_steal(QTree *tree,
 +                      gconstpointer key);
 +gpointer q_tree_lookup(QTree *tree,
 +                       gconstpointer key);
 +gboolean q_tree_lookup_extended(QTree *tree,
 +                                gconstpointer lookup_key,
 +                                gpointer *orig_key,
 +                                gpointer *value);
 +void q_tree_foreach(QTree *tree,
 +                    GTraverseFunc func,
 +                    gpointer user_data);
 +gpointer q_tree_search(QTree *tree,
 +                       GCompareFunc search_func,
 +                       gconstpointer user_data);
 +gint q_tree_height(QTree *tree);
 +gint q_tree_nnodes(QTree *tree);
 +
 +#else /* !HAVE_GLIB_WITH_SLICE_ALLOCATOR */
 +
 +typedef GTree QTree;
 +typedef GTreeNode QTreeNode;
 +typedef GTraverseNodeFunc QTraverseNodeFunc;
 +
 +static inline QTree *q_tree_new(GCompareFunc key_compare_func)
 +{
 +    return g_tree_new(key_compare_func);
 +}
 +
 +static inline QTree *q_tree_new_with_data(GCompareDataFunc key_compare_func,
 +                                          gpointer key_compare_data)
 +{
 +    return g_tree_new_with_data(key_compare_func, key_compare_data);
 +}
 +
 +static inline QTree *q_tree_new_full(GCompareDataFunc key_compare_func,
 +                                     gpointer key_compare_data,
 +                                     GDestroyNotify key_destroy_func,
 +                                     GDestroyNotify value_destroy_func)
 +{
 +    return g_tree_new_full(key_compare_func, key_compare_data,
 +                           key_destroy_func, value_destroy_func);
 +}
 +
 +static inline QTree *q_tree_ref(QTree *tree)
 +{
 +    return g_tree_ref(tree);
 +}
 +
 +static inline void q_tree_unref(QTree *tree)
 +{
 +    g_tree_unref(tree);
 +}
 +
 +static inline void q_tree_destroy(QTree *tree)
 +{
 +    g_tree_destroy(tree);
 +}
 +
 +static inline void q_tree_insert(QTree *tree,
 +                                 gpointer key,
 +                                 gpointer value)
 +{
 +    g_tree_insert(tree, key, value);
 +}
 +
 +static inline void q_tree_replace(QTree *tree,
 +                                  gpointer key,
 +                                  gpointer value)
 +{
 +    g_tree_replace(tree, key, value);
 +}
 +
 +static inline gboolean q_tree_remove(QTree *tree,
 +                                     gconstpointer key)
 +{
 +    return g_tree_remove(tree, key);
 +}
 +
 +static inline gboolean q_tree_steal(QTree *tree,
 +                                    gconstpointer key)
 +{
 +    return g_tree_steal(tree, key);
 +}
 +
 +static inline gpointer q_tree_lookup(QTree *tree,
 +                                     gconstpointer key)
 +{
 +    return g_tree_lookup(tree, key);
 +}
 +
 +static inline gboolean q_tree_lookup_extended(QTree *tree,
 +                                              gconstpointer lookup_key,
 +                                              gpointer *orig_key,
 +                                              gpointer *value)
 +{
 +    return g_tree_lookup_extended(tree, lookup_key, orig_key, value);
 +}
 +
 +static inline void q_tree_foreach(QTree *tree,
 +                                  GTraverseFunc func,
 +                                  gpointer user_data)
 +{
 +    return g_tree_foreach(tree, func, user_data);
 +}
 +
 +static inline gpointer q_tree_search(QTree *tree,
 +                                     GCompareFunc search_func,
 +                                     gconstpointer user_data)
 +{
 +    return g_tree_search(tree, search_func, user_data);
 +}
 +
 +static inline gint q_tree_height(QTree *tree)
 +{
 +    return g_tree_height(tree);
 +}
 +
 +static inline gint q_tree_nnodes(QTree *tree)
 +{
 +    return g_tree_nnodes(tree);
 +}
 +
 +#endif /* HAVE_GLIB_WITH_SLICE_ALLOCATOR */
 +
 +#endif /* QEMU_QTREE_H */
 diff --git a/tests/bench/qtree-bench.c b/tests/bench/qtree-bench.c
 new file mode 100644
 index XXXXXXX..XXXXXXX
 --- /dev/null
 +++ b/tests/bench/qtree-bench.c
@@ -XXX,XX +XXX,XX @@
 +/* SPDX-License-Identifier: GPL-2.0-or-later */
 +#include "qemu/osdep.h"
 +#include "qemu/qtree.h"
 +#include "qemu/timer.h"
 +
 +enum tree_op {
 +    OP_LOOKUP,
 +    OP_INSERT,
 +    OP_REMOVE,
 +    OP_REMOVE_ALL,
 +    OP_TRAVERSE,
 +};
 +
 +struct benchmark {
 +    const char * const name;
 +    enum tree_op op;
 +    bool fill_on_init;
 +};
 +
 +enum impl_type {
 +    IMPL_GTREE,
 +    IMPL_QTREE,
 +};
 +
 +struct tree_implementation {
 +    const char * const name;
 +    enum impl_type type;
 +};
 +
 +static const struct benchmark benchmarks[] = {
 +    {
 +        .name = "Lookup",
 +        .op = OP_LOOKUP,
 +        .fill_on_init = true,
 +    },
 +    {
 +        .name = "Insert",
 +        .op = OP_INSERT,
 +        .fill_on_init = false,
 +    },
 +    {
 +        .name = "Remove",
 +        .op = OP_REMOVE,
 +        .fill_on_init = true,
 +    },
 +    {
 +        .name = "RemoveAll",
 +        .op = OP_REMOVE_ALL,
 +        .fill_on_init = true,
 +    },
 +    {
 +        .name = "Traverse",
 +        .op = OP_TRAVERSE,
 +        .fill_on_init = true,
 +    },
 +};
 +
 +static const struct tree_implementation impls[] = {
 +    {
 +        .name = "GTree",
 +        .type = IMPL_GTREE,
 +    },
 +    {
 +        .name = "QTree",
 +        .type = IMPL_QTREE,
 +    },
 +};
 +
 +static int compare_func(const void *ap, const void *bp)
 +{
 +    const size_t *a = ap;
 +    const size_t *b = bp;
 +
 +    return *a - *b;
 +}
 +
 +static void init_empty_tree_and_keys(enum impl_type impl,
 +                                     void **ret_tree, size_t **ret_keys,
 +                                     size_t n_elems)
 +{
 +    size_t *keys = g_malloc_n(n_elems, sizeof(*keys));
 +    for (size_t i = 0; i < n_elems; i++) {
 +        keys[i] = i;
 +    }
 +
 +    void *tree;
 +    switch (impl) {
 +    case IMPL_GTREE:
 +        tree = g_tree_new(compare_func);
 +        break;
 +    case IMPL_QTREE:
 +        tree = q_tree_new(compare_func);
 +        break;
 +    default:
 +        g_assert_not_reached();
 +    }
 +
 +    *ret_tree = tree;
 +    *ret_keys = keys;
 +}
 +
 +static gboolean traverse_func(gpointer key, gpointer value, gpointer data)
 +{
 +    return FALSE;
 +}
 +
 +static inline void remove_all(void *tree, enum impl_type impl)
 +{
 +    switch (impl) {
 +    case IMPL_GTREE:
 +        g_tree_destroy(tree);
 +        break;
 +    case IMPL_QTREE:
 +        q_tree_destroy(tree);
 +        break;
 +    default:
 +        g_assert_not_reached();
 +    }
 +}
 +
 +static int64_t run_benchmark(const struct benchmark *bench,
 +                             enum impl_type impl,
 +                             size_t n_elems)
 +{
 +    void *tree;
 +    size_t *keys;
 +
 +    init_empty_tree_and_keys(impl, &tree, &keys, n_elems);
 +    if (bench->fill_on_init) {
 +        for (size_t i = 0; i < n_elems; i++) {
 +            switch (impl) {
 +            case IMPL_GTREE:
 +                g_tree_insert(tree, &keys[i], &keys[i]);
 +                break;
 +            case IMPL_QTREE:
 +                q_tree_insert(tree, &keys[i], &keys[i]);
 +                break;
 +            default:
 +                g_assert_not_reached();
 +            }
 +        }
 +    }
 +
 +    int64_t start_ns = get_clock();
 +    switch (bench->op) {
 +    case OP_LOOKUP:
 +        for (size_t i = 0; i < n_elems; i++) {
 +            void *value;
 +            switch (impl) {
 +            case IMPL_GTREE:
 +                value = g_tree_lookup(tree, &keys[i]);
 +                break;
 +            case IMPL_QTREE:
 +                value = q_tree_lookup(tree, &keys[i]);
 +                break;
 +            default:
 +                g_assert_not_reached();
 +            }
 +            (void)value;
 +        }
 +        break;
 +    case OP_INSERT:
 +        for (size_t i = 0; i < n_elems; i++) {
 +            switch (impl) {
 +            case IMPL_GTREE:
 +                g_tree_insert(tree, &keys[i], &keys[i]);
 +                break;
 +            case IMPL_QTREE:
 +                q_tree_insert(tree, &keys[i], &keys[i]);
 +                break;
 +            default:
 +                g_assert_not_reached();
 +            }
 +        }
 +        break;
 +    case OP_REMOVE:
 +        for (size_t i = 0; i < n_elems; i++) {
 +            switch (impl) {
 +            case IMPL_GTREE:
 +                g_tree_remove(tree, &keys[i]);
 +                break;
 +            case IMPL_QTREE:
 +                q_tree_remove(tree, &keys[i]);
 +                break;
 +            default:
 +                g_assert_not_reached();
 +            }
 +        }
 +        break;
 +    case OP_REMOVE_ALL:
 +        remove_all(tree, impl);
 +        break;
 +    case OP_TRAVERSE:
 +        switch (impl) {
 +        case IMPL_GTREE:
 +            g_tree_foreach(tree, traverse_func, NULL);
 +            break;
 +        case IMPL_QTREE:
 +            q_tree_foreach(tree, traverse_func, NULL);
 +            break;
 +        default:
 +            g_assert_not_reached();
 +        }
 +        break;
 +    default:
 +        g_assert_not_reached();
 +    }
 +    int64_t ns = get_clock() - start_ns;
 +
 +    if (bench->op != OP_REMOVE_ALL) {
 +        remove_all(tree, impl);
 +    }
 +    g_free(keys);
 +
 +    return ns;
 +}
 +
 +int main(int argc, char *argv[])
 +{
 +    size_t sizes[] = {
 +        32,
 +        1024,
 +        1024 * 4,
 +        1024 * 128,
 +        1024 * 1024,
 +    };
 +
 +    double res[ARRAY_SIZE(benchmarks)][ARRAY_SIZE(impls)][ARRAY_SIZE(sizes)];
 +    for (int i = 0; i < ARRAY_SIZE(sizes); i++) {
 +        size_t size = sizes[i];
 +        for (int j = 0; j < ARRAY_SIZE(impls); j++) {
 +            const struct tree_implementation *impl = &impls[j];
 +            for (int k = 0; k < ARRAY_SIZE(benchmarks); k++) {
 +                const struct benchmark *bench = &benchmarks[k];
 +
 +                /* warm-up run */
 +                run_benchmark(bench, impl->type, size);
 +
 +                int64_t total_ns = 0;
 +                int64_t n_runs = 0;
 +                while (total_ns < 2e8 || n_runs < 5) {
 +                    total_ns += run_benchmark(bench, impl->type, size);
 +                    n_runs++;
 +                }
 +                double ns_per_run = (double)total_ns / n_runs;
 +
 +                /* Throughput, in Mops/s */
 +                res[k][j][i] = size / ns_per_run * 1e3;
 +            }
 +        }
 +    }
 +
 +    printf("# Results' breakdown: Tree, Op and #Elements. Units: Mops/s\n");
 +    printf("%5s %10s ", "Tree", "Op");
 +    for (int i = 0; i < ARRAY_SIZE(sizes); i++) {
 +        printf("%7zu         ", sizes[i]);
 +    }
 +    printf("\n");
 +    char separator[97];
 +    for (int i = 0; i < ARRAY_SIZE(separator) - 1; i++) {
 +        separator[i] = '-';
 +    }
 +    separator[ARRAY_SIZE(separator) - 1] = '\0';
 +    printf("%s\n", separator);
 +    for (int i = 0; i < ARRAY_SIZE(benchmarks); i++) {
 +        for (int j = 0; j < ARRAY_SIZE(impls); j++) {
 +            printf("%5s %10s ", impls[j].name, benchmarks[i].name);
 +            for (int k = 0; k < ARRAY_SIZE(sizes); k++) {
 +                printf("%7.2f ", res[i][j][k]);
 +                if (j == 0) {
 +                    printf("        ");
 +                } else {
 +                    if (res[i][0][k] != 0) {
 +                        double speedup = res[i][j][k] / res[i][0][k];
 +                        printf("(%4.2fx) ", speedup);
 +                    } else {
 +                        printf("(     ) ");
 +                    }
 +                }
 +            }
 +            printf("\n");
 +        }
 +    }
 +    printf("%s\n", separator);
 +    return 0;
 +}
 diff --git a/tests/unit/test-qtree.c b/tests/unit/test-qtree.c
 new file mode 100644
 index XXXXXXX..XXXXXXX
 --- /dev/null
 +++ b/tests/unit/test-qtree.c
@@ -XXX,XX +XXX,XX @@
 +/*
 + * SPDX-License-Identifier: LGPL-2.1-or-later
 + *
 + * Tests for QTree.
 + * Original source: glib
 + *   https://gitlab.gnome.org/GNOME/glib/-/blob/main/glib/tests/tree.c
 + *   LGPL license.
 + *   Copyright (C) 1995-1997 Peter Mattis, Spencer Kimball and Josh MacDonald
 + */
 +
 +#include "qemu/osdep.h"
 +#include "qemu/qtree.h"
 +
 +static gint my_compare(gconstpointer a, gconstpointer b)
 +{
 +    const char *cha = a;
 +    const char *chb = b;
 +
 +    return *cha - *chb;
 +}
 +
 +static gint my_compare_with_data(gconstpointer a,
 +                                 gconstpointer b,
 +                                 gpointer user_data)
 +{
 +    const char *cha = a;
 +    const char *chb = b;
 +
 +    /* just check that we got the right data */
 +    g_assert(GPOINTER_TO_INT(user_data) == 123);
 +
 +    return *cha - *chb;
 +}
 +
 +static gint my_search(gconstpointer a, gconstpointer b)
 +{
 +    return my_compare(b, a);
 +}
 +
 +static gpointer destroyed_key;
 +static gpointer destroyed_value;
 +static guint destroyed_key_count;
 +static guint destroyed_value_count;
 +
 +static void my_key_destroy(gpointer key)
 +{
 +    destroyed_key = key;
 +    destroyed_key_count++;
 +}
 +
 +static void my_value_destroy(gpointer value)
 +{
 +    destroyed_value = value;
 +    destroyed_value_count++;
 +}
 +
 +static gint my_traverse(gpointer key, gpointer value, gpointer data)
 +{
 +    char *ch = key;
 +
 +    g_assert((*ch) > 0);
 +
 +    if (*ch == 'd') {
 +        return TRUE;
 +    }
 +
 +    return FALSE;
 +}
 +
 +char chars[] =
 +    "0123456789"
 +    "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 +    "abcdefghijklmnopqrstuvwxyz";
 +
 +char chars2[] =
 +    "0123456789"
 +    "abcdefghijklmnopqrstuvwxyz";
 +
 +static gint check_order(gpointer key, gpointer value, gpointer data)
 +{
 +    char **p = data;
 +    char *ch = key;
 +
 +    g_assert(**p == *ch);
 +
 +    (*p)++;
 +
 +    return FALSE;
 +}
 +
 +static void test_tree_search(void)
 +{
 +    gint i;
 +    QTree *tree;
 +    gboolean removed;
 +    gchar c;
 +    gchar *p, *d;
 +
 +    tree = q_tree_new_with_data(my_compare_with_data, GINT_TO_POINTER(123));
 +
 +    for (i = 0; chars[i]; i++) {
 +        q_tree_insert(tree, &chars[i], &chars[i]);
 +    }
 +
 +    q_tree_foreach(tree, my_traverse, NULL);
 +
 +    g_assert(q_tree_nnodes(tree) == strlen(chars));
 +    g_assert(q_tree_height(tree) == 6);
 +
 +    p = chars;
 +    q_tree_foreach(tree, check_order, &p);
 +
 +    for (i = 0; i < 26; i++) {
 +        removed = q_tree_remove(tree, &chars[i + 10]);
 +        g_assert(removed);
 +    }
 +
 +    c = '\0';
 +    removed = q_tree_remove(tree, &c);
 +    g_assert(!removed);
 +
 +    q_tree_foreach(tree, my_traverse, NULL);
 +
 +    g_assert(q_tree_nnodes(tree) == strlen(chars2));
 +    g_assert(q_tree_height(tree) == 6);
 +
 +    p = chars2;
 +    q_tree_foreach(tree, check_order, &p);
 +
 +    for (i = 25; i >= 0; i--) {
 +        q_tree_insert(tree, &chars[i + 10], &chars[i + 10]);
 +    }
 +
 +    p = chars;
 +    q_tree_foreach(tree, check_order, &p);
 +
 +    c = '0';
 +    p = q_tree_lookup(tree, &c);
 +    g_assert(p && *p == c);
 +    g_assert(q_tree_lookup_extended(tree, &c, (gpointer *)&d, (gpointer *)&p));
 +    g_assert(c == *d && c == *p);
 +
 +    c = 'A';
 +    p = q_tree_lookup(tree, &c);
 +    g_assert(p && *p == c);
 +
 +    c = 'a';
 +    p = q_tree_lookup(tree, &c);
 +    g_assert(p && *p == c);
 +
 +    c = 'z';
 +    p = q_tree_lookup(tree, &c);
 +    g_assert(p && *p == c);
 +
 +    c = '!';
 +    p = q_tree_lookup(tree, &c);
 +    g_assert(p == NULL);
 +
 +    c = '=';
 +    p = q_tree_lookup(tree, &c);
 +    g_assert(p == NULL);
 +
 +    c = '|';
 +    p = q_tree_lookup(tree, &c);
 +    g_assert(p == NULL);
 +
 +    c = '0';
 +    p = q_tree_search(tree, my_search, &c);
 +    g_assert(p && *p == c);
 +
 +    c = 'A';
 +    p = q_tree_search(tree, my_search, &c);
 +    g_assert(p && *p == c);
 +
 +    c = 'a';
 +    p = q_tree_search(tree, my_search, &c);
 +    g_assert(p && *p == c);
 +
 +    c = 'z';
 +    p = q_tree_search(tree, my_search, &c);
 +    g_assert(p && *p == c);
 +
 +    c = '!';
 +    p = q_tree_search(tree, my_search, &c);
 +    g_assert(p == NULL);
 +
 +    c = '=';
 +    p = q_tree_search(tree, my_search, &c);
 +    g_assert(p == NULL);
 +
 +    c = '|';
 +    p = q_tree_search(tree, my_search, &c);
 +    g_assert(p == NULL);
 +
 +    q_tree_destroy(tree);
 +}
 +
 +static void test_tree_remove(void)
 +{
 +    QTree *tree;
 +    char c, d;
 +    gint i;
 +    gboolean removed;
 +
 +    tree = q_tree_new_full((GCompareDataFunc)my_compare, NULL,
 +                           my_key_destroy,
 +                           my_value_destroy);
 +
 +    for (i = 0; chars[i]; i++) {
 +        q_tree_insert(tree, &chars[i], &chars[i]);
 +    }
 +
 +    c = '0';
 +    q_tree_insert(tree, &c, &c);
 +    g_assert(destroyed_key == &c);
 +    g_assert(destroyed_value == &chars[0]);
 +    destroyed_key = NULL;
 +    destroyed_value = NULL;
 +
 +    d = '1';
 +    q_tree_replace(tree, &d, &d);
 +    g_assert(destroyed_key == &chars[1]);
 +    g_assert(destroyed_value == &chars[1]);
 +    destroyed_key = NULL;
 +    destroyed_value = NULL;
 +
 +    c = '2';
 +    removed = q_tree_remove(tree, &c);
 +    g_assert(removed);
 +    g_assert(destroyed_key == &chars[2]);
 +    g_assert(destroyed_value == &chars[2]);
 +    destroyed_key = NULL;
 +    destroyed_value = NULL;
 +
 +    c = '3';
 +    removed = q_tree_steal(tree, &c);
 +    g_assert(removed);
 +    g_assert(destroyed_key == NULL);
 +    g_assert(destroyed_value == NULL);
 +
 +    const gchar *remove = "omkjigfedba";
 +    for (i = 0; remove[i]; i++) {
 +        removed = q_tree_remove(tree, &remove[i]);
 +        g_assert(removed);
 +    }
 +
 +    q_tree_destroy(tree);
 +}
 +
 +static void test_tree_destroy(void)
 +{
 +    QTree *tree;
 +    gint i;
 +
 +    tree = q_tree_new(my_compare);
 +
 +    for (i = 0; chars[i]; i++) {
 +        q_tree_insert(tree, &chars[i], &chars[i]);
 +    }
 +
 +    g_assert(q_tree_nnodes(tree) == strlen(chars));
 +
 +    g_test_message("nnodes: %d", q_tree_nnodes(tree));
 +    q_tree_ref(tree);
 +    q_tree_destroy(tree);
 +
 +    g_test_message("nnodes: %d", q_tree_nnodes(tree));
 +    g_assert(q_tree_nnodes(tree) == 0);
 +
 +    q_tree_unref(tree);
 +}
 +
 +static void test_tree_insert(void)
 +{
 +    QTree *tree;
 +    gchar *p;
 +    gint i;
 +    gchar *scrambled;
 +
 +    tree = q_tree_new(my_compare);
 +
 +    for (i = 0; chars[i]; i++) {
 +        q_tree_insert(tree, &chars[i], &chars[i]);
 +    }
 +    p = chars;
 +    q_tree_foreach(tree, check_order, &p);
 +
 +    q_tree_unref(tree);
 +    tree = q_tree_new(my_compare);
 +
 +    for (i = strlen(chars) - 1; i >= 0; i--) {
 +        q_tree_insert(tree, &chars[i], &chars[i]);
 +    }
 +    p = chars;
 +    q_tree_foreach(tree, check_order, &p);
 +
 +    q_tree_unref(tree);
 +    tree = q_tree_new(my_compare);
 +
 +    scrambled = g_strdup(chars);
 +
 +    for (i = 0; i < 30; i++) {
 +        gchar tmp;
 +        gint a, b;
 +
 +        a = g_random_int_range(0, strlen(scrambled));
 +        b = g_random_int_range(0, strlen(scrambled));
 +        tmp = scrambled[a];
 +        scrambled[a] = scrambled[b];
 +        scrambled[b] = tmp;
 +    }
 +
 +    for (i = 0; scrambled[i]; i++) {
 +        q_tree_insert(tree, &scrambled[i], &scrambled[i]);
 +    }
 +    p = chars;
 +    q_tree_foreach(tree, check_order, &p);
 +
 +    g_free(scrambled);
 +    q_tree_unref(tree);
 +}
 +
 +int main(int argc, char *argv[])
 +{
 +    g_test_init(&argc, &argv, NULL);
 +
 +    g_test_add_func("/qtree/search", test_tree_search);
 +    g_test_add_func("/qtree/remove", test_tree_remove);
 +    g_test_add_func("/qtree/destroy", test_tree_destroy);
 +    g_test_add_func("/qtree/insert", test_tree_insert);
 +
 +    return g_test_run();
 +}
 diff --git a/util/qtree.c b/util/qtree.c
 new file mode 100644
 index XXXXXXX..XXXXXXX
 --- /dev/null
 +++ b/util/qtree.c
@@ -XXX,XX +XXX,XX @@
 +/*
 + * GLIB - Library of useful routines for C programming
 + * Copyright (C) 1995-1997  Peter Mattis, Spencer Kimball and Josh MacDonald
 + *
 + * SPDX-License-Identifier: LGPL-2.1-or-later
 + *
 + * This library is free software; you can redistribute it and/or
 + * modify it under the terms of the GNU Lesser General Public
 + * License as published by the Free Software Foundation; either
 + * version 2.1 of the License, or (at your option) any later version.
 + *
 + * This library is distributed in the hope that it will be useful,
 + * but WITHOUT ANY WARRANTY; without even the implied warranty of
 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 + * Lesser General Public License for more details.
 + *
 + * You should have received a copy of the GNU Lesser General Public
 + * License along with this library; if not, see <http://www.gnu.org/licenses/>.
 + */
 +
 +/*
 + * Modified by the GLib Team and others 1997-2000.  See the AUTHORS
 + * file for a list of people on the GLib Team.  See the ChangeLog
 + * files for a list of changes.  These files are distributed with
 + * GLib at ftp://ftp.gtk.org/pub/gtk/.
 + */
 +
 +/*
 + * MT safe
 + */
 +
 +#include "qemu/osdep.h"
 +#include "qemu/qtree.h"
 +
 +/**
 + * SECTION:trees-binary
 + * @title: Balanced Binary Trees
 + * @short_description: a sorted collection of key/value pairs optimized
 + *                     for searching and traversing in order
 + *
 + * The #QTree structure and its associated functions provide a sorted
 + * collection of key/value pairs optimized for searching and traversing
 + * in order. This means that most of the operations  (access, search,
 + * insertion, deletion, ...) on #QTree are O(log(n)) in average and O(n)
 + * in worst case for time complexity. But, note that maintaining a
 + * balanced sorted #QTree of n elements is done in time O(n log(n)).
 + *
 + * To create a new #QTree use q_tree_new().
 + *
 + * To insert a key/value pair into a #QTree use q_tree_insert()
 + * (O(n log(n))).
 + *
 + * To remove a key/value pair use q_tree_remove() (O(n log(n))).
 + *
 + * To look up the value corresponding to a given key, use
 + * q_tree_lookup() and q_tree_lookup_extended().
 + *
 + * To find out the number of nodes in a #QTree, use q_tree_nnodes(). To
 + * get the height of a #QTree, use q_tree_height().
 + *
 + * To traverse a #QTree, calling a function for each node visited in
 + * the traversal, use q_tree_foreach().
 + *
 + * To destroy a #QTree, use q_tree_destroy().
 + **/
 +
 +#define MAX_GTREE_HEIGHT 40
 +
 +/**
 + * QTree:
 + *
 + * The QTree struct is an opaque data structure representing a
 + * [balanced binary tree][glib-Balanced-Binary-Trees]. It should be
 + * accessed only by using the following functions.
 + */
 +struct _QTree {
 +    QTreeNode        *root;
 +    GCompareDataFunc  key_compare;
 +    GDestroyNotify    key_destroy_func;
 +    GDestroyNotify    value_destroy_func;
 +    gpointer          key_compare_data;
 +    guint             nnodes;
 +    gint              ref_count;
 +};
 +
 +struct _QTreeNode {
 +    gpointer   key;         /* key for this node */
 +    gpointer   value;       /* value stored at this node */
 +    QTreeNode *left;        /* left subtree */
 +    QTreeNode *right;       /* right subtree */
 +    gint8      balance;     /* height (right) - height (left) */
 +    guint8     left_child;
 +    guint8     right_child;
 +};
 +
 +
 +static QTreeNode *q_tree_node_new(gpointer       key,
 +                                  gpointer       value);
 +static QTreeNode *q_tree_insert_internal(QTree *tree,
 +                                         gpointer key,
 +                                         gpointer value,
 +                                         gboolean replace);
 +static gboolean   q_tree_remove_internal(QTree         *tree,
 +                                         gconstpointer  key,
 +                                         gboolean       steal);
 +static QTreeNode *q_tree_node_balance(QTreeNode     *node);
 +static QTreeNode *q_tree_find_node(QTree         *tree,
 +                                   gconstpointer  key);
 +static QTreeNode *q_tree_node_search(QTreeNode *node,
 +                                     GCompareFunc search_func,
 +                                     gconstpointer data);
 +static QTreeNode *q_tree_node_rotate_left(QTreeNode     *node);
 +static QTreeNode *q_tree_node_rotate_right(QTreeNode     *node);
 +#ifdef Q_TREE_DEBUG
 +static void       q_tree_node_check(QTreeNode     *node);
 +#endif
 +
 +static QTreeNode*
 +q_tree_node_new(gpointer key,
 +                gpointer value)
 +{
 +    QTreeNode *node = g_new(QTreeNode, 1);
 +
 +    node->balance = 0;
 +    node->left = NULL;
 +    node->right = NULL;
 +    node->left_child = FALSE;
 +    node->right_child = FALSE;
 +    node->key = key;
 +    node->value = value;
 +
 +    return node;
 +}
 +
 +/**
 + * q_tree_new:
 + * @key_compare_func: the function used to order the nodes in the #QTree.
 + *   It should return values similar to the standard strcmp() function -
 + *   0 if the two arguments are equal, a negative value if the first argument
 + *   comes before the second, or a positive value if the first argument comes
 + *   after the second.
 + *
 + * Creates a new #QTree.
 + *
 + * Returns: a newly allocated #QTree
 + */
 +QTree *
 +q_tree_new(GCompareFunc key_compare_func)
 +{
 +    g_return_val_if_fail(key_compare_func != NULL, NULL);
 +
 +    return q_tree_new_full((GCompareDataFunc) key_compare_func, NULL,
 +                           NULL, NULL);
 +}
 +
 +/**
 + * q_tree_new_with_data:
 + * @key_compare_func: qsort()-style comparison function
 + * @key_compare_data: data to pass to comparison function
 + *
 + * Creates a new #QTree with a comparison function that accepts user data.
 + * See q_tree_new() for more details.
 + *
 + * Returns: a newly allocated #QTree
 + */
 +QTree *
 +q_tree_new_with_data(GCompareDataFunc key_compare_func,
 +                     gpointer         key_compare_data)
 +{
 +    g_return_val_if_fail(key_compare_func != NULL, NULL);
 +
 +    return q_tree_new_full(key_compare_func, key_compare_data,
 +                           NULL, NULL);
 +}
 +
 +/**
 + * q_tree_new_full:
 + * @key_compare_func: qsort()-style comparison function
 + * @key_compare_data: data to pass to comparison function
 + * @key_destroy_func: a function to free the memory allocated for the key
 + *   used when removing the entry from the #QTree or %NULL if you don't
 + *   want to supply such a function
 + * @value_destroy_func: a function to free the memory allocated for the
 + *   value used when removing the entry from the #QTree or %NULL if you
 + *   don't want to supply such a function
 + *
 + * Creates a new #QTree like q_tree_new() and allows to specify functions
 + * to free the memory allocated for the key and value that get called when
 + * removing the entry from the #QTree.
 + *
 + * Returns: a newly allocated #QTree
 + */
 +QTree *
 +q_tree_new_full(GCompareDataFunc key_compare_func,
 +                gpointer         key_compare_data,
 +                GDestroyNotify   key_destroy_func,
 +                GDestroyNotify   value_destroy_func)
 +{
 +    QTree *tree;
 +
 +    g_return_val_if_fail(key_compare_func != NULL, NULL);
 +
 +    tree = g_new(QTree, 1);
 +    tree->root               = NULL;
 +    tree->key_compare        = key_compare_func;
 +    tree->key_destroy_func   = key_destroy_func;
 +    tree->value_destroy_func = value_destroy_func;
 +    tree->key_compare_data   = key_compare_data;
 +    tree->nnodes             = 0;
 +    tree->ref_count          = 1;
 +
 +    return tree;
 +}
 +
 +/**
 + * q_tree_node_first:
 + * @tree: a #QTree
 + *
 + * Returns the first in-order node of the tree, or %NULL
 + * for an empty tree.
 + *
 + * Returns: (nullable) (transfer none): the first node in the tree
 + *
 + * Since: 2.68 in GLib. Internal in Qtree, i.e. not in the public API.
 + */
 +static QTreeNode *
 +q_tree_node_first(QTree *tree)
 +{
 +    QTreeNode *tmp;
 +
 +    g_return_val_if_fail(tree != NULL, NULL);
 +
 +    if (!tree->root) {
 +        return NULL;
 +    }
 +
 +    tmp = tree->root;
 +
 +    while (tmp->left_child) {
 +        tmp = tmp->left;
 +    }
 +
 +    return tmp;
 +}
 +
 +/**
 + * q_tree_node_previous
 + * @node: a #QTree node
 + *
 + * Returns the previous in-order node of the tree, or %NULL
 + * if the passed node was already the first one.
 + *
 + * Returns: (nullable) (transfer none): the previous node in the tree
 + *
 + * Since: 2.68 in GLib. Internal in Qtree, i.e. not in the public API.
 + */
 +static QTreeNode *
 +q_tree_node_previous(QTreeNode *node)
 +{
 +    QTreeNode *tmp;
 +
 +    g_return_val_if_fail(node != NULL, NULL);
 +
 +    tmp = node->left;
 +
 +    if (node->left_child) {
 +        while (tmp->right_child) {
 +            tmp = tmp->right;
 +        }
 +    }
 +
 +    return tmp;
 +}
 +
 +/**
 + * q_tree_node_next
 + * @node: a #QTree node
 + *
 + * Returns the next in-order node of the tree, or %NULL
 + * if the passed node was already the last one.
 + *
 + * Returns: (nullable) (transfer none): the next node in the tree
 + *
 + * Since: 2.68 in GLib. Internal in Qtree, i.e. not in the public API.
 + */
 +static QTreeNode *
 +q_tree_node_next(QTreeNode *node)
 +{
 +    QTreeNode *tmp;
 +
 +    g_return_val_if_fail(node != NULL, NULL);
 +
 +    tmp = node->right;
 +
 +    if (node->right_child) {
 +        while (tmp->left_child) {
 +            tmp = tmp->left;
 +        }
 +    }
 +
 +    return tmp;
 +}
 +
 +/**
 + * q_tree_remove_all:
 + * @tree: a #QTree
 + *
 + * Removes all nodes from a #QTree and destroys their keys and values,
 + * then resets the #QTree’s root to %NULL.
 + *
 + * Since: 2.70 in GLib. Internal in Qtree, i.e. not in the public API.
 + */
 +static void
 +q_tree_remove_all(QTree *tree)
 +{
 +    QTreeNode *node;
 +    QTreeNode *next;
 +
 +    g_return_if_fail(tree != NULL);
 +
 +    node = q_tree_node_first(tree);
 +
 +    while (node) {
 +        next = q_tree_node_next(node);
 +
 +        if (tree->key_destroy_func) {
 +            tree->key_destroy_func(node->key);
 +        }
 +        if (tree->value_destroy_func) {
 +            tree->value_destroy_func(node->value);
 +        }
 +        g_free(node);
 +
 +#ifdef Q_TREE_DEBUG
 +        g_assert(tree->nnodes > 0);
 +        tree->nnodes--;
 +#endif
 +
 +        node = next;
 +    }
 +
 +#ifdef Q_TREE_DEBUG
 +    g_assert(tree->nnodes == 0);
 +#endif
 +
 +    tree->root = NULL;
 +#ifndef Q_TREE_DEBUG
 +    tree->nnodes = 0;
 +#endif
 +}
 +
 +/**
 + * q_tree_ref:
 + * @tree: a #QTree
 + *
 + * Increments the reference count of @tree by one.
 + *
 + * It is safe to call this function from any thread.
 + *
 + * Returns: the passed in #QTree
 + *
 + * Since: 2.22
 + */
 +QTree *
 +q_tree_ref(QTree *tree)
 +{
 +    g_return_val_if_fail(tree != NULL, NULL);
 +
 +    g_atomic_int_inc(&tree->ref_count);
 +
 +    return tree;
 +}
 +
 +/**
 + * q_tree_unref:
 + * @tree: a #QTree
 + *
 + * Decrements the reference count of @tree by one.
 + * If the reference count drops to 0, all keys and values will
 + * be destroyed (if destroy functions were specified) and all
 + * memory allocated by @tree will be released.
 + *
 + * It is safe to call this function from any thread.
 + *
 + * Since: 2.22
 + */
 +void
 +q_tree_unref(QTree *tree)
 +{
 +    g_return_if_fail(tree != NULL);
 +
 +    if (g_atomic_int_dec_and_test(&tree->ref_count)) {
 +        q_tree_remove_all(tree);
 +        g_free(tree);
 +    }
 +}
 +
 +/**
 + * q_tree_destroy:
 + * @tree: a #QTree
 + *
 + * Removes all keys and values from the #QTree and decreases its
 + * reference count by one. If keys and/or values are dynamically
 + * allocated, you should either free them first or create the #QTree
 + * using q_tree_new_full(). In the latter case the destroy functions
 + * you supplied will be called on all keys and values before destroying
 + * the #QTree.
 + */
 +void
 +q_tree_destroy(QTree *tree)
 +{
 +    g_return_if_fail(tree != NULL);
 +
 +    q_tree_remove_all(tree);
 +    q_tree_unref(tree);
 +}
 +
 +/**
 + * q_tree_insert_node:
 + * @tree: a #QTree
 + * @key: the key to insert
 + * @value: the value corresponding to the key
 + *
 + * Inserts a key/value pair into a #QTree.
 + *
 + * If the given key already exists in the #QTree its corresponding value
 + * is set to the new value. If you supplied a @value_destroy_func when
 + * creating the #QTree, the old value is freed using that function. If
 + * you supplied a @key_destroy_func when creating the #QTree, the passed
 + * key is freed using that function.
 + *
 + * The tree is automatically 'balanced' as new key/value pairs are added,
 + * so that the distance from the root to every leaf is as small as possible.
 + * The cost of maintaining a balanced tree while inserting new key/value
 + * result in a O(n log(n)) operation where most of the other operations
 + * are O(log(n)).
 + *
 + * Returns: (transfer none): the inserted (or set) node.
 + *
 + * Since: 2.68 in GLib. Internal in Qtree, i.e. not in the public API.
 + */
 +static QTreeNode *
 +q_tree_insert_node(QTree    *tree,
 +                   gpointer  key,
 +                   gpointer  value)
 +{
 +    QTreeNode *node;
 +
 +    g_return_val_if_fail(tree != NULL, NULL);
 +
 +    node = q_tree_insert_internal(tree, key, value, FALSE);
 +
 +#ifdef Q_TREE_DEBUG
 +    q_tree_node_check(tree->root);
 +#endif
 +
 +    return node;
 +}
 +
 +/**
 + * q_tree_insert:
 + * @tree: a #QTree
 + * @key: the key to insert
 + * @value: the value corresponding to the key
 + *
 + * Inserts a key/value pair into a #QTree.
 + *
 + * Inserts a new key and value into a #QTree as q_tree_insert_node() does,
 + * only this function does not return the inserted or set node.
 + */
 +void
 +q_tree_insert(QTree    *tree,
 +              gpointer  key,
 +              gpointer  value)
 +{
 +    q_tree_insert_node(tree, key, value);
 +}
 +
 +/**
 + * q_tree_replace_node:
 + * @tree: a #QTree
 + * @key: the key to insert
 + * @value: the value corresponding to the key
 + *
 + * Inserts a new key and value into a #QTree similar to q_tree_insert_node().
 + * The difference is that if the key already exists in the #QTree, it gets
 + * replaced by the new key. If you supplied a @value_destroy_func when
 + * creating the #QTree, the old value is freed using that function. If you
 + * supplied a @key_destroy_func when creating the #QTree, the old key is
 + * freed using that function.
 + *
 + * The tree is automatically 'balanced' as new key/value pairs are added,
 + * so that the distance from the root to every leaf is as small as possible.
 + *
 + * Returns: (transfer none): the inserted (or set) node.
 + *
 + * Since: 2.68 in GLib. Internal in Qtree, i.e. not in the public API.
 + */
 +static QTreeNode *
 +q_tree_replace_node(QTree    *tree,
 +                    gpointer  key,
 +                    gpointer  value)
 +{
 +    QTreeNode *node;
 +
 +    g_return_val_if_fail(tree != NULL, NULL);
 +
 +    node = q_tree_insert_internal(tree, key, value, TRUE);
 +
 +#ifdef Q_TREE_DEBUG
 +    q_tree_node_check(tree->root);
 +#endif
 +
 +    return node;
 +}
 +
 +/**
 + * q_tree_replace:
 + * @tree: a #QTree
 + * @key: the key to insert
 + * @value: the value corresponding to the key
 + *
 + * Inserts a new key and value into a #QTree as q_tree_replace_node() does,
 + * only this function does not return the inserted or set node.
 + */
 +void
 +q_tree_replace(QTree    *tree,
 +               gpointer  key,
 +               gpointer  value)
 +{
 +    q_tree_replace_node(tree, key, value);
 +}
 +
 +/* internal insert routine */
 +static QTreeNode *
 +q_tree_insert_internal(QTree    *tree,
 +                       gpointer  key,
 +                       gpointer  value,
 +                       gboolean  replace)
 +{
 +    QTreeNode *node, *retnode;
 +    QTreeNode *path[MAX_GTREE_HEIGHT];
 +    int idx;
 +
 +    g_return_val_if_fail(tree != NULL, NULL);
 +
 +    if (!tree->root) {
 +        tree->root = q_tree_node_new(key, value);
 +        tree->nnodes++;
 +        return tree->root;
 +    }
 +
 +    idx = 0;
 +    path[idx++] = NULL;
 +    node = tree->root;
 +
 +    while (1) {
 +        int cmp = tree->key_compare(key, node->key, tree->key_compare_data);
 +
 +        if (cmp == 0) {
 +            if (tree->value_destroy_func) {
 +                tree->value_destroy_func(node->value);
 +            }
 +
 +            node->value = value;
 +
 +            if (replace) {
 +                if (tree->key_destroy_func) {
 +                    tree->key_destroy_func(node->key);
 +                }
 +
 +                node->key = key;
 +            } else {
 +                /* free the passed key */
 +                if (tree->key_destroy_func) {
 +                    tree->key_destroy_func(key);
 +                }
 +            }
 +
 +            return node;
 +        } else if (cmp < 0) {
 +            if (node->left_child) {
 +                path[idx++] = node;
 +                node = node->left;
 +            } else {
 +                QTreeNode *child = q_tree_node_new(key, value);
 +
 +                child->left = node->left;
 +                child->right = node;
 +                node->left = child;
 +                node->left_child = TRUE;
 +                node->balance -= 1;
 +
 +                tree->nnodes++;
 +
 +                retnode = child;
 +                break;
 +            }
 +        } else {
 +            if (node->right_child) {
 +                path[idx++] = node;
 +                node = node->right;
 +            } else {
 +                QTreeNode *child = q_tree_node_new(key, value);
 +
 +                child->right = node->right;
 +                child->left = node;
 +                node->right = child;
 +                node->right_child = TRUE;
 +                node->balance += 1;
 +
 +                tree->nnodes++;
 +
 +                retnode = child;
 +                break;
 +            }
 +        }
 +    }
 +
 +    /*
 +     * Restore balance. This is the goodness of a non-recursive
 +     * implementation, when we are done with balancing we 'break'
 +     * the loop and we are done.
 +     */
 +    while (1) {
 +        QTreeNode *bparent = path[--idx];
 +        gboolean left_node = (bparent && node == bparent->left);
 +        g_assert(!bparent || bparent->left == node || bparent->right == node);
 +
 +        if (node->balance < -1 || node->balance > 1) {
 +            node = q_tree_node_balance(node);
 +            if (bparent == NULL) {
 +                tree->root = node;
 +            } else if (left_node) {
 +                bparent->left = node;
 +            } else {
 +                bparent->right = node;
 +            }
 +        }
 +
 +        if (node->balance == 0 || bparent == NULL) {
 +            break;
 +        }
 +
 +        if (left_node) {
 +            bparent->balance -= 1;
 +        } else {
 +            bparent->balance += 1;
 +        }
 +
 +        node = bparent;
 +    }
 +
 +    return retnode;
 +}
 +
 +/**
 + * q_tree_remove:
 + * @tree: a #QTree
 + * @key: the key to remove
 + *
 + * Removes a key/value pair from a #QTree.
 + *
 + * If the #QTree was created using q_tree_new_full(), the key and value
 + * are freed using the supplied destroy functions, otherwise you have to
 + * make sure that any dynamically allocated values are freed yourself.
 + * If the key does not exist in the #QTree, the function does nothing.
 + *
 + * The cost of maintaining a balanced tree while removing a key/value
 + * result in a O(n log(n)) operation where most of the other operations
 + * are O(log(n)).
 + *
 + * Returns: %TRUE if the key was found (prior to 2.8, this function
 + *     returned nothing)
 + */
 +gboolean
 +q_tree_remove(QTree         *tree,
 +              gconstpointer  key)
 +{
 +    gboolean removed;
 +
 +    g_return_val_if_fail(tree != NULL, FALSE);
 +
 +    removed = q_tree_remove_internal(tree, key, FALSE);
 +
 +#ifdef Q_TREE_DEBUG
 +    q_tree_node_check(tree->root);
 +#endif
 +
 +    return removed;
 +}
 +
 +/**
 + * q_tree_steal:
 + * @tree: a #QTree
 + * @key: the key to remove
 + *
 + * Removes a key and its associated value from a #QTree without calling
 + * the key and value destroy functions.
 + *
 + * If the key does not exist in the #QTree, the function does nothing.
 + *
 + * Returns: %TRUE if the key was found (prior to 2.8, this function
 + *     returned nothing)
 + */
 +gboolean
 +q_tree_steal(QTree         *tree,
 +             gconstpointer  key)
 +{
 +    gboolean removed;
 +
 +    g_return_val_if_fail(tree != NULL, FALSE);
 +
 +    removed = q_tree_remove_internal(tree, key, TRUE);
 +
 +#ifdef Q_TREE_DEBUG
 +    q_tree_node_check(tree->root);
 +#endif
 +
 +    return removed;
 +}
 +
 +/* internal remove routine */
 +static gboolean
 +q_tree_remove_internal(QTree         *tree,
 +                       gconstpointer  key,
 +                       gboolean       steal)
 +{
 +    QTreeNode *node, *parent, *balance;
 +    QTreeNode *path[MAX_GTREE_HEIGHT];
 +    int idx;
 +    gboolean left_node;
 +
 +    g_return_val_if_fail(tree != NULL, FALSE);
 +
 +    if (!tree->root) {
 +        return FALSE;
 +    }
 +
 +    idx = 0;
 +    path[idx++] = NULL;
 +    node = tree->root;
 +
 +    while (1) {
 +        int cmp = tree->key_compare(key, node->key, tree->key_compare_data);
 +
 +        if (cmp == 0) {
 +            break;
 +        } else if (cmp < 0) {
 +            if (!node->left_child) {
 +                return FALSE;
 +            }
 +
 +            path[idx++] = node;
 +            node = node->left;
 +        } else {
 +            if (!node->right_child) {
 +                return FALSE;
 +            }
 +
 +            path[idx++] = node;
 +            node = node->right;
 +        }
 +    }
 +
 +    /*
 +     * The following code is almost equal to q_tree_remove_node,
 +     * except that we do not have to call q_tree_node_parent.
 +     */
 +    balance = parent = path[--idx];
 +    g_assert(!parent || parent->left == node || parent->right == node);
 +    left_node = (parent && node == parent->left);
 +
 +    if (!node->left_child) {
 +        if (!node->right_child) {
 +            if (!parent) {
 +                tree->root = NULL;
 +            } else if (left_node) {
 +                parent->left_child = FALSE;
 +                parent->left = node->left;
 +                parent->balance += 1;
 +            } else {
 +                parent->right_child = FALSE;
 +                parent->right = node->right;
 +                parent->balance -= 1;
 +            }
 +        } else {
 +            /* node has a right child */
 +            QTreeNode *tmp = q_tree_node_next(node);
 +            tmp->left = node->left;
 +
 +            if (!parent) {
 +                tree->root = node->right;
 +            } else if (left_node) {
 +                parent->left = node->right;
 +                parent->balance += 1;
 +            } else {
 +                parent->right = node->right;
 +                parent->balance -= 1;
 +            }
 +        }
 +    } else {
-+        insn = OPC_NOP;
++        /* node has a left child */
-+    }
++        if (!node->right_child) {
-+    qatomic_set((uint32_t *)jmp_rw, insn);
++            QTreeNode *tmp = q_tree_node_previous(node);
-+    flush_idcache_range(jmp_rx, jmp_rw, 4);
++            tmp->right = node->right;
- }
++
++            if (parent == NULL) {
- static void tcg_out_op(TCGContext *s, TCGOpcode opc,
++                tree->root = node->left;
 +            } else if (left_node) {
 +                parent->left = node->left;
 +                parent->balance += 1;
 +            } else {
 +                parent->right = node->left;
 +                parent->balance -= 1;
 +            }
 +        } else {
 +            /* node has a both children (pant, pant!) */
 +            QTreeNode *prev = node->left;
 +            QTreeNode *next = node->right;
 +            QTreeNode *nextp = node;
 +            int old_idx = idx + 1;
 +            idx++;
 +
 +            /* path[idx] == parent */
 +            /* find the immediately next node (and its parent) */
 +            while (next->left_child) {
 +                path[++idx] = nextp = next;
 +                next = next->left;
 +            }
 +
 +            path[old_idx] = next;
 +            balance = path[idx];
 +
 +            /* remove 'next' from the tree */
 +            if (nextp != node) {
 +                if (next->right_child) {
 +                    nextp->left = next->right;
 +                } else {
 +                    nextp->left_child = FALSE;
 +                }
 +                nextp->balance += 1;
 +
 +                next->right_child = TRUE;
 +                next->right = node->right;
 +            } else {
 +                node->balance -= 1;
 +            }
 +
 +            /* set the prev to point to the right place */
 +            while (prev->right_child) {
 +                prev = prev->right;
 +            }
 +            prev->right = next;
 +
 +            /* prepare 'next' to replace 'node' */
 +            next->left_child = TRUE;
 +            next->left = node->left;
 +            next->balance = node->balance;
 +
 +            if (!parent) {
 +                tree->root = next;
 +            } else if (left_node) {
 +                parent->left = next;
 +            } else {
 +                parent->right = next;
 +            }
 +        }
 +    }
 +
 +    /* restore balance */
 +    if (balance) {
 +        while (1) {
 +            QTreeNode *bparent = path[--idx];
 +            g_assert(!bparent ||
 +                     bparent->left == balance ||
 +                     bparent->right == balance);
 +            left_node = (bparent && balance == bparent->left);
 +
 +            if (balance->balance < -1 || balance->balance > 1) {
 +                balance = q_tree_node_balance(balance);
 +                if (!bparent) {
 +                    tree->root = balance;
 +                } else if (left_node) {
 +                    bparent->left = balance;
 +                } else {
 +                    bparent->right = balance;
 +                }
 +            }
 +
 +            if (balance->balance != 0 || !bparent) {
 +                break;
 +            }
 +
 +            if (left_node) {
 +                bparent->balance += 1;
 +            } else {
 +                bparent->balance -= 1;
 +            }
 +
 +            balance = bparent;
 +        }
 +    }
 +
 +    if (!steal) {
 +        if (tree->key_destroy_func) {
 +            tree->key_destroy_func(node->key);
 +        }
 +        if (tree->value_destroy_func) {
 +            tree->value_destroy_func(node->value);
 +        }
 +    }
 +
 +    g_free(node);
 +
 +    tree->nnodes--;
 +
 +    return TRUE;
 +}
 +
 +/**
 + * q_tree_lookup_node:
 + * @tree: a #QTree
 + * @key: the key to look up
 + *
 + * Gets the tree node corresponding to the given key. Since a #QTree is
 + * automatically balanced as key/value pairs are added, key lookup
 + * is O(log n) (where n is the number of key/value pairs in the tree).
 + *
 + * Returns: (nullable) (transfer none): the tree node corresponding to
 + *          the key, or %NULL if the key was not found
 + *
 + * Since: 2.68 in GLib. Internal in Qtree, i.e. not in the public API.
 + */
 +static QTreeNode *
 +q_tree_lookup_node(QTree         *tree,
 +                   gconstpointer  key)
 +{
 +    g_return_val_if_fail(tree != NULL, NULL);
 +
 +    return q_tree_find_node(tree, key);
 +}
 +
 +/**
 + * q_tree_lookup:
 + * @tree: a #QTree
 + * @key: the key to look up
 + *
 + * Gets the value corresponding to the given key. Since a #QTree is
 + * automatically balanced as key/value pairs are added, key lookup
 + * is O(log n) (where n is the number of key/value pairs in the tree).
 + *
 + * Returns: the value corresponding to the key, or %NULL
 + *     if the key was not found
 + */
 +gpointer
 +q_tree_lookup(QTree         *tree,
 +              gconstpointer  key)
 +{
 +    QTreeNode *node;
 +
 +    node = q_tree_lookup_node(tree, key);
 +
 +    return node ? node->value : NULL;
 +}
 +
 +/**
 + * q_tree_lookup_extended:
 + * @tree: a #QTree
 + * @lookup_key: the key to look up
 + * @orig_key: (out) (optional) (nullable): returns the original key
 + * @value: (out) (optional) (nullable): returns the value associated with
 + *         the key
 + *
 + * Looks up a key in the #QTree, returning the original key and the
 + * associated value. This is useful if you need to free the memory
 + * allocated for the original key, for example before calling
 + * q_tree_remove().
 + *
 + * Returns: %TRUE if the key was found in the #QTree
 + */
 +gboolean
 +q_tree_lookup_extended(QTree         *tree,
 +                       gconstpointer  lookup_key,
 +                       gpointer      *orig_key,
 +                       gpointer      *value)
 +{
 +    QTreeNode *node;
 +
 +    g_return_val_if_fail(tree != NULL, FALSE);
 +
 +    node = q_tree_find_node(tree, lookup_key);
 +
 +    if (node) {
 +        if (orig_key) {
 +            *orig_key = node->key;
 +        }
 +        if (value) {
 +            *value = node->value;
 +        }
 +        return TRUE;
 +    } else {
 +        return FALSE;
 +    }
 +}
 +
 +/**
 + * q_tree_foreach:
 + * @tree: a #QTree
 + * @func: the function to call for each node visited.
 + *     If this function returns %TRUE, the traversal is stopped.
 + * @user_data: user data to pass to the function
 + *
 + * Calls the given function for each of the key/value pairs in the #QTree.
 + * The function is passed the key and value of each pair, and the given
 + * @data parameter. The tree is traversed in sorted order.
 + *
 + * The tree may not be modified while iterating over it (you can't
 + * add/remove items). To remove all items matching a predicate, you need
 + * to add each item to a list in your #GTraverseFunc as you walk over
 + * the tree, then walk the list and remove each item.
 + */
 +void
 +q_tree_foreach(QTree         *tree,
 +               GTraverseFunc  func,
 +               gpointer       user_data)
 +{
 +    QTreeNode *node;
 +
 +    g_return_if_fail(tree != NULL);
 +
 +    if (!tree->root) {
 +        return;
 +    }
 +
 +    node = q_tree_node_first(tree);
 +
 +    while (node) {
 +        if ((*func)(node->key, node->value, user_data)) {
 +            break;
 +        }
 +
 +        node = q_tree_node_next(node);
 +    }
 +}
 +
 +/**
 + * q_tree_search_node:
 + * @tree: a #QTree
 + * @search_func: a function used to search the #QTree
 + * @user_data: the data passed as the second argument to @search_func
 + *
 + * Searches a #QTree using @search_func.
 + *
 + * The @search_func is called with a pointer to the key of a key/value
 + * pair in the tree, and the passed in @user_data. If @search_func returns
 + * 0 for a key/value pair, then the corresponding node is returned as
 + * the result of q_tree_search(). If @search_func returns -1, searching
 + * will proceed among the key/value pairs that have a smaller key; if
 + * @search_func returns 1, searching will proceed among the key/value
 + * pairs that have a larger key.
 + *
 + * Returns: (nullable) (transfer none): the node corresponding to the
 + *          found key, or %NULL if the key was not found
 + *
 + * Since: 2.68 in GLib. Internal in Qtree, i.e. not in the public API.
 + */
 +static QTreeNode *
 +q_tree_search_node(QTree         *tree,
 +                   GCompareFunc   search_func,
 +                   gconstpointer  user_data)
 +{
 +    g_return_val_if_fail(tree != NULL, NULL);
 +
 +    if (!tree->root) {
 +        return NULL;
 +    }
 +
 +    return q_tree_node_search(tree->root, search_func, user_data);
 +}
 +
 +/**
 + * q_tree_search:
 + * @tree: a #QTree
 + * @search_func: a function used to search the #QTree
 + * @user_data: the data passed as the second argument to @search_func
 + *
 + * Searches a #QTree using @search_func.
 + *
 + * The @search_func is called with a pointer to the key of a key/value
 + * pair in the tree, and the passed in @user_data. If @search_func returns
 + * 0 for a key/value pair, then the corresponding value is returned as
 + * the result of q_tree_search(). If @search_func returns -1, searching
 + * will proceed among the key/value pairs that have a smaller key; if
 + * @search_func returns 1, searching will proceed among the key/value
 + * pairs that have a larger key.
 + *
 + * Returns: the value corresponding to the found key, or %NULL
 + *     if the key was not found
 + */
 +gpointer
 +q_tree_search(QTree         *tree,
 +              GCompareFunc   search_func,
 +              gconstpointer  user_data)
 +{
 +    QTreeNode *node;
 +
 +    node = q_tree_search_node(tree, search_func, user_data);
 +
 +    return node ? node->value : NULL;
 +}
 +
 +/**
 + * q_tree_height:
 + * @tree: a #QTree
 + *
 + * Gets the height of a #QTree.
 + *
 + * If the #QTree contains no nodes, the height is 0.
 + * If the #QTree contains only one root node the height is 1.
 + * If the root node has children the height is 2, etc.
 + *
 + * Returns: the height of @tree
 + */
 +gint
 +q_tree_height(QTree *tree)
 +{
 +    QTreeNode *node;
 +    gint height;
 +
 +    g_return_val_if_fail(tree != NULL, 0);
 +
 +    if (!tree->root) {
 +        return 0;
 +    }
 +
 +    height = 0;
 +    node = tree->root;
 +
 +    while (1) {
 +        height += 1 + MAX(node->balance, 0);
 +
 +        if (!node->left_child) {
 +            return height;
 +        }
 +
 +        node = node->left;
 +    }
 +}
 +
 +/**
 + * q_tree_nnodes:
 + * @tree: a #QTree
 + *
 + * Gets the number of nodes in a #QTree.
 + *
 + * Returns: the number of nodes in @tree
 + */
 +gint
 +q_tree_nnodes(QTree *tree)
 +{
 +    g_return_val_if_fail(tree != NULL, 0);
 +
 +    return tree->nnodes;
 +}
 +
 +static QTreeNode *
 +q_tree_node_balance(QTreeNode *node)
 +{
 +    if (node->balance < -1) {
 +        if (node->left->balance > 0) {
 +            node->left = q_tree_node_rotate_left(node->left);
 +        }
 +        node = q_tree_node_rotate_right(node);
 +    } else if (node->balance > 1) {
 +        if (node->right->balance < 0) {
 +            node->right = q_tree_node_rotate_right(node->right);
 +        }
 +        node = q_tree_node_rotate_left(node);
 +    }
 +
 +    return node;
 +}
 +
 +static QTreeNode *
 +q_tree_find_node(QTree        *tree,
 +                 gconstpointer key)
 +{
 +    QTreeNode *node;
 +    gint cmp;
 +
 +    node = tree->root;
 +    if (!node) {
 +        return NULL;
 +    }
 +
 +    while (1) {
 +        cmp = tree->key_compare(key, node->key, tree->key_compare_data);
 +        if (cmp == 0) {
 +            return node;
 +        } else if (cmp < 0) {
 +            if (!node->left_child) {
 +                return NULL;
 +            }
 +
 +            node = node->left;
 +        } else {
 +            if (!node->right_child) {
 +                return NULL;
 +            }
 +
 +            node = node->right;
 +        }
 +    }
 +}
 +
 +static QTreeNode *
 +q_tree_node_search(QTreeNode     *node,
 +                   GCompareFunc   search_func,
 +                   gconstpointer  data)
 +{
 +    gint dir;
 +
 +    if (!node) {
 +        return NULL;
 +    }
 +
 +    while (1) {
 +        dir = (*search_func)(node->key, data);
 +        if (dir == 0) {
 +            return node;
 +        } else if (dir < 0) {
 +            if (!node->left_child) {
 +                return NULL;
 +            }
 +
 +            node = node->left;
 +        } else {
 +            if (!node->right_child) {
 +                return NULL;
 +            }
 +
 +            node = node->right;
 +        }
 +    }
 +}
 +
 +static QTreeNode *
 +q_tree_node_rotate_left(QTreeNode *node)
 +{
 +    QTreeNode *right;
 +    gint a_bal;
 +    gint b_bal;
 +
 +    right = node->right;
 +
 +    if (right->left_child) {
 +        node->right = right->left;
 +    } else {
 +        node->right_child = FALSE;
 +        right->left_child = TRUE;
 +    }
 +    right->left = node;
 +
 +    a_bal = node->balance;
 +    b_bal = right->balance;
 +
 +    if (b_bal <= 0) {
 +        if (a_bal >= 1) {
 +            right->balance = b_bal - 1;
 +        } else {
 +            right->balance = a_bal + b_bal - 2;
 +        }
 +        node->balance = a_bal - 1;
 +    } else {
 +        if (a_bal <= b_bal) {
 +            right->balance = a_bal - 2;
 +        } else {
 +            right->balance = b_bal - 1;
 +        }
 +        node->balance = a_bal - b_bal - 1;
 +    }
 +
 +    return right;
 +}
 +
 +static QTreeNode *
 +q_tree_node_rotate_right(QTreeNode *node)
 +{
 +    QTreeNode *left;
 +    gint a_bal;
 +    gint b_bal;
 +
 +    left = node->left;
 +
 +    if (left->right_child) {
 +        node->left = left->right;
 +    } else {
 +        node->left_child = FALSE;
 +        left->right_child = TRUE;
 +    }
 +    left->right = node;
 +
 +    a_bal = node->balance;
 +    b_bal = left->balance;
 +
 +    if (b_bal <= 0) {
 +        if (b_bal > a_bal) {
 +            left->balance = b_bal + 1;
 +        } else {
 +            left->balance = a_bal + 2;
 +        }
 +        node->balance = a_bal - b_bal + 1;
 +    } else {
 +        if (a_bal <= -1) {
 +            left->balance = b_bal + 1;
 +        } else {
 +            left->balance = a_bal + b_bal + 2;
 +        }
 +        node->balance = a_bal + 1;
 +    }
 +
 +    return left;
 +}
 +
 +#ifdef Q_TREE_DEBUG
 +static gint
 +q_tree_node_height(QTreeNode *node)
 +{
 +    gint left_height;
 +    gint right_height;
 +
 +    if (node) {
 +        left_height = 0;
 +        right_height = 0;
 +
 +        if (node->left_child) {
 +            left_height = q_tree_node_height(node->left);
 +        }
 +
 +        if (node->right_child) {
 +            right_height = q_tree_node_height(node->right);
 +        }
 +
 +        return MAX(left_height, right_height) + 1;
 +    }
 +
 +    return 0;
 +}
 +
 +static void q_tree_node_check(QTreeNode *node)
 +{
 +    gint left_height;
 +    gint right_height;
 +    gint balance;
 +    QTreeNode *tmp;
 +
 +    if (node) {
 +        if (node->left_child) {
 +            tmp = q_tree_node_previous(node);
 +            g_assert(tmp->right == node);
 +        }
 +
 +        if (node->right_child) {
 +            tmp = q_tree_node_next(node);
 +            g_assert(tmp->left == node);
 +        }
 +
 +        left_height = 0;
 +        right_height = 0;
 +
 +        if (node->left_child) {
 +            left_height = q_tree_node_height(node->left);
 +        }
 +        if (node->right_child) {
 +            right_height = q_tree_node_height(node->right);
 +        }
 +
 +        balance = right_height - left_height;
 +        g_assert(balance == node->balance);
 +
 +        if (node->left_child) {
 +            q_tree_node_check(node->left);
 +        }
 +        if (node->right_child) {
 +            q_tree_node_check(node->right);
 +        }
 +    }
 +}
 +#endif
 diff --git a/tests/bench/meson.build b/tests/bench/meson.build
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/bench/meson.build
 +++ b/tests/bench/meson.build
@@ -XXX,XX +XXX,XX @@ xbzrle_bench = executable('xbzrle-bench',
                         dependencies: [qemuutil,migration])
  endif
 +qtree_bench = executable('qtree-bench',
 +                         sources: 'qtree-bench.c',
 +                         dependencies: [qemuutil])
 +
  executable('atomic_add-bench',
             sources: files('atomic_add-bench.c'),
             dependencies: [qemuutil],
 diff --git a/tests/unit/meson.build b/tests/unit/meson.build
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/unit/meson.build
 +++ b/tests/unit/meson.build
@@ -XXX,XX +XXX,XX @@ tests = {
    'test-rcu-slist': [],
    'test-qdist': [],
    'test-qht': [],
 +  'test-qtree': [],
    'test-bitops': [],
    'test-bitcnt': [],
    'test-qgraph': ['../qtest/libqos/qgraph.c'],
 diff --git a/util/meson.build b/util/meson.build
 index XXXXXXX..XXXXXXX 100644
 --- a/util/meson.build
 +++ b/util/meson.build
@@ -XXX,XX +XXX,XX @@ util_ss.add(when: 'CONFIG_WIN32', if_true: files('oslib-win32.c'))
  util_ss.add(when: 'CONFIG_WIN32', if_true: files('qemu-thread-win32.c'))
  util_ss.add(when: 'CONFIG_WIN32', if_true: winmm)
  util_ss.add(when: 'CONFIG_WIN32', if_true: pathcch)
 +util_ss.add(when: 'HAVE_GLIB_WITH_SLICE_ALLOCATOR', if_true: files('qtree.c'))
  util_ss.add(files('envlist.c', 'path.c', 'module.c'))
  util_ss.add(files('host-utils.c'))
  util_ss.add(files('bitmap.c', 'bitops.c'))
 --
 .34.1

-[PULL 17/22] tcg/ppc: Reorg goto_tb implementation
+[PATCH for-8.0 02/11] tcg: use QTree instead of GTree
-The old ppc64 implementation replaces 2 or 4 insns, which leaves a race
+From: Emilio Cota <cota@braap.org>
-condition in which a thread could be stopped at a PC in the middle of
-the sequence, and when restarted does not see the complete address
+qemu-user can hang in a multi-threaded fork. One common
-computation and branches to nowhere.
+reason is that when creating a TB, between fork and exec
+we manipulate a GTree whose memory allocator (GSlice) is
-The new implemetation replaces only one insn, swapping between
+not fork-safe.
-    b       <dest>
+Although POSIX does not mandate it, the system's allocator
-and
+(e.g. tcmalloc, libc malloc) is probably fork-safe.
-    mtctr    r31
+Fix some of these hangs by using QTree, which uses the system's
-falling through to a general-case indirect branch.
+allocator regardless of the Glib version that we used at
+configuration time.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
 Tested with the test program in the original bug report, i.e.:
 ```
 void garble() {
   int pid = fork();
   if (pid == 0) {
     exit(0);
   } else {
     int wstatus;
     waitpid(pid, &wstatus, 0);
   }
 }
 void supragarble(unsigned depth) {
   if (depth == 0)
     return ;
   std::thread a(supragarble, depth-1);
   std::thread b(supragarble, depth-1);
   garble();
   a.join();
   b.join();
 }
 int main() {
   supragarble(10);
 }
 ```
 Resolves: https://gitlab.com/qemu-project/qemu/-/issues/285
 Reported-by: Valentin David <me@valentindavid.com>
 Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Signed-off-by: Emilio Cota <cota@braap.org>
 Message-Id: <20230205163758.416992-3-cota@braap.org>
 [rth: Add QEMU_DISABLE_CFI for all callback using functions.]
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- tcg/ppc/tcg-target.h     |   3 +-
+ accel/tcg/tb-maint.c | 17 +++++++++--------
- tcg/ppc/tcg-target.c.inc | 158 +++++++++++----------------------------
+ tcg/region.c         | 19 ++++++++++---------
-files changed, 44 insertions(+), 117 deletions(-)
+ util/qtree.c         |  8 ++++----
+files changed, 23 insertions(+), 21 deletions(-)
-diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
 diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c
 index XXXXXXX..XXXXXXX 100644
---- a/tcg/ppc/tcg-target.h
+--- a/accel/tcg/tb-maint.c
-+++ b/tcg/ppc/tcg-target.h
++++ b/accel/tcg/tb-maint.c
 @@ -XXX,XX +XXX,XX @@
- #ifdef _ARCH_PPC64
+ #include "qemu/osdep.h"
- # define TCG_TARGET_REG_BITS  64
+ #include "qemu/interval-tree.h"
--# define MAX_CODE_GEN_BUFFER_SIZE  (2 * GiB)
++#include "qemu/qtree.h"
- #else
+ #include "exec/cputlb.h"
- # define TCG_TARGET_REG_BITS  32
+ #include "exec/log.h"
--# define MAX_CODE_GEN_BUFFER_SIZE  (32 * MiB)
+ #include "exec/exec-all.h"
- #endif
+@@ -XXX,XX +XXX,XX @@ struct page_entry {
-+#define MAX_CODE_GEN_BUFFER_SIZE  ((size_t)-1)
+  * See also: page_collection_lock().
+  */
- #define TCG_TARGET_NB_REGS 64
+ struct page_collection {
- #define TCG_TARGET_INSN_UNIT_SIZE 4
+-    GTree *tree;
-diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
++    QTree *tree;
      struct page_entry *max;
  };
@@ -XXX,XX +XXX,XX @@ static bool page_trylock_add(struct page_collection *set, tb_page_addr_t addr)
      struct page_entry *pe;
      PageDesc *pd;
 -    pe = g_tree_lookup(set->tree, &index);
 +    pe = q_tree_lookup(set->tree, &index);
      if (pe) {
          return false;
      }
@@ -XXX,XX +XXX,XX @@ static bool page_trylock_add(struct page_collection *set, tb_page_addr_t addr)
      }
      pe = page_entry_new(pd, index);
 -    g_tree_insert(set->tree, &pe->index, pe);
 +    q_tree_insert(set->tree, &pe->index, pe);
      /*
       * If this is either (1) the first insertion or (2) a page whose index
@@ -XXX,XX +XXX,XX @@ static struct page_collection *page_collection_lock(tb_page_addr_t start,
      end   >>= TARGET_PAGE_BITS;
      g_assert(start <= end);
 -    set->tree = g_tree_new_full(tb_page_addr_cmp, NULL, NULL,
 +    set->tree = q_tree_new_full(tb_page_addr_cmp, NULL, NULL,
                                  page_entry_destroy);
      set->max = NULL;
      assert_no_pages_locked();
   retry:
 -    g_tree_foreach(set->tree, page_entry_lock, NULL);
 +    q_tree_foreach(set->tree, page_entry_lock, NULL);
      for (index = start; index <= end; index++) {
          TranslationBlock *tb;
@@ -XXX,XX +XXX,XX @@ static struct page_collection *page_collection_lock(tb_page_addr_t start,
              continue;
          }
          if (page_trylock_add(set, index << TARGET_PAGE_BITS)) {
 -            g_tree_foreach(set->tree, page_entry_unlock, NULL);
 +            q_tree_foreach(set->tree, page_entry_unlock, NULL);
              goto retry;
          }
          assert_page_locked(pd);
@@ -XXX,XX +XXX,XX @@ static struct page_collection *page_collection_lock(tb_page_addr_t start,
                  (tb_page_addr1(tb) != -1 &&
                   page_trylock_add(set, tb_page_addr1(tb)))) {
                  /* drop all locks, and reacquire in order */
 -                g_tree_foreach(set->tree, page_entry_unlock, NULL);
 +                q_tree_foreach(set->tree, page_entry_unlock, NULL);
                  goto retry;
              }
          }
@@ -XXX,XX +XXX,XX @@ static struct page_collection *page_collection_lock(tb_page_addr_t start,
  static void page_collection_unlock(struct page_collection *set)
  {
      /* entries are unlocked and freed via page_entry_destroy */
 -    g_tree_destroy(set->tree);
 +    q_tree_destroy(set->tree);
      g_free(set);
  }
 diff --git a/tcg/region.c b/tcg/region.c
 index XXXXXXX..XXXXXXX 100644
---- a/tcg/ppc/tcg-target.c.inc
+--- a/tcg/region.c
-+++ b/tcg/ppc/tcg-target.c.inc
++++ b/tcg/region.c
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_mb(TCGContext *s, TCGArg a0)
+@@ -XXX,XX +XXX,XX @@
-     tcg_out32(s, insn);
+ #include "qemu/mprotect.h"
- }
+ #include "qemu/memalign.h"
+ #include "qemu/cacheinfo.h"
--static inline uint64_t make_pair(tcg_insn_unit i1, tcg_insn_unit i2)
++#include "qemu/qtree.h"
--{
+ #include "qapi/error.h"
--    if (HOST_BIG_ENDIAN) {
+ #include "exec/exec-all.h"
--        return (uint64_t)i1 << 32 | i2;
+ #include "tcg/tcg.h"
--    }
+@@ -XXX,XX +XXX,XX @@
--    return (uint64_t)i2 << 32 | i1;
--}
+ struct tcg_region_tree {
--
+     QemuMutex lock;
--static inline void ppc64_replace2(uintptr_t rx, uintptr_t rw,
+-    GTree *tree;
--                                  tcg_insn_unit i0, tcg_insn_unit i1)
++    QTree *tree;
--{
+     /* padding to avoid false sharing is computed at run-time */
--#if TCG_TARGET_REG_BITS == 64
+ };
--    qatomic_set((uint64_t *)rw, make_pair(i0, i1));
--    flush_idcache_range(rx, rw, 8);
+@@ -XXX,XX +XXX,XX @@ static void tcg_region_trees_init(void)
--#else
+         struct tcg_region_tree *rt = region_trees + i * tree_size;
--    qemu_build_not_reached();
--#endif
+         qemu_mutex_init(&rt->lock);
--}
+-        rt->tree = g_tree_new_full(tb_tc_cmp, NULL, NULL, tb_destroy);
--
++        rt->tree = q_tree_new_full(tb_tc_cmp, NULL, NULL, tb_destroy);
--static inline void ppc64_replace4(uintptr_t rx, uintptr_t rw,
+     }
--                                  tcg_insn_unit i0, tcg_insn_unit i1,
+ }
--                                  tcg_insn_unit i2, tcg_insn_unit i3)
--{
+@@ -XXX,XX +XXX,XX @@ void tcg_tb_insert(TranslationBlock *tb)
--    uint64_t p[2];
--
+     g_assert(rt != NULL);
--    p[!HOST_BIG_ENDIAN] = make_pair(i0, i1);
+     qemu_mutex_lock(&rt->lock);
--    p[HOST_BIG_ENDIAN] = make_pair(i2, i3);
+-    g_tree_insert(rt->tree, &tb->tc, tb);
--
++    q_tree_insert(rt->tree, &tb->tc, tb);
--    /*
+     qemu_mutex_unlock(&rt->lock);
--     * There's no convenient way to get the compiler to allocate a pair
+ }
--     * of registers at an even index, so copy into r6/r7 and clobber.
--     */
+@@ -XXX,XX +XXX,XX @@ void tcg_tb_remove(TranslationBlock *tb)
--    asm("mr  %%r6, %1\n\t"
--        "mr  %%r7, %2\n\t"
+     g_assert(rt != NULL);
--        "stq %%r6, %0"
+     qemu_mutex_lock(&rt->lock);
--        : "=Q"(*(__int128 *)rw) : "r"(p[0]), "r"(p[1]) : "r6", "r7");
+-    g_tree_remove(rt->tree, &tb->tc);
--    flush_idcache_range(rx, rw, 16);
++    q_tree_remove(rt->tree, &tb->tc);
--}
+     qemu_mutex_unlock(&rt->lock);
--
+ }
--void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
--                              uintptr_t jmp_rx, uintptr_t jmp_rw)
+@@ -XXX,XX +XXX,XX @@ TranslationBlock *tcg_tb_lookup(uintptr_t tc_ptr)
--{
+     }
--    tcg_insn_unit i0, i1, i2, i3;
--    uintptr_t addr = tb->jmp_target_addr[n];
+     qemu_mutex_lock(&rt->lock);
--    intptr_t tb_diff = addr - (uintptr_t)tb->tc.ptr;
+-    tb = g_tree_lookup(rt->tree, &s);
--    intptr_t br_diff = addr - (jmp_rx + 4);
++    tb = q_tree_lookup(rt->tree, &s);
--    intptr_t lo, hi;
+     qemu_mutex_unlock(&rt->lock);
--
+     return tb;
--    if (TCG_TARGET_REG_BITS == 32) {
+ }
--        intptr_t diff = addr - jmp_rx;
+@@ -XXX,XX +XXX,XX @@ void tcg_tb_foreach(GTraverseFunc func, gpointer user_data)
--        tcg_debug_assert(in_range_b(diff));
+     for (i = 0; i < region.n; i++) {
--        qatomic_set((uint32_t *)jmp_rw, B | (diff & 0x3fffffc));
+         struct tcg_region_tree *rt = region_trees + i * tree_size;
--        flush_idcache_range(jmp_rx, jmp_rw, 4);
--        return;
+-        g_tree_foreach(rt->tree, func, user_data);
--    }
++        q_tree_foreach(rt->tree, func, user_data);
--
+     }
--    /*
+     tcg_region_tree_unlock_all();
--     * For 16-bit displacements, we can use a single add + branch.
+ }
--     * This happens quite often.
+@@ -XXX,XX +XXX,XX @@ size_t tcg_nb_tbs(void)
--     */
+     for (i = 0; i < region.n; i++) {
--    if (tb_diff == (int16_t)tb_diff) {
+         struct tcg_region_tree *rt = region_trees + i * tree_size;
--        i0 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, tb_diff);
--        i1 = B | (br_diff & 0x3fffffc);
+-        nb_tbs += g_tree_nnodes(rt->tree);
--        ppc64_replace2(jmp_rx, jmp_rw, i0, i1);
++        nb_tbs += q_tree_nnodes(rt->tree);
--        return;
+     }
--    }
+     tcg_region_tree_unlock_all();
--
+     return nb_tbs;
--    lo = (int16_t)tb_diff;
+@@ -XXX,XX +XXX,XX @@ static void tcg_region_tree_reset_all(void)
--    hi = (int32_t)(tb_diff - lo);
+         struct tcg_region_tree *rt = region_trees + i * tree_size;
--    assert(tb_diff == hi + lo);
--    i0 = ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, hi >> 16);
+         /* Increment the refcount first so that destroy acts as a reset */
--    i1 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, lo);
+-        g_tree_ref(rt->tree);
--
+-        g_tree_destroy(rt->tree);
--    /*
++        q_tree_ref(rt->tree);
--     * Without stq from 2.07, we can only update two insns,
++        q_tree_destroy(rt->tree);
--     * and those must be the ones that load the target address.
+     }
--     */
+     tcg_region_tree_unlock_all();
--    if (!have_isa_2_07) {
+ }
--        ppc64_replace2(jmp_rx, jmp_rw, i0, i1);
+diff --git a/util/qtree.c b/util/qtree.c
--        return;
+index XXXXXXX..XXXXXXX 100644
--    }
+--- a/util/qtree.c
--
++++ b/util/qtree.c
--    /*
+@@ -XXX,XX +XXX,XX @@ q_tree_node_next(QTreeNode *node)
--     * For 26-bit displacements, we can use a direct branch.
+  *
--     * Otherwise we still need the indirect branch, which we
+  * Since: 2.70 in GLib. Internal in Qtree, i.e. not in the public API.
--     * must restore after a potential direct branch write.
+  */
--     */
+-static void
--    br_diff -= 4;
++static void QEMU_DISABLE_CFI
--    if (in_range_b(br_diff)) {
+ q_tree_remove_all(QTree *tree)
 -        i2 = B | (br_diff & 0x3fffffc);
 -        i3 = NOP;
 -    } else {
 -        i2 = MTSPR | RS(TCG_REG_TB) | CTR;
 -        i3 = BCCTR | BO_ALWAYS;
 -    }
 -    ppc64_replace4(jmp_rx, jmp_rw, i0, i1, i2, i3);
 -}
 -
  static void tcg_out_call_int(TCGContext *s, int lk,
                               const tcg_insn_unit *target)
  {
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
+     QTreeNode *node;
+@@ -XXX,XX +XXX,XX @@ q_tree_replace(QTree    *tree,
- static void tcg_out_goto_tb(TCGContext *s, int which)
+ }
  /* internal insert routine */
 -static QTreeNode *
 +static QTreeNode * QEMU_DISABLE_CFI
  q_tree_insert_internal(QTree    *tree,
                         gpointer  key,
                         gpointer  value,
@@ -XXX,XX +XXX,XX @@ q_tree_steal(QTree         *tree,
  }
  /* internal remove routine */
 -static gboolean
 +static gboolean QEMU_DISABLE_CFI
  q_tree_remove_internal(QTree         *tree,
                         gconstpointer  key,
                         gboolean       steal)
@@ -XXX,XX +XXX,XX @@ q_tree_node_balance(QTreeNode *node)
      return node;
  }
 -static QTreeNode *
 +static QTreeNode * QEMU_DISABLE_CFI
  q_tree_find_node(QTree        *tree,
                   gconstpointer key)
  {
--    /* Direct jump. */
--    if (TCG_TARGET_REG_BITS == 64) {
--        /* Ensure the next insns are 8 or 16-byte aligned. */
--        while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) {
--            tcg_out32(s, NOP);
--        }
-+    uintptr_t ptr = get_jmp_target_addr(s, which);
-+
-+    if (USE_REG_TB) {
-+        ptrdiff_t offset = tcg_tbrel_diff(s, (void *)ptr);
-+        tcg_out_mem_long(s, LD, LDX, TCG_REG_TB, TCG_REG_TB, offset);
-+
-+        /* Direct branch will be patched by tb_target_set_jmp_target. */
-         set_jmp_insn_offset(s, which);
--        tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0));
--        tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0));
-         tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR);
-+
-+        /* When branch is out of range, fall through to indirect. */
-+        tcg_out32(s, BCCTR | BO_ALWAYS);
-+
-+        /* For the unlinked case, need to reset TCG_REG_TB.  */
-+        set_jmp_reset_offset(s, which);
-+        tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB,
-+                         -tcg_current_code_size(s));
-+    } else {
-+        /* Direct branch will be patched by tb_target_set_jmp_target. */
-+        set_jmp_insn_offset(s, which);
-+        tcg_out32(s, NOP);
-+
-+        /* When branch is out of range, fall through to indirect. */
-+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, ptr - (int16_t)ptr);
-+        tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_REG_TMP1, (int16_t)ptr);
-+        tcg_out32(s, MTSPR | RS(TCG_REG_TMP1) | CTR);
-         tcg_out32(s, BCCTR | BO_ALWAYS);
-         set_jmp_reset_offset(s, which);
--        if (USE_REG_TB) {
--            /* For the unlinked case, need to reset TCG_REG_TB.  */
--            tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB,
--                             -tcg_current_code_size(s));
--        }
--    } else {
--        set_jmp_insn_offset(s, which);
--        tcg_out32(s, B);
--        set_jmp_reset_offset(s, which);
-     }
- }
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
-+{
-+    uintptr_t addr = tb->jmp_target_addr[n];
-+    intptr_t diff = addr - jmp_rx;
-+    tcg_insn_unit insn;
-+
-+    if (in_range_b(diff)) {
-+        insn = B | (diff & 0x3fffffc);
-+    } else if (USE_REG_TB) {
-+        insn = MTSPR | RS(TCG_REG_TB) | CTR;
-+    } else {
-+        insn = NOP;
-+    }
-+
-+    qatomic_set((uint32_t *)jmp_rw, insn);
-+    flush_idcache_range(jmp_rx, jmp_rw, 4);
-+}
-+
- static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-                        const TCGArg args[TCG_MAX_OP_ARGS],
-                        const int const_args[TCG_MAX_OP_ARGS])
 --
 .34.1

-[PULL 13/22] tcg: Move tb_target_set_jmp_target declaration to tcg.h
+[PATCH for-8.0 03/11] linux-user: Diagnose misaligned -R size
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
+We have been enforcing host page alignment for the non-R
 fallback of MAX_RESERVED_VA, but failing to enforce for -R.
 Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- include/tcg/tcg.h            | 3 +++
+ linux-user/main.c | 6 ++++++
- tcg/aarch64/tcg-target.h     | 4 ----
+file changed, 6 insertions(+)
  tcg/arm/tcg-target.h         | 5 -----
  tcg/i386/tcg-target.h        | 3 ---
  tcg/loongarch64/tcg-target.h | 3 ---
  tcg/mips/tcg-target.h        | 5 -----
  tcg/ppc/tcg-target.h         | 4 ----
  tcg/riscv/tcg-target.h       | 4 ----
  tcg/s390x/tcg-target.h       | 4 ----
  tcg/sparc64/tcg-target.h     | 4 ----
  tcg/tci/tcg-target.h         | 4 ----
 files changed, 3 insertions(+), 40 deletions(-)
-diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h
+diff --git a/linux-user/main.c b/linux-user/main.c
 index XXXXXXX..XXXXXXX 100644
---- a/include/tcg/tcg.h
+--- a/linux-user/main.c
-+++ b/include/tcg/tcg.h
++++ b/linux-user/main.c
-@@ -XXX,XX +XXX,XX @@ void tcg_func_start(TCGContext *s);
+@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
+      */
- int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start);
+     max_reserved_va = MAX_RESERVED_VA(cpu);
+     if (reserved_va != 0) {
-+void tb_target_set_jmp_target(const TranslationBlock *, int,
++        if (reserved_va % qemu_host_page_size) {
-+                              uintptr_t, uintptr_t);
++            char *s = size_to_str(qemu_host_page_size);
-+
++            fprintf(stderr, "Reserved virtual address not aligned mod %s\n", s);
- void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size);
++            g_free(s);
++            exit(EXIT_FAILURE);
- TCGTemp *tcg_global_mem_new_internal(TCGType, TCGv_ptr,
++        }
-diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h
+         if (max_reserved_va && reserved_va > max_reserved_va) {
-index XXXXXXX..XXXXXXX 100644
+             fprintf(stderr, "Reserved virtual address too big\n");
---- a/tcg/aarch64/tcg-target.h
+             exit(EXIT_FAILURE);
 +++ b/tcg/aarch64/tcg-target.h
@@ -XXX,XX +XXX,XX @@ typedef enum {
  #define TCG_TARGET_DEFAULT_MO (0)
  #define TCG_TARGET_HAS_MEMORY_BSWAP     0
 -
 -void tb_target_set_jmp_target(const TranslationBlock *, int,
 -                              uintptr_t, uintptr_t);
 -
  #define TCG_TARGET_NEED_LDST_LABELS
  #define TCG_TARGET_NEED_POOL_LABELS
 diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/arm/tcg-target.h
 +++ b/tcg/arm/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool use_neon_instructions;
  #define TCG_TARGET_DEFAULT_MO (0)
  #define TCG_TARGET_HAS_MEMORY_BSWAP     0
 -
 -/* not defined -- call should be eliminated at compile time */
 -void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
 -                              uintptr_t, uintptr_t);
 -
  #define TCG_TARGET_NEED_LDST_LABELS
  #define TCG_TARGET_NEED_POOL_LABELS
 diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/i386/tcg-target.h
 +++ b/tcg/i386/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool have_movbe;
  #define TCG_TARGET_extract_i64_valid(ofs, len) \
      (((ofs) == 8 && (len) == 8) || ((ofs) + (len)) == 32)
 -void tb_target_set_jmp_target(const TranslationBlock *, int,
 -                              uintptr_t, uintptr_t);
 -
  /* This defines the natural memory order supported by this
   * architecture before guarantees made by various barrier
   * instructions.
 diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/loongarch64/tcg-target.h
 +++ b/tcg/loongarch64/tcg-target.h
@@ -XXX,XX +XXX,XX @@ typedef enum {
  #define TCG_TARGET_HAS_muluh_i64        1
  #define TCG_TARGET_HAS_mulsh_i64        1
 -void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
 -                              uintptr_t, uintptr_t);
 -
  #define TCG_TARGET_DEFAULT_MO (0)
  #define TCG_TARGET_NEED_LDST_LABELS
 diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/mips/tcg-target.h
 +++ b/tcg/mips/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions;
  #define TCG_TARGET_DEFAULT_MO (0)
  #define TCG_TARGET_HAS_MEMORY_BSWAP     1
 -/* not defined -- call should be eliminated at compile time */
 -void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
 -                              uintptr_t, uintptr_t)
 -    QEMU_ERROR("code path is reachable");
 -
  #define TCG_TARGET_NEED_LDST_LABELS
  #endif
 diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/ppc/tcg-target.h
 +++ b/tcg/ppc/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool have_vsx;
  #define TCG_TARGET_HAS_bitsel_vec       have_vsx
  #define TCG_TARGET_HAS_cmpsel_vec       0
 -void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
 -                              uintptr_t, uintptr_t);
 -
  #define TCG_TARGET_DEFAULT_MO (0)
  #define TCG_TARGET_HAS_MEMORY_BSWAP     1
 -
  #define TCG_TARGET_NEED_LDST_LABELS
  #define TCG_TARGET_NEED_POOL_LABELS
 diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/riscv/tcg-target.h
 +++ b/tcg/riscv/tcg-target.h
@@ -XXX,XX +XXX,XX @@ typedef enum {
  #define TCG_TARGET_HAS_mulsh_i64        1
  #endif
 -/* not defined -- call should be eliminated at compile time */
 -void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
 -                              uintptr_t, uintptr_t);
 -
  #define TCG_TARGET_DEFAULT_MO (0)
  #define TCG_TARGET_NEED_LDST_LABELS
 diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/s390x/tcg-target.h
 +++ b/tcg/s390x/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern uint64_t s390_facilities[3];
  #define TCG_TARGET_HAS_MEMORY_BSWAP   1
  #define TCG_TARGET_DEFAULT_MO (TCG_MO_ALL & ~TCG_MO_ST_LD)
 -
 -void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
 -                              uintptr_t jmp_rx, uintptr_t jmp_rw);
 -
  #define TCG_TARGET_NEED_LDST_LABELS
  #define TCG_TARGET_NEED_POOL_LABELS
 diff --git a/tcg/sparc64/tcg-target.h b/tcg/sparc64/tcg-target.h
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/sparc64/tcg-target.h
 +++ b/tcg/sparc64/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions;
  #define TCG_TARGET_DEFAULT_MO (0)
  #define TCG_TARGET_HAS_MEMORY_BSWAP     1
 -
 -void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
 -                              uintptr_t, uintptr_t);
 -
  #define TCG_TARGET_NEED_POOL_LABELS
  #endif
 diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/tci/tcg-target.h
 +++ b/tcg/tci/tcg-target.h
@@ -XXX,XX +XXX,XX @@ typedef enum {
  #define TCG_TARGET_HAS_MEMORY_BSWAP     1
 -/* not defined -- call should be eliminated at compile time */
 -void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
 -                              uintptr_t, uintptr_t);
 -
  #endif /* TCG_TARGET_H */
 --
 .34.1

-[PULL 21/22] tcg/riscv: Introduce OPC_NOP
+[PATCH for-8.0 04/11] include/exec: Change reserved_va semantics to last byte
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
+Change the semantics to be the last byte of the guest va, rather
 than the following byte.  This avoids some overflow conditions.
 Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- tcg/riscv/tcg-target.c.inc | 3 ++-
+ include/exec/cpu-all.h      | 11 ++++++++++-
-file changed, 2 insertions(+), 1 deletion(-)
+ linux-user/arm/target_cpu.h |  2 +-
+ bsd-user/main.c             | 10 +++-------
-diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
+ bsd-user/mmap.c             |  4 ++--
-index XXXXXXX..XXXXXXX 100644
+ linux-user/elfload.c        | 21 +++++++++++----------
---- a/tcg/riscv/tcg-target.c.inc
+ linux-user/main.c           | 27 +++++++++++++--------------
-+++ b/tcg/riscv/tcg-target.c.inc
+ linux-user/mmap.c           |  4 ++--
-@@ -XXX,XX +XXX,XX @@ typedef enum {
+files changed, 42 insertions(+), 37 deletions(-)
- #endif
+diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h
-     OPC_FENCE = 0x0000000f,
+index XXXXXXX..XXXXXXX 100644
-+    OPC_NOP   = OPC_ADDI,   /* nop = addi r0,r0,0 */
+--- a/include/exec/cpu-all.h
- } RISCVInsn;
++++ b/include/exec/cpu-all.h
@@ -XXX,XX +XXX,XX @@ static inline void tswap64s(uint64_t *s)
   */
  extern uintptr_t guest_base;
  extern bool have_guest_base;
 +
 +/*
 + * If non-zero, the guest virtual address space is a contiguous subset
 + * of the host virtual address space, i.e. '-R reserved_va' is in effect
 + * either from the command-line or by default.  The value is the last
 + * byte of the guest address space e.g. UINT32_MAX.
 + *
 + * If zero, the host and guest virtual address spaces are intermingled.
 + */
  extern unsigned long reserved_va;
  /*
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
+@@ -XXX,XX +XXX,XX @@ extern unsigned long reserved_va;
  #define GUEST_ADDR_MAX_                                                 \
      ((MIN_CONST(TARGET_VIRT_ADDR_SPACE_BITS, TARGET_ABI_BITS) <= 32) ?  \
       UINT32_MAX : ~0ul)
 -#define GUEST_ADDR_MAX    (reserved_va ? reserved_va - 1 : GUEST_ADDR_MAX_)
 +#define GUEST_ADDR_MAX    (reserved_va ? : GUEST_ADDR_MAX_)
  #else
 diff --git a/linux-user/arm/target_cpu.h b/linux-user/arm/target_cpu.h
 index XXXXXXX..XXXXXXX 100644
 --- a/linux-user/arm/target_cpu.h
 +++ b/linux-user/arm/target_cpu.h
@@ -XXX,XX +XXX,XX @@ static inline unsigned long arm_max_reserved_va(CPUState *cs)
           * the high addresses.  Restrict linux-user to the
           * cached write-back RAM in the system map.
           */
 -        return 0x80000000ul;
 +        return 0x7ffffffful;
      } else {
          /*
           * We need to be able to map the commpage.
 diff --git a/bsd-user/main.c b/bsd-user/main.c
 index XXXXXXX..XXXXXXX 100644
 --- a/bsd-user/main.c
 +++ b/bsd-user/main.c
@@ -XXX,XX +XXX,XX @@ bool have_guest_base;
  # if HOST_LONG_BITS > TARGET_VIRT_ADDR_SPACE_BITS
  #  if TARGET_VIRT_ADDR_SPACE_BITS == 32 && \
        (TARGET_LONG_BITS == 32 || defined(TARGET_ABI32))
 -/*
 - * There are a number of places where we assign reserved_va to a variable
 - * of type abi_ulong and expect it to fit.  Avoid the last page.
 - */
 -#   define MAX_RESERVED_VA  (0xfffffffful & TARGET_PAGE_MASK)
 +#   define MAX_RESERVED_VA  0xfffffffful
  #  else
 -#   define MAX_RESERVED_VA  (1ul << TARGET_VIRT_ADDR_SPACE_BITS)
 +#   define MAX_RESERVED_VA  ((1ul << TARGET_VIRT_ADDR_SPACE_BITS) - 1)
  #  endif
  # else
  #  define MAX_RESERVED_VA  0
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
      envlist_free(envlist);
      if (reserved_va) {
 -            mmap_next_start = reserved_va;
 +        mmap_next_start = reserved_va + 1;
      }
      {
 diff --git a/bsd-user/mmap.c b/bsd-user/mmap.c
 index XXXXXXX..XXXXXXX 100644
 --- a/bsd-user/mmap.c
 +++ b/bsd-user/mmap.c
@@ -XXX,XX +XXX,XX @@ static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size,
      size = HOST_PAGE_ALIGN(size) + alignment;
      end_addr = start + size;
      if (end_addr > reserved_va) {
 -        end_addr = reserved_va;
 +        end_addr = reserved_va + 1;
      }
      addr = end_addr - qemu_host_page_size;
@@ -XXX,XX +XXX,XX @@ static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size,
              if (looped) {
                  return (abi_ulong)-1;
              }
 -            end_addr = reserved_va;
 +            end_addr = reserved_va + 1;
              addr = end_addr - qemu_host_page_size;
              looped = 1;
              continue;
 diff --git a/linux-user/elfload.c b/linux-user/elfload.c
 index XXXXXXX..XXXXXXX 100644
 --- a/linux-user/elfload.c
 +++ b/linux-user/elfload.c
@@ -XXX,XX +XXX,XX @@ static bool init_guest_commpage(void)
       * has specified -R reserved_va, which would trigger an assert().
       */
      if (reserved_va != 0 &&
 -        TARGET_VSYSCALL_PAGE + TARGET_PAGE_SIZE >= reserved_va) {
 +        TARGET_VSYSCALL_PAGE + TARGET_PAGE_SIZE - 1 > reserved_va) {
          error_report("Cannot allocate vsyscall page");
          exit(EXIT_FAILURE);
      }
@@ -XXX,XX +XXX,XX @@ static void pgb_have_guest_base(const char *image_name, abi_ulong guest_loaddr,
      /* Sanity check the guest binary. */
      if (reserved_va) {
 -        if (guest_hiaddr > reserved_va) {
 +        if (guest_hiaddr - 1 > reserved_va) {
              error_report("%s: requires more than reserved virtual "
                           "address space (0x%" PRIx64 " > 0x%lx)",
 -                         image_name, (uint64_t)guest_hiaddr, reserved_va);
 +                         image_name, (uint64_t)guest_hiaddr - 1,
 +                         reserved_va);
              exit(EXIT_FAILURE);
          }
      } else {
@@ -XXX,XX +XXX,XX @@ static void pgb_have_guest_base(const char *image_name, abi_ulong guest_loaddr,
      if (reserved_va) {
          guest_loaddr = (guest_base >= mmap_min_addr ? 0
                          : mmap_min_addr - guest_base);
 -        guest_hiaddr = reserved_va;
 +        guest_hiaddr = reserved_va + 1;
      }
      /* Reserve the address space for the binary, or reserved_va. */
@@ -XXX,XX +XXX,XX @@ static void pgb_reserved_va(const char *image_name, abi_ulong guest_loaddr,
      int flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE;
      void *addr, *test;
 -    if (guest_hiaddr > reserved_va) {
 +    if (guest_hiaddr - 1 > reserved_va) {
          error_report("%s: requires more than reserved virtual "
                       "address space (0x%" PRIx64 " > 0x%lx)",
 -                     image_name, (uint64_t)guest_hiaddr, reserved_va);
 +                     image_name, (uint64_t)guest_hiaddr - 1, reserved_va);
          exit(EXIT_FAILURE);
      }
      /* Widen the "image" to the entire reserved address space. */
 -    pgb_static(image_name, 0, reserved_va, align);
 +    pgb_static(image_name, 0, reserved_va + 1, align);
      /* osdep.h defines this as 0 if it's missing */
      flags |= MAP_FIXED_NOREPLACE;
@@ -XXX,XX +XXX,XX @@ static void pgb_reserved_va(const char *image_name, abi_ulong guest_loaddr,
      /* Reserve the memory on the host. */
      assert(guest_base != 0);
      test = g2h_untagged(0);
 -    addr = mmap(test, reserved_va, PROT_NONE, flags, -1, 0);
 +    addr = mmap(test, reserved_va + 1, PROT_NONE, flags, -1, 0);
      if (addr == MAP_FAILED || addr != test) {
          error_report("Unable to reserve 0x%lx bytes of virtual address "
                       "space at %p (%s) for use as guest address space (check your "
                       "virtual memory ulimit setting, min_mmap_addr or reserve less "
 -                     "using -R option)", reserved_va, test, strerror(errno));
 +                     "using -R option)", reserved_va + 1, test, strerror(errno));
          exit(EXIT_FAILURE);
      }
      qemu_log_mask(CPU_LOG_PAGE, "%s: base @ %p for %lu bytes\n",
 -                  __func__, addr, reserved_va);
 +                  __func__, addr, reserved_va + 1);
  }
  void probe_guest_base(const char *image_name, abi_ulong guest_loaddr,
 diff --git a/linux-user/main.c b/linux-user/main.c
 index XXXXXXX..XXXXXXX 100644
 --- a/linux-user/main.c
 +++ b/linux-user/main.c
@@ -XXX,XX +XXX,XX @@ static const char *last_log_filename;
  # if HOST_LONG_BITS > TARGET_VIRT_ADDR_SPACE_BITS
  #  if TARGET_VIRT_ADDR_SPACE_BITS == 32 && \
        (TARGET_LONG_BITS == 32 || defined(TARGET_ABI32))
 -/* There are a number of places where we assign reserved_va to a variable
 -   of type abi_ulong and expect it to fit.  Avoid the last page.  */
 -#   define MAX_RESERVED_VA(CPU)  (0xfffffffful & TARGET_PAGE_MASK)
 +#   define MAX_RESERVED_VA(CPU)  0xfffffffful
  #  else
 -#   define MAX_RESERVED_VA(CPU)  (1ul << TARGET_VIRT_ADDR_SPACE_BITS)
 +#   define MAX_RESERVED_VA(CPU)  ((1ul << TARGET_VIRT_ADDR_SPACE_BITS) - 1)
  #  endif
  # else
  #  define MAX_RESERVED_VA(CPU)  0
@@ -XXX,XX +XXX,XX @@ static void handle_arg_reserved_va(const char *arg)
  {
-     int i;
+     char *p;
-     for (i = 0; i < count; ++i) {
+     int shift = 0;
--        p[i] = encode_i(OPC_ADDI, TCG_REG_ZERO, TCG_REG_ZERO, 0);
+-    reserved_va = strtoul(arg, &p, 0);
-+        p[i] = OPC_NOP;
++    unsigned long val;
-     }
++
 +    val = strtoul(arg, &p, 0);
      switch (*p) {
      case 'k':
      case 'K':
@@ -XXX,XX +XXX,XX @@ static void handle_arg_reserved_va(const char *arg)
          break;
      }
      if (shift) {
 -        unsigned long unshifted = reserved_va;
 +        unsigned long unshifted = val;
          p++;
 -        reserved_va <<= shift;
 -        if (reserved_va >> shift != unshifted) {
 +        val <<= shift;
 +        if (val >> shift != unshifted) {
              fprintf(stderr, "Reserved virtual address too big\n");
              exit(EXIT_FAILURE);
          }
@@ -XXX,XX +XXX,XX @@ static void handle_arg_reserved_va(const char *arg)
          fprintf(stderr, "Unrecognised -R size suffix '%s'\n", p);
          exit(EXIT_FAILURE);
      }
 +    /* The representation is size - 1, with 0 remaining "default". */
 +    reserved_va = val ? val - 1 : 0;
  }
+ static void handle_arg_singlestep(const char *arg)
+@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
+      */
+     max_reserved_va = MAX_RESERVED_VA(cpu);
+     if (reserved_va != 0) {
+-        if (reserved_va % qemu_host_page_size) {
++        if ((reserved_va + 1) % qemu_host_page_size) {
+             char *s = size_to_str(qemu_host_page_size);
+             fprintf(stderr, "Reserved virtual address not aligned mod %s\n", s);
+             g_free(s);
+@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
+             exit(EXIT_FAILURE);
+         }
+     } else if (HOST_LONG_BITS == 64 && TARGET_VIRT_ADDR_SPACE_BITS <= 32) {
+-        /*
+-         * reserved_va must be aligned with the host page size
+-         * as it is used with mmap()
+-         */
+-        reserved_va = max_reserved_va & qemu_host_page_mask;
++        /* MAX_RESERVED_VA + 1 is a large power of 2, so is aligned. */
++        reserved_va = max_reserved_va;
+     }
+     {
+diff --git a/linux-user/mmap.c b/linux-user/mmap.c
+index XXXXXXX..XXXXXXX 100644
+--- a/linux-user/mmap.c
++++ b/linux-user/mmap.c
+@@ -XXX,XX +XXX,XX @@ static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size,
+     end_addr = start + size;
+     if (start > reserved_va - size) {
+         /* Start at the top of the address space.  */
+-        end_addr = ((reserved_va - size) & -align) + size;
++        end_addr = ((reserved_va + 1 - size) & -align) + size;
+         looped = true;
+     }
+@@ -XXX,XX +XXX,XX @@ static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size,
+                 return (abi_ulong)-1;
+             }
+             /* Re-start at the top of the address space.  */
+-            addr = end_addr = ((reserved_va - size) & -align) + size;
++            addr = end_addr = ((reserved_va + 1 - size) & -align) + size;
+             looped = true;
+         } else {
+             prot = page_get_flags(addr);
 --
 .34.1

-[PULL 01/22] tcg: Split out tcg_out_exit_tb
+[PATCH for-8.0 05/11] accel/tcg: Pass last not end to page_set_flags
-The INDEX_op_exit_tb opcode needs no register allocation.
+Pass the address of the last byte to be changed, rather than
-Split out a dedicated helper function for it.
+the first address past the last byte.  This avoids overflow
+when the last page of the address space is involved.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
 Resolves: https://gitlab.com/qemu-project/qemu/-/issues/1528
 Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- tcg/tcg.c                        |  4 ++++
+ include/exec/cpu-all.h |  2 +-
- tcg/aarch64/tcg-target.c.inc     | 22 ++++++++++--------
+ accel/tcg/user-exec.c  | 16 +++++++---------
- tcg/arm/tcg-target.c.inc         | 11 +++++----
+ bsd-user/mmap.c        |  6 +++---
- tcg/i386/tcg-target.c.inc        | 21 +++++++++--------
+ linux-user/elfload.c   | 11 ++++++-----
- tcg/loongarch64/tcg-target.c.inc | 22 ++++++++++--------
+ linux-user/mmap.c      | 16 ++++++++--------
- tcg/mips/tcg-target.c.inc        | 33 +++++++++++++--------------
+ linux-user/syscall.c   |  4 ++--
- tcg/ppc/tcg-target.c.inc         | 11 +++++----
+files changed, 27 insertions(+), 28 deletions(-)
- tcg/riscv/tcg-target.c.inc       | 22 ++++++++++--------
- tcg/s390x/tcg-target.c.inc       | 23 ++++++++++---------
+diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h
- tcg/sparc64/tcg-target.c.inc     | 39 +++++++++++++++++---------------
+index XXXXXXX..XXXXXXX 100644
- tcg/tci/tcg-target.c.inc         | 10 ++++----
+--- a/include/exec/cpu-all.h
-files changed, 121 insertions(+), 97 deletions(-)
++++ b/include/exec/cpu-all.h
+@@ -XXX,XX +XXX,XX @@ typedef int (*walk_memory_regions_fn)(void *, target_ulong,
-diff --git a/tcg/tcg.c b/tcg/tcg.c
+ int walk_memory_regions(void *, walk_memory_regions_fn);
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/tcg.c
+ int page_get_flags(target_ulong address);
-+++ b/tcg/tcg.c
+-void page_set_flags(target_ulong start, target_ulong end, int flags);
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
++void page_set_flags(target_ulong start, target_ulong last, int flags);
- static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
+ void page_reset_target_data(target_ulong start, target_ulong end);
- static void tcg_out_movi(TCGContext *s, TCGType type,
+ int page_check_range(target_ulong start, target_ulong len, int flags);
-                          TCGReg ret, tcg_target_long arg);
-+static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
+diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c
- static void tcg_out_op(TCGContext *s, TCGOpcode opc,
+index XXXXXXX..XXXXXXX 100644
-                        const TCGArg args[TCG_MAX_OP_ARGS],
+--- a/accel/tcg/user-exec.c
-                        const int const_args[TCG_MAX_OP_ARGS]);
++++ b/accel/tcg/user-exec.c
-@@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
+@@ -XXX,XX +XXX,XX @@ static bool pageflags_set_clear(target_ulong start, target_ulong last,
-         case INDEX_op_call:
+  * The flag PAGE_WRITE_ORG is positioned automatically depending
-             tcg_reg_alloc_call(s, op);
+  * on PAGE_WRITE.  The mmap_lock should already be held.
   */
 -void page_set_flags(target_ulong start, target_ulong end, int flags)
 +void page_set_flags(target_ulong start, target_ulong last, int flags)
  {
 -    target_ulong last;
      bool reset = false;
      bool inval_tb = false;
      /* This function should never be called with addresses outside the
         guest address space.  If this assert fires, it probably indicates
         a missing call to h2g_valid.  */
 -    assert(start < end);
 -    assert(end - 1 <= GUEST_ADDR_MAX);
 +    assert(start <= last);
 +    assert(last <= GUEST_ADDR_MAX);
      /* Only set PAGE_ANON with new mappings. */
      assert(!(flags & PAGE_ANON) || (flags & PAGE_RESET));
      assert_memory_lock();
 -    start = start & TARGET_PAGE_MASK;
 -    end = TARGET_PAGE_ALIGN(end);
 -    last = end - 1;
 +    start &= TARGET_PAGE_MASK;
 +    last |= ~TARGET_PAGE_MASK;
      if (!(flags & PAGE_VALID)) {
          flags = 0;
@@ -XXX,XX +XXX,XX @@ void page_set_flags(target_ulong start, target_ulong end, int flags)
      }
      if (!flags || reset) {
 -        page_reset_target_data(start, end);
 +        page_reset_target_data(start, last + 1);
          inval_tb |= pageflags_unset(start, last);
      }
      if (flags) {
@@ -XXX,XX +XXX,XX @@ void page_set_flags(target_ulong start, target_ulong end, int flags)
                                          ~(reset ? 0 : PAGE_STICKY));
      }
      if (inval_tb) {
 -        tb_invalidate_phys_range(start, end);
 +        tb_invalidate_phys_range(start, last + 1);
      }
  }
 diff --git a/bsd-user/mmap.c b/bsd-user/mmap.c
 index XXXXXXX..XXXXXXX 100644
 --- a/bsd-user/mmap.c
 +++ b/bsd-user/mmap.c
@@ -XXX,XX +XXX,XX @@ int target_mprotect(abi_ulong start, abi_ulong len, int prot)
          if (ret != 0)
              goto error;
      }
 -    page_set_flags(start, start + len, prot | PAGE_VALID);
 +    page_set_flags(start, start + len - 1, prot | PAGE_VALID);
      mmap_unlock();
      return 0;
  error:
@@ -XXX,XX +XXX,XX @@ abi_long target_mmap(abi_ulong start, abi_ulong len, int prot,
          }
      }
   the_end1:
 -    page_set_flags(start, start + len, prot | PAGE_VALID);
 +    page_set_flags(start, start + len - 1, prot | PAGE_VALID);
   the_end:
  #ifdef DEBUG_MMAP
      printf("ret=0x" TARGET_ABI_FMT_lx "\n", start);
@@ -XXX,XX +XXX,XX @@ int target_munmap(abi_ulong start, abi_ulong len)
      }
      if (ret == 0) {
 -        page_set_flags(start, start + len, 0);
 +        page_set_flags(start, start + len - 1, 0);
      }
      mmap_unlock();
      return ret;
 diff --git a/linux-user/elfload.c b/linux-user/elfload.c
 index XXXXXXX..XXXXXXX 100644
 --- a/linux-user/elfload.c
 +++ b/linux-user/elfload.c
@@ -XXX,XX +XXX,XX @@ static bool init_guest_commpage(void)
          exit(EXIT_FAILURE);
      }
      page_set_flags(TARGET_VSYSCALL_PAGE,
 -                   TARGET_VSYSCALL_PAGE + TARGET_PAGE_SIZE,
 +                   TARGET_VSYSCALL_PAGE | ~TARGET_PAGE_MASK,
                     PAGE_EXEC | PAGE_VALID);
      return true;
  }
@@ -XXX,XX +XXX,XX @@ static bool init_guest_commpage(void)
          exit(EXIT_FAILURE);
      }
 -    page_set_flags(commpage, commpage + qemu_host_page_size,
 +    page_set_flags(commpage, commpage | ~qemu_host_page_mask,
                     PAGE_READ | PAGE_EXEC | PAGE_VALID);
      return true;
  }
@@ -XXX,XX +XXX,XX @@ static bool init_guest_commpage(void)
          exit(EXIT_FAILURE);
      }
 -    page_set_flags(LO_COMMPAGE, LO_COMMPAGE + TARGET_PAGE_SIZE,
 +    page_set_flags(LO_COMMPAGE, LO_COMMPAGE | ~TARGET_PAGE_MASK,
                     PAGE_READ | PAGE_EXEC | PAGE_VALID);
      return true;
  }
@@ -XXX,XX +XXX,XX @@ static bool init_guest_commpage(void)
       * and implement syscalls.  Here, simply mark the page executable.
       * Special case the entry points during translation (see do_page_zero).
       */
 -    page_set_flags(LO_COMMPAGE, LO_COMMPAGE + TARGET_PAGE_SIZE,
 +    page_set_flags(LO_COMMPAGE, LO_COMMPAGE | ~TARGET_PAGE_MASK,
                     PAGE_EXEC | PAGE_VALID);
      return true;
  }
@@ -XXX,XX +XXX,XX @@ static void zero_bss(abi_ulong elf_bss, abi_ulong last_bss, int prot)
      /* Ensure that the bss page(s) are valid */
      if ((page_get_flags(last_bss-1) & prot) != prot) {
 -        page_set_flags(elf_bss & TARGET_PAGE_MASK, last_bss, prot | PAGE_VALID);
 +        page_set_flags(elf_bss & TARGET_PAGE_MASK, last_bss - 1,
 +                       prot | PAGE_VALID);
      }
      if (host_start < host_map_start) {
 diff --git a/linux-user/mmap.c b/linux-user/mmap.c
 index XXXXXXX..XXXXXXX 100644
 --- a/linux-user/mmap.c
 +++ b/linux-user/mmap.c
@@ -XXX,XX +XXX,XX @@ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot)
          }
      }
 -    page_set_flags(start, start + len, page_flags);
 +    page_set_flags(start, start + len - 1, page_flags);
      ret = 0;
  error:
@@ -XXX,XX +XXX,XX @@ abi_long target_mmap(abi_ulong start, abi_ulong len, int target_prot,
      }
      page_flags |= PAGE_RESET;
      if (passthrough_start == passthrough_end) {
 -        page_set_flags(start, start + len, page_flags);
 +        page_set_flags(start, start + len - 1, page_flags);
      } else {
          if (start < passthrough_start) {
 -            page_set_flags(start, passthrough_start, page_flags);
 +            page_set_flags(start, passthrough_start - 1, page_flags);
          }
 -        page_set_flags(passthrough_start, passthrough_end,
 +        page_set_flags(passthrough_start, passthrough_end - 1,
                         page_flags | PAGE_PASSTHROUGH);
          if (passthrough_end < start + len) {
 -            page_set_flags(passthrough_end, start + len, page_flags);
 +            page_set_flags(passthrough_end, start + len - 1, page_flags);
          }
      }
   the_end:
@@ -XXX,XX +XXX,XX @@ int target_munmap(abi_ulong start, abi_ulong len)
      }
      if (ret == 0) {
 -        page_set_flags(start, start + len, 0);
 +        page_set_flags(start, start + len - 1, 0);
      }
      mmap_unlock();
      return ret;
@@ -XXX,XX +XXX,XX @@ abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size,
      } else {
          new_addr = h2g(host_addr);
          prot = page_get_flags(old_addr);
 -        page_set_flags(old_addr, old_addr + old_size, 0);
 -        page_set_flags(new_addr, new_addr + new_size,
 +        page_set_flags(old_addr, old_addr + old_size - 1, 0);
 +        page_set_flags(new_addr, new_addr + new_size - 1,
                         prot | PAGE_VALID | PAGE_RESET);
      }
      mmap_unlock();
 diff --git a/linux-user/syscall.c b/linux-user/syscall.c
 index XXXXXXX..XXXXXXX 100644
 --- a/linux-user/syscall.c
 +++ b/linux-user/syscall.c
@@ -XXX,XX +XXX,XX @@ static inline abi_ulong do_shmat(CPUArchState *cpu_env,
      }
      raddr=h2g((unsigned long)host_raddr);
 -    page_set_flags(raddr, raddr + shm_info.shm_segsz,
 +    page_set_flags(raddr, raddr + shm_info.shm_segsz - 1,
                     PAGE_VALID | PAGE_RESET | PAGE_READ |
                     (shmflg & SHM_RDONLY ? 0 : PAGE_WRITE));
@@ -XXX,XX +XXX,XX @@ static inline abi_long do_shmdt(abi_ulong shmaddr)
      for (i = 0; i < N_SHM_REGIONS; ++i) {
          if (shm_regions[i].in_use && shm_regions[i].start == shmaddr) {
              shm_regions[i].in_use = false;
 -            page_set_flags(shmaddr, shmaddr + shm_regions[i].size, 0);
 +            page_set_flags(shmaddr, shmaddr + shm_regions[i].size - 1, 0);
              break;
-+        case INDEX_op_exit_tb:
+         }
 +            tcg_out_exit_tb(s, op->args[0]);
 +            break;
          case INDEX_op_dup2_vec:
              if (tcg_reg_alloc_dup2(s, op)) {
                  break;
 diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/aarch64/tcg-target.c.inc
 +++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
  static const tcg_insn_unit *tb_ret_addr;
 +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
 +{
 +    /* Reuse the zeroing that exists for goto_ptr.  */
 +    if (a0 == 0) {
 +        tcg_out_goto_long(s, tcg_code_gen_epilogue);
 +    } else {
 +        tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0);
 +        tcg_out_goto_long(s, tb_ret_addr);
 +    }
 +}
 +
  static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         const TCGArg args[TCG_MAX_OP_ARGS],
                         const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
  #define REG0(I)  (const_args[I] ? TCG_REG_XZR : (TCGReg)args[I])
      switch (opc) {
 -    case INDEX_op_exit_tb:
 -        /* Reuse the zeroing that exists for goto_ptr.  */
 -        if (a0 == 0) {
 -            tcg_out_goto_long(s, tcg_code_gen_epilogue);
 -        } else {
 -            tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0);
 -            tcg_out_goto_long(s, tb_ret_addr);
 -        }
 -        break;
 -
      case INDEX_op_goto_tb:
          tcg_debug_assert(s->tb_jmp_insn_offset != NULL);
          /*
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
      case INDEX_op_mov_i64:
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
 +    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/arm/tcg-target.c.inc
 +++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
  static void tcg_out_epilogue(TCGContext *s);
 +static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
 +{
 +    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, arg);
 +    tcg_out_epilogue(s);
 +}
 +
  static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         const TCGArg args[TCG_MAX_OP_ARGS],
                         const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      int c;
      switch (opc) {
 -    case INDEX_op_exit_tb:
 -        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, args[0]);
 -        tcg_out_epilogue(s);
 -        break;
      case INDEX_op_goto_tb:
          {
              /* Indirect jump method */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
 +    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      default:
          tcg_abort();
      }
 diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/i386/tcg-target.c.inc
 +++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
  #endif
  }
 +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
 +{
 +    /* Reuse the zeroing that exists for goto_ptr.  */
 +    if (a0 == 0) {
 +        tcg_out_jmp(s, tcg_code_gen_epilogue);
 +    } else {
 +        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0);
 +        tcg_out_jmp(s, tb_ret_addr);
 +    }
 +}
 +
  static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
                                const TCGArg args[TCG_MAX_OP_ARGS],
                                const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      const_a2 = const_args[2];
      switch (opc) {
 -    case INDEX_op_exit_tb:
 -        /* Reuse the zeroing that exists for goto_ptr.  */
 -        if (a0 == 0) {
 -            tcg_out_jmp(s, tcg_code_gen_epilogue);
 -        } else {
 -            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0);
 -            tcg_out_jmp(s, tb_ret_addr);
 -        }
 -        break;
      case INDEX_op_goto_tb:
          if (s->tb_jmp_insn_offset) {
              /* direct jump method */
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
      case INDEX_op_mov_i64:
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
 +    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      default:
          tcg_abort();
      }
 diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/loongarch64/tcg-target.c.inc
 +++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
  static const tcg_insn_unit *tb_ret_addr;
 +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
 +{
 +    /* Reuse the zeroing that exists for goto_ptr.  */
 +    if (a0 == 0) {
 +        tcg_out_call_int(s, tcg_code_gen_epilogue, true);
 +    } else {
 +        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0);
 +        tcg_out_call_int(s, tb_ret_addr, true);
 +    }
 +}
 +
  static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         const TCGArg args[TCG_MAX_OP_ARGS],
                         const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      int c2 = const_args[2];
      switch (opc) {
 -    case INDEX_op_exit_tb:
 -        /* Reuse the zeroing that exists for goto_ptr.  */
 -        if (a0 == 0) {
 -            tcg_out_call_int(s, tcg_code_gen_epilogue, true);
 -        } else {
 -            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0);
 -            tcg_out_call_int(s, tb_ret_addr, true);
 -        }
 -        break;
 -
      case INDEX_op_goto_tb:
          tcg_debug_assert(s->tb_jmp_insn_offset != NULL);
          /*
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
      case INDEX_op_mov_i64:
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
 +    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/mips/tcg-target.c.inc
 +++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_clz(TCGContext *s, MIPSInsn opcv2, MIPSInsn opcv6,
      }
  }
 +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
 +{
 +    TCGReg b0 = TCG_REG_ZERO;
 +
 +    if (a0 & ~0xffff) {
 +        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_V0, a0 & ~0xffff);
 +        b0 = TCG_REG_V0;
 +    }
 +    if (!tcg_out_opc_jmp(s, OPC_J, tb_ret_addr)) {
 +        tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)tb_ret_addr);
 +        tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
 +    }
 +    tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff);
 +}
 +
  static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         const TCGArg args[TCG_MAX_OP_ARGS],
                         const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      c2 = const_args[2];
      switch (opc) {
 -    case INDEX_op_exit_tb:
 -        {
 -            TCGReg b0 = TCG_REG_ZERO;
 -
 -            a0 = (intptr_t)a0;
 -            if (a0 & ~0xffff) {
 -                tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_V0, a0 & ~0xffff);
 -                b0 = TCG_REG_V0;
 -            }
 -            if (!tcg_out_opc_jmp(s, OPC_J, tb_ret_addr)) {
 -                tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0,
 -                             (uintptr_t)tb_ret_addr);
 -                tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
 -            }
 -            tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff);
 -        }
 -        break;
      case INDEX_op_goto_tb:
          /* indirect jump method */
          tcg_debug_assert(s->tb_jmp_insn_offset == 0);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
      case INDEX_op_mov_i64:
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
 +    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      default:
          tcg_abort();
      }
 diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/ppc/tcg-target.c.inc
 +++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s)
      tcg_out32(s, BCLR | BO_ALWAYS);
  }
 +static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
 +{
 +    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, arg);
 +    tcg_out_b(s, 0, tcg_code_gen_epilogue);
 +}
 +
  static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         const TCGArg args[TCG_MAX_OP_ARGS],
                         const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      TCGArg a0, a1, a2;
      switch (opc) {
 -    case INDEX_op_exit_tb:
 -        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]);
 -        tcg_out_b(s, 0, tcg_code_gen_epilogue);
 -        break;
      case INDEX_op_goto_tb:
          if (s->tb_jmp_insn_offset) {
              /* Direct jump. */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i32:   /* Always emitted via tcg_out_mov.  */
      case INDEX_op_mov_i64:
      case INDEX_op_call:      /* Always emitted via tcg_out_call.  */
 +    case INDEX_op_exit_tb:   /* Always emitted via tcg_out_exit_tb.  */
      default:
          tcg_abort();
      }
 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/riscv/tcg-target.c.inc
 +++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
  static const tcg_insn_unit *tb_ret_addr;
 +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
 +{
 +    /* Reuse the zeroing that exists for goto_ptr.  */
 +    if (a0 == 0) {
 +        tcg_out_call_int(s, tcg_code_gen_epilogue, true);
 +    } else {
 +        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0);
 +        tcg_out_call_int(s, tb_ret_addr, true);
 +    }
 +}
 +
  static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         const TCGArg args[TCG_MAX_OP_ARGS],
                         const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      int c2 = const_args[2];
      switch (opc) {
 -    case INDEX_op_exit_tb:
 -        /* Reuse the zeroing that exists for goto_ptr.  */
 -        if (a0 == 0) {
 -            tcg_out_call_int(s, tcg_code_gen_epilogue, true);
 -        } else {
 -            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0);
 -            tcg_out_call_int(s, tb_ret_addr, true);
 -        }
 -        break;
 -
      case INDEX_op_goto_tb:
          assert(s->tb_jmp_insn_offset == 0);
          /* indirect jump method */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
      case INDEX_op_mov_i64:
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
 +    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/s390x/tcg-target.c.inc
 +++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg,
  #endif
  }
 +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
 +{
 +    /* Reuse the zeroing that exists for goto_ptr.  */
 +    if (a0 == 0) {
 +        tgen_gotoi(s, S390_CC_ALWAYS, tcg_code_gen_epilogue);
 +    } else {
 +        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, a0);
 +        tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr);
 +    }
 +}
 +
  # define OP_32_64(x) \
          case glue(glue(INDEX_op_,x),_i32): \
          case glue(glue(INDEX_op_,x),_i64)
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      TCGArg a0, a1, a2;
      switch (opc) {
 -    case INDEX_op_exit_tb:
 -        /* Reuse the zeroing that exists for goto_ptr.  */
 -        a0 = args[0];
 -        if (a0 == 0) {
 -            tgen_gotoi(s, S390_CC_ALWAYS, tcg_code_gen_epilogue);
 -        } else {
 -            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, a0);
 -            tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr);
 -        }
 -        break;
 -
      case INDEX_op_goto_tb:
          a0 = args[0];
          /*
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
      case INDEX_op_mov_i64:
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
 +    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      default:
          tcg_abort();
      }
 diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/sparc64/tcg-target.c.inc
 +++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr,
  #endif /* CONFIG_SOFTMMU */
  }
 +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
 +{
 +    if (check_fit_ptr(a0, 13)) {
 +        tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
 +        tcg_out_movi_imm13(s, TCG_REG_O0, a0);
 +        return;
 +    } else if (USE_REG_TB) {
 +        intptr_t tb_diff = tcg_tbrel_diff(s, (void *)a0);
 +        if (check_fit_ptr(tb_diff, 13)) {
 +            tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
 +            /* Note that TCG_REG_TB has been unwound to O1.  */
 +            tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O1, tb_diff, ARITH_ADD);
 +            return;
 +        }
 +    }
 +    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_I0, a0 & ~0x3ff);
 +    tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
 +    tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, a0 & 0x3ff, ARITH_OR);
 +}
 +
  static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         const TCGArg args[TCG_MAX_OP_ARGS],
                         const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      c2 = const_args[2];
      switch (opc) {
 -    case INDEX_op_exit_tb:
 -        if (check_fit_ptr(a0, 13)) {
 -            tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
 -            tcg_out_movi_imm13(s, TCG_REG_O0, a0);
 -            break;
 -        } else if (USE_REG_TB) {
 -            intptr_t tb_diff = tcg_tbrel_diff(s, (void *)a0);
 -            if (check_fit_ptr(tb_diff, 13)) {
 -                tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
 -                /* Note that TCG_REG_TB has been unwound to O1.  */
 -                tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O1, tb_diff, ARITH_ADD);
 -                break;
 -            }
 -        }
 -        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_I0, a0 & ~0x3ff);
 -        tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
 -        tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, a0 & 0x3ff, ARITH_OR);
 -        break;
      case INDEX_op_goto_tb:
          if (s->tb_jmp_insn_offset) {
              /* direct jump method */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
      case INDEX_op_mov_i64:
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
 +    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      default:
          tcg_abort();
      }
 diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/tci/tcg-target.c.inc
 +++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *func,
  # define CASE_64(x)
  #endif
 +static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
 +{
 +    tcg_out_op_p(s, INDEX_op_exit_tb, (void *)arg);
 +}
 +
  static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         const TCGArg args[TCG_MAX_OP_ARGS],
                         const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      TCGOpcode exts;
      switch (opc) {
 -    case INDEX_op_exit_tb:
 -        tcg_out_op_p(s, opc, (void *)args[0]);
 -        break;
 -
      case INDEX_op_goto_tb:
          tcg_debug_assert(s->tb_jmp_insn_offset == 0);
          /* indirect jump method. */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
      case INDEX_op_mov_i64:
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
 +    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      default:
          tcg_abort();
      }
 --
 .34.1

-[PULL 02/22] tcg/i386: Remove unused goto_tb code for indirect jump
+Deleted patch
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
-Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
----
- tcg/i386/tcg-target.c.inc | 14 +++++---------
-file changed, 5 insertions(+), 9 deletions(-)
-diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/i386/tcg-target.c.inc
-+++ b/tcg/i386/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
-     switch (opc) {
-     case INDEX_op_goto_tb:
--        if (s->tb_jmp_insn_offset) {
--            /* direct jump method */
--            int gap;
--            /* jump displacement must be aligned for atomic patching;
-+        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
-+        {
-+            /*
-+             * Jump displacement must be aligned for atomic patching;
-              * see if we need to add extra nops before jump
-              */
--            gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr;
-+            int gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr;
-             if (gap != 1) {
-                 tcg_out_nopn(s, gap - 1);
-             }
-             tcg_out8(s, OPC_JMP_long); /* jmp im */
-             s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
-             tcg_out32(s, 0);
--        } else {
--            /* indirect jump method */
--            tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, -1,
--                                 (intptr_t)(s->tb_jmp_target_addr + a0));
-         }
-         set_jmp_reset_offset(s, a0);
-         break;
---
-.34.1

-[PULL 03/22] tcg/ppc: Remove unused goto_tb code for indirect jump
+Deleted patch
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
-Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
----
- tcg/ppc/tcg-target.c.inc | 32 +++++++++++++-------------------
-file changed, 13 insertions(+), 19 deletions(-)
-diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/ppc/tcg-target.c.inc
-+++ b/tcg/ppc/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-     switch (opc) {
-     case INDEX_op_goto_tb:
--        if (s->tb_jmp_insn_offset) {
--            /* Direct jump. */
--            if (TCG_TARGET_REG_BITS == 64) {
--                /* Ensure the next insns are 8 or 16-byte aligned. */
--                while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) {
--                    tcg_out32(s, NOP);
--                }
--                s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
--                tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0));
--                tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0));
--            } else {
--                s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
--                tcg_out32(s, B);
--                s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
--                break;
-+        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
-+        /* Direct jump. */
-+        if (TCG_TARGET_REG_BITS == 64) {
-+            /* Ensure the next insns are 8 or 16-byte aligned. */
-+            while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) {
-+                tcg_out32(s, NOP);
-             }
-+            s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
-+            tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0));
-+            tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0));
-         } else {
--            /* Indirect jump. */
--            tcg_debug_assert(s->tb_jmp_insn_offset == NULL);
--            tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TB, 0,
--                       (intptr_t)(s->tb_jmp_insn_offset + args[0]));
-+            s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
-+            tcg_out32(s, B);
-+            s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
-+            break;
-         }
-         tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR);
-         tcg_out32(s, BCCTR | BO_ALWAYS);
---
-.34.1

-[PULL 04/22] tcg/sparc64: Remove unused goto_tb code for indirect jump
+Deleted patch
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
-Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
----
- tcg/sparc64/tcg-target.c.inc | 41 +++++++++++-------------------------
-file changed, 12 insertions(+), 29 deletions(-)
-diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/sparc64/tcg-target.c.inc
-+++ b/tcg/sparc64/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
-     return false;
- }
--static void tcg_out_ld_ptr(TCGContext *s, TCGReg ret, const void *arg)
--{
--    intptr_t diff = tcg_tbrel_diff(s, arg);
--    if (USE_REG_TB && check_fit_ptr(diff, 13)) {
--        tcg_out_ld(s, TCG_TYPE_PTR, ret, TCG_REG_TB, diff);
--        return;
--    }
--    tcg_out_movi(s, TCG_TYPE_PTR, ret, (uintptr_t)arg & ~0x3ff);
--    tcg_out_ld(s, TCG_TYPE_PTR, ret, ret, (uintptr_t)arg & 0x3ff);
--}
--
- static void tcg_out_sety(TCGContext *s, TCGReg rs)
- {
-     tcg_out32(s, WRY | INSN_RS1(TCG_REG_G0) | INSN_RS2(rs));
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-     switch (opc) {
-     case INDEX_op_goto_tb:
--        if (s->tb_jmp_insn_offset) {
--            /* direct jump method */
--            if (USE_REG_TB) {
--                /* make sure the patch is 8-byte aligned.  */
--                if ((intptr_t)s->code_ptr & 4) {
--                    tcg_out_nop(s);
--                }
--                s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
--                tcg_out_sethi(s, TCG_REG_T1, 0);
--                tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR);
--                tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL);
--                tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
--            } else {
--                s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
--                tcg_out32(s, CALL);
-+        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
-+        /* Direct jump. */
-+        if (USE_REG_TB) {
-+            /* make sure the patch is 8-byte aligned.  */
-+            if ((intptr_t)s->code_ptr & 4) {
-                 tcg_out_nop(s);
-             }
-+            s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
-+            tcg_out_sethi(s, TCG_REG_T1, 0);
-+            tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR);
-+            tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL);
-+            tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
-         } else {
--            /* indirect jump method */
--            tcg_out_ld_ptr(s, TCG_REG_TB, s->tb_jmp_target_addr + a0);
--            tcg_out_arithi(s, TCG_REG_G0, TCG_REG_TB, 0, JMPL);
-+            s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
-+            tcg_out32(s, CALL);
-             tcg_out_nop(s);
-         }
-         set_jmp_reset_offset(s, a0);
---
-.34.1

-[PULL 05/22] tcg: Replace asserts on tcg_jmp_insn_offset
+Deleted patch
-Test TCG_TARGET_HAS_direct_jump instead of testing an
-implementation pointer.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
-Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
-Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
----
- tcg/aarch64/tcg-target.c.inc     | 2 +-
- tcg/arm/tcg-target.c.inc         | 2 +-
- tcg/loongarch64/tcg-target.c.inc | 2 +-
- tcg/mips/tcg-target.c.inc        | 2 +-
- tcg/riscv/tcg-target.c.inc       | 2 +-
- tcg/tci/tcg-target.c.inc         | 2 +-
-files changed, 6 insertions(+), 6 deletions(-)
-diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/aarch64/tcg-target.c.inc
-+++ b/tcg/aarch64/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-     switch (opc) {
-     case INDEX_op_goto_tb:
--        tcg_debug_assert(s->tb_jmp_insn_offset != NULL);
-+        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
-         /*
-          * Ensure that ADRP+ADD are 8-byte aligned so that an atomic
-          * write can be used to patch the target address.
-diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/arm/tcg-target.c.inc
-+++ b/tcg/arm/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-             intptr_t ptr, dif, dil;
-             TCGReg base = TCG_REG_PC;
--            tcg_debug_assert(s->tb_jmp_insn_offset == 0);
-+            qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
-             ptr = (intptr_t)tcg_splitwx_to_rx(s->tb_jmp_target_addr + args[0]);
-             dif = tcg_pcrel_diff(s, (void *)ptr) - 8;
-             dil = sextract32(dif, 0, 12);
-diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/loongarch64/tcg-target.c.inc
-+++ b/tcg/loongarch64/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-     switch (opc) {
-     case INDEX_op_goto_tb:
--        tcg_debug_assert(s->tb_jmp_insn_offset != NULL);
-+        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
-         /*
-          * Ensure that patch area is 8-byte aligned so that an
-          * atomic write can be used to patch the target address.
-diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/mips/tcg-target.c.inc
-+++ b/tcg/mips/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-     switch (opc) {
-     case INDEX_op_goto_tb:
-         /* indirect jump method */
--        tcg_debug_assert(s->tb_jmp_insn_offset == 0);
-+        qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
-         tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO,
-                    (uintptr_t)(s->tb_jmp_target_addr + a0));
-         tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
-diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/riscv/tcg-target.c.inc
-+++ b/tcg/riscv/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-     switch (opc) {
-     case INDEX_op_goto_tb:
--        assert(s->tb_jmp_insn_offset == 0);
-+        qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
-         /* indirect jump method */
-         tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO,
-                    (uintptr_t)(s->tb_jmp_target_addr + a0));
-diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/tci/tcg-target.c.inc
-+++ b/tcg/tci/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-     switch (opc) {
-     case INDEX_op_goto_tb:
--        tcg_debug_assert(s->tb_jmp_insn_offset == 0);
-+        qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
-         /* indirect jump method. */
-         tcg_out_op_p(s, opc, s->tb_jmp_target_addr + args[0]);
-         set_jmp_reset_offset(s, args[0]);
---
-.34.1

-[PULL 06/22] tcg: Introduce set_jmp_insn_offset
+Deleted patch
-Similar to the existing set_jmp_reset_offset.  Move any assert for
-TCG_TARGET_HAS_direct_jump into the new function (which now cannot
-be build-time).  Will be unused if TCG_TARGET_HAS_direct_jump is
-constant 0, but we can't test for constant in the preprocessor,
-so just mark it G_GNUC_UNUSED.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
-Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
----
- tcg/tcg.c                        | 10 ++++++++++
- tcg/aarch64/tcg-target.c.inc     |  3 +--
- tcg/i386/tcg-target.c.inc        |  3 +--
- tcg/loongarch64/tcg-target.c.inc |  3 +--
- tcg/ppc/tcg-target.c.inc         |  7 +++----
- tcg/s390x/tcg-target.c.inc       |  2 +-
- tcg/sparc64/tcg-target.c.inc     |  5 ++---
-files changed, 19 insertions(+), 14 deletions(-)
-diff --git a/tcg/tcg.c b/tcg/tcg.c
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/tcg.c
-+++ b/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@ static void set_jmp_reset_offset(TCGContext *s, int which)
-     s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
- }
-+static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
-+{
-+    /*
-+     * We will check for overflow at the end of the opcode loop in
-+     * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
-+     */
-+    tcg_debug_assert(TCG_TARGET_HAS_direct_jump);
-+    s->tb_jmp_insn_offset[which] = tcg_current_code_size(s);
-+}
-+
- /* Signal overflow, starting over with fewer guest insns. */
- static G_NORETURN
- void tcg_raise_tb_overflow(TCGContext *s)
-diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/aarch64/tcg-target.c.inc
-+++ b/tcg/aarch64/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-     switch (opc) {
-     case INDEX_op_goto_tb:
--        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
-         /*
-          * Ensure that ADRP+ADD are 8-byte aligned so that an atomic
-          * write can be used to patch the target address.
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-         if ((uintptr_t)s->code_ptr & 7) {
-             tcg_out32(s, NOP);
-         }
--        s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
-+        set_jmp_insn_offset(s, a0);
-         /*
-          * actual branch destination will be patched by
-          * tb_target_set_jmp_target later
-diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/i386/tcg-target.c.inc
-+++ b/tcg/i386/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
-     switch (opc) {
-     case INDEX_op_goto_tb:
--        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
-         {
-             /*
-              * Jump displacement must be aligned for atomic patching;
-@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
-                 tcg_out_nopn(s, gap - 1);
-             }
-             tcg_out8(s, OPC_JMP_long); /* jmp im */
--            s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
-+            set_jmp_insn_offset(s, a0);
-             tcg_out32(s, 0);
-         }
-         set_jmp_reset_offset(s, a0);
-diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/loongarch64/tcg-target.c.inc
-+++ b/tcg/loongarch64/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-     switch (opc) {
-     case INDEX_op_goto_tb:
--        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
-         /*
-          * Ensure that patch area is 8-byte aligned so that an
-          * atomic write can be used to patch the target address.
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-         if ((uintptr_t)s->code_ptr & 7) {
-             tcg_out_nop(s);
-         }
--        s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
-+        set_jmp_insn_offset(s, a0);
-         /*
-          * actual branch destination will be patched by
-          * tb_target_set_jmp_target later
-diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/ppc/tcg-target.c.inc
-+++ b/tcg/ppc/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-     switch (opc) {
-     case INDEX_op_goto_tb:
--        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
-         /* Direct jump. */
-         if (TCG_TARGET_REG_BITS == 64) {
-             /* Ensure the next insns are 8 or 16-byte aligned. */
-             while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) {
-                 tcg_out32(s, NOP);
-             }
--            s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
-+            set_jmp_insn_offset(s, args[0]);
-             tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0));
-             tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0));
-         } else {
--            s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
-+            set_jmp_insn_offset(s, args[0]);
-             tcg_out32(s, B);
--            s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
-+            set_jmp_reset_offset(s, args[0]);
-             break;
-         }
-         tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR);
-diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/s390x/tcg-target.c.inc
-+++ b/tcg/s390x/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
-             tcg_out16(s, NOP);
-         }
-         tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4));
--        s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
-+        set_jmp_insn_offset(s, a0);
-         s->code_ptr += 2;
-         set_jmp_reset_offset(s, a0);
-         break;
-diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/sparc64/tcg-target.c.inc
-+++ b/tcg/sparc64/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-     switch (opc) {
-     case INDEX_op_goto_tb:
--        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
-         /* Direct jump. */
-         if (USE_REG_TB) {
-             /* make sure the patch is 8-byte aligned.  */
-             if ((intptr_t)s->code_ptr & 4) {
-                 tcg_out_nop(s);
-             }
--            s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
-+            set_jmp_insn_offset(s, a0);
-             tcg_out_sethi(s, TCG_REG_T1, 0);
-             tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR);
-             tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL);
-             tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
-         } else {
--            s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
-+            set_jmp_insn_offset(s, a0);
-             tcg_out32(s, CALL);
-             tcg_out_nop(s);
-         }
---
-.34.1

-[PULL 07/22] tcg: Introduce get_jmp_target_addr
+Deleted patch
-Similar to the existing set_jmp_reset_offset.  Include the
-rw->rx address space conversion done by arm and s390x, and
-forgotten by mips and riscv.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
-Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
-Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
----
- tcg/tcg.c                  | 9 +++++++++
- tcg/arm/tcg-target.c.inc   | 2 +-
- tcg/mips/tcg-target.c.inc  | 2 +-
- tcg/riscv/tcg-target.c.inc | 2 +-
- tcg/tci/tcg-target.c.inc   | 2 +-
-files changed, 13 insertions(+), 4 deletions(-)
-diff --git a/tcg/tcg.c b/tcg/tcg.c
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/tcg.c
-+++ b/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@ static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
-     s->tb_jmp_insn_offset[which] = tcg_current_code_size(s);
- }
-+static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
-+{
-+    /*
-+     * Return the read-execute version of the pointer, for the benefit
-+     * of any pc-relative addressing mode.
-+     */
-+    return (uintptr_t)tcg_splitwx_to_rx(&s->tb_jmp_target_addr[which]);
-+}
-+
- /* Signal overflow, starting over with fewer guest insns. */
- static G_NORETURN
- void tcg_raise_tb_overflow(TCGContext *s)
-diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/arm/tcg-target.c.inc
-+++ b/tcg/arm/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-             TCGReg base = TCG_REG_PC;
-             qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
--            ptr = (intptr_t)tcg_splitwx_to_rx(s->tb_jmp_target_addr + args[0]);
-+            ptr = get_jmp_target_addr(s, args[0]);
-             dif = tcg_pcrel_diff(s, (void *)ptr) - 8;
-             dil = sextract32(dif, 0, 12);
-             if (dif != dil) {
-diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/mips/tcg-target.c.inc
-+++ b/tcg/mips/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-         /* indirect jump method */
-         qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
-         tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO,
--                   (uintptr_t)(s->tb_jmp_target_addr + a0));
-+                   get_jmp_target_addr(s, a0));
-         tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
-         tcg_out_nop(s);
-         set_jmp_reset_offset(s, a0);
-diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/riscv/tcg-target.c.inc
-+++ b/tcg/riscv/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-         qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
-         /* indirect jump method */
-         tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO,
--                   (uintptr_t)(s->tb_jmp_target_addr + a0));
-+                   get_jmp_target_addr(s, a0));
-         tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0);
-         set_jmp_reset_offset(s, a0);
-         break;
-diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/tci/tcg-target.c.inc
-+++ b/tcg/tci/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-     case INDEX_op_goto_tb:
-         qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
-         /* indirect jump method. */
--        tcg_out_op_p(s, opc, s->tb_jmp_target_addr + args[0]);
-+        tcg_out_op_p(s, opc, (void *)get_jmp_target_addr(s, args[0]));
-         set_jmp_reset_offset(s, args[0]);
-         break;
---
-.34.1

-[PULL 08/22] tcg: Split out tcg_out_goto_tb
+[PATCH for-8.0 06/11] accel/tcg: Pass last not end to page_reset_target_data
-The INDEX_op_goto_tb opcode needs no register allocation.
+Pass the address of the last byte to be changed, rather than
-Split out a dedicated helper function for it.
+the first address past the last byte.  This avoids overflow
 when the last page of the address space is involved.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
 Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- tcg/tcg.c                        |  4 ++
+ include/exec/cpu-all.h |  2 +-
- tcg/aarch64/tcg-target.c.inc     | 40 ++++++++++---------
+ accel/tcg/user-exec.c  | 11 +++++------
- tcg/arm/tcg-target.c.inc         | 49 ++++++++++++-----------
+ linux-user/mmap.c      |  2 +-
- tcg/i386/tcg-target.c.inc        | 33 ++++++++--------
+files changed, 7 insertions(+), 8 deletions(-)
  tcg/loongarch64/tcg-target.c.inc | 38 +++++++++---------
  tcg/mips/tcg-target.c.inc        | 21 +++++-----
  tcg/ppc/tcg-target.c.inc         | 52 ++++++++++++------------
  tcg/riscv/tcg-target.c.inc       | 20 +++++-----
  tcg/s390x/tcg-target.c.inc       | 31 ++++++++-------
  tcg/sparc64/tcg-target.c.inc     | 68 +++++++++++++++++---------------
  tcg/tci/tcg-target.c.inc         | 16 ++++----
 files changed, 199 insertions(+), 173 deletions(-)
-diff --git a/tcg/tcg.c b/tcg/tcg.c
+diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h
 index XXXXXXX..XXXXXXX 100644
---- a/tcg/tcg.c
+--- a/include/exec/cpu-all.h
-+++ b/tcg/tcg.c
++++ b/include/exec/cpu-all.h
-@@ -XXX,XX +XXX,XX @@ static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
+@@ -XXX,XX +XXX,XX @@ int walk_memory_regions(void *, walk_memory_regions_fn);
- static void tcg_out_movi(TCGContext *s, TCGType type,
-                          TCGReg ret, tcg_target_long arg);
+ int page_get_flags(target_ulong address);
- static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
+ void page_set_flags(target_ulong start, target_ulong last, int flags);
-+static void tcg_out_goto_tb(TCGContext *s, int which);
+-void page_reset_target_data(target_ulong start, target_ulong end);
- static void tcg_out_op(TCGContext *s, TCGOpcode opc,
++void page_reset_target_data(target_ulong start, target_ulong last);
-                        const TCGArg args[TCG_MAX_OP_ARGS],
+ int page_check_range(target_ulong start, target_ulong len, int flags);
-                        const int const_args[TCG_MAX_OP_ARGS]);
-@@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
+ /**
-         case INDEX_op_exit_tb:
+diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c
              tcg_out_exit_tb(s, op->args[0]);
              break;
 +        case INDEX_op_goto_tb:
 +            tcg_out_goto_tb(s, op->args[0]);
 +            break;
          case INDEX_op_dup2_vec:
              if (tcg_reg_alloc_dup2(s, op)) {
                  break;
 diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
---- a/tcg/aarch64/tcg-target.c.inc
+--- a/accel/tcg/user-exec.c
-+++ b/tcg/aarch64/tcg-target.c.inc
++++ b/accel/tcg/user-exec.c
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
+@@ -XXX,XX +XXX,XX @@ void page_set_flags(target_ulong start, target_ulong last, int flags)
      }
+     if (!flags || reset) {
+-        page_reset_target_data(start, last + 1);
++        page_reset_target_data(start, last);
+         inval_tb |= pageflags_unset(start, last);
+     }
+     if (flags) {
+@@ -XXX,XX +XXX,XX @@ typedef struct TargetPageDataNode {
+ static IntervalTreeRoot targetdata_root;
+-void page_reset_target_data(target_ulong start, target_ulong end)
++void page_reset_target_data(target_ulong start, target_ulong last)
+ {
+     IntervalTreeNode *n, *next;
+-    target_ulong last;
+     assert_memory_lock();
+-    start = start & TARGET_PAGE_MASK;
+-    last = TARGET_PAGE_ALIGN(end) - 1;
++    start &= TARGET_PAGE_MASK;
++    last |= ~TARGET_PAGE_MASK;
+     for (n = interval_tree_iter_first(&targetdata_root, start, last),
+          next = n ? interval_tree_iter_next(n, start, last) : NULL;
+@@ -XXX,XX +XXX,XX @@ void *page_get_target_data(target_ulong address)
+     return t->data[(page - region) >> TARGET_PAGE_BITS];
  }
+ #else
-+static void tcg_out_goto_tb(TCGContext *s, int which)
+-void page_reset_target_data(target_ulong start, target_ulong end) { }
-+{
++void page_reset_target_data(target_ulong start, target_ulong last) { }
-+    /*
+ #endif /* TARGET_PAGE_DATA_SIZE */
-+     * Ensure that ADRP+ADD are 8-byte aligned so that an atomic
-+     * write can be used to patch the target address.
+ /* The softmmu versions of these helpers are in cputlb.c.  */
-+     */
+diff --git a/linux-user/mmap.c b/linux-user/mmap.c
 +    if ((uintptr_t)s->code_ptr & 7) {
 +        tcg_out32(s, NOP);
 +    }
 +    set_jmp_insn_offset(s, which);
 +    /*
 +     * actual branch destination will be patched by
 +     * tb_target_set_jmp_target later
 +     */
 +    tcg_out_insn(s, 3406, ADRP, TCG_REG_TMP, 0);
 +    tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_TMP, TCG_REG_TMP, 0);
 +    tcg_out_insn(s, 3207, BR, TCG_REG_TMP);
 +    set_jmp_reset_offset(s, which);
 +}
 +
  static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         const TCGArg args[TCG_MAX_OP_ARGS],
                         const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
  #define REG0(I)  (const_args[I] ? TCG_REG_XZR : (TCGReg)args[I])
      switch (opc) {
 -    case INDEX_op_goto_tb:
 -        /*
 -         * Ensure that ADRP+ADD are 8-byte aligned so that an atomic
 -         * write can be used to patch the target address.
 -         */
 -        if ((uintptr_t)s->code_ptr & 7) {
 -            tcg_out32(s, NOP);
 -        }
 -        set_jmp_insn_offset(s, a0);
 -        /*
 -         * actual branch destination will be patched by
 -         * tb_target_set_jmp_target later
 -         */
 -        tcg_out_insn(s, 3406, ADRP, TCG_REG_TMP, 0);
 -        tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_TMP, TCG_REG_TMP, 0);
 -        tcg_out_insn(s, 3207, BR, TCG_REG_TMP);
 -        set_jmp_reset_offset(s, a0);
 -        break;
 -
      case INDEX_op_goto_ptr:
          tcg_out_insn(s, 3207, BR, a0);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i64:
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
 +    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
---- a/tcg/arm/tcg-target.c.inc
+--- a/linux-user/mmap.c
-+++ b/tcg/arm/tcg-target.c.inc
++++ b/linux-user/mmap.c
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
+@@ -XXX,XX +XXX,XX @@ abi_long target_madvise(abi_ulong start, abi_ulong len_in, int advice)
-     tcg_out_epilogue(s);
+         if (can_passthrough_madvise(start, end)) {
- }
+             ret = get_errno(madvise(g2h_untagged(start), len, advice));
+             if ((advice == MADV_DONTNEED) && (ret == 0)) {
-+static void tcg_out_goto_tb(TCGContext *s, int which)
+-                page_reset_target_data(start, start + len);
-+{
++                page_reset_target_data(start, start + len - 1);
-+    /* Indirect jump method */
+             }
-+    intptr_t ptr, dif, dil;
+         }
 +    TCGReg base = TCG_REG_PC;
 +
 +    qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
 +    ptr = get_jmp_target_addr(s, which);
 +    dif = tcg_pcrel_diff(s, (void *)ptr) - 8;
 +    dil = sextract32(dif, 0, 12);
 +    if (dif != dil) {
 +        /*
 +         * The TB is close, but outside the 12 bits addressable by
 +         * the load.  We can extend this to 20 bits with a sub of a
 +         * shifted immediate from pc.  In the vastly unlikely event
 +         * the code requires more than 1MB, we'll use 2 insns and
 +         * be no worse off.
 +         */
 +        base = TCG_REG_R0;
 +        tcg_out_movi32(s, COND_AL, base, ptr - dil);
 +    }
 +    tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, base, dil);
 +    set_jmp_reset_offset(s, which);
 +}
 +
  static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         const TCGArg args[TCG_MAX_OP_ARGS],
                         const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      int c;
      switch (opc) {
 -    case INDEX_op_goto_tb:
 -        {
 -            /* Indirect jump method */
 -            intptr_t ptr, dif, dil;
 -            TCGReg base = TCG_REG_PC;
 -
 -            qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
 -            ptr = get_jmp_target_addr(s, args[0]);
 -            dif = tcg_pcrel_diff(s, (void *)ptr) - 8;
 -            dil = sextract32(dif, 0, 12);
 -            if (dif != dil) {
 -                /* The TB is close, but outside the 12 bits addressable by
 -                   the load.  We can extend this to 20 bits with a sub of a
 -                   shifted immediate from pc.  In the vastly unlikely event
 -                   the code requires more than 1MB, we'll use 2 insns and
 -                   be no worse off.  */
 -                base = TCG_REG_R0;
 -                tcg_out_movi32(s, COND_AL, base, ptr - dil);
 -            }
 -            tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, base, dil);
 -            set_jmp_reset_offset(s, args[0]);
 -        }
 -        break;
      case INDEX_op_goto_ptr:
          tcg_out_b_reg(s, COND_AL, args[0]);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
 +    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      default:
          tcg_abort();
      }
 diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/i386/tcg-target.c.inc
 +++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
      }
  }
 +static void tcg_out_goto_tb(TCGContext *s, int which)
 +{
 +    /*
 +     * Jump displacement must be aligned for atomic patching;
 +     * see if we need to add extra nops before jump
 +     */
 +    int gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr;
 +    if (gap != 1) {
 +        tcg_out_nopn(s, gap - 1);
 +    }
 +    tcg_out8(s, OPC_JMP_long); /* jmp im */
 +    set_jmp_insn_offset(s, which);
 +    tcg_out32(s, 0);
 +    set_jmp_reset_offset(s, which);
 +}
 +
  static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
                                const TCGArg args[TCG_MAX_OP_ARGS],
                                const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      const_a2 = const_args[2];
      switch (opc) {
 -    case INDEX_op_goto_tb:
 -        {
 -            /*
 -             * Jump displacement must be aligned for atomic patching;
 -             * see if we need to add extra nops before jump
 -             */
 -            int gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr;
 -            if (gap != 1) {
 -                tcg_out_nopn(s, gap - 1);
 -            }
 -            tcg_out8(s, OPC_JMP_long); /* jmp im */
 -            set_jmp_insn_offset(s, a0);
 -            tcg_out32(s, 0);
 -        }
 -        set_jmp_reset_offset(s, a0);
 -        break;
      case INDEX_op_goto_ptr:
          /* jmp to the given host address (could be epilogue) */
          tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, a0);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i64:
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
 +    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      default:
          tcg_abort();
      }
 diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/loongarch64/tcg-target.c.inc
 +++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
      }
  }
 +static void tcg_out_goto_tb(TCGContext *s, int which)
 +{
 +    /*
 +     * Ensure that patch area is 8-byte aligned so that an
 +     * atomic write can be used to patch the target address.
 +     */
 +    if ((uintptr_t)s->code_ptr & 7) {
 +        tcg_out_nop(s);
 +    }
 +    set_jmp_insn_offset(s, which);
 +    /*
 +     * actual branch destination will be patched by
 +     * tb_target_set_jmp_target later
 +     */
 +    tcg_out_opc_pcaddu18i(s, TCG_REG_TMP0, 0);
 +    tcg_out_opc_jirl(s, TCG_REG_ZERO, TCG_REG_TMP0, 0);
 +    set_jmp_reset_offset(s, which);
 +}
 +
  static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         const TCGArg args[TCG_MAX_OP_ARGS],
                         const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      int c2 = const_args[2];
      switch (opc) {
 -    case INDEX_op_goto_tb:
 -        /*
 -         * Ensure that patch area is 8-byte aligned so that an
 -         * atomic write can be used to patch the target address.
 -         */
 -        if ((uintptr_t)s->code_ptr & 7) {
 -            tcg_out_nop(s);
 -        }
 -        set_jmp_insn_offset(s, a0);
 -        /*
 -         * actual branch destination will be patched by
 -         * tb_target_set_jmp_target later
 -         */
 -        tcg_out_opc_pcaddu18i(s, TCG_REG_TMP0, 0);
 -        tcg_out_opc_jirl(s, TCG_REG_ZERO, TCG_REG_TMP0, 0);
 -        set_jmp_reset_offset(s, a0);
 -        break;
 -
      case INDEX_op_mb:
          tcg_out_mb(s, a0);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i64:
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
 +    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/mips/tcg-target.c.inc
 +++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
      tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff);
  }
 +static void tcg_out_goto_tb(TCGContext *s, int which)
 +{
 +    /* indirect jump method */
 +    qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
 +    tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO,
 +               get_jmp_target_addr(s, which));
 +    tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
 +    tcg_out_nop(s);
 +    set_jmp_reset_offset(s, which);
 +}
 +
  static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         const TCGArg args[TCG_MAX_OP_ARGS],
                         const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      c2 = const_args[2];
      switch (opc) {
 -    case INDEX_op_goto_tb:
 -        /* indirect jump method */
 -        qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
 -        tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO,
 -                   get_jmp_target_addr(s, a0));
 -        tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
 -        tcg_out_nop(s);
 -        set_jmp_reset_offset(s, a0);
 -        break;
      case INDEX_op_goto_ptr:
          /* jmp to the given host address (could be epilogue) */
          tcg_out_opc_reg(s, OPC_JR, 0, a0, 0);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i64:
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
 +    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      default:
          tcg_abort();
      }
 diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/ppc/tcg-target.c.inc
 +++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
      tcg_out_b(s, 0, tcg_code_gen_epilogue);
  }
 +static void tcg_out_goto_tb(TCGContext *s, int which)
 +{
 +    /* Direct jump. */
 +    if (TCG_TARGET_REG_BITS == 64) {
 +        /* Ensure the next insns are 8 or 16-byte aligned. */
 +        while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) {
 +            tcg_out32(s, NOP);
 +        }
 +        set_jmp_insn_offset(s, which);
 +        tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0));
 +        tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0));
 +        tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR);
 +        tcg_out32(s, BCCTR | BO_ALWAYS);
 +        set_jmp_reset_offset(s, which);
 +        if (USE_REG_TB) {
 +            /* For the unlinked case, need to reset TCG_REG_TB.  */
 +            tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB,
 +                             -tcg_current_code_size(s));
 +        }
 +    } else {
 +        set_jmp_insn_offset(s, which);
 +        tcg_out32(s, B);
 +        set_jmp_reset_offset(s, which);
 +    }
 +}
 +
  static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         const TCGArg args[TCG_MAX_OP_ARGS],
                         const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      TCGArg a0, a1, a2;
      switch (opc) {
 -    case INDEX_op_goto_tb:
 -        /* Direct jump. */
 -        if (TCG_TARGET_REG_BITS == 64) {
 -            /* Ensure the next insns are 8 or 16-byte aligned. */
 -            while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) {
 -                tcg_out32(s, NOP);
 -            }
 -            set_jmp_insn_offset(s, args[0]);
 -            tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0));
 -            tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0));
 -        } else {
 -            set_jmp_insn_offset(s, args[0]);
 -            tcg_out32(s, B);
 -            set_jmp_reset_offset(s, args[0]);
 -            break;
 -        }
 -        tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR);
 -        tcg_out32(s, BCCTR | BO_ALWAYS);
 -        set_jmp_reset_offset(s, args[0]);
 -        if (USE_REG_TB) {
 -            /* For the unlinked case, need to reset TCG_REG_TB.  */
 -            tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB,
 -                             -tcg_current_code_size(s));
 -        }
 -        break;
      case INDEX_op_goto_ptr:
          tcg_out32(s, MTSPR | RS(args[0]) | CTR);
          if (USE_REG_TB) {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i64:
      case INDEX_op_call:      /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:   /* Always emitted via tcg_out_exit_tb.  */
 +    case INDEX_op_goto_tb:   /* Always emitted via tcg_out_goto_tb.  */
      default:
          tcg_abort();
      }
 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/riscv/tcg-target.c.inc
 +++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
      }
  }
 +static void tcg_out_goto_tb(TCGContext *s, int which)
 +{
 +    qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
 +    /* indirect jump method */
 +    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO,
 +               get_jmp_target_addr(s, which));
 +    tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0);
 +    set_jmp_reset_offset(s, which);
 +}
 +
  static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         const TCGArg args[TCG_MAX_OP_ARGS],
                         const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      int c2 = const_args[2];
      switch (opc) {
 -    case INDEX_op_goto_tb:
 -        qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
 -        /* indirect jump method */
 -        tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO,
 -                   get_jmp_target_addr(s, a0));
 -        tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0);
 -        set_jmp_reset_offset(s, a0);
 -        break;
 -
      case INDEX_op_goto_ptr:
          tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, a0, 0);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i64:
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
 +    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/s390x/tcg-target.c.inc
 +++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
      }
  }
 +static void tcg_out_goto_tb(TCGContext *s, int which)
 +{
 +    /*
 +     * Branch displacement must be aligned for atomic patching;
 +     * see if we need to add extra nop before branch
 +     */
 +    if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) {
 +        tcg_out16(s, NOP);
 +    }
 +    tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4));
 +    set_jmp_insn_offset(s, which);
 +    s->code_ptr += 2;
 +    set_jmp_reset_offset(s, which);
 +}
 +
  # define OP_32_64(x) \
          case glue(glue(INDEX_op_,x),_i32): \
          case glue(glue(INDEX_op_,x),_i64)
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      TCGArg a0, a1, a2;
      switch (opc) {
 -    case INDEX_op_goto_tb:
 -        a0 = args[0];
 -        /*
 -         * branch displacement must be aligned for atomic patching;
 -         * see if we need to add extra nop before branch
 -         */
 -        if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) {
 -            tcg_out16(s, NOP);
 -        }
 -        tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4));
 -        set_jmp_insn_offset(s, a0);
 -        s->code_ptr += 2;
 -        set_jmp_reset_offset(s, a0);
 -        break;
 -
      case INDEX_op_goto_ptr:
          a0 = args[0];
          tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, a0);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i64:
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
 +    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      default:
          tcg_abort();
      }
 diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/sparc64/tcg-target.c.inc
 +++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
      tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, a0 & 0x3ff, ARITH_OR);
  }
 +static void tcg_out_goto_tb(TCGContext *s, int which)
 +{
 +    /* Direct jump. */
 +    if (USE_REG_TB) {
 +        /* make sure the patch is 8-byte aligned.  */
 +        if ((intptr_t)s->code_ptr & 4) {
 +            tcg_out_nop(s);
 +        }
 +        set_jmp_insn_offset(s, which);
 +        tcg_out_sethi(s, TCG_REG_T1, 0);
 +        tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR);
 +        tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL);
 +        tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
 +    } else {
 +        set_jmp_insn_offset(s, which);
 +        tcg_out32(s, CALL);
 +        tcg_out_nop(s);
 +    }
 +    set_jmp_reset_offset(s, which);
 +
 +    /*
 +     * For the unlinked path of goto_tb, we need to reset TCG_REG_TB
 +     * to the beginning of this TB.
 +     */
 +    if (USE_REG_TB) {
 +        int c = -tcg_current_code_size(s);
 +        if (check_fit_i32(c, 13)) {
 +            tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD);
 +        } else {
 +            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c);
 +            tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
 +        }
 +    }
 +}
 +
  static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         const TCGArg args[TCG_MAX_OP_ARGS],
                         const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      c2 = const_args[2];
      switch (opc) {
 -    case INDEX_op_goto_tb:
 -        /* Direct jump. */
 -        if (USE_REG_TB) {
 -            /* make sure the patch is 8-byte aligned.  */
 -            if ((intptr_t)s->code_ptr & 4) {
 -                tcg_out_nop(s);
 -            }
 -            set_jmp_insn_offset(s, a0);
 -            tcg_out_sethi(s, TCG_REG_T1, 0);
 -            tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR);
 -            tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL);
 -            tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
 -        } else {
 -            set_jmp_insn_offset(s, a0);
 -            tcg_out32(s, CALL);
 -            tcg_out_nop(s);
 -        }
 -        set_jmp_reset_offset(s, a0);
 -
 -        /* For the unlinked path of goto_tb, we need to reset
 -           TCG_REG_TB to the beginning of this TB.  */
 -        if (USE_REG_TB) {
 -            c = -tcg_current_code_size(s);
 -            if (check_fit_i32(c, 13)) {
 -                tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD);
 -            } else {
 -                tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c);
 -                tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB,
 -                              TCG_REG_T1, ARITH_ADD);
 -            }
 -        }
 -        break;
      case INDEX_op_goto_ptr:
          tcg_out_arithi(s, TCG_REG_G0, a0, 0, JMPL);
          if (USE_REG_TB) {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i64:
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
 +    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      default:
          tcg_abort();
      }
 diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/tci/tcg-target.c.inc
 +++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
      tcg_out_op_p(s, INDEX_op_exit_tb, (void *)arg);
  }
 +static void tcg_out_goto_tb(TCGContext *s, int which)
 +{
 +    qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
 +    /* indirect jump method. */
 +    tcg_out_op_p(s, INDEX_op_goto_tb, (void *)get_jmp_target_addr(s, which));
 +    set_jmp_reset_offset(s, which);
 +}
 +
  static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         const TCGArg args[TCG_MAX_OP_ARGS],
                         const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      TCGOpcode exts;
      switch (opc) {
 -    case INDEX_op_goto_tb:
 -        qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
 -        /* indirect jump method. */
 -        tcg_out_op_p(s, opc, (void *)get_jmp_target_addr(s, args[0]));
 -        set_jmp_reset_offset(s, args[0]);
 -        break;
 -
      case INDEX_op_goto_ptr:
          tcg_out_op_r(s, opc, args[0]);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_mov_i64:
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
 +    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      default:
          tcg_abort();
      }
 --
 .34.1

-[PULL 14/22] tcg: Always define tb_target_set_jmp_target
+[PATCH for-8.0 07/11] accel/tcg: Pass last not end to PAGE_FOR_EACH_TB
-Install empty versions for !TCG_TARGET_HAS_direct_jump hosts.
+Pass the address of the last byte to be changed, rather than
 the first address past the last byte.  This avoids overflow
 when the last page of the address space is involved.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
 Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- tcg/arm/tcg-target.c.inc   | 6 ++++++
+ accel/tcg/tb-maint.c | 28 ++++++++++++++++------------
- tcg/mips/tcg-target.c.inc  | 6 ++++++
+file changed, 16 insertions(+), 12 deletions(-)
  tcg/riscv/tcg-target.c.inc | 6 ++++++
  tcg/tci/tcg-target.c.inc   | 6 ++++++
 files changed, 24 insertions(+)
-diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
+diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c
 index XXXXXXX..XXXXXXX 100644
---- a/tcg/arm/tcg-target.c.inc
+--- a/accel/tcg/tb-maint.c
-+++ b/tcg/arm/tcg-target.c.inc
++++ b/accel/tcg/tb-maint.c
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
+@@ -XXX,XX +XXX,XX @@ static void tb_remove(TranslationBlock *tb)
      set_jmp_reset_offset(s, which);
  }
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+ /* TODO: For now, still shared with translate-all.c for system mode. */
-+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
+-#define PAGE_FOR_EACH_TB(start, end, pagedesc, T, N)    \
-+{
+-    for (T = foreach_tb_first(start, end),              \
-+    /* Always indirect, nothing to do */
+-         N = foreach_tb_next(T, start, end);            \
-+}
++#define PAGE_FOR_EACH_TB(start, last, pagedesc, T, N)   \
-+
++    for (T = foreach_tb_first(start, last),             \
- static void tcg_out_op(TCGContext *s, TCGOpcode opc,
++         N = foreach_tb_next(T, start, last);           \
-                        const TCGArg args[TCG_MAX_OP_ARGS],
+          T != NULL;                                     \
-                        const int const_args[TCG_MAX_OP_ARGS])
+-         T = N, N = foreach_tb_next(N, start, end))
-diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
++         T = N, N = foreach_tb_next(N, start, last))
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/mips/tcg-target.c.inc
+ typedef TranslationBlock *PageForEachNext;
-+++ b/tcg/mips/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
+ static PageForEachNext foreach_tb_first(tb_page_addr_t start,
-     set_jmp_reset_offset(s, which);
+-                                        tb_page_addr_t end)
 +                                        tb_page_addr_t last)
  {
 -    IntervalTreeNode *n = interval_tree_iter_first(&tb_root, start, end - 1);
 +    IntervalTreeNode *n = interval_tree_iter_first(&tb_root, start, last);
      return n ? container_of(n, TranslationBlock, itree) : NULL;
  }
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+ static PageForEachNext foreach_tb_next(PageForEachNext tb,
-+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
+                                        tb_page_addr_t start,
-+{
+-                                       tb_page_addr_t end)
-+    /* Always indirect, nothing to do */
++                                       tb_page_addr_t last)
-+}
+ {
-+
+     IntervalTreeNode *n;
- static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-                        const TCGArg args[TCG_MAX_OP_ARGS],
+     if (tb) {
-                        const int const_args[TCG_MAX_OP_ARGS])
+-        n = interval_tree_iter_next(&tb->itree, start, end - 1);
-diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
++        n = interval_tree_iter_next(&tb->itree, start, last);
-index XXXXXXX..XXXXXXX 100644
+         if (n) {
---- a/tcg/riscv/tcg-target.c.inc
+             return container_of(n, TranslationBlock, itree);
-+++ b/tcg/riscv/tcg-target.c.inc
+         }
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
+@@ -XXX,XX +XXX,XX @@ struct page_collection {
-     set_jmp_reset_offset(s, which);
+ };
  typedef int PageForEachNext;
 -#define PAGE_FOR_EACH_TB(start, end, pagedesc, tb, n) \
 +#define PAGE_FOR_EACH_TB(start, last, pagedesc, tb, n) \
      TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next)
  #ifdef CONFIG_DEBUG_TCG
@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
  {
      TranslationBlock *tb;
      PageForEachNext n;
 +    tb_page_addr_t last = end - 1;
      assert_memory_lock();
 -    PAGE_FOR_EACH_TB(start, end, unused, tb, n) {
 +    PAGE_FOR_EACH_TB(start, last, unused, tb, n) {
          tb_phys_invalidate__locked(tb);
      }
  }
+@@ -XXX,XX +XXX,XX @@ bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc)
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+     bool current_tb_modified;
-+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
+     TranslationBlock *tb;
-+{
+     PageForEachNext n;
-+    /* Always indirect, nothing to do */
++    tb_page_addr_t last;
-+}
-+
+     /*
- static void tcg_out_op(TCGContext *s, TCGOpcode opc,
+      * Without precise smc semantics, or when outside of a TB,
-                        const TCGArg args[TCG_MAX_OP_ARGS],
+@@ -XXX,XX +XXX,XX @@ bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc)
-                        const int const_args[TCG_MAX_OP_ARGS])
+     assert_memory_lock();
-diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
+     current_tb = tcg_tb_lookup(pc);
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/tci/tcg-target.c.inc
++    last = addr | ~TARGET_PAGE_MASK;
-+++ b/tcg/tci/tcg-target.c.inc
+     addr &= TARGET_PAGE_MASK;
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
+     current_tb_modified = false;
-     set_jmp_reset_offset(s, which);
- }
+-    PAGE_FOR_EACH_TB(addr, addr + TARGET_PAGE_SIZE, unused, tb, n) {
++    PAGE_FOR_EACH_TB(addr, last, unused, tb, n) {
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+         if (current_tb == tb &&
-+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
+             (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
-+{
+             /*
-+    /* Always indirect, nothing to do */
+@@ -XXX,XX +XXX,XX @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages,
-+}
+     bool current_tb_modified = false;
-+
+     TranslationBlock *current_tb = retaddr ? tcg_tb_lookup(retaddr) : NULL;
- static void tcg_out_op(TCGContext *s, TCGOpcode opc,
+ #endif /* TARGET_HAS_PRECISE_SMC */
-                        const TCGArg args[TCG_MAX_OP_ARGS],
++    tb_page_addr_t last G_GNUC_UNUSED = end - 1;
-                        const int const_args[TCG_MAX_OP_ARGS])
      /*
       * We remove all the TBs in the range [start, end[.
       * XXX: see if in some cases it could be faster to invalidate all the code
       */
 -    PAGE_FOR_EACH_TB(start, end, p, tb, n) {
 +    PAGE_FOR_EACH_TB(start, last, p, tb, n) {
          /* NOTE: this is subtle as a TB may span two physical pages */
          if (n == 0) {
              /* NOTE: tb_end may be after the end of the page, but
 --
 .34.1

-[PULL 16/22] tcg/aarch64: Reorg goto_tb implementation
+[PATCH for-8.0 08/11] accel/tcg: Pass last not end to page_collection_lock
-The old implementation replaces two insns, swapping between
+Pass the address of the last byte to be changed, rather than
 the first address past the last byte.  This avoids overflow
 when the last page of the address space is involved.
-    b    <dest>
+Fixes a bug in the loop comparision where "<= end" would lock
-    nop
+one more page than required.
     br    x30
 and
     adrp    x30, <dest>
     addi    x30, x30, lo12:<dest>
     br    x30
-There is a race condition in which a thread could be stopped at
+Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 the PC of the second insn, and when restarted does not see the
 complete address computation and branches to nowhere.
 The new implemetation replaces only one insn, swapping between
     b    <dest>
     br    tmp
 and
     ldr    tmp, <jmp_addr>
     br    tmp
 Reported-by: hev <r@hev.cc>
 Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- tcg/aarch64/tcg-target.h     |  2 +-
+ accel/tcg/tb-maint.c | 22 +++++++++++-----------
- tcg/aarch64/tcg-target.c.inc | 66 +++++++++++++++---------------------
+file changed, 11 insertions(+), 11 deletions(-)
 files changed, 29 insertions(+), 39 deletions(-)
-diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h
+diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c
 index XXXXXXX..XXXXXXX 100644
---- a/tcg/aarch64/tcg-target.h
+--- a/accel/tcg/tb-maint.c
-+++ b/tcg/aarch64/tcg-target.h
++++ b/accel/tcg/tb-maint.c
-@@ -XXX,XX +XXX,XX @@
+@@ -XXX,XX +XXX,XX @@ static gint tb_page_addr_cmp(gconstpointer ap, gconstpointer bp, gpointer udata)
  #define TCG_TARGET_INSN_UNIT_SIZE  4
  #define TCG_TARGET_TLB_DISPLACEMENT_BITS 24
 -#define MAX_CODE_GEN_BUFFER_SIZE  (2 * GiB)
 +#define MAX_CODE_GEN_BUFFER_SIZE  ((size_t)-1)
  typedef enum {
      TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3,
 diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/aarch64/tcg-target.c.inc
 +++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
      tcg_out_call_int(s, target);
  }
--void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+ /*
--                              uintptr_t jmp_rx, uintptr_t jmp_rw)
+- * Lock a range of pages ([@start,@end[) as well as the pages of all
--{
++ * Lock a range of pages ([@start,@last]) as well as the pages of all
--    uintptr_t addr = tb->jmp_target_addr[n];
+  * intersecting TBs.
--    tcg_insn_unit i1, i2;
+  * Locking order: acquire locks in ascending order of page index.
--    TCGType rt = TCG_TYPE_I64;
+  */
--    TCGReg  rd = TCG_REG_TMP;
+ static struct page_collection *page_collection_lock(tb_page_addr_t start,
--    uint64_t pair;
+-                                                    tb_page_addr_t end)
--
++                                                    tb_page_addr_t last)
 -    ptrdiff_t offset = addr - jmp_rx;
 -
 -    if (offset == sextract64(offset, 0, 26)) {
 -        i1 = I3206_B | ((offset >> 2) & 0x3ffffff);
 -        i2 = NOP;
 -    } else {
 -        offset = (addr >> 12) - (jmp_rx >> 12);
 -
 -        /* patch ADRP */
 -        i1 = I3406_ADRP | (offset & 3) << 29 | (offset & 0x1ffffc) << (5 - 2) | rd;
 -        /* patch ADDI */
 -        i2 = I3401_ADDI | rt << 31 | (addr & 0xfff) << 10 | rd << 5 | rd;
 -    }
 -    pair = (uint64_t)i2 << 32 | i1;
 -    qatomic_set((uint64_t *)jmp_rw, pair);
 -    flush_idcache_range(jmp_rx, jmp_rw, 8);
 -}
 -
  static inline void tcg_out_goto_label(TCGContext *s, TCGLabel *l)
  {
-     if (!l->has_value) {
+     struct page_collection *set = g_malloc(sizeof(*set));
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
+     tb_page_addr_t index;
- static void tcg_out_goto_tb(TCGContext *s, int which)
+     PageDesc *pd;
      start >>= TARGET_PAGE_BITS;
 -    end   >>= TARGET_PAGE_BITS;
 -    g_assert(start <= end);
 +    last >>= TARGET_PAGE_BITS;
 +    g_assert(start <= last);
      set->tree = q_tree_new_full(tb_page_addr_cmp, NULL, NULL,
                                  page_entry_destroy);
@@ -XXX,XX +XXX,XX @@ static struct page_collection *page_collection_lock(tb_page_addr_t start,
   retry:
      q_tree_foreach(set->tree, page_entry_lock, NULL);
 -    for (index = start; index <= end; index++) {
 +    for (index = start; index <= last; index++) {
          TranslationBlock *tb;
          PageForEachNext n;
@@ -XXX,XX +XXX,XX @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages,
  void tb_invalidate_phys_page(tb_page_addr_t addr)
  {
-     /*
+     struct page_collection *pages;
--     * Ensure that ADRP+ADD are 8-byte aligned so that an atomic
+-    tb_page_addr_t start, end;
--     * write can be used to patch the target address.
++    tb_page_addr_t start, last;
-+     * Direct branch, or indirect address load, will be patched
+     PageDesc *p;
-+     * by tb_target_set_jmp_target.  Assert indirect load offset
-+     * in range early, regardless of direct branch distance.
+     p = page_find(addr >> TARGET_PAGE_BITS);
-      */
+@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_page(tb_page_addr_t addr)
--    if ((uintptr_t)s->code_ptr & 7) {
+     }
--        tcg_out32(s, NOP);
--    }
+     start = addr & TARGET_PAGE_MASK;
-+    intptr_t i_off = tcg_pcrel_diff(s, (void *)get_jmp_target_addr(s, which));
+-    end = start + TARGET_PAGE_SIZE;
-+    tcg_debug_assert(i_off == sextract64(i_off, 0, 21));
+-    pages = page_collection_lock(start, end);
-+
+-    tb_invalidate_phys_page_range__locked(pages, p, start, end, 0);
-     set_jmp_insn_offset(s, which);
++    last = addr | ~TARGET_PAGE_MASK;
--    /*
++    pages = page_collection_lock(start, last);
--     * actual branch destination will be patched by
++    tb_invalidate_phys_page_range__locked(pages, p, start, last + 1, 0);
--     * tb_target_set_jmp_target later
+     page_collection_unlock(pages);
 -     */
 -    tcg_out_insn(s, 3406, ADRP, TCG_REG_TMP, 0);
 -    tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_TMP, TCG_REG_TMP, 0);
 +    tcg_out32(s, I3206_B);
      tcg_out_insn(s, 3207, BR, TCG_REG_TMP);
      set_jmp_reset_offset(s, which);
  }
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
-+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
+     struct page_collection *pages;
-+{
+     tb_page_addr_t next;
-+    uintptr_t d_addr = tb->jmp_target_addr[n];
-+    ptrdiff_t d_offset = d_addr - jmp_rx;
+-    pages = page_collection_lock(start, end);
-+    tcg_insn_unit insn;
++    pages = page_collection_lock(start, end - 1);
-+
+     for (next = (start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-+    /* Either directly branch, or indirect branch load. */
+          start < end;
-+    if (d_offset == sextract64(d_offset, 0, 28)) {
+          start = next, next += TARGET_PAGE_SIZE) {
-+        insn = deposit32(I3206_B, 0, 26, d_offset >> 2);
+@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_range_fast(ram_addr_t ram_addr,
-+    } else {
+ {
-+        uintptr_t i_addr = (uintptr_t)&tb->jmp_target_addr[n];
+     struct page_collection *pages;
-+        ptrdiff_t i_offset = i_addr - jmp_rx;
-+
+-    pages = page_collection_lock(ram_addr, ram_addr + size);
-+        /* Note that we asserted this in range in tcg_out_goto_tb. */
++    pages = page_collection_lock(ram_addr, ram_addr + size - 1);
-+        insn = deposit32(I3305_LDR | TCG_REG_TMP, 0, 5, i_offset >> 2);
+     tb_invalidate_phys_page_fast__locked(pages, ram_addr, size, retaddr);
-+    }
+     page_collection_unlock(pages);
-+    qatomic_set((uint32_t *)jmp_rw, insn);
+ }
 +    flush_idcache_range(jmp_rx, jmp_rw, 4);
 +}
 +
  static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         const TCGArg args[TCG_MAX_OP_ARGS],
                         const int const_args[TCG_MAX_OP_ARGS])
 --
 .34.1

-[PULL 18/22] tcg/sparc64: Remove USE_REG_TB
+[PATCH for-8.0 09/11] accel/tcg: Pass last not end to tb_invalidate_phys_page_range__locked
-This is always true for sparc64, so this is dead since 3a5f6805c7ca.
+Pass the address of the last byte to be changed, rather than
 the first address past the last byte.  This avoids overflow
 when the last page of the address space is involved.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
+Properly truncate tb_last to the end of the page; the comment about
 tb_end being past the end of the page being ok is not correct,
 considering overflow.
 Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- tcg/sparc64/tcg-target.c.inc | 62 ++++++++++++------------------------
+ accel/tcg/tb-maint.c | 26 ++++++++++++--------------
-file changed, 21 insertions(+), 41 deletions(-)
+file changed, 12 insertions(+), 14 deletions(-)
-diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
+diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c
 index XXXXXXX..XXXXXXX 100644
---- a/tcg/sparc64/tcg-target.c.inc
+--- a/accel/tcg/tb-maint.c
-+++ b/tcg/sparc64/tcg-target.c.inc
++++ b/accel/tcg/tb-maint.c
-@@ -XXX,XX +XXX,XX @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
+@@ -XXX,XX +XXX,XX @@ bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc)
- #endif
+ static void
+ tb_invalidate_phys_page_range__locked(struct page_collection *pages,
- #define TCG_REG_TB  TCG_REG_I1
+                                       PageDesc *p, tb_page_addr_t start,
--#define USE_REG_TB  (sizeof(void *) > 4)
+-                                      tb_page_addr_t end,
++                                      tb_page_addr_t last,
- static const int tcg_target_reg_alloc_order[] = {
+                                       uintptr_t retaddr)
-     TCG_REG_L0,
+ {
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret,
+     TranslationBlock *tb;
 -    tb_page_addr_t tb_start, tb_end;
      PageForEachNext n;
  #ifdef TARGET_HAS_PRECISE_SMC
      bool current_tb_modified = false;
      TranslationBlock *current_tb = retaddr ? tcg_tb_lookup(retaddr) : NULL;
  #endif /* TARGET_HAS_PRECISE_SMC */
 -    tb_page_addr_t last G_GNUC_UNUSED = end - 1;
      /*
 -     * We remove all the TBs in the range [start, end[.
 +     * We remove all the TBs in the range [start, last].
       * XXX: see if in some cases it could be faster to invalidate all the code
       */
      PAGE_FOR_EACH_TB(start, last, p, tb, n) {
 +        tb_page_addr_t tb_start, tb_last;
 +
          /* NOTE: this is subtle as a TB may span two physical pages */
 +        tb_start = tb_page_addr0(tb);
 +        tb_last = tb_start + tb->size - 1;
          if (n == 0) {
 -            /* NOTE: tb_end may be after the end of the page, but
 -               it is not a problem */
 -            tb_start = tb_page_addr0(tb);
 -            tb_end = tb_start + tb->size;
 +            tb_last = MIN(tb_last, tb_start | ~TARGET_PAGE_MASK);
          } else {
              tb_start = tb_page_addr1(tb);
 -            tb_end = tb_start + ((tb_page_addr0(tb) + tb->size)
 -                                 & ~TARGET_PAGE_MASK);
 +            tb_last = tb_start + (tb_last & ~TARGET_PAGE_MASK);
          }
 -        if (!(tb_end <= start || tb_start >= end)) {
 +        if (!(tb_last < start || tb_start > last)) {
  #ifdef TARGET_HAS_PRECISE_SMC
              if (current_tb == tb &&
                  (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_page(tb_page_addr_t addr)
      start = addr & TARGET_PAGE_MASK;
      last = addr | ~TARGET_PAGE_MASK;
      pages = page_collection_lock(start, last);
 -    tb_invalidate_phys_page_range__locked(pages, p, start, last + 1, 0);
 +    tb_invalidate_phys_page_range__locked(pages, p, start, last, 0);
      page_collection_unlock(pages);
  }
@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
              continue;
          }
          assert_page_locked(pd);
 -        tb_invalidate_phys_page_range__locked(pages, pd, start, bound, 0);
 +        tb_invalidate_phys_page_range__locked(pages, pd, start, bound - 1, 0);
      }
+     page_collection_unlock(pages);
-     /* A 13-bit constant relative to the TB.  */
+ }
--    if (!in_prologue && USE_REG_TB) {
+@@ -XXX,XX +XXX,XX @@ static void tb_invalidate_phys_page_fast__locked(struct page_collection *pages,
 +    if (!in_prologue) {
          test = tcg_tbrel_diff(s, (void *)arg);
          if (check_fit_ptr(test, 13)) {
              tcg_out_arithi(s, ret, TCG_REG_TB, test, ARITH_ADD);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret,
      }
-     /* Use the constant pool, if possible. */
+     assert_page_locked(p);
--    if (!in_prologue && USE_REG_TB) {
+-    tb_invalidate_phys_page_range__locked(pages, p, start, start + len, ra);
-+    if (!in_prologue) {
++    tb_invalidate_phys_page_range__locked(pages, p, start, start + len - 1, ra);
          new_pool_label(s, arg, R_SPARC_13, s->code_ptr,
                         tcg_tbrel_diff(s, NULL));
          tcg_out32(s, LDX | INSN_RD(ret) | INSN_RS1(TCG_REG_TB));
@@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s)
  #endif
      /* We choose TCG_REG_TB such that no move is required.  */
 -    if (USE_REG_TB) {
 -        QEMU_BUILD_BUG_ON(TCG_REG_TB != TCG_REG_I1);
 -        tcg_regset_set_reg(s->reserved_regs, TCG_REG_TB);
 -    }
 +    QEMU_BUILD_BUG_ON(TCG_REG_TB != TCG_REG_I1);
 +    tcg_regset_set_reg(s->reserved_regs, TCG_REG_TB);
      tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I1, 0, JMPL);
      /* delay slot */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
          tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
          tcg_out_movi_imm13(s, TCG_REG_O0, a0);
          return;
 -    } else if (USE_REG_TB) {
 +    } else {
          intptr_t tb_diff = tcg_tbrel_diff(s, (void *)a0);
          if (check_fit_ptr(tb_diff, 13)) {
              tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
  static void tcg_out_goto_tb(TCGContext *s, int which)
  {
 +    int c;
 +
      /* Direct jump. */
 -    if (USE_REG_TB) {
 -        /* make sure the patch is 8-byte aligned.  */
 -        if ((intptr_t)s->code_ptr & 4) {
 -            tcg_out_nop(s);
 -        }
 -        set_jmp_insn_offset(s, which);
 -        tcg_out_sethi(s, TCG_REG_T1, 0);
 -        tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR);
 -        tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL);
 -        tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
 -    } else {
 -        set_jmp_insn_offset(s, which);
 -        tcg_out32(s, CALL);
 +    /* make sure the patch is 8-byte aligned.  */
 +    if ((intptr_t)s->code_ptr & 4) {
          tcg_out_nop(s);
      }
 +    set_jmp_insn_offset(s, which);
 +    tcg_out_sethi(s, TCG_REG_T1, 0);
 +    tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR);
 +    tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL);
 +    tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
      set_jmp_reset_offset(s, which);
      /*
       * For the unlinked path of goto_tb, we need to reset TCG_REG_TB
       * to the beginning of this TB.
       */
 -    if (USE_REG_TB) {
 -        int c = -tcg_current_code_size(s);
 -        if (check_fit_i32(c, 13)) {
 -            tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD);
 -        } else {
 -            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c);
 -            tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
 -        }
 +    c = -tcg_current_code_size(s);
 +    if (check_fit_i32(c, 13)) {
 +        tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD);
 +    } else {
 +        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c);
 +        tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
      }
  }
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
+ /*
      switch (opc) {
      case INDEX_op_goto_ptr:
          tcg_out_arithi(s, TCG_REG_G0, a0, 0, JMPL);
 -        if (USE_REG_TB) {
 -            tcg_out_mov_delay(s, TCG_REG_TB, a0);
 -        } else {
 -            tcg_out_nop(s);
 -        }
 +        tcg_out_mov_delay(s, TCG_REG_TB, a0);
          break;
      case INDEX_op_br:
          tcg_out_bpcc(s, COND_A, BPCC_PT, arg_label(a0));
@@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
      tcg_debug_assert(tb_disp == (int32_t)tb_disp);
      tcg_debug_assert(br_disp == (int32_t)br_disp);
 -    if (!USE_REG_TB) {
 -        qatomic_set((uint32_t *)jmp_rw,
 -            deposit32(CALL, 0, 30, br_disp >> 2));
 -        flush_idcache_range(jmp_rx, jmp_rw, 4);
 -        return;
 -    }
 -
      /* This does not exercise the range of the branch, but we do
         still need to be able to load the new value of TCG_REG_TB.
         But this does still happen quite often.  */
 --
 .34.1

-[PULL 09/22] tcg: Rename TB_JMP_RESET_OFFSET_INVALID to TB_JMP_OFFSET_INVALID
+[PATCH for-8.0 10/11] accel/tcg: Pass last not end to tb_invalidate_phys_range
-This will shortly be used for more than reset.
+Pass the address of the last byte to be changed, rather than
 the first address past the last byte.  This avoids overflow
 when the last page of the address space is involved.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
 Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- include/exec/exec-all.h   | 2 +-
+ include/exec/exec-all.h   |  2 +-
- accel/tcg/translate-all.c | 8 ++++----
+ accel/tcg/tb-maint.c      | 31 ++++++++++++++++---------------
- tcg/tcg.c                 | 4 ++--
+ accel/tcg/translate-all.c |  2 +-
-files changed, 7 insertions(+), 7 deletions(-)
+ accel/tcg/user-exec.c     |  2 +-
  softmmu/physmem.c         |  2 +-
 files changed, 20 insertions(+), 19 deletions(-)
 diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
 index XXXXXXX..XXXXXXX 100644
 --- a/include/exec/exec-all.h
 +++ b/include/exec/exec-all.h
-@@ -XXX,XX +XXX,XX @@ struct TranslationBlock {
+@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_addr(target_ulong addr);
-      * setting one of the jump targets (or patching the jump instruction). Only
+ void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs);
-      * two of such jumps are supported.
+ #endif
-      */
+ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr);
-+#define TB_JMP_OFFSET_INVALID 0xffff /* indicates no jump generated */
+-void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end);
-     uint16_t jmp_reset_offset[2]; /* offset of original jump target */
++void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t last);
--#define TB_JMP_RESET_OFFSET_INVALID 0xffff /* indicates no jump generated */
+ void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr);
-     uintptr_t jmp_target_arg[2];  /* target address or offset */
+ /* GETPC is the true target of the return instruction that we'll execute.  */
-     /*
+diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c
 index XXXXXXX..XXXXXXX 100644
 --- a/accel/tcg/tb-maint.c
 +++ b/accel/tcg/tb-maint.c
@@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
   * Called with mmap_lock held for user-mode emulation.
   * NOTE: this function must not be called while a TB is running.
   */
 -void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
 +void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t last)
  {
      TranslationBlock *tb;
      PageForEachNext n;
 -    tb_page_addr_t last = end - 1;
      assert_memory_lock();
@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
   */
  void tb_invalidate_phys_page(tb_page_addr_t addr)
  {
 -    tb_page_addr_t start, end;
 +    tb_page_addr_t start, last;
      start = addr & TARGET_PAGE_MASK;
 -    end = start + TARGET_PAGE_SIZE;
 -    tb_invalidate_phys_range(start, end);
 +    last = addr | ~TARGET_PAGE_MASK;
 +    tb_invalidate_phys_range(start, last);
  }
  /*
@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_page(tb_page_addr_t addr)
  /*
   * Invalidate all TBs which intersect with the target physical address range
 - * [start;end[. NOTE: start and end may refer to *different* physical pages.
 + * [start;last]. NOTE: start and end may refer to *different* physical pages.
   * 'is_cpu_write_access' should be true if called from a real cpu write
   * access: the virtual CPU will exit the current TB if code is modified inside
   * this TB.
   */
 -void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
 +void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t last)
  {
      struct page_collection *pages;
 -    tb_page_addr_t next;
 +    tb_page_addr_t index, index_last;
 -    pages = page_collection_lock(start, end - 1);
 -    for (next = (start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
 -         start < end;
 -         start = next, next += TARGET_PAGE_SIZE) {
 -        PageDesc *pd = page_find(start >> TARGET_PAGE_BITS);
 -        tb_page_addr_t bound = MIN(next, end);
 +    pages = page_collection_lock(start, last);
 +
 +    index_last = last >> TARGET_PAGE_BITS;
 +    for (index = start >> TARGET_PAGE_BITS; index <= index_last; index++) {
 +        PageDesc *pd = page_find(index);
 +        tb_page_addr_t bound;
          if (pd == NULL) {
              continue;
          }
          assert_page_locked(pd);
 -        tb_invalidate_phys_page_range__locked(pages, pd, start, bound - 1, 0);
 +        bound = (index << TARGET_PAGE_BITS) | ~TARGET_PAGE_MASK;
 +        bound = MIN(bound, last);
 +        tb_invalidate_phys_page_range__locked(pages, pd, start, bound, 0);
      }
      page_collection_unlock(pages);
  }
 diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c
 index XXXXXXX..XXXXXXX 100644
 --- a/accel/tcg/translate-all.c
 +++ b/accel/tcg/translate-all.c
-@@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu,
+@@ -XXX,XX +XXX,XX @@ void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr)
-     tb->jmp_dest[1] = (uintptr_t)NULL;
+         cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
+         addr = get_page_addr_code(env, pc);
-     /* init original jump addresses which have been set during tcg_gen_code() */
+         if (addr != -1) {
--    if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
+-            tb_invalidate_phys_range(addr, addr + 1);
-+    if (tb->jmp_reset_offset[0] != TB_JMP_OFFSET_INVALID) {
++            tb_invalidate_phys_range(addr, addr);
          tb_reset_jump(tb, 0);
      }
 -    if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
 +    if (tb->jmp_reset_offset[1] != TB_JMP_OFFSET_INVALID) {
          tb_reset_jump(tb, 1);
      }
@@ -XXX,XX +XXX,XX @@ static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data)
      if (tb_page_addr1(tb) != -1) {
          tst->cross_page++;
      }
 -    if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
 +    if (tb->jmp_reset_offset[0] != TB_JMP_OFFSET_INVALID) {
          tst->direct_jmp_count++;
 -        if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
 +        if (tb->jmp_reset_offset[1] != TB_JMP_OFFSET_INVALID) {
              tst->direct_jmp2_count++;
          }
      }
-diff --git a/tcg/tcg.c b/tcg/tcg.c
+ }
 diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c
 index XXXXXXX..XXXXXXX 100644
---- a/tcg/tcg.c
+--- a/accel/tcg/user-exec.c
-+++ b/tcg/tcg.c
++++ b/accel/tcg/user-exec.c
-@@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
+@@ -XXX,XX +XXX,XX @@ void page_set_flags(target_ulong start, target_ulong last, int flags)
- #endif
+                                         ~(reset ? 0 : PAGE_STICKY));
+     }
-     /* Initialize goto_tb jump offsets. */
+     if (inval_tb) {
--    tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
+-        tb_invalidate_phys_range(start, last + 1);
--    tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
++        tb_invalidate_phys_range(start, last);
-+    tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID;
+     }
-+    tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID;
+ }
-     tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset;
-     if (TCG_TARGET_HAS_direct_jump) {
+diff --git a/softmmu/physmem.c b/softmmu/physmem.c
-         tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg;
+index XXXXXXX..XXXXXXX 100644
 --- a/softmmu/physmem.c
 +++ b/softmmu/physmem.c
@@ -XXX,XX +XXX,XX @@ static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
      }
      if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
          assert(tcg_enabled());
 -        tb_invalidate_phys_range(addr, addr + length);
 +        tb_invalidate_phys_range(addr, addr + length - 1);
          dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
      }
      cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
 --
 .34.1

-[PULL 10/22] tcg: Add gen_tb to TCGContext
+Deleted patch
-This can replace four other variables that are references
-into the TranslationBlock structure.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
-Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
----
- include/tcg/tcg.h         | 11 +++--------
- accel/tcg/translate-all.c |  2 +-
- tcg/tcg-op.c              | 14 +++++++-------
- tcg/tcg.c                 | 14 +++-----------
-files changed, 14 insertions(+), 27 deletions(-)
-diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h
-index XXXXXXX..XXXXXXX 100644
---- a/include/tcg/tcg.h
-+++ b/include/tcg/tcg.h
-@@ -XXX,XX +XXX,XX @@ struct TCGContext {
-     int nb_indirects;
-     int nb_ops;
--    /* goto_tb support */
--    tcg_insn_unit *code_buf;
--    uint16_t *tb_jmp_reset_offset; /* tb->jmp_reset_offset */
--    uintptr_t *tb_jmp_insn_offset; /* tb->jmp_target_arg if direct_jump */
--    uintptr_t *tb_jmp_target_addr; /* tb->jmp_target_arg if !direct_jump */
--
-     TCGRegSet reserved_regs;
--    uint32_t tb_cflags; /* cflags of the current TB */
-     intptr_t current_frame_offset;
-     intptr_t frame_start;
-     intptr_t frame_end;
-     TCGTemp *frame_temp;
--    tcg_insn_unit *code_ptr;
-+    TranslationBlock *gen_tb;     /* tb for which code is being generated */
-+    tcg_insn_unit *code_buf;      /* pointer for start of tb */
-+    tcg_insn_unit *code_ptr;      /* pointer for running end of tb */
- #ifdef CONFIG_PROFILER
-     TCGProfile prof;
-diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c
-index XXXXXXX..XXXXXXX 100644
---- a/accel/tcg/translate-all.c
-+++ b/accel/tcg/translate-all.c
-@@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu,
-     tb->trace_vcpu_dstate = *cpu->trace_dstate;
-     tb_set_page_addr0(tb, phys_pc);
-     tb_set_page_addr1(tb, -1);
--    tcg_ctx->tb_cflags = cflags;
-+    tcg_ctx->gen_tb = tb;
-  tb_overflow:
- #ifdef CONFIG_PROFILER
-diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/tcg-op.c
-+++ b/tcg/tcg-op.c
-@@ -XXX,XX +XXX,XX @@ void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
- void tcg_gen_mb(TCGBar mb_type)
- {
--    if (tcg_ctx->tb_cflags & CF_PARALLEL) {
-+    if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) {
-         tcg_gen_op1(INDEX_op_mb, mb_type);
-     }
- }
-@@ -XXX,XX +XXX,XX @@ void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx)
- void tcg_gen_goto_tb(unsigned idx)
- {
-     /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */
--    tcg_debug_assert(!(tcg_ctx->tb_cflags & CF_NO_GOTO_TB));
-+    tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB));
-     /* We only support two chained exits.  */
-     tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
- #ifdef CONFIG_DEBUG_TCG
-@@ -XXX,XX +XXX,XX @@ void tcg_gen_lookup_and_goto_ptr(void)
- {
-     TCGv_ptr ptr;
--    if (tcg_ctx->tb_cflags & CF_NO_GOTO_PTR) {
-+    if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) {
-         tcg_gen_exit_tb(NULL, 0);
-         return;
-     }
-@@ -XXX,XX +XXX,XX @@ void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
- {
-     memop = tcg_canonicalize_memop(memop, 0, 0);
--    if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
-+    if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
-         TCGv_i32 t1 = tcg_temp_new_i32();
-         TCGv_i32 t2 = tcg_temp_new_i32();
-@@ -XXX,XX +XXX,XX @@ void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
- {
-     memop = tcg_canonicalize_memop(memop, 1, 0);
--    if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
-+    if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
-         TCGv_i64 t1 = tcg_temp_new_i64();
-         TCGv_i64 t2 = tcg_temp_new_i64();
-@@ -XXX,XX +XXX,XX @@ static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = {          \
- void tcg_gen_atomic_##NAME##_i32                                        \
-     (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop)    \
- {                                                                       \
--    if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
-+    if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) {                        \
-         do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME);     \
-     } else {                                                            \
-         do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW,            \
-@@ -XXX,XX +XXX,XX @@ void tcg_gen_atomic_##NAME##_i32                                        \
- void tcg_gen_atomic_##NAME##_i64                                        \
-     (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop)    \
- {                                                                       \
--    if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
-+    if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) {                        \
-         do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME);     \
-     } else {                                                            \
-         do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW,            \
-diff --git a/tcg/tcg.c b/tcg/tcg.c
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/tcg.c
-+++ b/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@ static void set_jmp_reset_offset(TCGContext *s, int which)
-      * We will check for overflow at the end of the opcode loop in
-      * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
-      */
--    s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
-+    s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s);
- }
- static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
-@@ -XXX,XX +XXX,XX @@ static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
-      * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
-      */
-     tcg_debug_assert(TCG_TARGET_HAS_direct_jump);
--    s->tb_jmp_insn_offset[which] = tcg_current_code_size(s);
-+    s->gen_tb->jmp_target_arg[which] = tcg_current_code_size(s);
- }
- static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
-@@ -XXX,XX +XXX,XX @@ static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
-      * Return the read-execute version of the pointer, for the benefit
-      * of any pc-relative addressing mode.
-      */
--    return (uintptr_t)tcg_splitwx_to_rx(&s->tb_jmp_target_addr[which]);
-+    return (uintptr_t)tcg_splitwx_to_rx(s->gen_tb->jmp_target_arg + which);
- }
- /* Signal overflow, starting over with fewer guest insns. */
-@@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
-     /* Initialize goto_tb jump offsets. */
-     tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID;
-     tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID;
--    tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset;
--    if (TCG_TARGET_HAS_direct_jump) {
--        tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg;
--        tcg_ctx->tb_jmp_target_addr = NULL;
--    } else {
--        tcg_ctx->tb_jmp_insn_offset = NULL;
--        tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg;
--    }
-     tcg_reg_alloc_start(s);
---
-.34.1

-[PULL 11/22] tcg: Add TranslationBlock.jmp_insn_offset
+Deleted patch
-Stop overloading jmp_target_arg for both offset and address,
-depending on TCG_TARGET_HAS_direct_jump.  Instead, add a new
-field to hold the jump insn offset and always set the target
-address in jmp_target_addr[].  This will allow a tcg backend
-to use either direct or indirect depending on displacement.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
-Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
----
- include/exec/exec-all.h | 3 ++-
- accel/tcg/cpu-exec.c    | 5 ++---
- tcg/tcg.c               | 6 ++++--
-files changed, 8 insertions(+), 6 deletions(-)
-diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
-index XXXXXXX..XXXXXXX 100644
---- a/include/exec/exec-all.h
-+++ b/include/exec/exec-all.h
-@@ -XXX,XX +XXX,XX @@ struct TranslationBlock {
-      */
- #define TB_JMP_OFFSET_INVALID 0xffff /* indicates no jump generated */
-     uint16_t jmp_reset_offset[2]; /* offset of original jump target */
--    uintptr_t jmp_target_arg[2];  /* target address or offset */
-+    uint16_t jmp_insn_offset[2];  /* offset of direct jump insn */
-+    uintptr_t jmp_target_addr[2]; /* target address */
-     /*
-      * Each TB has a NULL-terminated list (jmp_list_head) of incoming jumps.
-diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c
-index XXXXXXX..XXXXXXX 100644
---- a/accel/tcg/cpu-exec.c
-+++ b/accel/tcg/cpu-exec.c
-@@ -XXX,XX +XXX,XX @@ void cpu_exec_step_atomic(CPUState *cpu)
- void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr)
- {
-+    tb->jmp_target_addr[n] = addr;
-     if (TCG_TARGET_HAS_direct_jump) {
--        uintptr_t offset = tb->jmp_target_arg[n];
-+        uintptr_t offset = tb->jmp_insn_offset[n];
-         uintptr_t tc_ptr = (uintptr_t)tb->tc.ptr;
-         uintptr_t jmp_rx = tc_ptr + offset;
-         uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff;
-         tb_target_set_jmp_target(tc_ptr, jmp_rx, jmp_rw, addr);
--    } else {
--        tb->jmp_target_arg[n] = addr;
-     }
- }
-diff --git a/tcg/tcg.c b/tcg/tcg.c
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/tcg.c
-+++ b/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@ static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
-      * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
-      */
-     tcg_debug_assert(TCG_TARGET_HAS_direct_jump);
--    s->gen_tb->jmp_target_arg[which] = tcg_current_code_size(s);
-+    s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s);
- }
- static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
-@@ -XXX,XX +XXX,XX @@ static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
-      * Return the read-execute version of the pointer, for the benefit
-      * of any pc-relative addressing mode.
-      */
--    return (uintptr_t)tcg_splitwx_to_rx(s->gen_tb->jmp_target_arg + which);
-+    return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]);
- }
- /* Signal overflow, starting over with fewer guest insns. */
-@@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
-     /* Initialize goto_tb jump offsets. */
-     tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID;
-     tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID;
-+    tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID;
-+    tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID;
-     tcg_reg_alloc_start(s);
---
-.34.1

-[PULL 12/22] tcg: Change tb_target_set_jmp_target arguments
+Deleted patch
-Replace 'tc_ptr' and 'addr' with 'tb' and 'n'.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
-Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
----
- tcg/aarch64/tcg-target.h         |  3 ++-
- tcg/arm/tcg-target.h             |  3 ++-
- tcg/i386/tcg-target.h            |  9 ++-------
- tcg/loongarch64/tcg-target.h     |  3 ++-
- tcg/mips/tcg-target.h            |  3 ++-
- tcg/ppc/tcg-target.h             |  3 ++-
- tcg/riscv/tcg-target.h           |  3 ++-
- tcg/s390x/tcg-target.h           | 10 ++--------
- tcg/sparc64/tcg-target.h         |  3 ++-
- tcg/tci/tcg-target.h             |  3 ++-
- accel/tcg/cpu-exec.c             | 11 ++++++++---
- tcg/aarch64/tcg-target.c.inc     |  5 +++--
- tcg/i386/tcg-target.c.inc        |  9 +++++++++
- tcg/loongarch64/tcg-target.c.inc |  5 +++--
- tcg/ppc/tcg-target.c.inc         |  7 ++++---
- tcg/s390x/tcg-target.c.inc       | 10 ++++++++++
- tcg/sparc64/tcg-target.c.inc     |  7 ++++---
-files changed, 61 insertions(+), 36 deletions(-)
-diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/aarch64/tcg-target.h
-+++ b/tcg/aarch64/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ typedef enum {
- #define TCG_TARGET_DEFAULT_MO (0)
- #define TCG_TARGET_HAS_MEMORY_BSWAP     0
--void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-+void tb_target_set_jmp_target(const TranslationBlock *, int,
-+                              uintptr_t, uintptr_t);
- #define TCG_TARGET_NEED_LDST_LABELS
- #define TCG_TARGET_NEED_POOL_LABELS
-diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/arm/tcg-target.h
-+++ b/tcg/arm/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ extern bool use_neon_instructions;
- #define TCG_TARGET_HAS_MEMORY_BSWAP     0
- /* not defined -- call should be eliminated at compile time */
--void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-+                              uintptr_t, uintptr_t);
- #define TCG_TARGET_NEED_LDST_LABELS
- #define TCG_TARGET_NEED_POOL_LABELS
-diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/i386/tcg-target.h
-+++ b/tcg/i386/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ extern bool have_movbe;
- #define TCG_TARGET_extract_i64_valid(ofs, len) \
-     (((ofs) == 8 && (len) == 8) || ((ofs) + (len)) == 32)
--static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
--                                            uintptr_t jmp_rw, uintptr_t addr)
--{
--    /* patch the branch destination */
--    qatomic_set((int32_t *)jmp_rw, addr - (jmp_rx + 4));
--    /* no need to flush icache explicitly */
--}
-+void tb_target_set_jmp_target(const TranslationBlock *, int,
-+                              uintptr_t, uintptr_t);
- /* This defines the natural memory order supported by this
-  * architecture before guarantees made by various barrier
-diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/loongarch64/tcg-target.h
-+++ b/tcg/loongarch64/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ typedef enum {
- #define TCG_TARGET_HAS_muluh_i64        1
- #define TCG_TARGET_HAS_mulsh_i64        1
--void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-+                              uintptr_t, uintptr_t);
- #define TCG_TARGET_DEFAULT_MO (0)
-diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/mips/tcg-target.h
-+++ b/tcg/mips/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions;
- #define TCG_TARGET_HAS_MEMORY_BSWAP     1
- /* not defined -- call should be eliminated at compile time */
--void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t)
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-+                              uintptr_t, uintptr_t)
-     QEMU_ERROR("code path is reachable");
- #define TCG_TARGET_NEED_LDST_LABELS
-diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/ppc/tcg-target.h
-+++ b/tcg/ppc/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ extern bool have_vsx;
- #define TCG_TARGET_HAS_bitsel_vec       have_vsx
- #define TCG_TARGET_HAS_cmpsel_vec       0
--void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-+                              uintptr_t, uintptr_t);
- #define TCG_TARGET_DEFAULT_MO (0)
- #define TCG_TARGET_HAS_MEMORY_BSWAP     1
-diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/riscv/tcg-target.h
-+++ b/tcg/riscv/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ typedef enum {
- #endif
- /* not defined -- call should be eliminated at compile time */
--void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-+                              uintptr_t, uintptr_t);
- #define TCG_TARGET_DEFAULT_MO (0)
-diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/s390x/tcg-target.h
-+++ b/tcg/s390x/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ extern uint64_t s390_facilities[3];
- #define TCG_TARGET_DEFAULT_MO (TCG_MO_ALL & ~TCG_MO_ST_LD)
--static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
--                                            uintptr_t jmp_rw, uintptr_t addr)
--{
--    /* patch the branch destination */
--    intptr_t disp = addr - (jmp_rx - 2);
--    qatomic_set((int32_t *)jmp_rw, disp / 2);
--    /* no need to flush icache explicitly */
--}
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-+                              uintptr_t jmp_rx, uintptr_t jmp_rw);
- #define TCG_TARGET_NEED_LDST_LABELS
- #define TCG_TARGET_NEED_POOL_LABELS
-diff --git a/tcg/sparc64/tcg-target.h b/tcg/sparc64/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/sparc64/tcg-target.h
-+++ b/tcg/sparc64/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions;
- #define TCG_TARGET_DEFAULT_MO (0)
- #define TCG_TARGET_HAS_MEMORY_BSWAP     1
--void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-+                              uintptr_t, uintptr_t);
- #define TCG_TARGET_NEED_POOL_LABELS
-diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/tci/tcg-target.h
-+++ b/tcg/tci/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ typedef enum {
- #define TCG_TARGET_HAS_MEMORY_BSWAP     1
- /* not defined -- call should be eliminated at compile time */
--void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-+                              uintptr_t, uintptr_t);
- #endif /* TCG_TARGET_H */
-diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c
-index XXXXXXX..XXXXXXX 100644
---- a/accel/tcg/cpu-exec.c
-+++ b/accel/tcg/cpu-exec.c
-@@ -XXX,XX +XXX,XX @@ void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr)
- {
-     tb->jmp_target_addr[n] = addr;
-     if (TCG_TARGET_HAS_direct_jump) {
-+        /*
-+         * Get the rx view of the structure, from which we find the
-+         * executable code address, and tb_target_set_jmp_target can
-+         * produce a pc-relative displacement to jmp_target_addr[n].
-+         */
-+        const TranslationBlock *c_tb = tcg_splitwx_to_rx(tb);
-         uintptr_t offset = tb->jmp_insn_offset[n];
--        uintptr_t tc_ptr = (uintptr_t)tb->tc.ptr;
--        uintptr_t jmp_rx = tc_ptr + offset;
-+        uintptr_t jmp_rx = (uintptr_t)tb->tc.ptr + offset;
-         uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff;
--        tb_target_set_jmp_target(tc_ptr, jmp_rx, jmp_rw, addr);
-+        tb_target_set_jmp_target(c_tb, n, jmp_rx, jmp_rw);
-     }
- }
-diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/aarch64/tcg-target.c.inc
-+++ b/tcg/aarch64/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
-     tcg_out_call_int(s, target);
- }
--void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
--                              uintptr_t jmp_rw, uintptr_t addr)
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
- {
-+    uintptr_t addr = tb->jmp_target_addr[n];
-     tcg_insn_unit i1, i2;
-     TCGType rt = TCG_TYPE_I64;
-     TCGReg  rd = TCG_REG_TMP;
-diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/i386/tcg-target.c.inc
-+++ b/tcg/i386/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
-     set_jmp_reset_offset(s, which);
- }
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
-+{
-+    /* patch the branch destination */
-+    uintptr_t addr = tb->jmp_target_addr[n];
-+    qatomic_set((int32_t *)jmp_rw, addr - (jmp_rx + 4));
-+    /* no need to flush icache explicitly */
-+}
-+
- static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
-                               const TCGArg args[TCG_MAX_OP_ARGS],
-                               const int const_args[TCG_MAX_OP_ARGS])
-diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/loongarch64/tcg-target.c.inc
-+++ b/tcg/loongarch64/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_nop(TCGContext *s)
-     tcg_out32(s, NOP);
- }
--void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
--                              uintptr_t jmp_rw, uintptr_t addr)
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
- {
-     tcg_insn_unit i1, i2;
-     ptrdiff_t upper, lower;
-+    uintptr_t addr = tb->jmp_target_addr[n];
-     ptrdiff_t offset = (ptrdiff_t)(addr - jmp_rx) >> 2;
-     if (offset == sextreg(offset, 0, 26)) {
-diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/ppc/tcg-target.c.inc
-+++ b/tcg/ppc/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static inline void ppc64_replace4(uintptr_t rx, uintptr_t rw,
-     flush_idcache_range(rx, rw, 16);
- }
--void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
--                              uintptr_t jmp_rw, uintptr_t addr)
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
- {
-     tcg_insn_unit i0, i1, i2, i3;
--    intptr_t tb_diff = addr - tc_ptr;
-+    uintptr_t addr = tb->jmp_target_addr[n];
-+    intptr_t tb_diff = addr - (uintptr_t)tb->tc.ptr;
-     intptr_t br_diff = addr - (jmp_rx + 4);
-     intptr_t lo, hi;
-diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/s390x/tcg-target.c.inc
-+++ b/tcg/s390x/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
-     set_jmp_reset_offset(s, which);
- }
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
-+{
-+    /* patch the branch destination */
-+    uintptr_t addr = tb->jmp_target_addr[n];
-+    intptr_t disp = addr - (jmp_rx - 2);
-+    qatomic_set((int32_t *)jmp_rw, disp / 2);
-+    /* no need to flush icache explicitly */
-+}
-+
- # define OP_32_64(x) \
-         case glue(glue(INDEX_op_,x),_i32): \
-         case glue(glue(INDEX_op_,x),_i64)
-diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/sparc64/tcg-target.c.inc
-+++ b/tcg/sparc64/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ void tcg_register_jit(const void *buf, size_t buf_size)
-     tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
- }
--void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
--                              uintptr_t jmp_rw, uintptr_t addr)
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
- {
--    intptr_t tb_disp = addr - tc_ptr;
-+    uintptr_t addr = tb->jmp_target_addr[n];
-+    intptr_t tb_disp = addr - (uintptr_t)tb->tc.ptr;
-     intptr_t br_disp = addr - jmp_rx;
-     tcg_insn_unit i1, i2;
---
-.34.1

-[PULL 15/22] tcg: Remove TCG_TARGET_HAS_direct_jump
+Deleted patch
-We now have the option to generate direct or indirect
-goto_tb depending on the dynamic displacement, thus
-the define is no longer necessary or completely accurate.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
-Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
----
- tcg/aarch64/tcg-target.h     |  1 -
- tcg/arm/tcg-target.h         |  1 -
- tcg/i386/tcg-target.h        |  1 -
- tcg/loongarch64/tcg-target.h |  1 -
- tcg/mips/tcg-target.h        |  1 -
- tcg/ppc/tcg-target.h         |  1 -
- tcg/riscv/tcg-target.h       |  1 -
- tcg/s390x/tcg-target.h       |  1 -
- tcg/sparc64/tcg-target.h     |  1 -
- tcg/tci/tcg-target.h         |  1 -
- accel/tcg/cpu-exec.c         | 23 +++++++++++------------
- tcg/tcg.c                    |  1 -
- tcg/arm/tcg-target.c.inc     |  1 -
- tcg/mips/tcg-target.c.inc    |  1 -
- tcg/riscv/tcg-target.c.inc   |  1 -
- tcg/s390x/tcg-target.c.inc   |  3 +++
- tcg/tci/tcg-target.c.inc     |  1 -
-files changed, 14 insertions(+), 27 deletions(-)
-diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/aarch64/tcg-target.h
-+++ b/tcg/aarch64/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ typedef enum {
- #define TCG_TARGET_HAS_muls2_i64        0
- #define TCG_TARGET_HAS_muluh_i64        1
- #define TCG_TARGET_HAS_mulsh_i64        1
--#define TCG_TARGET_HAS_direct_jump      1
- #define TCG_TARGET_HAS_v64              1
- #define TCG_TARGET_HAS_v128             1
-diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/arm/tcg-target.h
-+++ b/tcg/arm/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ extern bool use_neon_instructions;
- #define TCG_TARGET_HAS_mulsh_i32        0
- #define TCG_TARGET_HAS_div_i32          use_idiv_instructions
- #define TCG_TARGET_HAS_rem_i32          0
--#define TCG_TARGET_HAS_direct_jump      0
- #define TCG_TARGET_HAS_qemu_st8_i32     0
- #define TCG_TARGET_HAS_v64              use_neon_instructions
-diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/i386/tcg-target.h
-+++ b/tcg/i386/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ extern bool have_movbe;
- #define TCG_TARGET_HAS_muls2_i32        1
- #define TCG_TARGET_HAS_muluh_i32        0
- #define TCG_TARGET_HAS_mulsh_i32        0
--#define TCG_TARGET_HAS_direct_jump      1
- #if TCG_TARGET_REG_BITS == 64
- /* Keep target addresses zero-extended in a register.  */
-diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/loongarch64/tcg-target.h
-+++ b/tcg/loongarch64/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ typedef enum {
- #define TCG_TARGET_HAS_clz_i32          1
- #define TCG_TARGET_HAS_ctz_i32          1
- #define TCG_TARGET_HAS_ctpop_i32        0
--#define TCG_TARGET_HAS_direct_jump      1
- #define TCG_TARGET_HAS_brcond2          0
- #define TCG_TARGET_HAS_setcond2         0
- #define TCG_TARGET_HAS_qemu_st8_i32     0
-diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/mips/tcg-target.h
-+++ b/tcg/mips/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions;
- #define TCG_TARGET_HAS_muluh_i32        1
- #define TCG_TARGET_HAS_mulsh_i32        1
- #define TCG_TARGET_HAS_bswap32_i32      1
--#define TCG_TARGET_HAS_direct_jump      0
- #if TCG_TARGET_REG_BITS == 64
- #define TCG_TARGET_HAS_add2_i32         0
-diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/ppc/tcg-target.h
-+++ b/tcg/ppc/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ extern bool have_vsx;
- #define TCG_TARGET_HAS_muls2_i32        0
- #define TCG_TARGET_HAS_muluh_i32        1
- #define TCG_TARGET_HAS_mulsh_i32        1
--#define TCG_TARGET_HAS_direct_jump      1
- #define TCG_TARGET_HAS_qemu_st8_i32     0
- #if TCG_TARGET_REG_BITS == 64
-diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/riscv/tcg-target.h
-+++ b/tcg/riscv/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ typedef enum {
- #define TCG_TARGET_HAS_clz_i32          0
- #define TCG_TARGET_HAS_ctz_i32          0
- #define TCG_TARGET_HAS_ctpop_i32        0
--#define TCG_TARGET_HAS_direct_jump      0
- #define TCG_TARGET_HAS_brcond2          1
- #define TCG_TARGET_HAS_setcond2         1
- #define TCG_TARGET_HAS_qemu_st8_i32     0
-diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/s390x/tcg-target.h
-+++ b/tcg/s390x/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ extern uint64_t s390_facilities[3];
- #define TCG_TARGET_HAS_mulsh_i32      0
- #define TCG_TARGET_HAS_extrl_i64_i32  0
- #define TCG_TARGET_HAS_extrh_i64_i32  0
--#define TCG_TARGET_HAS_direct_jump    1
- #define TCG_TARGET_HAS_qemu_st8_i32   0
- #define TCG_TARGET_HAS_div2_i64       1
-diff --git a/tcg/sparc64/tcg-target.h b/tcg/sparc64/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/sparc64/tcg-target.h
-+++ b/tcg/sparc64/tcg-target.h
-@@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions;
- #define TCG_TARGET_HAS_muls2_i32        1
- #define TCG_TARGET_HAS_muluh_i32        0
- #define TCG_TARGET_HAS_mulsh_i32        0
--#define TCG_TARGET_HAS_direct_jump      1
- #define TCG_TARGET_HAS_qemu_st8_i32     0
- #define TCG_TARGET_HAS_extrl_i64_i32    1
-diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/tci/tcg-target.h
-+++ b/tcg/tci/tcg-target.h
-@@ -XXX,XX +XXX,XX @@
- #define TCG_TARGET_HAS_muls2_i32        1
- #define TCG_TARGET_HAS_muluh_i32        0
- #define TCG_TARGET_HAS_mulsh_i32        0
--#define TCG_TARGET_HAS_direct_jump      0
- #define TCG_TARGET_HAS_qemu_st8_i32     0
- #if TCG_TARGET_REG_BITS == 64
-diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c
-index XXXXXXX..XXXXXXX 100644
---- a/accel/tcg/cpu-exec.c
-+++ b/accel/tcg/cpu-exec.c
-@@ -XXX,XX +XXX,XX @@ void cpu_exec_step_atomic(CPUState *cpu)
- void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr)
- {
-+    /*
-+     * Get the rx view of the structure, from which we find the
-+     * executable code address, and tb_target_set_jmp_target can
-+     * produce a pc-relative displacement to jmp_target_addr[n].
-+     */
-+    const TranslationBlock *c_tb = tcg_splitwx_to_rx(tb);
-+    uintptr_t offset = tb->jmp_insn_offset[n];
-+    uintptr_t jmp_rx = (uintptr_t)tb->tc.ptr + offset;
-+    uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff;
-+
-     tb->jmp_target_addr[n] = addr;
--    if (TCG_TARGET_HAS_direct_jump) {
--        /*
--         * Get the rx view of the structure, from which we find the
--         * executable code address, and tb_target_set_jmp_target can
--         * produce a pc-relative displacement to jmp_target_addr[n].
--         */
--        const TranslationBlock *c_tb = tcg_splitwx_to_rx(tb);
--        uintptr_t offset = tb->jmp_insn_offset[n];
--        uintptr_t jmp_rx = (uintptr_t)tb->tc.ptr + offset;
--        uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff;
--        tb_target_set_jmp_target(c_tb, n, jmp_rx, jmp_rw);
--    }
-+    tb_target_set_jmp_target(c_tb, n, jmp_rx, jmp_rw);
- }
- static inline void tb_add_jump(TranslationBlock *tb, int n,
-diff --git a/tcg/tcg.c b/tcg/tcg.c
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/tcg.c
-+++ b/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@ static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
-      * We will check for overflow at the end of the opcode loop in
-      * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
-      */
--    tcg_debug_assert(TCG_TARGET_HAS_direct_jump);
-     s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s);
- }
-diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/arm/tcg-target.c.inc
-+++ b/tcg/arm/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
-     intptr_t ptr, dif, dil;
-     TCGReg base = TCG_REG_PC;
--    qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
-     ptr = get_jmp_target_addr(s, which);
-     dif = tcg_pcrel_diff(s, (void *)ptr) - 8;
-     dil = sextract32(dif, 0, 12);
-diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/mips/tcg-target.c.inc
-+++ b/tcg/mips/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
- static void tcg_out_goto_tb(TCGContext *s, int which)
- {
-     /* indirect jump method */
--    qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
-     tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO,
-                get_jmp_target_addr(s, which));
-     tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
-diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/riscv/tcg-target.c.inc
-+++ b/tcg/riscv/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
- static void tcg_out_goto_tb(TCGContext *s, int which)
- {
--    qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
-     /* indirect jump method */
-     tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO,
-                get_jmp_target_addr(s, which));
-diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/s390x/tcg-target.c.inc
-+++ b/tcg/s390x/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
- void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-                               uintptr_t jmp_rx, uintptr_t jmp_rw)
- {
-+    if (!HAVE_FACILITY(GEN_INST_EXT)) {
-+        return;
-+    }
-     /* patch the branch destination */
-     uintptr_t addr = tb->jmp_target_addr[n];
-     intptr_t disp = addr - (jmp_rx - 2);
-diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/tci/tcg-target.c.inc
-+++ b/tcg/tci/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
- static void tcg_out_goto_tb(TCGContext *s, int which)
- {
--    qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
-     /* indirect jump method. */
-     tcg_out_op_p(s, INDEX_op_goto_tb, (void *)get_jmp_target_addr(s, which));
-     set_jmp_reset_offset(s, which);
---
-.34.1

-[PULL 19/22] tcg/sparc64: Reorg goto_tb implementation
+Deleted patch
-The old sparc64 implementation may replace two insns, which leaves
-a race condition in which a thread could be stopped at a PC in the
-middle of the sequence, and when restarted does not see the complete
-address computation and branches to nowhere.
-The new implemetation replaces only one insn, swapping between a
-direct branch and a direct call.  The TCG_REG_TB register is loaded
-from tb->jmp_target_addr[] in the delay slot.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
-Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
----
- tcg/sparc64/tcg-target.c.inc | 87 +++++++++++++++---------------------
-file changed, 37 insertions(+), 50 deletions(-)
-diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/sparc64/tcg-target.c.inc
-+++ b/tcg/sparc64/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
- static void tcg_out_goto_tb(TCGContext *s, int which)
- {
--    int c;
-+    ptrdiff_t off = tcg_tbrel_diff(s, (void *)get_jmp_target_addr(s, which));
--    /* Direct jump. */
--    /* make sure the patch is 8-byte aligned.  */
--    if ((intptr_t)s->code_ptr & 4) {
--        tcg_out_nop(s);
--    }
-+    /* Direct branch will be patched by tb_target_set_jmp_target. */
-     set_jmp_insn_offset(s, which);
--    tcg_out_sethi(s, TCG_REG_T1, 0);
--    tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR);
--    tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL);
--    tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
-+    tcg_out32(s, CALL);
-+    /* delay slot */
-+    tcg_debug_assert(check_fit_ptr(off, 13));
-+    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TB, TCG_REG_TB, off);
-     set_jmp_reset_offset(s, which);
-     /*
-      * For the unlinked path of goto_tb, we need to reset TCG_REG_TB
-      * to the beginning of this TB.
-      */
--    c = -tcg_current_code_size(s);
--    if (check_fit_i32(c, 13)) {
--        tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD);
-+    off = -tcg_current_code_size(s);
-+    if (check_fit_i32(off, 13)) {
-+        tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, off, ARITH_ADD);
-     } else {
--        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c);
-+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, off);
-         tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
-     }
- }
-+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
-+{
-+    uintptr_t addr = tb->jmp_target_addr[n];
-+    intptr_t br_disp = (intptr_t)(addr - jmp_rx) >> 2;
-+    tcg_insn_unit insn;
-+
-+    br_disp >>= 2;
-+    if (check_fit_ptr(br_disp, 19)) {
-+        /* ba,pt %icc, addr */
-+        insn = deposit32(INSN_OP(0) | INSN_OP2(1) | INSN_COND(COND_A)
-+                         | BPCC_ICC | BPCC_PT, 0, 19, br_disp);
-+    } else if (check_fit_ptr(br_disp, 22)) {
-+        /* ba addr */
-+        insn = deposit32(INSN_OP(0) | INSN_OP2(2) | INSN_COND(COND_A),
-+                         0, 22, br_disp);
-+    } else {
-+        /* The code_gen_buffer can't be larger than 2GB.  */
-+        tcg_debug_assert(check_fit_ptr(br_disp, 30));
-+        /* call addr */
-+        insn = deposit32(CALL, 0, 30, br_disp);
-+    }
-+
-+    qatomic_set((uint32_t *)jmp_rw, insn);
-+    flush_idcache_range(jmp_rx, jmp_rw, 4);
-+}
-+
- static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-                        const TCGArg args[TCG_MAX_OP_ARGS],
-                        const int const_args[TCG_MAX_OP_ARGS])
-@@ -XXX,XX +XXX,XX @@ void tcg_register_jit(const void *buf, size_t buf_size)
- {
-     tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
- }
--
--void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
--                              uintptr_t jmp_rx, uintptr_t jmp_rw)
--{
--    uintptr_t addr = tb->jmp_target_addr[n];
--    intptr_t tb_disp = addr - (uintptr_t)tb->tc.ptr;
--    intptr_t br_disp = addr - jmp_rx;
--    tcg_insn_unit i1, i2;
--
--    /* We can reach the entire address space for ILP32.
--       For LP64, the code_gen_buffer can't be larger than 2GB.  */
--    tcg_debug_assert(tb_disp == (int32_t)tb_disp);
--    tcg_debug_assert(br_disp == (int32_t)br_disp);
--
--    /* This does not exercise the range of the branch, but we do
--       still need to be able to load the new value of TCG_REG_TB.
--       But this does still happen quite often.  */
--    if (check_fit_ptr(tb_disp, 13)) {
--        /* ba,pt %icc, addr */
--        i1 = (INSN_OP(0) | INSN_OP2(1) | INSN_COND(COND_A)
--              | BPCC_ICC | BPCC_PT | INSN_OFF19(br_disp));
--        i2 = (ARITH_ADD | INSN_RD(TCG_REG_TB) | INSN_RS1(TCG_REG_TB)
--              | INSN_IMM13(tb_disp));
--    } else if (tb_disp >= 0) {
--        i1 = SETHI | INSN_RD(TCG_REG_T1) | ((tb_disp & 0xfffffc00) >> 10);
--        i2 = (ARITH_OR | INSN_RD(TCG_REG_T1) | INSN_RS1(TCG_REG_T1)
--              | INSN_IMM13(tb_disp & 0x3ff));
--    } else {
--        i1 = SETHI | INSN_RD(TCG_REG_T1) | ((~tb_disp & 0xfffffc00) >> 10);
--        i2 = (ARITH_XOR | INSN_RD(TCG_REG_T1) | INSN_RS1(TCG_REG_T1)
--              | INSN_IMM13((tb_disp & 0x3ff) | -0x400));
--    }
--
--    qatomic_set((uint64_t *)jmp_rw, deposit64(i2, 32, 32, i1));
--    flush_idcache_range(jmp_rx, jmp_rw, 8);
--}
---
-.34.1

-[PULL 20/22] tcg/arm: Implement direct branch for goto_tb
+[PATCH for-8.0 11/11] linux-user/arm: Take more care allocating commpage
-Now that tcg can handle direct and indirect goto_tb
+User setting of -R reserved_va can lead to an assertion
-simultaneously, we can optimistically leave space for
+failure in page_set_flags.  Sanity check the value of
-a direct branch and fall back to loading the pointer
+reserved_va and print an error message instead.  Do not
-from the TB for an indirect branch.
+allocate a commpage at all for m-profile cpus.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- tcg/arm/tcg-target.c.inc | 52 ++++++++++++++++++++++++++++------------
+ linux-user/elfload.c | 37 +++++++++++++++++++++++++++----------
-file changed, 37 insertions(+), 15 deletions(-)
+file changed, 27 insertions(+), 10 deletions(-)
-diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
+diff --git a/linux-user/elfload.c b/linux-user/elfload.c
 index XXXXXXX..XXXXXXX 100644
---- a/tcg/arm/tcg-target.c.inc
+--- a/linux-user/elfload.c
-+++ b/tcg/arm/tcg-target.c.inc
++++ b/linux-user/elfload.c
-@@ -XXX,XX +XXX,XX @@ typedef enum {
+@@ -XXX,XX +XXX,XX @@ enum {
-     ARITH_BIC = 0xe << 21,
-     ARITH_MVN = 0xf << 21,
+ static bool init_guest_commpage(void)
 +    INSN_B         = 0x0a000000,
 +
      INSN_CLZ       = 0x016f0f10,
      INSN_RBIT      = 0x06ff0f30,
@@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct)
  static void tcg_out_b_imm(TCGContext *s, ARMCond cond, int32_t offset)
  {
--    tcg_out32(s, (cond << 28) | 0x0a000000 |
+-    abi_ptr commpage = HI_COMMPAGE & -qemu_host_page_size;
-+    tcg_out32(s, (cond << 28) | INSN_B |
+-    void *want = g2h_untagged(commpage);
-                     (((offset - 8) >> 2) & 0x00ffffff));
+-    void *addr = mmap(want, qemu_host_page_size, PROT_READ | PROT_WRITE,
- }
+-                      MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0);
++    ARMCPU *cpu = ARM_CPU(thread_cpu);
-@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
++    abi_ptr want = HI_COMMPAGE & TARGET_PAGE_MASK;
++    abi_ptr addr;
- static void tcg_out_goto_tb(TCGContext *s, int which)
- {
+-    if (addr == MAP_FAILED) {
--    /* Indirect jump method */
++    /*
--    intptr_t ptr, dif, dil;
++     * M-profile allocates maximum of 2GB address space, so can never
--    TCGReg base = TCG_REG_PC;
++     * allocate the commpage.  Skip it.
-+    uintptr_t i_addr;
++     */
-+    intptr_t i_disp;
++    if (arm_feature(&cpu->env, ARM_FEATURE_M)) {
++        return true;
 -    ptr = get_jmp_target_addr(s, which);
 -    dif = tcg_pcrel_diff(s, (void *)ptr) - 8;
 -    dil = sextract32(dif, 0, 12);
 -    if (dif != dil) {
 +    /* Direct branch will be patched by tb_target_set_jmp_target. */
 +    set_jmp_insn_offset(s, which);
 +    tcg_out32(s, INSN_NOP);
 +
 +    /* When branch is out of range, fall through to indirect. */
 +    i_addr = get_jmp_target_addr(s, which);
 +    i_disp = tcg_pcrel_diff(s, (void *)i_addr) - 8;
 +    tcg_debug_assert(i_disp < 0);
 +    if (i_disp >= -0xfff) {
 +        tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_PC, i_disp);
 +    } else {
          /*
           * The TB is close, but outside the 12 bits addressable by
           * the load.  We can extend this to 20 bits with a sub of a
 -         * shifted immediate from pc.  In the vastly unlikely event
 -         * the code requires more than 1MB, we'll use 2 insns and
 -         * be no worse off.
 +         * shifted immediate from pc.
           */
 -        base = TCG_REG_R0;
 -        tcg_out_movi32(s, COND_AL, base, ptr - dil);
 +        int h = -i_disp;
 +        int l = h & 0xfff;
 +
 +        h = encode_imm_nofail(h - l);
 +        tcg_out_dat_imm(s, COND_AL, ARITH_SUB, TCG_REG_R0, TCG_REG_PC, h);
 +        tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_R0, l);
      }
 -    tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, base, dil);
      set_jmp_reset_offset(s, which);
  }
  void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
                                uintptr_t jmp_rx, uintptr_t jmp_rw)
  {
 -    /* Always indirect, nothing to do */
 +    uintptr_t addr = tb->jmp_target_addr[n];
 +    ptrdiff_t offset = addr - (jmp_rx + 8);
 +    tcg_insn_unit insn;
 +
 +    /* Either directly branch, or fall through to indirect branch. */
 +    if (offset == sextract64(offset, 0, 26)) {
 +        /* B <addr> */
 +        insn = deposit32((COND_AL << 28) | INSN_B, 0, 24, offset >> 2);
 +    } else {
 +        insn = INSN_NOP;
 +    }
 +
-+    qatomic_set((uint32_t *)jmp_rw, insn);
++    /*
-+    flush_idcache_range(jmp_rx, jmp_rw, 4);
++     * If reserved_va does not cover the commpage, we get an assert
 +     * in page_set_flags.  Produce an intelligent error instead.
 +     */
 +    if (reserved_va != 0 && want + TARGET_PAGE_SIZE - 1 > reserved_va) {
 +        error_report("Allocating guest commpage: -R 0x%" PRIx64 " too small",
 +                     (uint64_t)reserved_va + 1);
 +        exit(EXIT_FAILURE);
 +    }
 +
 +    addr = target_mmap(want, TARGET_PAGE_SIZE, PROT_READ | PROT_WRITE,
 +                       MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0);
 +
 +    if (addr == -1) {
          perror("Allocating guest commpage");
          exit(EXIT_FAILURE);
      }
@@ -XXX,XX +XXX,XX @@ static bool init_guest_commpage(void)
      }
      /* Set kernel helper versions; rest of page is 0.  */
 -    __put_user(5, (uint32_t *)g2h_untagged(0xffff0ffcu));
 +    put_user_u32(5, 0xffff0ffcu);
 -    if (mprotect(addr, qemu_host_page_size, PROT_READ)) {
 +    if (target_mprotect(addr, qemu_host_page_size, PROT_READ | PROT_EXEC)) {
          perror("Protecting guest commpage");
          exit(EXIT_FAILURE);
      }
 -
 -    page_set_flags(commpage, commpage | ~qemu_host_page_mask,
 -                   PAGE_READ | PAGE_EXEC | PAGE_VALID);
      return true;
  }
- static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 --
 .34.1

Second pull for this week, since this set is large enough by itself.

The following changes since commit 7c9236d6d61f30583d5d860097d88dbf0fe487bf:

Merge tag 'pull-tcg-20230116' of https://gitlab.com/rth7680/qemu into staging (2023-01-17 10:24:16 +0000)

are available in the Git repository at:

https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20230117

for you to fetch changes up to 493c9b19a7fb7f387c4fcf57d3836504d5242bf5:

tcg/riscv: Implement direct branch for goto_tb (2023-01-17 22:36:17 +0000)

----------------------------------------------------------------
tcg: Fix race conditions in (most) goto_tb implementations

----------------------------------------------------------------
Richard Henderson (22):
      tcg: Split out tcg_out_exit_tb
      tcg/i386: Remove unused goto_tb code for indirect jump
      tcg/ppc: Remove unused goto_tb code for indirect jump
      tcg/sparc64: Remove unused goto_tb code for indirect jump
      tcg: Replace asserts on tcg_jmp_insn_offset
      tcg: Introduce set_jmp_insn_offset
      tcg: Introduce get_jmp_target_addr
      tcg: Split out tcg_out_goto_tb
      tcg: Rename TB_JMP_RESET_OFFSET_INVALID to TB_JMP_OFFSET_INVALID
      tcg: Add gen_tb to TCGContext
      tcg: Add TranslationBlock.jmp_insn_offset
      tcg: Change tb_target_set_jmp_target arguments
      tcg: Move tb_target_set_jmp_target declaration to tcg.h
      tcg: Always define tb_target_set_jmp_target
      tcg: Remove TCG_TARGET_HAS_direct_jump
      tcg/aarch64: Reorg goto_tb implementation
      tcg/ppc: Reorg goto_tb implementation
      tcg/sparc64: Remove USE_REG_TB
      tcg/sparc64: Reorg goto_tb implementation
      tcg/arm: Implement direct branch for goto_tb
      tcg/riscv: Introduce OPC_NOP
      tcg/riscv: Implement direct branch for goto_tb

The INDEX_op_exit_tb opcode needs no register allocation.
Split out a dedicated helper function for it.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/tcg.c                        |  4 ++++
 tcg/aarch64/tcg-target.c.inc     | 22 ++++++++++--------
 tcg/arm/tcg-target.c.inc         | 11 +++++----
 tcg/i386/tcg-target.c.inc        | 21 +++++++++--------
 tcg/loongarch64/tcg-target.c.inc | 22 ++++++++++--------
 tcg/mips/tcg-target.c.inc        | 33 +++++++++++++--------------
 tcg/ppc/tcg-target.c.inc         | 11 +++++----
 tcg/riscv/tcg-target.c.inc       | 22 ++++++++++--------
 tcg/s390x/tcg-target.c.inc       | 23 ++++++++++---------
 tcg/sparc64/tcg-target.c.inc     | 39 +++++++++++++++++---------------
 tcg/tci/tcg-target.c.inc         | 10 ++++----
 11 files changed, 121 insertions(+), 97 deletions(-)

diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
 static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
 static void tcg_out_movi(TCGContext *s, TCGType type,
                          TCGReg ret, tcg_target_long arg);
+static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS]);
@@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
         case INDEX_op_call:
             tcg_reg_alloc_call(s, op);
             break;
+        case INDEX_op_exit_tb:
+            tcg_out_exit_tb(s, op->args[0]);
+            break;
         case INDEX_op_dup2_vec:
             if (tcg_reg_alloc_dup2(s, op)) {
                 break;
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
 
 static const tcg_insn_unit *tb_ret_addr;
 
+static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
+{
+    /* Reuse the zeroing that exists for goto_ptr.  */
+    if (a0 == 0) {
+        tcg_out_goto_long(s, tcg_code_gen_epilogue);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0);
+        tcg_out_goto_long(s, tb_ret_addr);
+    }
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 #define REG0(I)  (const_args[I] ? TCG_REG_XZR : (TCGReg)args[I])
 
     switch (opc) {
-    case INDEX_op_exit_tb:
-        /* Reuse the zeroing that exists for goto_ptr.  */
-        if (a0 == 0) {
-            tcg_out_goto_long(s, tcg_code_gen_epilogue);
-        } else {
-            tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0);
-            tcg_out_goto_long(s, tb_ret_addr);
-        }
-        break;
-
     case INDEX_op_goto_tb:
         tcg_debug_assert(s->tb_jmp_insn_offset != NULL);
         /*
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
     case INDEX_op_mov_i64:
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
+    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
 
 static void tcg_out_epilogue(TCGContext *s);
 
+static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
+{
+    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, arg);
+    tcg_out_epilogue(s);
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     int c;
 
     switch (opc) {
-    case INDEX_op_exit_tb:
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, args[0]);
-        tcg_out_epilogue(s);
-        break;
     case INDEX_op_goto_tb:
         {
             /* Indirect jump method */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
+    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     default:
         tcg_abort();
     }
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
 #endif
 }
 
+static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
+{
+    /* Reuse the zeroing that exists for goto_ptr.  */
+    if (a0 == 0) {
+        tcg_out_jmp(s, tcg_code_gen_epilogue);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0);
+        tcg_out_jmp(s, tb_ret_addr);
+    }
+}
+
 static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
                               const TCGArg args[TCG_MAX_OP_ARGS],
                               const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     const_a2 = const_args[2];
 
     switch (opc) {
-    case INDEX_op_exit_tb:
-        /* Reuse the zeroing that exists for goto_ptr.  */
-        if (a0 == 0) {
-            tcg_out_jmp(s, tcg_code_gen_epilogue);
-        } else {
-            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0);
-            tcg_out_jmp(s, tb_ret_addr);
-        }
-        break;
     case INDEX_op_goto_tb:
         if (s->tb_jmp_insn_offset) {
             /* direct jump method */
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
     case INDEX_op_mov_i64:
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
+    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     default:
         tcg_abort();
     }
diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.c.inc
+++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
 
 static const tcg_insn_unit *tb_ret_addr;
 
+static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
+{
+    /* Reuse the zeroing that exists for goto_ptr.  */
+    if (a0 == 0) {
+        tcg_out_call_int(s, tcg_code_gen_epilogue, true);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0);
+        tcg_out_call_int(s, tb_ret_addr, true);
+    }
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     int c2 = const_args[2];
 
     switch (opc) {
-    case INDEX_op_exit_tb:
-        /* Reuse the zeroing that exists for goto_ptr.  */
-        if (a0 == 0) {
-            tcg_out_call_int(s, tcg_code_gen_epilogue, true);
-        } else {
-            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0);
-            tcg_out_call_int(s, tb_ret_addr, true);
-        }
-        break;
-
     case INDEX_op_goto_tb:
         tcg_debug_assert(s->tb_jmp_insn_offset != NULL);
         /*
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
     case INDEX_op_mov_i64:
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
+    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_clz(TCGContext *s, MIPSInsn opcv2, MIPSInsn opcv6,
     }
 }
 
+static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
+{
+    TCGReg b0 = TCG_REG_ZERO;
+
+    if (a0 & ~0xffff) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_V0, a0 & ~0xffff);
+        b0 = TCG_REG_V0;
+    }
+    if (!tcg_out_opc_jmp(s, OPC_J, tb_ret_addr)) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)tb_ret_addr);
+        tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
+    }
+    tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff);
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     c2 = const_args[2];
 
     switch (opc) {
-    case INDEX_op_exit_tb:
-        {
-            TCGReg b0 = TCG_REG_ZERO;
-
-            a0 = (intptr_t)a0;
-            if (a0 & ~0xffff) {
-                tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_V0, a0 & ~0xffff);
-                b0 = TCG_REG_V0;
-            }
-            if (!tcg_out_opc_jmp(s, OPC_J, tb_ret_addr)) {
-                tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0,
-                             (uintptr_t)tb_ret_addr);
-                tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
-            }
-            tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff);
-        }
-        break;
     case INDEX_op_goto_tb:
         /* indirect jump method */
         tcg_debug_assert(s->tb_jmp_insn_offset == 0);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
     case INDEX_op_mov_i64:
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
+    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     default:
         tcg_abort();
     }
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s)
     tcg_out32(s, BCLR | BO_ALWAYS);
 }
 
+static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
+{
+    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, arg);
+    tcg_out_b(s, 0, tcg_code_gen_epilogue);
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     TCGArg a0, a1, a2;
 
     switch (opc) {
-    case INDEX_op_exit_tb:
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]);
-        tcg_out_b(s, 0, tcg_code_gen_epilogue);
-        break;
     case INDEX_op_goto_tb:
         if (s->tb_jmp_insn_offset) {
             /* Direct jump. */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i32:   /* Always emitted via tcg_out_mov.  */
     case INDEX_op_mov_i64:
     case INDEX_op_call:      /* Always emitted via tcg_out_call.  */
+    case INDEX_op_exit_tb:   /* Always emitted via tcg_out_exit_tb.  */
     default:
         tcg_abort();
     }
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
 
 static const tcg_insn_unit *tb_ret_addr;
 
+static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
+{
+    /* Reuse the zeroing that exists for goto_ptr.  */
+    if (a0 == 0) {
+        tcg_out_call_int(s, tcg_code_gen_epilogue, true);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0);
+        tcg_out_call_int(s, tb_ret_addr, true);
+    }
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     int c2 = const_args[2];
 
     switch (opc) {
-    case INDEX_op_exit_tb:
-        /* Reuse the zeroing that exists for goto_ptr.  */
-        if (a0 == 0) {
-            tcg_out_call_int(s, tcg_code_gen_epilogue, true);
-        } else {
-            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0);
-            tcg_out_call_int(s, tb_ret_addr, true);
-        }
-        break;
-
     case INDEX_op_goto_tb:
         assert(s->tb_jmp_insn_offset == 0);
         /* indirect jump method */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
     case INDEX_op_mov_i64:
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
+    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg,
 #endif
 }
 
+static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
+{
+    /* Reuse the zeroing that exists for goto_ptr.  */
+    if (a0 == 0) {
+        tgen_gotoi(s, S390_CC_ALWAYS, tcg_code_gen_epilogue);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, a0);
+        tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr);
+    }
+}
+
 # define OP_32_64(x) \
         case glue(glue(INDEX_op_,x),_i32): \
         case glue(glue(INDEX_op_,x),_i64)
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     TCGArg a0, a1, a2;
 
     switch (opc) {
-    case INDEX_op_exit_tb:
-        /* Reuse the zeroing that exists for goto_ptr.  */
-        a0 = args[0];
-        if (a0 == 0) {
-            tgen_gotoi(s, S390_CC_ALWAYS, tcg_code_gen_epilogue);
-        } else {
-            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, a0);
-            tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr);
-        }
-        break;
-
     case INDEX_op_goto_tb:
         a0 = args[0];
         /*
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
     case INDEX_op_mov_i64:
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
+    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     default:
         tcg_abort();
     }
diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr,
 #endif /* CONFIG_SOFTMMU */
 }
 
+static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
+{
+    if (check_fit_ptr(a0, 13)) {
+        tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
+        tcg_out_movi_imm13(s, TCG_REG_O0, a0);
+        return;
+    } else if (USE_REG_TB) {
+        intptr_t tb_diff = tcg_tbrel_diff(s, (void *)a0);
+        if (check_fit_ptr(tb_diff, 13)) {
+            tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
+            /* Note that TCG_REG_TB has been unwound to O1.  */
+            tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O1, tb_diff, ARITH_ADD);
+            return;
+        }
+    }
+    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_I0, a0 & ~0x3ff);
+    tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
+    tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, a0 & 0x3ff, ARITH_OR);
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     c2 = const_args[2];
 
     switch (opc) {
-    case INDEX_op_exit_tb:
-        if (check_fit_ptr(a0, 13)) {
-            tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
-            tcg_out_movi_imm13(s, TCG_REG_O0, a0);
-            break;
-        } else if (USE_REG_TB) {
-            intptr_t tb_diff = tcg_tbrel_diff(s, (void *)a0);
-            if (check_fit_ptr(tb_diff, 13)) {
-                tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
-                /* Note that TCG_REG_TB has been unwound to O1.  */
-                tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O1, tb_diff, ARITH_ADD);
-                break;
-            }
-        }
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_I0, a0 & ~0x3ff);
-        tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
-        tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, a0 & 0x3ff, ARITH_OR);
-        break;
     case INDEX_op_goto_tb:
         if (s->tb_jmp_insn_offset) {
             /* direct jump method */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
     case INDEX_op_mov_i64:
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
+    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     default:
         tcg_abort();
     }
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *func,
 # define CASE_64(x)
 #endif
 
+static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
+{
+    tcg_out_op_p(s, INDEX_op_exit_tb, (void *)arg);
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     TCGOpcode exts;
 
     switch (opc) {
-    case INDEX_op_exit_tb:
-        tcg_out_op_p(s, opc, (void *)args[0]);
-        break;
-
     case INDEX_op_goto_tb:
         tcg_debug_assert(s->tb_jmp_insn_offset == 0);
         /* indirect jump method. */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
     case INDEX_op_mov_i64:
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
+    case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     default:
         tcg_abort();
     }
-- 
2.34.1

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/i386/tcg-target.c.inc | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     switch (opc) {
     case INDEX_op_goto_tb:
-        if (s->tb_jmp_insn_offset) {
-            /* direct jump method */
-            int gap;
-            /* jump displacement must be aligned for atomic patching;
+        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
+        {
+            /*
+             * Jump displacement must be aligned for atomic patching;
              * see if we need to add extra nops before jump
              */
-            gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr;
+            int gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr;
             if (gap != 1) {
                 tcg_out_nopn(s, gap - 1);
             }
             tcg_out8(s, OPC_JMP_long); /* jmp im */
             s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
             tcg_out32(s, 0);
-        } else {
-            /* indirect jump method */
-            tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, -1,
-                                 (intptr_t)(s->tb_jmp_target_addr + a0));
         }
         set_jmp_reset_offset(s, a0);
         break;
-- 
2.34.1

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/ppc/tcg-target.c.inc | 32 +++++++++++++-------------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     switch (opc) {
     case INDEX_op_goto_tb:
-        if (s->tb_jmp_insn_offset) {
-            /* Direct jump. */
-            if (TCG_TARGET_REG_BITS == 64) {
-                /* Ensure the next insns are 8 or 16-byte aligned. */
-                while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) {
-                    tcg_out32(s, NOP);
-                }
-                s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
-                tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0));
-                tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0));
-            } else {
-                s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
-                tcg_out32(s, B);
-                s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
-                break;
+        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
+        /* Direct jump. */
+        if (TCG_TARGET_REG_BITS == 64) {
+            /* Ensure the next insns are 8 or 16-byte aligned. */
+            while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) {
+                tcg_out32(s, NOP);
             }
+            s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
+            tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0));
+            tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0));
         } else {
-            /* Indirect jump. */
-            tcg_debug_assert(s->tb_jmp_insn_offset == NULL);
-            tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TB, 0,
-                       (intptr_t)(s->tb_jmp_insn_offset + args[0]));
+            s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
+            tcg_out32(s, B);
+            s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
+            break;
         }
         tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR);
         tcg_out32(s, BCCTR | BO_ALWAYS);
-- 
2.34.1

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/sparc64/tcg-target.c.inc | 41 +++++++++++-------------------------
 1 file changed, 12 insertions(+), 29 deletions(-)

diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
     return false;
 }
 
-static void tcg_out_ld_ptr(TCGContext *s, TCGReg ret, const void *arg)
-{
-    intptr_t diff = tcg_tbrel_diff(s, arg);
-    if (USE_REG_TB && check_fit_ptr(diff, 13)) {
-        tcg_out_ld(s, TCG_TYPE_PTR, ret, TCG_REG_TB, diff);
-        return;
-    }
-    tcg_out_movi(s, TCG_TYPE_PTR, ret, (uintptr_t)arg & ~0x3ff);
-    tcg_out_ld(s, TCG_TYPE_PTR, ret, ret, (uintptr_t)arg & 0x3ff);
-}
-
 static void tcg_out_sety(TCGContext *s, TCGReg rs)
 {
     tcg_out32(s, WRY | INSN_RS1(TCG_REG_G0) | INSN_RS2(rs));
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     switch (opc) {
     case INDEX_op_goto_tb:
-        if (s->tb_jmp_insn_offset) {
-            /* direct jump method */
-            if (USE_REG_TB) {
-                /* make sure the patch is 8-byte aligned.  */
-                if ((intptr_t)s->code_ptr & 4) {
-                    tcg_out_nop(s);
-                }
-                s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
-                tcg_out_sethi(s, TCG_REG_T1, 0);
-                tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR);
-                tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL);
-                tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
-            } else {
-                s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
-                tcg_out32(s, CALL);
+        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
+        /* Direct jump. */
+        if (USE_REG_TB) {
+            /* make sure the patch is 8-byte aligned.  */
+            if ((intptr_t)s->code_ptr & 4) {
                 tcg_out_nop(s);
             }
+            s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
+            tcg_out_sethi(s, TCG_REG_T1, 0);
+            tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR);
+            tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL);
+            tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
         } else {
-            /* indirect jump method */
-            tcg_out_ld_ptr(s, TCG_REG_TB, s->tb_jmp_target_addr + a0);
-            tcg_out_arithi(s, TCG_REG_G0, TCG_REG_TB, 0, JMPL);
+            s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
+            tcg_out32(s, CALL);
             tcg_out_nop(s);
         }
         set_jmp_reset_offset(s, a0);
-- 
2.34.1

Test TCG_TARGET_HAS_direct_jump instead of testing an
implementation pointer.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/aarch64/tcg-target.c.inc     | 2 +-
 tcg/arm/tcg-target.c.inc         | 2 +-
 tcg/loongarch64/tcg-target.c.inc | 2 +-
 tcg/mips/tcg-target.c.inc        | 2 +-
 tcg/riscv/tcg-target.c.inc       | 2 +-
 tcg/tci/tcg-target.c.inc         | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     switch (opc) {
     case INDEX_op_goto_tb:
-        tcg_debug_assert(s->tb_jmp_insn_offset != NULL);
+        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
         /*
          * Ensure that ADRP+ADD are 8-byte aligned so that an atomic
          * write can be used to patch the target address.
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
             intptr_t ptr, dif, dil;
             TCGReg base = TCG_REG_PC;
 
-            tcg_debug_assert(s->tb_jmp_insn_offset == 0);
+            qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
             ptr = (intptr_t)tcg_splitwx_to_rx(s->tb_jmp_target_addr + args[0]);
             dif = tcg_pcrel_diff(s, (void *)ptr) - 8;
             dil = sextract32(dif, 0, 12);
diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.c.inc
+++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     switch (opc) {
     case INDEX_op_goto_tb:
-        tcg_debug_assert(s->tb_jmp_insn_offset != NULL);
+        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
         /*
          * Ensure that patch area is 8-byte aligned so that an
          * atomic write can be used to patch the target address.
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     switch (opc) {
     case INDEX_op_goto_tb:
         /* indirect jump method */
-        tcg_debug_assert(s->tb_jmp_insn_offset == 0);
+        qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
         tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO,
                    (uintptr_t)(s->tb_jmp_target_addr + a0));
         tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     switch (opc) {
     case INDEX_op_goto_tb:
-        assert(s->tb_jmp_insn_offset == 0);
+        qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
         /* indirect jump method */
         tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO,
                    (uintptr_t)(s->tb_jmp_target_addr + a0));
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     switch (opc) {
     case INDEX_op_goto_tb:
-        tcg_debug_assert(s->tb_jmp_insn_offset == 0);
+        qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
         /* indirect jump method. */
         tcg_out_op_p(s, opc, s->tb_jmp_target_addr + args[0]);
         set_jmp_reset_offset(s, args[0]);
-- 
2.34.1

Similar to the existing set_jmp_reset_offset.  Move any assert for
TCG_TARGET_HAS_direct_jump into the new function (which now cannot
be build-time).  Will be unused if TCG_TARGET_HAS_direct_jump is
constant 0, but we can't test for constant in the preprocessor,
so just mark it G_GNUC_UNUSED.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/tcg.c                        | 10 ++++++++++
 tcg/aarch64/tcg-target.c.inc     |  3 +--
 tcg/i386/tcg-target.c.inc        |  3 +--
 tcg/loongarch64/tcg-target.c.inc |  3 +--
 tcg/ppc/tcg-target.c.inc         |  7 +++----
 tcg/s390x/tcg-target.c.inc       |  2 +-
 tcg/sparc64/tcg-target.c.inc     |  5 ++---
 7 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static void set_jmp_reset_offset(TCGContext *s, int which)
     s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
 }
 
+static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
+{
+    /*
+     * We will check for overflow at the end of the opcode loop in
+     * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
+     */
+    tcg_debug_assert(TCG_TARGET_HAS_direct_jump);
+    s->tb_jmp_insn_offset[which] = tcg_current_code_size(s);
+}
+
 /* Signal overflow, starting over with fewer guest insns. */
 static G_NORETURN
 void tcg_raise_tb_overflow(TCGContext *s)
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     switch (opc) {
     case INDEX_op_goto_tb:
-        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
         /*
          * Ensure that ADRP+ADD are 8-byte aligned so that an atomic
          * write can be used to patch the target address.
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         if ((uintptr_t)s->code_ptr & 7) {
             tcg_out32(s, NOP);
         }
-        s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
+        set_jmp_insn_offset(s, a0);
         /*
          * actual branch destination will be patched by
          * tb_target_set_jmp_target later
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     switch (opc) {
     case INDEX_op_goto_tb:
-        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
         {
             /*
              * Jump displacement must be aligned for atomic patching;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
                 tcg_out_nopn(s, gap - 1);
             }
             tcg_out8(s, OPC_JMP_long); /* jmp im */
-            s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
+            set_jmp_insn_offset(s, a0);
             tcg_out32(s, 0);
         }
         set_jmp_reset_offset(s, a0);
diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.c.inc
+++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     switch (opc) {
     case INDEX_op_goto_tb:
-        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
         /*
          * Ensure that patch area is 8-byte aligned so that an
          * atomic write can be used to patch the target address.
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         if ((uintptr_t)s->code_ptr & 7) {
             tcg_out_nop(s);
         }
-        s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
+        set_jmp_insn_offset(s, a0);
         /*
          * actual branch destination will be patched by
          * tb_target_set_jmp_target later
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     switch (opc) {
     case INDEX_op_goto_tb:
-        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
         /* Direct jump. */
         if (TCG_TARGET_REG_BITS == 64) {
             /* Ensure the next insns are 8 or 16-byte aligned. */
             while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) {
                 tcg_out32(s, NOP);
             }
-            s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
+            set_jmp_insn_offset(s, args[0]);
             tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0));
             tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0));
         } else {
-            s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
+            set_jmp_insn_offset(s, args[0]);
             tcg_out32(s, B);
-            s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
+            set_jmp_reset_offset(s, args[0]);
             break;
         }
         tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR);
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
             tcg_out16(s, NOP);
         }
         tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4));
-        s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
+        set_jmp_insn_offset(s, a0);
         s->code_ptr += 2;
         set_jmp_reset_offset(s, a0);
         break;
diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     switch (opc) {
     case INDEX_op_goto_tb:
-        qemu_build_assert(TCG_TARGET_HAS_direct_jump);
         /* Direct jump. */
         if (USE_REG_TB) {
             /* make sure the patch is 8-byte aligned.  */
             if ((intptr_t)s->code_ptr & 4) {
                 tcg_out_nop(s);
             }
-            s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
+            set_jmp_insn_offset(s, a0);
             tcg_out_sethi(s, TCG_REG_T1, 0);
             tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR);
             tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL);
             tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
         } else {
-            s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
+            set_jmp_insn_offset(s, a0);
             tcg_out32(s, CALL);
             tcg_out_nop(s);
         }
-- 
2.34.1

Similar to the existing set_jmp_reset_offset.  Include the
rw->rx address space conversion done by arm and s390x, and
forgotten by mips and riscv.

diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
     s->tb_jmp_insn_offset[which] = tcg_current_code_size(s);
 }
 
+static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
+{
+    /*
+     * Return the read-execute version of the pointer, for the benefit
+     * of any pc-relative addressing mode.
+     */
+    return (uintptr_t)tcg_splitwx_to_rx(&s->tb_jmp_target_addr[which]);
+}
+
 /* Signal overflow, starting over with fewer guest insns. */
 static G_NORETURN
 void tcg_raise_tb_overflow(TCGContext *s)
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
             TCGReg base = TCG_REG_PC;
 
             qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
-            ptr = (intptr_t)tcg_splitwx_to_rx(s->tb_jmp_target_addr + args[0]);
+            ptr = get_jmp_target_addr(s, args[0]);
             dif = tcg_pcrel_diff(s, (void *)ptr) - 8;
             dil = sextract32(dif, 0, 12);
             if (dif != dil) {
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         /* indirect jump method */
         qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
         tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO,
-                   (uintptr_t)(s->tb_jmp_target_addr + a0));
+                   get_jmp_target_addr(s, a0));
         tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
         tcg_out_nop(s);
         set_jmp_reset_offset(s, a0);
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
         /* indirect jump method */
         tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO,
-                   (uintptr_t)(s->tb_jmp_target_addr + a0));
+                   get_jmp_target_addr(s, a0));
         tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0);
         set_jmp_reset_offset(s, a0);
         break;
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_goto_tb:
         qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
         /* indirect jump method. */
-        tcg_out_op_p(s, opc, s->tb_jmp_target_addr + args[0]);
+        tcg_out_op_p(s, opc, (void *)get_jmp_target_addr(s, args[0]));
         set_jmp_reset_offset(s, args[0]);
         break;
 
-- 
2.34.1

The INDEX_op_goto_tb opcode needs no register allocation.
Split out a dedicated helper function for it.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/tcg.c                        |  4 ++
 tcg/aarch64/tcg-target.c.inc     | 40 ++++++++++---------
 tcg/arm/tcg-target.c.inc         | 49 ++++++++++++-----------
 tcg/i386/tcg-target.c.inc        | 33 ++++++++--------
 tcg/loongarch64/tcg-target.c.inc | 38 +++++++++---------
 tcg/mips/tcg-target.c.inc        | 21 +++++-----
 tcg/ppc/tcg-target.c.inc         | 52 ++++++++++++------------
 tcg/riscv/tcg-target.c.inc       | 20 +++++-----
 tcg/s390x/tcg-target.c.inc       | 31 ++++++++-------
 tcg/sparc64/tcg-target.c.inc     | 68 +++++++++++++++++---------------
 tcg/tci/tcg-target.c.inc         | 16 ++++----
 11 files changed, 199 insertions(+), 173 deletions(-)

diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
 static void tcg_out_movi(TCGContext *s, TCGType type,
                          TCGReg ret, tcg_target_long arg);
 static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
+static void tcg_out_goto_tb(TCGContext *s, int which);
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS]);
@@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
         case INDEX_op_exit_tb:
             tcg_out_exit_tb(s, op->args[0]);
             break;
+        case INDEX_op_goto_tb:
+            tcg_out_goto_tb(s, op->args[0]);
+            break;
         case INDEX_op_dup2_vec:
             if (tcg_reg_alloc_dup2(s, op)) {
                 break;
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
     }
 }
 
+static void tcg_out_goto_tb(TCGContext *s, int which)
+{
+    /*
+     * Ensure that ADRP+ADD are 8-byte aligned so that an atomic
+     * write can be used to patch the target address.
+     */
+    if ((uintptr_t)s->code_ptr & 7) {
+        tcg_out32(s, NOP);
+    }
+    set_jmp_insn_offset(s, which);
+    /*
+     * actual branch destination will be patched by
+     * tb_target_set_jmp_target later
+     */
+    tcg_out_insn(s, 3406, ADRP, TCG_REG_TMP, 0);
+    tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_TMP, TCG_REG_TMP, 0);
+    tcg_out_insn(s, 3207, BR, TCG_REG_TMP);
+    set_jmp_reset_offset(s, which);
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 #define REG0(I)  (const_args[I] ? TCG_REG_XZR : (TCGReg)args[I])
 
     switch (opc) {
-    case INDEX_op_goto_tb:
-        /*
-         * Ensure that ADRP+ADD are 8-byte aligned so that an atomic
-         * write can be used to patch the target address.
-         */
-        if ((uintptr_t)s->code_ptr & 7) {
-            tcg_out32(s, NOP);
-        }
-        set_jmp_insn_offset(s, a0);
-        /*
-         * actual branch destination will be patched by
-         * tb_target_set_jmp_target later
-         */
-        tcg_out_insn(s, 3406, ADRP, TCG_REG_TMP, 0);
-        tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_TMP, TCG_REG_TMP, 0);
-        tcg_out_insn(s, 3207, BR, TCG_REG_TMP);
-        set_jmp_reset_offset(s, a0);
-        break;
-
     case INDEX_op_goto_ptr:
         tcg_out_insn(s, 3207, BR, a0);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i64:
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
+    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
     tcg_out_epilogue(s);
 }
 
+static void tcg_out_goto_tb(TCGContext *s, int which)
+{
+    /* Indirect jump method */
+    intptr_t ptr, dif, dil;
+    TCGReg base = TCG_REG_PC;
+
+    qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
+    ptr = get_jmp_target_addr(s, which);
+    dif = tcg_pcrel_diff(s, (void *)ptr) - 8;
+    dil = sextract32(dif, 0, 12);
+    if (dif != dil) {
+        /*
+         * The TB is close, but outside the 12 bits addressable by
+         * the load.  We can extend this to 20 bits with a sub of a
+         * shifted immediate from pc.  In the vastly unlikely event
+         * the code requires more than 1MB, we'll use 2 insns and
+         * be no worse off.
+         */
+        base = TCG_REG_R0;
+        tcg_out_movi32(s, COND_AL, base, ptr - dil);
+    }
+    tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, base, dil);
+    set_jmp_reset_offset(s, which);
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     int c;
 
     switch (opc) {
-    case INDEX_op_goto_tb:
-        {
-            /* Indirect jump method */
-            intptr_t ptr, dif, dil;
-            TCGReg base = TCG_REG_PC;
-
-            qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
-            ptr = get_jmp_target_addr(s, args[0]);
-            dif = tcg_pcrel_diff(s, (void *)ptr) - 8;
-            dil = sextract32(dif, 0, 12);
-            if (dif != dil) {
-                /* The TB is close, but outside the 12 bits addressable by
-                   the load.  We can extend this to 20 bits with a sub of a
-                   shifted immediate from pc.  In the vastly unlikely event
-                   the code requires more than 1MB, we'll use 2 insns and
-                   be no worse off.  */
-                base = TCG_REG_R0;
-                tcg_out_movi32(s, COND_AL, base, ptr - dil);
-            }
-            tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, base, dil);
-            set_jmp_reset_offset(s, args[0]);
-        }
-        break;
     case INDEX_op_goto_ptr:
         tcg_out_b_reg(s, COND_AL, args[0]);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
+    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     default:
         tcg_abort();
     }
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
     }
 }
 
+static void tcg_out_goto_tb(TCGContext *s, int which)
+{
+    /*
+     * Jump displacement must be aligned for atomic patching;
+     * see if we need to add extra nops before jump
+     */
+    int gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr;
+    if (gap != 1) {
+        tcg_out_nopn(s, gap - 1);
+    }
+    tcg_out8(s, OPC_JMP_long); /* jmp im */
+    set_jmp_insn_offset(s, which);
+    tcg_out32(s, 0);
+    set_jmp_reset_offset(s, which);
+}
+
 static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
                               const TCGArg args[TCG_MAX_OP_ARGS],
                               const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     const_a2 = const_args[2];
 
     switch (opc) {
-    case INDEX_op_goto_tb:
-        {
-            /*
-             * Jump displacement must be aligned for atomic patching;
-             * see if we need to add extra nops before jump
-             */
-            int gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr;
-            if (gap != 1) {
-                tcg_out_nopn(s, gap - 1);
-            }
-            tcg_out8(s, OPC_JMP_long); /* jmp im */
-            set_jmp_insn_offset(s, a0);
-            tcg_out32(s, 0);
-        }
-        set_jmp_reset_offset(s, a0);
-        break;
     case INDEX_op_goto_ptr:
         /* jmp to the given host address (could be epilogue) */
         tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, a0);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i64:
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
+    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     default:
         tcg_abort();
     }
diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.c.inc
+++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
     }
 }
 
+static void tcg_out_goto_tb(TCGContext *s, int which)
+{
+    /*
+     * Ensure that patch area is 8-byte aligned so that an
+     * atomic write can be used to patch the target address.
+     */
+    if ((uintptr_t)s->code_ptr & 7) {
+        tcg_out_nop(s);
+    }
+    set_jmp_insn_offset(s, which);
+    /*
+     * actual branch destination will be patched by
+     * tb_target_set_jmp_target later
+     */
+    tcg_out_opc_pcaddu18i(s, TCG_REG_TMP0, 0);
+    tcg_out_opc_jirl(s, TCG_REG_ZERO, TCG_REG_TMP0, 0);
+    set_jmp_reset_offset(s, which);
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     int c2 = const_args[2];
 
     switch (opc) {
-    case INDEX_op_goto_tb:
-        /*
-         * Ensure that patch area is 8-byte aligned so that an
-         * atomic write can be used to patch the target address.
-         */
-        if ((uintptr_t)s->code_ptr & 7) {
-            tcg_out_nop(s);
-        }
-        set_jmp_insn_offset(s, a0);
-        /*
-         * actual branch destination will be patched by
-         * tb_target_set_jmp_target later
-         */
-        tcg_out_opc_pcaddu18i(s, TCG_REG_TMP0, 0);
-        tcg_out_opc_jirl(s, TCG_REG_ZERO, TCG_REG_TMP0, 0);
-        set_jmp_reset_offset(s, a0);
-        break;
-
     case INDEX_op_mb:
         tcg_out_mb(s, a0);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i64:
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
+    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
     tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff);
 }
 
+static void tcg_out_goto_tb(TCGContext *s, int which)
+{
+    /* indirect jump method */
+    qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
+    tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO,
+               get_jmp_target_addr(s, which));
+    tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
+    tcg_out_nop(s);
+    set_jmp_reset_offset(s, which);
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     c2 = const_args[2];
 
     switch (opc) {
-    case INDEX_op_goto_tb:
-        /* indirect jump method */
-        qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
-        tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO,
-                   get_jmp_target_addr(s, a0));
-        tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
-        tcg_out_nop(s);
-        set_jmp_reset_offset(s, a0);
-        break;
     case INDEX_op_goto_ptr:
         /* jmp to the given host address (could be epilogue) */
         tcg_out_opc_reg(s, OPC_JR, 0, a0, 0);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i64:
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
+    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     default:
         tcg_abort();
     }
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
     tcg_out_b(s, 0, tcg_code_gen_epilogue);
 }
 
+static void tcg_out_goto_tb(TCGContext *s, int which)
+{
+    /* Direct jump. */
+    if (TCG_TARGET_REG_BITS == 64) {
+        /* Ensure the next insns are 8 or 16-byte aligned. */
+        while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) {
+            tcg_out32(s, NOP);
+        }
+        set_jmp_insn_offset(s, which);
+        tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0));
+        tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0));
+        tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR);
+        tcg_out32(s, BCCTR | BO_ALWAYS);
+        set_jmp_reset_offset(s, which);
+        if (USE_REG_TB) {
+            /* For the unlinked case, need to reset TCG_REG_TB.  */
+            tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB,
+                             -tcg_current_code_size(s));
+        }
+    } else {
+        set_jmp_insn_offset(s, which);
+        tcg_out32(s, B);
+        set_jmp_reset_offset(s, which);
+    }
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     TCGArg a0, a1, a2;
 
     switch (opc) {
-    case INDEX_op_goto_tb:
-        /* Direct jump. */
-        if (TCG_TARGET_REG_BITS == 64) {
-            /* Ensure the next insns are 8 or 16-byte aligned. */
-            while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) {
-                tcg_out32(s, NOP);
-            }
-            set_jmp_insn_offset(s, args[0]);
-            tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0));
-            tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0));
-        } else {
-            set_jmp_insn_offset(s, args[0]);
-            tcg_out32(s, B);
-            set_jmp_reset_offset(s, args[0]);
-            break;
-        }
-        tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR);
-        tcg_out32(s, BCCTR | BO_ALWAYS);
-        set_jmp_reset_offset(s, args[0]);
-        if (USE_REG_TB) {
-            /* For the unlinked case, need to reset TCG_REG_TB.  */
-            tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB,
-                             -tcg_current_code_size(s));
-        }
-        break;
     case INDEX_op_goto_ptr:
         tcg_out32(s, MTSPR | RS(args[0]) | CTR);
         if (USE_REG_TB) {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i64:
     case INDEX_op_call:      /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:   /* Always emitted via tcg_out_exit_tb.  */
+    case INDEX_op_goto_tb:   /* Always emitted via tcg_out_goto_tb.  */
     default:
         tcg_abort();
     }
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
     }
 }
 
+static void tcg_out_goto_tb(TCGContext *s, int which)
+{
+    qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
+    /* indirect jump method */
+    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO,
+               get_jmp_target_addr(s, which));
+    tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0);
+    set_jmp_reset_offset(s, which);
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     int c2 = const_args[2];
 
     switch (opc) {
-    case INDEX_op_goto_tb:
-        qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
-        /* indirect jump method */
-        tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO,
-                   get_jmp_target_addr(s, a0));
-        tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0);
-        set_jmp_reset_offset(s, a0);
-        break;
-
     case INDEX_op_goto_ptr:
         tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, a0, 0);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i64:
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
+    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
     }
 }
 
+static void tcg_out_goto_tb(TCGContext *s, int which)
+{
+    /*
+     * Branch displacement must be aligned for atomic patching;
+     * see if we need to add extra nop before branch
+     */
+    if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) {
+        tcg_out16(s, NOP);
+    }
+    tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4));
+    set_jmp_insn_offset(s, which);
+    s->code_ptr += 2;
+    set_jmp_reset_offset(s, which);
+}
+
 # define OP_32_64(x) \
         case glue(glue(INDEX_op_,x),_i32): \
         case glue(glue(INDEX_op_,x),_i64)
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     TCGArg a0, a1, a2;
 
     switch (opc) {
-    case INDEX_op_goto_tb:
-        a0 = args[0];
-        /*
-         * branch displacement must be aligned for atomic patching;
-         * see if we need to add extra nop before branch
-         */
-        if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) {
-            tcg_out16(s, NOP);
-        }
-        tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4));
-        set_jmp_insn_offset(s, a0);
-        s->code_ptr += 2;
-        set_jmp_reset_offset(s, a0);
-        break;
-
     case INDEX_op_goto_ptr:
         a0 = args[0];
         tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, a0);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i64:
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
+    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     default:
         tcg_abort();
     }
diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
     tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, a0 & 0x3ff, ARITH_OR);
 }
 
+static void tcg_out_goto_tb(TCGContext *s, int which)
+{
+    /* Direct jump. */
+    if (USE_REG_TB) {
+        /* make sure the patch is 8-byte aligned.  */
+        if ((intptr_t)s->code_ptr & 4) {
+            tcg_out_nop(s);
+        }
+        set_jmp_insn_offset(s, which);
+        tcg_out_sethi(s, TCG_REG_T1, 0);
+        tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR);
+        tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL);
+        tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
+    } else {
+        set_jmp_insn_offset(s, which);
+        tcg_out32(s, CALL);
+        tcg_out_nop(s);
+    }
+    set_jmp_reset_offset(s, which);
+
+    /*
+     * For the unlinked path of goto_tb, we need to reset TCG_REG_TB
+     * to the beginning of this TB.
+     */
+    if (USE_REG_TB) {
+        int c = -tcg_current_code_size(s);
+        if (check_fit_i32(c, 13)) {
+            tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD);
+        } else {
+            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c);
+            tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
+        }
+    }
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     c2 = const_args[2];
 
     switch (opc) {
-    case INDEX_op_goto_tb:
-        /* Direct jump. */
-        if (USE_REG_TB) {
-            /* make sure the patch is 8-byte aligned.  */
-            if ((intptr_t)s->code_ptr & 4) {
-                tcg_out_nop(s);
-            }
-            set_jmp_insn_offset(s, a0);
-            tcg_out_sethi(s, TCG_REG_T1, 0);
-            tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR);
-            tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL);
-            tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
-        } else {
-            set_jmp_insn_offset(s, a0);
-            tcg_out32(s, CALL);
-            tcg_out_nop(s);
-        }
-        set_jmp_reset_offset(s, a0);
-
-        /* For the unlinked path of goto_tb, we need to reset
-           TCG_REG_TB to the beginning of this TB.  */
-        if (USE_REG_TB) {
-            c = -tcg_current_code_size(s);
-            if (check_fit_i32(c, 13)) {
-                tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD);
-            } else {
-                tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c);
-                tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB,
-                              TCG_REG_T1, ARITH_ADD);
-            }
-        }
-        break;
     case INDEX_op_goto_ptr:
         tcg_out_arithi(s, TCG_REG_G0, a0, 0, JMPL);
         if (USE_REG_TB) {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i64:
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
+    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     default:
         tcg_abort();
     }
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
     tcg_out_op_p(s, INDEX_op_exit_tb, (void *)arg);
 }
 
+static void tcg_out_goto_tb(TCGContext *s, int which)
+{
+    qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
+    /* indirect jump method. */
+    tcg_out_op_p(s, INDEX_op_goto_tb, (void *)get_jmp_target_addr(s, which));
+    set_jmp_reset_offset(s, which);
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     TCGOpcode exts;
 
     switch (opc) {
-    case INDEX_op_goto_tb:
-        qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
-        /* indirect jump method. */
-        tcg_out_op_p(s, opc, (void *)get_jmp_target_addr(s, args[0]));
-        set_jmp_reset_offset(s, args[0]);
-        break;
-
     case INDEX_op_goto_ptr:
         tcg_out_op_r(s, opc, args[0]);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mov_i64:
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
+    case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     default:
         tcg_abort();
     }
-- 
2.34.1

This will shortly be used for more than reset.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 include/exec/exec-all.h   | 2 +-
 accel/tcg/translate-all.c | 8 ++++----
 tcg/tcg.c                 | 4 ++--
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
index XXXXXXX..XXXXXXX 100644
--- a/include/exec/exec-all.h
+++ b/include/exec/exec-all.h
@@ -XXX,XX +XXX,XX @@ struct TranslationBlock {
      * setting one of the jump targets (or patching the jump instruction). Only
      * two of such jumps are supported.
      */
+#define TB_JMP_OFFSET_INVALID 0xffff /* indicates no jump generated */
     uint16_t jmp_reset_offset[2]; /* offset of original jump target */
-#define TB_JMP_RESET_OFFSET_INVALID 0xffff /* indicates no jump generated */
     uintptr_t jmp_target_arg[2];  /* target address or offset */
 
     /*
diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/translate-all.c
+++ b/accel/tcg/translate-all.c
@@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu,
     tb->jmp_dest[1] = (uintptr_t)NULL;
 
     /* init original jump addresses which have been set during tcg_gen_code() */
-    if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
+    if (tb->jmp_reset_offset[0] != TB_JMP_OFFSET_INVALID) {
         tb_reset_jump(tb, 0);
     }
-    if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
+    if (tb->jmp_reset_offset[1] != TB_JMP_OFFSET_INVALID) {
         tb_reset_jump(tb, 1);
     }
 
@@ -XXX,XX +XXX,XX @@ static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data)
     if (tb_page_addr1(tb) != -1) {
         tst->cross_page++;
     }
-    if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
+    if (tb->jmp_reset_offset[0] != TB_JMP_OFFSET_INVALID) {
         tst->direct_jmp_count++;
-        if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
+        if (tb->jmp_reset_offset[1] != TB_JMP_OFFSET_INVALID) {
             tst->direct_jmp2_count++;
         }
     }
diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
 #endif
 
     /* Initialize goto_tb jump offsets. */
-    tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
-    tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
+    tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID;
+    tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID;
     tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset;
     if (TCG_TARGET_HAS_direct_jump) {
         tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg;
-- 
2.34.1

This can replace four other variables that are references
into the TranslationBlock structure.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 include/tcg/tcg.h         | 11 +++--------
 accel/tcg/translate-all.c |  2 +-
 tcg/tcg-op.c              | 14 +++++++-------
 tcg/tcg.c                 | 14 +++-----------
 4 files changed, 14 insertions(+), 27 deletions(-)

diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h
index XXXXXXX..XXXXXXX 100644
--- a/include/tcg/tcg.h
+++ b/include/tcg/tcg.h
@@ -XXX,XX +XXX,XX @@ struct TCGContext {
     int nb_indirects;
     int nb_ops;
 
-    /* goto_tb support */
-    tcg_insn_unit *code_buf;
-    uint16_t *tb_jmp_reset_offset; /* tb->jmp_reset_offset */
-    uintptr_t *tb_jmp_insn_offset; /* tb->jmp_target_arg if direct_jump */
-    uintptr_t *tb_jmp_target_addr; /* tb->jmp_target_arg if !direct_jump */
-
     TCGRegSet reserved_regs;
-    uint32_t tb_cflags; /* cflags of the current TB */
     intptr_t current_frame_offset;
     intptr_t frame_start;
     intptr_t frame_end;
     TCGTemp *frame_temp;
 
-    tcg_insn_unit *code_ptr;
+    TranslationBlock *gen_tb;     /* tb for which code is being generated */
+    tcg_insn_unit *code_buf;      /* pointer for start of tb */
+    tcg_insn_unit *code_ptr;      /* pointer for running end of tb */
 
 #ifdef CONFIG_PROFILER
     TCGProfile prof;
diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/translate-all.c
+++ b/accel/tcg/translate-all.c
@@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu,
     tb->trace_vcpu_dstate = *cpu->trace_dstate;
     tb_set_page_addr0(tb, phys_pc);
     tb_set_page_addr1(tb, -1);
-    tcg_ctx->tb_cflags = cflags;
+    tcg_ctx->gen_tb = tb;
  tb_overflow:
 
 #ifdef CONFIG_PROFILER
diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg-op.c
+++ b/tcg/tcg-op.c
@@ -XXX,XX +XXX,XX @@ void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
 
 void tcg_gen_mb(TCGBar mb_type)
 {
-    if (tcg_ctx->tb_cflags & CF_PARALLEL) {
+    if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) {
         tcg_gen_op1(INDEX_op_mb, mb_type);
     }
 }
@@ -XXX,XX +XXX,XX @@ void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx)
 void tcg_gen_goto_tb(unsigned idx)
 {
     /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */
-    tcg_debug_assert(!(tcg_ctx->tb_cflags & CF_NO_GOTO_TB));
+    tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB));
     /* We only support two chained exits.  */
     tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
 #ifdef CONFIG_DEBUG_TCG
@@ -XXX,XX +XXX,XX @@ void tcg_gen_lookup_and_goto_ptr(void)
 {
     TCGv_ptr ptr;
 
-    if (tcg_ctx->tb_cflags & CF_NO_GOTO_PTR) {
+    if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) {
         tcg_gen_exit_tb(NULL, 0);
         return;
     }
@@ -XXX,XX +XXX,XX @@ void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
 {
     memop = tcg_canonicalize_memop(memop, 0, 0);
 
-    if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
+    if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
         TCGv_i32 t1 = tcg_temp_new_i32();
         TCGv_i32 t2 = tcg_temp_new_i32();
 
@@ -XXX,XX +XXX,XX @@ void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
 {
     memop = tcg_canonicalize_memop(memop, 1, 0);
 
-    if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
+    if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
         TCGv_i64 t1 = tcg_temp_new_i64();
         TCGv_i64 t2 = tcg_temp_new_i64();
 
@@ -XXX,XX +XXX,XX @@ static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = {          \
 void tcg_gen_atomic_##NAME##_i32                                        \
     (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop)    \
 {                                                                       \
-    if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
+    if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) {                        \
         do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME);     \
     } else {                                                            \
         do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW,            \
@@ -XXX,XX +XXX,XX @@ void tcg_gen_atomic_##NAME##_i32                                        \
 void tcg_gen_atomic_##NAME##_i64                                        \
     (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop)    \
 {                                                                       \
-    if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
+    if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) {                        \
         do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME);     \
     } else {                                                            \
         do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW,            \
diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static void set_jmp_reset_offset(TCGContext *s, int which)
      * We will check for overflow at the end of the opcode loop in
      * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
      */
-    s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
+    s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s);
 }
 
 static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
@@ -XXX,XX +XXX,XX @@ static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
      * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
      */
     tcg_debug_assert(TCG_TARGET_HAS_direct_jump);
-    s->tb_jmp_insn_offset[which] = tcg_current_code_size(s);
+    s->gen_tb->jmp_target_arg[which] = tcg_current_code_size(s);
 }
 
 static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
@@ -XXX,XX +XXX,XX @@ static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
      * Return the read-execute version of the pointer, for the benefit
      * of any pc-relative addressing mode.
      */
-    return (uintptr_t)tcg_splitwx_to_rx(&s->tb_jmp_target_addr[which]);
+    return (uintptr_t)tcg_splitwx_to_rx(s->gen_tb->jmp_target_arg + which);
 }
 
 /* Signal overflow, starting over with fewer guest insns. */
@@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
     /* Initialize goto_tb jump offsets. */
     tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID;
     tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID;
-    tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset;
-    if (TCG_TARGET_HAS_direct_jump) {
-        tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg;
-        tcg_ctx->tb_jmp_target_addr = NULL;
-    } else {
-        tcg_ctx->tb_jmp_insn_offset = NULL;
-        tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg;
-    }
 
     tcg_reg_alloc_start(s);
 
-- 
2.34.1

Stop overloading jmp_target_arg for both offset and address,
depending on TCG_TARGET_HAS_direct_jump.  Instead, add a new
field to hold the jump insn offset and always set the target
address in jmp_target_addr[].  This will allow a tcg backend
to use either direct or indirect depending on displacement.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 include/exec/exec-all.h | 3 ++-
 accel/tcg/cpu-exec.c    | 5 ++---
 tcg/tcg.c               | 6 ++++--
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
index XXXXXXX..XXXXXXX 100644
--- a/include/exec/exec-all.h
+++ b/include/exec/exec-all.h
@@ -XXX,XX +XXX,XX @@ struct TranslationBlock {
      */
 #define TB_JMP_OFFSET_INVALID 0xffff /* indicates no jump generated */
     uint16_t jmp_reset_offset[2]; /* offset of original jump target */
-    uintptr_t jmp_target_arg[2];  /* target address or offset */
+    uint16_t jmp_insn_offset[2];  /* offset of direct jump insn */
+    uintptr_t jmp_target_addr[2]; /* target address */
 
     /*
      * Each TB has a NULL-terminated list (jmp_list_head) of incoming jumps.
diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/cpu-exec.c
+++ b/accel/tcg/cpu-exec.c
@@ -XXX,XX +XXX,XX @@ void cpu_exec_step_atomic(CPUState *cpu)
 
 void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr)
 {
+    tb->jmp_target_addr[n] = addr;
     if (TCG_TARGET_HAS_direct_jump) {
-        uintptr_t offset = tb->jmp_target_arg[n];
+        uintptr_t offset = tb->jmp_insn_offset[n];
         uintptr_t tc_ptr = (uintptr_t)tb->tc.ptr;
         uintptr_t jmp_rx = tc_ptr + offset;
         uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff;
         tb_target_set_jmp_target(tc_ptr, jmp_rx, jmp_rw, addr);
-    } else {
-        tb->jmp_target_arg[n] = addr;
     }
 }
 
diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
      * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
      */
     tcg_debug_assert(TCG_TARGET_HAS_direct_jump);
-    s->gen_tb->jmp_target_arg[which] = tcg_current_code_size(s);
+    s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s);
 }
 
 static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
@@ -XXX,XX +XXX,XX @@ static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
      * Return the read-execute version of the pointer, for the benefit
      * of any pc-relative addressing mode.
      */
-    return (uintptr_t)tcg_splitwx_to_rx(s->gen_tb->jmp_target_arg + which);
+    return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]);
 }
 
 /* Signal overflow, starting over with fewer guest insns. */
@@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
     /* Initialize goto_tb jump offsets. */
     tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID;
     tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID;
+    tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID;
+    tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID;
 
     tcg_reg_alloc_start(s);
 
-- 
2.34.1

Replace 'tc_ptr' and 'addr' with 'tb' and 'n'.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/aarch64/tcg-target.h         |  3 ++-
 tcg/arm/tcg-target.h             |  3 ++-
 tcg/i386/tcg-target.h            |  9 ++-------
 tcg/loongarch64/tcg-target.h     |  3 ++-
 tcg/mips/tcg-target.h            |  3 ++-
 tcg/ppc/tcg-target.h             |  3 ++-
 tcg/riscv/tcg-target.h           |  3 ++-
 tcg/s390x/tcg-target.h           | 10 ++--------
 tcg/sparc64/tcg-target.h         |  3 ++-
 tcg/tci/tcg-target.h             |  3 ++-
 accel/tcg/cpu-exec.c             | 11 ++++++++---
 tcg/aarch64/tcg-target.c.inc     |  5 +++--
 tcg/i386/tcg-target.c.inc        |  9 +++++++++
 tcg/loongarch64/tcg-target.c.inc |  5 +++--
 tcg/ppc/tcg-target.c.inc         |  7 ++++---
 tcg/s390x/tcg-target.c.inc       | 10 ++++++++++
 tcg/sparc64/tcg-target.c.inc     |  7 ++++---
 17 files changed, 61 insertions(+), 36 deletions(-)

diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.h
+++ b/tcg/aarch64/tcg-target.h
@@ -XXX,XX +XXX,XX @@ typedef enum {
 #define TCG_TARGET_DEFAULT_MO (0)
 #define TCG_TARGET_HAS_MEMORY_BSWAP     0
 
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(const TranslationBlock *, int,
+                              uintptr_t, uintptr_t);
 
 #define TCG_TARGET_NEED_LDST_LABELS
 #define TCG_TARGET_NEED_POOL_LABELS
diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.h
+++ b/tcg/arm/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool use_neon_instructions;
 #define TCG_TARGET_HAS_MEMORY_BSWAP     0
 
 /* not defined -- call should be eliminated at compile time */
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t, uintptr_t);
 
 #define TCG_TARGET_NEED_LDST_LABELS
 #define TCG_TARGET_NEED_POOL_LABELS
diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.h
+++ b/tcg/i386/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool have_movbe;
 #define TCG_TARGET_extract_i64_valid(ofs, len) \
     (((ofs) == 8 && (len) == 8) || ((ofs) + (len)) == 32)
 
-static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
-                                            uintptr_t jmp_rw, uintptr_t addr)
-{
-    /* patch the branch destination */
-    qatomic_set((int32_t *)jmp_rw, addr - (jmp_rx + 4));
-    /* no need to flush icache explicitly */
-}
+void tb_target_set_jmp_target(const TranslationBlock *, int,
+                              uintptr_t, uintptr_t);
 
 /* This defines the natural memory order supported by this
  * architecture before guarantees made by various barrier
diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.h
+++ b/tcg/loongarch64/tcg-target.h
@@ -XXX,XX +XXX,XX @@ typedef enum {
 #define TCG_TARGET_HAS_muluh_i64        1
 #define TCG_TARGET_HAS_mulsh_i64        1
 
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t, uintptr_t);
 
 #define TCG_TARGET_DEFAULT_MO (0)
 
diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.h
+++ b/tcg/mips/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions;
 #define TCG_TARGET_HAS_MEMORY_BSWAP     1
 
 /* not defined -- call should be eliminated at compile time */
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t)
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t, uintptr_t)
     QEMU_ERROR("code path is reachable");
 
 #define TCG_TARGET_NEED_LDST_LABELS
diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.h
+++ b/tcg/ppc/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool have_vsx;
 #define TCG_TARGET_HAS_bitsel_vec       have_vsx
 #define TCG_TARGET_HAS_cmpsel_vec       0
 
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t, uintptr_t);
 
 #define TCG_TARGET_DEFAULT_MO (0)
 #define TCG_TARGET_HAS_MEMORY_BSWAP     1
diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.h
+++ b/tcg/riscv/tcg-target.h
@@ -XXX,XX +XXX,XX @@ typedef enum {
 #endif
 
 /* not defined -- call should be eliminated at compile time */
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t, uintptr_t);
 
 #define TCG_TARGET_DEFAULT_MO (0)
 
diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.h
+++ b/tcg/s390x/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern uint64_t s390_facilities[3];
 
 #define TCG_TARGET_DEFAULT_MO (TCG_MO_ALL & ~TCG_MO_ST_LD)
 
-static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
-                                            uintptr_t jmp_rw, uintptr_t addr)
-{
-    /* patch the branch destination */
-    intptr_t disp = addr - (jmp_rx - 2);
-    qatomic_set((int32_t *)jmp_rw, disp / 2);
-    /* no need to flush icache explicitly */
-}
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t jmp_rx, uintptr_t jmp_rw);
 
 #define TCG_TARGET_NEED_LDST_LABELS
 #define TCG_TARGET_NEED_POOL_LABELS
diff --git a/tcg/sparc64/tcg-target.h b/tcg/sparc64/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.h
+++ b/tcg/sparc64/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions;
 #define TCG_TARGET_DEFAULT_MO (0)
 #define TCG_TARGET_HAS_MEMORY_BSWAP     1
 
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t, uintptr_t);
 
 #define TCG_TARGET_NEED_POOL_LABELS
 
diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.h
+++ b/tcg/tci/tcg-target.h
@@ -XXX,XX +XXX,XX @@ typedef enum {
 #define TCG_TARGET_HAS_MEMORY_BSWAP     1
 
 /* not defined -- call should be eliminated at compile time */
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t, uintptr_t);
 
 #endif /* TCG_TARGET_H */
diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/cpu-exec.c
+++ b/accel/tcg/cpu-exec.c
@@ -XXX,XX +XXX,XX @@ void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr)
 {
     tb->jmp_target_addr[n] = addr;
     if (TCG_TARGET_HAS_direct_jump) {
+        /*
+         * Get the rx view of the structure, from which we find the
+         * executable code address, and tb_target_set_jmp_target can
+         * produce a pc-relative displacement to jmp_target_addr[n].
+         */
+        const TranslationBlock *c_tb = tcg_splitwx_to_rx(tb);
         uintptr_t offset = tb->jmp_insn_offset[n];
-        uintptr_t tc_ptr = (uintptr_t)tb->tc.ptr;
-        uintptr_t jmp_rx = tc_ptr + offset;
+        uintptr_t jmp_rx = (uintptr_t)tb->tc.ptr + offset;
         uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff;
-        tb_target_set_jmp_target(tc_ptr, jmp_rx, jmp_rw, addr);
+        tb_target_set_jmp_target(c_tb, n, jmp_rx, jmp_rw);
     }
 }
 
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
     tcg_out_call_int(s, target);
 }
 
-void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
-                              uintptr_t jmp_rw, uintptr_t addr)
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
 {
+    uintptr_t addr = tb->jmp_target_addr[n];
     tcg_insn_unit i1, i2;
     TCGType rt = TCG_TYPE_I64;
     TCGReg  rd = TCG_REG_TMP;
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
     set_jmp_reset_offset(s, which);
 }
 
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
+{
+    /* patch the branch destination */
+    uintptr_t addr = tb->jmp_target_addr[n];
+    qatomic_set((int32_t *)jmp_rw, addr - (jmp_rx + 4));
+    /* no need to flush icache explicitly */
+}
+
 static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
                               const TCGArg args[TCG_MAX_OP_ARGS],
                               const int const_args[TCG_MAX_OP_ARGS])
diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.c.inc
+++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_nop(TCGContext *s)
     tcg_out32(s, NOP);
 }
 
-void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
-                              uintptr_t jmp_rw, uintptr_t addr)
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
 {
     tcg_insn_unit i1, i2;
     ptrdiff_t upper, lower;
+    uintptr_t addr = tb->jmp_target_addr[n];
     ptrdiff_t offset = (ptrdiff_t)(addr - jmp_rx) >> 2;
 
     if (offset == sextreg(offset, 0, 26)) {
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static inline void ppc64_replace4(uintptr_t rx, uintptr_t rw,
     flush_idcache_range(rx, rw, 16);
 }
 
-void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
-                              uintptr_t jmp_rw, uintptr_t addr)
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
 {
     tcg_insn_unit i0, i1, i2, i3;
-    intptr_t tb_diff = addr - tc_ptr;
+    uintptr_t addr = tb->jmp_target_addr[n];
+    intptr_t tb_diff = addr - (uintptr_t)tb->tc.ptr;
     intptr_t br_diff = addr - (jmp_rx + 4);
     intptr_t lo, hi;
 
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
     set_jmp_reset_offset(s, which);
 }
 
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
+{
+    /* patch the branch destination */
+    uintptr_t addr = tb->jmp_target_addr[n];
+    intptr_t disp = addr - (jmp_rx - 2);
+    qatomic_set((int32_t *)jmp_rw, disp / 2);
+    /* no need to flush icache explicitly */
+}
+
 # define OP_32_64(x) \
         case glue(glue(INDEX_op_,x),_i32): \
         case glue(glue(INDEX_op_,x),_i64)
diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ void tcg_register_jit(const void *buf, size_t buf_size)
     tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
 }
 
-void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
-                              uintptr_t jmp_rw, uintptr_t addr)
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
 {
-    intptr_t tb_disp = addr - tc_ptr;
+    uintptr_t addr = tb->jmp_target_addr[n];
+    intptr_t tb_disp = addr - (uintptr_t)tb->tc.ptr;
     intptr_t br_disp = addr - jmp_rx;
     tcg_insn_unit i1, i2;
 
-- 
2.34.1

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 include/tcg/tcg.h            | 3 +++
 tcg/aarch64/tcg-target.h     | 4 ----
 tcg/arm/tcg-target.h         | 5 -----
 tcg/i386/tcg-target.h        | 3 ---
 tcg/loongarch64/tcg-target.h | 3 ---
 tcg/mips/tcg-target.h        | 5 -----
 tcg/ppc/tcg-target.h         | 4 ----
 tcg/riscv/tcg-target.h       | 4 ----
 tcg/s390x/tcg-target.h       | 4 ----
 tcg/sparc64/tcg-target.h     | 4 ----
 tcg/tci/tcg-target.h         | 4 ----
 11 files changed, 3 insertions(+), 40 deletions(-)

diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h
index XXXXXXX..XXXXXXX 100644
--- a/include/tcg/tcg.h
+++ b/include/tcg/tcg.h
@@ -XXX,XX +XXX,XX @@ void tcg_func_start(TCGContext *s);
 
 int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start);
 
+void tb_target_set_jmp_target(const TranslationBlock *, int,
+                              uintptr_t, uintptr_t);
+
 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size);
 
 TCGTemp *tcg_global_mem_new_internal(TCGType, TCGv_ptr,
diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.h
+++ b/tcg/aarch64/tcg-target.h
@@ -XXX,XX +XXX,XX @@ typedef enum {
 
 #define TCG_TARGET_DEFAULT_MO (0)
 #define TCG_TARGET_HAS_MEMORY_BSWAP     0
-
-void tb_target_set_jmp_target(const TranslationBlock *, int,
-                              uintptr_t, uintptr_t);
-
 #define TCG_TARGET_NEED_LDST_LABELS
 #define TCG_TARGET_NEED_POOL_LABELS
 
diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.h
+++ b/tcg/arm/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool use_neon_instructions;
 
 #define TCG_TARGET_DEFAULT_MO (0)
 #define TCG_TARGET_HAS_MEMORY_BSWAP     0
-
-/* not defined -- call should be eliminated at compile time */
-void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-                              uintptr_t, uintptr_t);
-
 #define TCG_TARGET_NEED_LDST_LABELS
 #define TCG_TARGET_NEED_POOL_LABELS
 
diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.h
+++ b/tcg/i386/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool have_movbe;
 #define TCG_TARGET_extract_i64_valid(ofs, len) \
     (((ofs) == 8 && (len) == 8) || ((ofs) + (len)) == 32)
 
-void tb_target_set_jmp_target(const TranslationBlock *, int,
-                              uintptr_t, uintptr_t);
-
 /* This defines the natural memory order supported by this
  * architecture before guarantees made by various barrier
  * instructions.
diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.h
+++ b/tcg/loongarch64/tcg-target.h
@@ -XXX,XX +XXX,XX @@ typedef enum {
 #define TCG_TARGET_HAS_muluh_i64        1
 #define TCG_TARGET_HAS_mulsh_i64        1
 
-void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-                              uintptr_t, uintptr_t);
-
 #define TCG_TARGET_DEFAULT_MO (0)
 
 #define TCG_TARGET_NEED_LDST_LABELS
diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.h
+++ b/tcg/mips/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions;
 #define TCG_TARGET_DEFAULT_MO (0)
 #define TCG_TARGET_HAS_MEMORY_BSWAP     1
 
-/* not defined -- call should be eliminated at compile time */
-void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-                              uintptr_t, uintptr_t)
-    QEMU_ERROR("code path is reachable");
-
 #define TCG_TARGET_NEED_LDST_LABELS
 
 #endif
diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.h
+++ b/tcg/ppc/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool have_vsx;
 #define TCG_TARGET_HAS_bitsel_vec       have_vsx
 #define TCG_TARGET_HAS_cmpsel_vec       0
 
-void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-                              uintptr_t, uintptr_t);
-
 #define TCG_TARGET_DEFAULT_MO (0)
 #define TCG_TARGET_HAS_MEMORY_BSWAP     1
-
 #define TCG_TARGET_NEED_LDST_LABELS
 #define TCG_TARGET_NEED_POOL_LABELS
 
diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.h
+++ b/tcg/riscv/tcg-target.h
@@ -XXX,XX +XXX,XX @@ typedef enum {
 #define TCG_TARGET_HAS_mulsh_i64        1
 #endif
 
-/* not defined -- call should be eliminated at compile time */
-void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-                              uintptr_t, uintptr_t);
-
 #define TCG_TARGET_DEFAULT_MO (0)
 
 #define TCG_TARGET_NEED_LDST_LABELS
diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.h
+++ b/tcg/s390x/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern uint64_t s390_facilities[3];
 #define TCG_TARGET_HAS_MEMORY_BSWAP   1
 
 #define TCG_TARGET_DEFAULT_MO (TCG_MO_ALL & ~TCG_MO_ST_LD)
-
-void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-                              uintptr_t jmp_rx, uintptr_t jmp_rw);
-
 #define TCG_TARGET_NEED_LDST_LABELS
 #define TCG_TARGET_NEED_POOL_LABELS
 
diff --git a/tcg/sparc64/tcg-target.h b/tcg/sparc64/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.h
+++ b/tcg/sparc64/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions;
 
 #define TCG_TARGET_DEFAULT_MO (0)
 #define TCG_TARGET_HAS_MEMORY_BSWAP     1
-
-void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-                              uintptr_t, uintptr_t);
-
 #define TCG_TARGET_NEED_POOL_LABELS
 
 #endif
diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.h
+++ b/tcg/tci/tcg-target.h
@@ -XXX,XX +XXX,XX @@ typedef enum {
 
 #define TCG_TARGET_HAS_MEMORY_BSWAP     1
 
-/* not defined -- call should be eliminated at compile time */
-void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-                              uintptr_t, uintptr_t);
-
 #endif /* TCG_TARGET_H */
-- 
2.34.1

Install empty versions for !TCG_TARGET_HAS_direct_jump hosts.

diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
     set_jmp_reset_offset(s, which);
 }
 
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
+{
+    /* Always indirect, nothing to do */
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
     set_jmp_reset_offset(s, which);
 }
 
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
+{
+    /* Always indirect, nothing to do */
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
     set_jmp_reset_offset(s, which);
 }
 
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
+{
+    /* Always indirect, nothing to do */
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
     set_jmp_reset_offset(s, which);
 }
 
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
+{
+    /* Always indirect, nothing to do */
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
-- 
2.34.1

We now have the option to generate direct or indirect
goto_tb depending on the dynamic displacement, thus
the define is no longer necessary or completely accurate.

diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.h
+++ b/tcg/aarch64/tcg-target.h
@@ -XXX,XX +XXX,XX @@ typedef enum {
 #define TCG_TARGET_HAS_muls2_i64        0
 #define TCG_TARGET_HAS_muluh_i64        1
 #define TCG_TARGET_HAS_mulsh_i64        1
-#define TCG_TARGET_HAS_direct_jump      1
 
 #define TCG_TARGET_HAS_v64              1
 #define TCG_TARGET_HAS_v128             1
diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.h
+++ b/tcg/arm/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool use_neon_instructions;
 #define TCG_TARGET_HAS_mulsh_i32        0
 #define TCG_TARGET_HAS_div_i32          use_idiv_instructions
 #define TCG_TARGET_HAS_rem_i32          0
-#define TCG_TARGET_HAS_direct_jump      0
 #define TCG_TARGET_HAS_qemu_st8_i32     0
 
 #define TCG_TARGET_HAS_v64              use_neon_instructions
diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.h
+++ b/tcg/i386/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool have_movbe;
 #define TCG_TARGET_HAS_muls2_i32        1
 #define TCG_TARGET_HAS_muluh_i32        0
 #define TCG_TARGET_HAS_mulsh_i32        0
-#define TCG_TARGET_HAS_direct_jump      1
 
 #if TCG_TARGET_REG_BITS == 64
 /* Keep target addresses zero-extended in a register.  */
diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.h
+++ b/tcg/loongarch64/tcg-target.h
@@ -XXX,XX +XXX,XX @@ typedef enum {
 #define TCG_TARGET_HAS_clz_i32          1
 #define TCG_TARGET_HAS_ctz_i32          1
 #define TCG_TARGET_HAS_ctpop_i32        0
-#define TCG_TARGET_HAS_direct_jump      1
 #define TCG_TARGET_HAS_brcond2          0
 #define TCG_TARGET_HAS_setcond2         0
 #define TCG_TARGET_HAS_qemu_st8_i32     0
diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.h
+++ b/tcg/mips/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions;
 #define TCG_TARGET_HAS_muluh_i32        1
 #define TCG_TARGET_HAS_mulsh_i32        1
 #define TCG_TARGET_HAS_bswap32_i32      1
-#define TCG_TARGET_HAS_direct_jump      0
 
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_add2_i32         0
diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.h
+++ b/tcg/ppc/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool have_vsx;
 #define TCG_TARGET_HAS_muls2_i32        0
 #define TCG_TARGET_HAS_muluh_i32        1
 #define TCG_TARGET_HAS_mulsh_i32        1
-#define TCG_TARGET_HAS_direct_jump      1
 #define TCG_TARGET_HAS_qemu_st8_i32     0
 
 #if TCG_TARGET_REG_BITS == 64
diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.h
+++ b/tcg/riscv/tcg-target.h
@@ -XXX,XX +XXX,XX @@ typedef enum {
 #define TCG_TARGET_HAS_clz_i32          0
 #define TCG_TARGET_HAS_ctz_i32          0
 #define TCG_TARGET_HAS_ctpop_i32        0
-#define TCG_TARGET_HAS_direct_jump      0
 #define TCG_TARGET_HAS_brcond2          1
 #define TCG_TARGET_HAS_setcond2         1
 #define TCG_TARGET_HAS_qemu_st8_i32     0
diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.h
+++ b/tcg/s390x/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern uint64_t s390_facilities[3];
 #define TCG_TARGET_HAS_mulsh_i32      0
 #define TCG_TARGET_HAS_extrl_i64_i32  0
 #define TCG_TARGET_HAS_extrh_i64_i32  0
-#define TCG_TARGET_HAS_direct_jump    1
 #define TCG_TARGET_HAS_qemu_st8_i32   0
 
 #define TCG_TARGET_HAS_div2_i64       1
diff --git a/tcg/sparc64/tcg-target.h b/tcg/sparc64/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.h
+++ b/tcg/sparc64/tcg-target.h
@@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions;
 #define TCG_TARGET_HAS_muls2_i32        1
 #define TCG_TARGET_HAS_muluh_i32        0
 #define TCG_TARGET_HAS_mulsh_i32        0
-#define TCG_TARGET_HAS_direct_jump      1
 #define TCG_TARGET_HAS_qemu_st8_i32     0
 
 #define TCG_TARGET_HAS_extrl_i64_i32    1
diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.h
+++ b/tcg/tci/tcg-target.h
@@ -XXX,XX +XXX,XX @@
 #define TCG_TARGET_HAS_muls2_i32        1
 #define TCG_TARGET_HAS_muluh_i32        0
 #define TCG_TARGET_HAS_mulsh_i32        0
-#define TCG_TARGET_HAS_direct_jump      0
 #define TCG_TARGET_HAS_qemu_st8_i32     0
 
 #if TCG_TARGET_REG_BITS == 64
diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/cpu-exec.c
+++ b/accel/tcg/cpu-exec.c
@@ -XXX,XX +XXX,XX @@ void cpu_exec_step_atomic(CPUState *cpu)
 
 void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr)
 {
+    /*
+     * Get the rx view of the structure, from which we find the
+     * executable code address, and tb_target_set_jmp_target can
+     * produce a pc-relative displacement to jmp_target_addr[n].
+     */
+    const TranslationBlock *c_tb = tcg_splitwx_to_rx(tb);
+    uintptr_t offset = tb->jmp_insn_offset[n];
+    uintptr_t jmp_rx = (uintptr_t)tb->tc.ptr + offset;
+    uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff;
+
     tb->jmp_target_addr[n] = addr;
-    if (TCG_TARGET_HAS_direct_jump) {
-        /*
-         * Get the rx view of the structure, from which we find the
-         * executable code address, and tb_target_set_jmp_target can
-         * produce a pc-relative displacement to jmp_target_addr[n].
-         */
-        const TranslationBlock *c_tb = tcg_splitwx_to_rx(tb);
-        uintptr_t offset = tb->jmp_insn_offset[n];
-        uintptr_t jmp_rx = (uintptr_t)tb->tc.ptr + offset;
-        uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff;
-        tb_target_set_jmp_target(c_tb, n, jmp_rx, jmp_rw);
-    }
+    tb_target_set_jmp_target(c_tb, n, jmp_rx, jmp_rw);
 }
 
 static inline void tb_add_jump(TranslationBlock *tb, int n,
diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
      * We will check for overflow at the end of the opcode loop in
      * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
      */
-    tcg_debug_assert(TCG_TARGET_HAS_direct_jump);
     s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s);
 }
 
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
     intptr_t ptr, dif, dil;
     TCGReg base = TCG_REG_PC;
 
-    qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
     ptr = get_jmp_target_addr(s, which);
     dif = tcg_pcrel_diff(s, (void *)ptr) - 8;
     dil = sextract32(dif, 0, 12);
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
 static void tcg_out_goto_tb(TCGContext *s, int which)
 {
     /* indirect jump method */
-    qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
     tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO,
                get_jmp_target_addr(s, which));
     tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
 
 static void tcg_out_goto_tb(TCGContext *s, int which)
 {
-    qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
     /* indirect jump method */
     tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO,
                get_jmp_target_addr(s, which));
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
 void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
                               uintptr_t jmp_rx, uintptr_t jmp_rw)
 {
+    if (!HAVE_FACILITY(GEN_INST_EXT)) {
+        return;
+    }
     /* patch the branch destination */
     uintptr_t addr = tb->jmp_target_addr[n];
     intptr_t disp = addr - (jmp_rx - 2);
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
 
 static void tcg_out_goto_tb(TCGContext *s, int which)
 {
-    qemu_build_assert(!TCG_TARGET_HAS_direct_jump);
     /* indirect jump method. */
     tcg_out_op_p(s, INDEX_op_goto_tb, (void *)get_jmp_target_addr(s, which));
     set_jmp_reset_offset(s, which);
-- 
2.34.1

The old implementation replaces two insns, swapping between

b	<dest>
	nop
	br	x30
and
	adrp	x30, <dest>
	addi	x30, x30, lo12:<dest>
	br	x30

There is a race condition in which a thread could be stopped at
the PC of the second insn, and when restarted does not see the
complete address computation and branches to nowhere.

The new implemetation replaces only one insn, swapping between

b	<dest>
	br	tmp
and
	ldr	tmp, <jmp_addr>
	br	tmp

Reported-by: hev <r@hev.cc>
Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/aarch64/tcg-target.h     |  2 +-
 tcg/aarch64/tcg-target.c.inc | 66 +++++++++++++++---------------------
 2 files changed, 29 insertions(+), 39 deletions(-)

diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.h
+++ b/tcg/aarch64/tcg-target.h
@@ -XXX,XX +XXX,XX @@
 
 #define TCG_TARGET_INSN_UNIT_SIZE  4
 #define TCG_TARGET_TLB_DISPLACEMENT_BITS 24
-#define MAX_CODE_GEN_BUFFER_SIZE  (2 * GiB)
+#define MAX_CODE_GEN_BUFFER_SIZE  ((size_t)-1)
 
 typedef enum {
     TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3,
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
     tcg_out_call_int(s, target);
 }
 
-void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-                              uintptr_t jmp_rx, uintptr_t jmp_rw)
-{
-    uintptr_t addr = tb->jmp_target_addr[n];
-    tcg_insn_unit i1, i2;
-    TCGType rt = TCG_TYPE_I64;
-    TCGReg  rd = TCG_REG_TMP;
-    uint64_t pair;
-
-    ptrdiff_t offset = addr - jmp_rx;
-
-    if (offset == sextract64(offset, 0, 26)) {
-        i1 = I3206_B | ((offset >> 2) & 0x3ffffff);
-        i2 = NOP;
-    } else {
-        offset = (addr >> 12) - (jmp_rx >> 12);
-
-        /* patch ADRP */
-        i1 = I3406_ADRP | (offset & 3) << 29 | (offset & 0x1ffffc) << (5 - 2) | rd;
-        /* patch ADDI */
-        i2 = I3401_ADDI | rt << 31 | (addr & 0xfff) << 10 | rd << 5 | rd;
-    }
-    pair = (uint64_t)i2 << 32 | i1;
-    qatomic_set((uint64_t *)jmp_rw, pair);
-    flush_idcache_range(jmp_rx, jmp_rw, 8);
-}
-
 static inline void tcg_out_goto_label(TCGContext *s, TCGLabel *l)
 {
     if (!l->has_value) {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
 static void tcg_out_goto_tb(TCGContext *s, int which)
 {
     /*
-     * Ensure that ADRP+ADD are 8-byte aligned so that an atomic
-     * write can be used to patch the target address.
+     * Direct branch, or indirect address load, will be patched
+     * by tb_target_set_jmp_target.  Assert indirect load offset
+     * in range early, regardless of direct branch distance.
      */
-    if ((uintptr_t)s->code_ptr & 7) {
-        tcg_out32(s, NOP);
-    }
+    intptr_t i_off = tcg_pcrel_diff(s, (void *)get_jmp_target_addr(s, which));
+    tcg_debug_assert(i_off == sextract64(i_off, 0, 21));
+
     set_jmp_insn_offset(s, which);
-    /*
-     * actual branch destination will be patched by
-     * tb_target_set_jmp_target later
-     */
-    tcg_out_insn(s, 3406, ADRP, TCG_REG_TMP, 0);
-    tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_TMP, TCG_REG_TMP, 0);
+    tcg_out32(s, I3206_B);
     tcg_out_insn(s, 3207, BR, TCG_REG_TMP);
     set_jmp_reset_offset(s, which);
 }
 
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
+{
+    uintptr_t d_addr = tb->jmp_target_addr[n];
+    ptrdiff_t d_offset = d_addr - jmp_rx;
+    tcg_insn_unit insn;
+
+    /* Either directly branch, or indirect branch load. */
+    if (d_offset == sextract64(d_offset, 0, 28)) {
+        insn = deposit32(I3206_B, 0, 26, d_offset >> 2);
+    } else {
+        uintptr_t i_addr = (uintptr_t)&tb->jmp_target_addr[n];
+        ptrdiff_t i_offset = i_addr - jmp_rx;
+
+        /* Note that we asserted this in range in tcg_out_goto_tb. */
+        insn = deposit32(I3305_LDR | TCG_REG_TMP, 0, 5, i_offset >> 2);
+    }
+    qatomic_set((uint32_t *)jmp_rw, insn);
+    flush_idcache_range(jmp_rx, jmp_rw, 4);
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
-- 
2.34.1

The old ppc64 implementation replaces 2 or 4 insns, which leaves a race
condition in which a thread could be stopped at a PC in the middle of
the sequence, and when restarted does not see the complete address
computation and branches to nowhere.

The new implemetation replaces only one insn, swapping between

b       <dest>
and
	mtctr	r31

falling through to a general-case indirect branch.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/ppc/tcg-target.h     |   3 +-
 tcg/ppc/tcg-target.c.inc | 158 +++++++++++----------------------------
 2 files changed, 44 insertions(+), 117 deletions(-)

diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.h
+++ b/tcg/ppc/tcg-target.h
@@ -XXX,XX +XXX,XX @@
 
 #ifdef _ARCH_PPC64
 # define TCG_TARGET_REG_BITS  64
-# define MAX_CODE_GEN_BUFFER_SIZE  (2 * GiB)
 #else
 # define TCG_TARGET_REG_BITS  32
-# define MAX_CODE_GEN_BUFFER_SIZE  (32 * MiB)
 #endif
+#define MAX_CODE_GEN_BUFFER_SIZE  ((size_t)-1)
 
 #define TCG_TARGET_NB_REGS 64
 #define TCG_TARGET_INSN_UNIT_SIZE 4
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_mb(TCGContext *s, TCGArg a0)
     tcg_out32(s, insn);
 }
 
-static inline uint64_t make_pair(tcg_insn_unit i1, tcg_insn_unit i2)
-{
-    if (HOST_BIG_ENDIAN) {
-        return (uint64_t)i1 << 32 | i2;
-    }
-    return (uint64_t)i2 << 32 | i1;
-}
-
-static inline void ppc64_replace2(uintptr_t rx, uintptr_t rw,
-                                  tcg_insn_unit i0, tcg_insn_unit i1)
-{
-#if TCG_TARGET_REG_BITS == 64
-    qatomic_set((uint64_t *)rw, make_pair(i0, i1));
-    flush_idcache_range(rx, rw, 8);
-#else
-    qemu_build_not_reached();
-#endif
-}
-
-static inline void ppc64_replace4(uintptr_t rx, uintptr_t rw,
-                                  tcg_insn_unit i0, tcg_insn_unit i1,
-                                  tcg_insn_unit i2, tcg_insn_unit i3)
-{
-    uint64_t p[2];
-
-    p[!HOST_BIG_ENDIAN] = make_pair(i0, i1);
-    p[HOST_BIG_ENDIAN] = make_pair(i2, i3);
-
-    /*
-     * There's no convenient way to get the compiler to allocate a pair
-     * of registers at an even index, so copy into r6/r7 and clobber.
-     */
-    asm("mr  %%r6, %1\n\t"
-        "mr  %%r7, %2\n\t"
-        "stq %%r6, %0"
-        : "=Q"(*(__int128 *)rw) : "r"(p[0]), "r"(p[1]) : "r6", "r7");
-    flush_idcache_range(rx, rw, 16);
-}
-
-void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-                              uintptr_t jmp_rx, uintptr_t jmp_rw)
-{
-    tcg_insn_unit i0, i1, i2, i3;
-    uintptr_t addr = tb->jmp_target_addr[n];
-    intptr_t tb_diff = addr - (uintptr_t)tb->tc.ptr;
-    intptr_t br_diff = addr - (jmp_rx + 4);
-    intptr_t lo, hi;
-
-    if (TCG_TARGET_REG_BITS == 32) {
-        intptr_t diff = addr - jmp_rx;
-        tcg_debug_assert(in_range_b(diff));
-        qatomic_set((uint32_t *)jmp_rw, B | (diff & 0x3fffffc));
-        flush_idcache_range(jmp_rx, jmp_rw, 4);
-        return;
-    }
-
-    /*
-     * For 16-bit displacements, we can use a single add + branch.
-     * This happens quite often.
-     */
-    if (tb_diff == (int16_t)tb_diff) {
-        i0 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, tb_diff);
-        i1 = B | (br_diff & 0x3fffffc);
-        ppc64_replace2(jmp_rx, jmp_rw, i0, i1);
-        return;
-    }
-
-    lo = (int16_t)tb_diff;
-    hi = (int32_t)(tb_diff - lo);
-    assert(tb_diff == hi + lo);
-    i0 = ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, hi >> 16);
-    i1 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, lo);
-
-    /*
-     * Without stq from 2.07, we can only update two insns,
-     * and those must be the ones that load the target address.
-     */
-    if (!have_isa_2_07) {
-        ppc64_replace2(jmp_rx, jmp_rw, i0, i1);
-        return;
-    }
-
-    /*
-     * For 26-bit displacements, we can use a direct branch.
-     * Otherwise we still need the indirect branch, which we
-     * must restore after a potential direct branch write.
-     */
-    br_diff -= 4;
-    if (in_range_b(br_diff)) {
-        i2 = B | (br_diff & 0x3fffffc);
-        i3 = NOP;
-    } else {
-        i2 = MTSPR | RS(TCG_REG_TB) | CTR;
-        i3 = BCCTR | BO_ALWAYS;
-    }
-    ppc64_replace4(jmp_rx, jmp_rw, i0, i1, i2, i3);
-}
-
 static void tcg_out_call_int(TCGContext *s, int lk,
                              const tcg_insn_unit *target)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
 
 static void tcg_out_goto_tb(TCGContext *s, int which)
 {
-    /* Direct jump. */
-    if (TCG_TARGET_REG_BITS == 64) {
-        /* Ensure the next insns are 8 or 16-byte aligned. */
-        while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) {
-            tcg_out32(s, NOP);
-        }
+    uintptr_t ptr = get_jmp_target_addr(s, which);
+
+    if (USE_REG_TB) {
+        ptrdiff_t offset = tcg_tbrel_diff(s, (void *)ptr);
+        tcg_out_mem_long(s, LD, LDX, TCG_REG_TB, TCG_REG_TB, offset);
+    
+        /* Direct branch will be patched by tb_target_set_jmp_target. */
         set_jmp_insn_offset(s, which);
-        tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0));
-        tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0));
         tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR);
+
+        /* When branch is out of range, fall through to indirect. */
+        tcg_out32(s, BCCTR | BO_ALWAYS);
+
+        /* For the unlinked case, need to reset TCG_REG_TB.  */
+        set_jmp_reset_offset(s, which);
+        tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB,
+                         -tcg_current_code_size(s));
+    } else {
+        /* Direct branch will be patched by tb_target_set_jmp_target. */
+        set_jmp_insn_offset(s, which);
+        tcg_out32(s, NOP);
+
+        /* When branch is out of range, fall through to indirect. */
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, ptr - (int16_t)ptr);
+        tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_REG_TMP1, (int16_t)ptr);
+        tcg_out32(s, MTSPR | RS(TCG_REG_TMP1) | CTR);
         tcg_out32(s, BCCTR | BO_ALWAYS);
         set_jmp_reset_offset(s, which);
-        if (USE_REG_TB) {
-            /* For the unlinked case, need to reset TCG_REG_TB.  */
-            tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB,
-                             -tcg_current_code_size(s));
-        }
-    } else {
-        set_jmp_insn_offset(s, which);
-        tcg_out32(s, B);
-        set_jmp_reset_offset(s, which);
     }
 }
 
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
+{
+    uintptr_t addr = tb->jmp_target_addr[n];
+    intptr_t diff = addr - jmp_rx;
+    tcg_insn_unit insn;
+
+    if (in_range_b(diff)) {
+        insn = B | (diff & 0x3fffffc);
+    } else if (USE_REG_TB) {
+        insn = MTSPR | RS(TCG_REG_TB) | CTR;
+    } else {
+        insn = NOP;
+    }
+
+    qatomic_set((uint32_t *)jmp_rw, insn);
+    flush_idcache_range(jmp_rx, jmp_rw, 4);
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
-- 
2.34.1

This is always true for sparc64, so this is dead since 3a5f6805c7ca.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/sparc64/tcg-target.c.inc | 62 ++++++++++++------------------------
 1 file changed, 21 insertions(+), 41 deletions(-)

diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
 #endif
 
 #define TCG_REG_TB  TCG_REG_I1
-#define USE_REG_TB  (sizeof(void *) > 4)
 
 static const int tcg_target_reg_alloc_order[] = {
     TCG_REG_L0,
@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret,
     }
 
     /* A 13-bit constant relative to the TB.  */
-    if (!in_prologue && USE_REG_TB) {
+    if (!in_prologue) {
         test = tcg_tbrel_diff(s, (void *)arg);
         if (check_fit_ptr(test, 13)) {
             tcg_out_arithi(s, ret, TCG_REG_TB, test, ARITH_ADD);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret,
     }
 
     /* Use the constant pool, if possible. */
-    if (!in_prologue && USE_REG_TB) {
+    if (!in_prologue) {
         new_pool_label(s, arg, R_SPARC_13, s->code_ptr,
                        tcg_tbrel_diff(s, NULL));
         tcg_out32(s, LDX | INSN_RD(ret) | INSN_RS1(TCG_REG_TB));
@@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s)
 #endif
 
     /* We choose TCG_REG_TB such that no move is required.  */
-    if (USE_REG_TB) {
-        QEMU_BUILD_BUG_ON(TCG_REG_TB != TCG_REG_I1);
-        tcg_regset_set_reg(s->reserved_regs, TCG_REG_TB);
-    }
+    QEMU_BUILD_BUG_ON(TCG_REG_TB != TCG_REG_I1);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_TB);
 
     tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I1, 0, JMPL);
     /* delay slot */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
         tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
         tcg_out_movi_imm13(s, TCG_REG_O0, a0);
         return;
-    } else if (USE_REG_TB) {
+    } else {
         intptr_t tb_diff = tcg_tbrel_diff(s, (void *)a0);
         if (check_fit_ptr(tb_diff, 13)) {
             tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
 
 static void tcg_out_goto_tb(TCGContext *s, int which)
 {
+    int c;
+
     /* Direct jump. */
-    if (USE_REG_TB) {
-        /* make sure the patch is 8-byte aligned.  */
-        if ((intptr_t)s->code_ptr & 4) {
-            tcg_out_nop(s);
-        }
-        set_jmp_insn_offset(s, which);
-        tcg_out_sethi(s, TCG_REG_T1, 0);
-        tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR);
-        tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL);
-        tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
-    } else {
-        set_jmp_insn_offset(s, which);
-        tcg_out32(s, CALL);
+    /* make sure the patch is 8-byte aligned.  */
+    if ((intptr_t)s->code_ptr & 4) {
         tcg_out_nop(s);
     }
+    set_jmp_insn_offset(s, which);
+    tcg_out_sethi(s, TCG_REG_T1, 0);
+    tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR);
+    tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL);
+    tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
     set_jmp_reset_offset(s, which);
 
     /*
      * For the unlinked path of goto_tb, we need to reset TCG_REG_TB
      * to the beginning of this TB.
      */
-    if (USE_REG_TB) {
-        int c = -tcg_current_code_size(s);
-        if (check_fit_i32(c, 13)) {
-            tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD);
-        } else {
-            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c);
-            tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
-        }
+    c = -tcg_current_code_size(s);
+    if (check_fit_i32(c, 13)) {
+        tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c);
+        tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     switch (opc) {
     case INDEX_op_goto_ptr:
         tcg_out_arithi(s, TCG_REG_G0, a0, 0, JMPL);
-        if (USE_REG_TB) {
-            tcg_out_mov_delay(s, TCG_REG_TB, a0);
-        } else {
-            tcg_out_nop(s);
-        }
+        tcg_out_mov_delay(s, TCG_REG_TB, a0);
         break;
     case INDEX_op_br:
         tcg_out_bpcc(s, COND_A, BPCC_PT, arg_label(a0));
@@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
     tcg_debug_assert(tb_disp == (int32_t)tb_disp);
     tcg_debug_assert(br_disp == (int32_t)br_disp);
 
-    if (!USE_REG_TB) {
-        qatomic_set((uint32_t *)jmp_rw,
-		    deposit32(CALL, 0, 30, br_disp >> 2));
-        flush_idcache_range(jmp_rx, jmp_rw, 4);
-        return;
-    }
-
     /* This does not exercise the range of the branch, but we do
        still need to be able to load the new value of TCG_REG_TB.
        But this does still happen quite often.  */
-- 
2.34.1

The old sparc64 implementation may replace two insns, which leaves
a race condition in which a thread could be stopped at a PC in the
middle of the sequence, and when restarted does not see the complete
address computation and branches to nowhere.

The new implemetation replaces only one insn, swapping between a
direct branch and a direct call.  The TCG_REG_TB register is loaded
from tb->jmp_target_addr[] in the delay slot.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/sparc64/tcg-target.c.inc | 87 +++++++++++++++---------------------
 1 file changed, 37 insertions(+), 50 deletions(-)

diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
 
 static void tcg_out_goto_tb(TCGContext *s, int which)
 {
-    int c;
+    ptrdiff_t off = tcg_tbrel_diff(s, (void *)get_jmp_target_addr(s, which));
 
-    /* Direct jump. */
-    /* make sure the patch is 8-byte aligned.  */
-    if ((intptr_t)s->code_ptr & 4) {
-        tcg_out_nop(s);
-    }
+    /* Direct branch will be patched by tb_target_set_jmp_target. */
     set_jmp_insn_offset(s, which);
-    tcg_out_sethi(s, TCG_REG_T1, 0);
-    tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR);
-    tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL);
-    tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
+    tcg_out32(s, CALL);
+    /* delay slot */
+    tcg_debug_assert(check_fit_ptr(off, 13));
+    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TB, TCG_REG_TB, off);
     set_jmp_reset_offset(s, which);
 
     /*
      * For the unlinked path of goto_tb, we need to reset TCG_REG_TB
      * to the beginning of this TB.
      */
-    c = -tcg_current_code_size(s);
-    if (check_fit_i32(c, 13)) {
-        tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD);
+    off = -tcg_current_code_size(s);
+    if (check_fit_i32(off, 13)) {
+        tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, off, ARITH_ADD);
     } else {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c);
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, off);
         tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
     }
 }
 
+void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
+                              uintptr_t jmp_rx, uintptr_t jmp_rw)
+{
+    uintptr_t addr = tb->jmp_target_addr[n];
+    intptr_t br_disp = (intptr_t)(addr - jmp_rx) >> 2;
+    tcg_insn_unit insn;
+
+    br_disp >>= 2;
+    if (check_fit_ptr(br_disp, 19)) {
+        /* ba,pt %icc, addr */
+        insn = deposit32(INSN_OP(0) | INSN_OP2(1) | INSN_COND(COND_A)
+                         | BPCC_ICC | BPCC_PT, 0, 19, br_disp);
+    } else if (check_fit_ptr(br_disp, 22)) {
+        /* ba addr */
+        insn = deposit32(INSN_OP(0) | INSN_OP2(2) | INSN_COND(COND_A),
+                         0, 22, br_disp);
+    } else {
+        /* The code_gen_buffer can't be larger than 2GB.  */
+        tcg_debug_assert(check_fit_ptr(br_disp, 30));
+        /* call addr */
+        insn = deposit32(CALL, 0, 30, br_disp);
+    }
+
+    qatomic_set((uint32_t *)jmp_rw, insn);
+    flush_idcache_range(jmp_rx, jmp_rw, 4);
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg args[TCG_MAX_OP_ARGS],
                        const int const_args[TCG_MAX_OP_ARGS])
@@ -XXX,XX +XXX,XX @@ void tcg_register_jit(const void *buf, size_t buf_size)
 {
     tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
 }
-
-void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
-                              uintptr_t jmp_rx, uintptr_t jmp_rw)
-{
-    uintptr_t addr = tb->jmp_target_addr[n];
-    intptr_t tb_disp = addr - (uintptr_t)tb->tc.ptr;
-    intptr_t br_disp = addr - jmp_rx;
-    tcg_insn_unit i1, i2;
-
-    /* We can reach the entire address space for ILP32.
-       For LP64, the code_gen_buffer can't be larger than 2GB.  */
-    tcg_debug_assert(tb_disp == (int32_t)tb_disp);
-    tcg_debug_assert(br_disp == (int32_t)br_disp);
-
-    /* This does not exercise the range of the branch, but we do
-       still need to be able to load the new value of TCG_REG_TB.
-       But this does still happen quite often.  */
-    if (check_fit_ptr(tb_disp, 13)) {
-        /* ba,pt %icc, addr */
-        i1 = (INSN_OP(0) | INSN_OP2(1) | INSN_COND(COND_A)
-              | BPCC_ICC | BPCC_PT | INSN_OFF19(br_disp));
-        i2 = (ARITH_ADD | INSN_RD(TCG_REG_TB) | INSN_RS1(TCG_REG_TB)
-              | INSN_IMM13(tb_disp));
-    } else if (tb_disp >= 0) {
-        i1 = SETHI | INSN_RD(TCG_REG_T1) | ((tb_disp & 0xfffffc00) >> 10);
-        i2 = (ARITH_OR | INSN_RD(TCG_REG_T1) | INSN_RS1(TCG_REG_T1)
-              | INSN_IMM13(tb_disp & 0x3ff));
-    } else {
-        i1 = SETHI | INSN_RD(TCG_REG_T1) | ((~tb_disp & 0xfffffc00) >> 10);
-        i2 = (ARITH_XOR | INSN_RD(TCG_REG_T1) | INSN_RS1(TCG_REG_T1)
-              | INSN_IMM13((tb_disp & 0x3ff) | -0x400));
-    }
-
-    qatomic_set((uint64_t *)jmp_rw, deposit64(i2, 32, 32, i1));
-    flush_idcache_range(jmp_rx, jmp_rw, 8);
-}
-- 
2.34.1

Now that tcg can handle direct and indirect goto_tb
simultaneously, we can optimistically leave space for
a direct branch and fall back to loading the pointer
from the TB for an indirect branch.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/arm/tcg-target.c.inc | 52 ++++++++++++++++++++++++++++------------
 1 file changed, 37 insertions(+), 15 deletions(-)

diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ typedef enum {
     ARITH_BIC = 0xe << 21,
     ARITH_MVN = 0xf << 21,
 
+    INSN_B         = 0x0a000000,
+
     INSN_CLZ       = 0x016f0f10,
     INSN_RBIT      = 0x06ff0f30,
 
@@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct)
 
 static void tcg_out_b_imm(TCGContext *s, ARMCond cond, int32_t offset)
 {
-    tcg_out32(s, (cond << 28) | 0x0a000000 |
+    tcg_out32(s, (cond << 28) | INSN_B |
                     (((offset - 8) >> 2) & 0x00ffffff));
 }
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
 
 static void tcg_out_goto_tb(TCGContext *s, int which)
 {
-    /* Indirect jump method */
-    intptr_t ptr, dif, dil;
-    TCGReg base = TCG_REG_PC;
+    uintptr_t i_addr;
+    intptr_t i_disp;
 
-    ptr = get_jmp_target_addr(s, which);
-    dif = tcg_pcrel_diff(s, (void *)ptr) - 8;
-    dil = sextract32(dif, 0, 12);
-    if (dif != dil) {
+    /* Direct branch will be patched by tb_target_set_jmp_target. */
+    set_jmp_insn_offset(s, which);
+    tcg_out32(s, INSN_NOP);
+
+    /* When branch is out of range, fall through to indirect. */
+    i_addr = get_jmp_target_addr(s, which);
+    i_disp = tcg_pcrel_diff(s, (void *)i_addr) - 8;
+    tcg_debug_assert(i_disp < 0);
+    if (i_disp >= -0xfff) {
+        tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_PC, i_disp);
+    } else {
         /*
          * The TB is close, but outside the 12 bits addressable by
          * the load.  We can extend this to 20 bits with a sub of a
-         * shifted immediate from pc.  In the vastly unlikely event
-         * the code requires more than 1MB, we'll use 2 insns and
-         * be no worse off.
+         * shifted immediate from pc.
          */
-        base = TCG_REG_R0;
-        tcg_out_movi32(s, COND_AL, base, ptr - dil);
+        int h = -i_disp;
+        int l = h & 0xfff;
+
+        h = encode_imm_nofail(h - l);
+        tcg_out_dat_imm(s, COND_AL, ARITH_SUB, TCG_REG_R0, TCG_REG_PC, h);
+        tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_R0, l);
     }
-    tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, base, dil);
     set_jmp_reset_offset(s, which);
 }
 
 void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
                               uintptr_t jmp_rx, uintptr_t jmp_rw)
 {
-    /* Always indirect, nothing to do */
+    uintptr_t addr = tb->jmp_target_addr[n];
+    ptrdiff_t offset = addr - (jmp_rx + 8);
+    tcg_insn_unit insn;
+
+    /* Either directly branch, or fall through to indirect branch. */
+    if (offset == sextract64(offset, 0, 26)) {
+        /* B <addr> */
+        insn = deposit32((COND_AL << 28) | INSN_B, 0, 24, offset >> 2);
+    } else {
+        insn = INSN_NOP;
+    }
+
+    qatomic_set((uint32_t *)jmp_rw, insn);
+    flush_idcache_range(jmp_rx, jmp_rw, 4);
 }
 
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-- 
2.34.1

Now that tcg can handle direct and indirect goto_tb simultaneously,
we can optimistically leave space for a direct branch and fall back
to loading the pointer from the TB for an indirect branch.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/riscv/tcg-target.c.inc | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
 
 static void tcg_out_goto_tb(TCGContext *s, int which)
 {
-    /* indirect jump method */
+    /* Direct branch will be patched by tb_target_set_jmp_target. */
+    set_jmp_insn_offset(s, which);
+    tcg_out32(s, OPC_JAL);
+
+    /* When branch is out of range, fall through to indirect. */
     tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO,
                get_jmp_target_addr(s, which));
     tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which)
 void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
                               uintptr_t jmp_rx, uintptr_t jmp_rw)
 {
-    /* Always indirect, nothing to do */
+    uintptr_t addr = tb->jmp_target_addr[n];
+    ptrdiff_t offset = addr - jmp_rx;
+    tcg_insn_unit insn;
+
+    /* Either directly branch, or fall through to indirect branch. */
+    if (offset == sextreg(offset, 0, 20)) {
+        insn = encode_uj(OPC_JAL, TCG_REG_ZERO, offset);
+    } else {
+        insn = OPC_NOP;
+    }
+    qatomic_set((uint32_t *)jmp_rw, insn);
+    flush_idcache_range(jmp_rx, jmp_rw, 4);
 }
 
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-- 
2.34.1

Posting pre-PR because I had to adjust Emilio's QTree patch [1],
and added a new patch to avoid an assert that can be generated
with incorrect -R reserved_va values vs the ARM commpage.

[1] https://gitlab.com/rth7680/qemu/-/jobs/3975817279#L92

Emilio Cota (2):
  util: import GTree as QTree
  tcg: use QTree instead of GTree

Richard Henderson (9):
  linux-user: Diagnose misaligned -R size
  include/exec: Change reserved_va semantics to last byte
  accel/tcg: Pass last not end to page_set_flags
  accel/tcg: Pass last not end to page_reset_target_data
  accel/tcg: Pass last not end to PAGE_FOR_EACH_TB
  accel/tcg: Pass last not end to page_collection_lock
  accel/tcg: Pass last not end to tb_invalidate_phys_page_range__locked
  accel/tcg: Pass last not end to tb_invalidate_phys_range
  linux-user/arm: Take more care allocating commpage

-- 
2.34.1

From: Emilio Cota <cota@braap.org>

The only reason to add this implementation is to control the memory allocator
used. Some users (e.g. TCG) cannot work reliably in multi-threaded
environments (e.g. forking in user-mode) with GTree's allocator, GSlice.
See https://gitlab.com/qemu-project/qemu/-/issues/285 for details.

Importing GTree is a temporary workaround until GTree migrates away
from GSlice.

This implementation is identical to that in glib v2.75.0, except that
we don't import recent additions to the API nor deprecated API calls,
none of which are used in QEMU.

I've imported tests from glib and added a benchmark just to
make sure that performance is similar. Note: it cannot be identical
because (1) we are not using GSlice, (2) we use different compilation flags
(e.g. -fPIC) and (3) we're linking statically.

$ cat /proc/cpuinfo| grep 'model name' | head -1
model name      : AMD Ryzen 7 PRO 5850U with Radeon Graphics
$ echo '0' | sudo tee /sys/devices/system/cpu/cpufreq/boost
$ tests/bench/qtree-bench

Tree         Op      32            1024            4096          131072         1048576
------------------------------------------------------------------------------------------------
GTree     Lookup   83.23           43.08           25.31           19.40           16.22
QTree     Lookup  113.42 (1.36x)   53.83 (1.25x)   28.38 (1.12x)   17.64 (0.91x)   13.04 (0.80x)
GTree     Insert   44.23           29.37           25.83           19.49           17.03
QTree     Insert   46.87 (1.06x)   25.62 (0.87x)   24.29 (0.94x)   16.83 (0.86x)   12.97 (0.76x)
GTree     Remove   53.27           35.15           31.43           24.64           16.70
QTree     Remove   57.32 (1.08x)   41.76 (1.19x)   38.37 (1.22x)   29.30 (1.19x)   15.07 (0.90x)
GTree  RemoveAll  135.44          127.52          126.72          120.11           64.34
QTree  RemoveAll  127.15 (0.94x)  110.37 (0.87x)  107.97 (0.85x)   97.13 (0.81x)   55.10 (0.86x)
GTree   Traverse  277.71          276.09          272.78          246.72           98.47
QTree   Traverse  370.33 (1.33x)  411.97 (1.49x)  400.23 (1.47x)  262.82 (1.07x)   78.52 (0.80x)
------------------------------------------------------------------------------------------------

As a sanity check, the same benchmark when Glib's version
is >= $glib_dropped_gslice_version (i.e. QTree == GTree):

Tree         Op      32            1024            4096          131072         1048576
------------------------------------------------------------------------------------------------
GTree     Lookup   82.72           43.09           24.18           19.73           16.09
QTree     Lookup   81.82 (0.99x)   43.10 (1.00x)   24.20 (1.00x)   19.76 (1.00x)   16.26 (1.01x)
GTree     Insert   45.07           29.62           26.34           19.90           17.18
QTree     Insert   45.72 (1.01x)   29.60 (1.00x)   26.38 (1.00x)   19.71 (0.99x)   17.20 (1.00x)
GTree     Remove   54.48           35.36           31.77           24.97           16.95
QTree     Remove   54.46 (1.00x)   35.32 (1.00x)   31.77 (1.00x)   24.91 (1.00x)   17.15 (1.01x)
GTree  RemoveAll  140.68          127.36          125.43          121.45           68.20
QTree  RemoveAll  140.65 (1.00x)  127.64 (1.00x)  125.01 (1.00x)  121.73 (1.00x)   67.06 (0.98x)
GTree   Traverse  278.68          276.05          266.75          251.65          104.93
QTree   Traverse  278.31 (1.00x)  275.78 (1.00x)  266.42 (1.00x)  247.89 (0.99x)  104.58 (1.00x)
------------------------------------------------------------------------------------------------

Signed-off-by: Emilio Cota <cota@braap.org>
Message-Id: <20230205163758.416992-2-cota@braap.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 configure                 |   15 +
 meson.build               |    4 +
 include/qemu/qtree.h      |  201 ++++++
 tests/bench/qtree-bench.c |  286 ++++++++
 tests/unit/test-qtree.c   |  333 +++++++++
 util/qtree.c              | 1390 +++++++++++++++++++++++++++++++++++++
 tests/bench/meson.build   |    4 +
 tests/unit/meson.build    |    1 +
 util/meson.build          |    1 +
 9 files changed, 2235 insertions(+)
 create mode 100644 include/qemu/qtree.h
 create mode 100644 tests/bench/qtree-bench.c
 create mode 100644 tests/unit/test-qtree.c
 create mode 100644 util/qtree.c

diff --git a/configure b/configure
index XXXXXXX..XXXXXXX 100755
--- a/configure
+++ b/configure
@@ -XXX,XX +XXX,XX @@ safe_stack=""
 use_containers="yes"
 gdb_bin=$(command -v "gdb-multiarch" || command -v "gdb")
 gdb_arches=""
+glib_has_gslice="no"
 
 if test -e "$source_path/.git"
 then
@@ -XXX,XX +XXX,XX @@ for i in $glib_modules; do
     fi
 done
 
+# Check whether glib has gslice, which we have to avoid for correctness.
+# TODO: remove this check and the corresponding workaround (qtree) when
+# the minimum supported glib is >= $glib_dropped_gslice_version.
+glib_dropped_gslice_version=2.75.3
+for i in $glib_modules; do
+    if ! $pkg_config --atleast-version=$glib_dropped_gslice_version $i; then
+        glib_has_gslice="yes"
+	break
+    fi
+done
+
 glib_bindir="$($pkg_config --variable=bindir glib-2.0)"
 if test -z "$glib_bindir" ; then
 	glib_bindir="$($pkg_config --variable=prefix glib-2.0)"/bin
@@ -XXX,XX +XXX,XX @@ echo "GLIB_CFLAGS=$glib_cflags" >> $config_host_mak
 echo "GLIB_LIBS=$glib_libs" >> $config_host_mak
 echo "GLIB_BINDIR=$glib_bindir" >> $config_host_mak
 echo "GLIB_VERSION=$($pkg_config --modversion glib-2.0)" >> $config_host_mak
+if test "$glib_has_gslice" = "yes" ; then
+    echo "HAVE_GLIB_WITH_SLICE_ALLOCATOR=y" >> $config_host_mak
+fi
 echo "QEMU_LDFLAGS=$QEMU_LDFLAGS" >> $config_host_mak
 echo "EXESUF=$EXESUF" >> $config_host_mak
 
diff --git a/meson.build b/meson.build
index XXXXXXX..XXXXXXX 100644
--- a/meson.build
+++ b/meson.build
@@ -XXX,XX +XXX,XX @@ glib = declare_dependency(compile_args: config_host['GLIB_CFLAGS'].split(),
                           })
 # override glib dep with the configure results (for subprojects)
 meson.override_dependency('glib-2.0', glib)
+# pass down whether Glib has the slice allocator
+if config_host.has_key('HAVE_GLIB_WITH_SLICE_ALLOCATOR')
+  config_host_data.set('HAVE_GLIB_WITH_SLICE_ALLOCATOR', true)
+endif
 
 gio = not_found
 gdbus_codegen = not_found
diff --git a/include/qemu/qtree.h b/include/qemu/qtree.h
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/include/qemu/qtree.h
@@ -XXX,XX +XXX,XX @@
+/*
+ * GLIB - Library of useful routines for C programming
+ * Copyright (C) 1995-1997  Peter Mattis, Spencer Kimball and Josh MacDonald
+ *
+ * SPDX-License-Identifier: LGPL-2.1-or-later
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * Modified by the GLib Team and others 1997-2000.  See the AUTHORS
+ * file for a list of people on the GLib Team.  See the ChangeLog
+ * files for a list of changes.  These files are distributed with
+ * GLib at ftp://ftp.gtk.org/pub/gtk/.
+ */
+
+/*
+ * QTree is a partial import of Glib's GTree. The parts excluded correspond
+ * to API calls either deprecated (e.g. g_tree_traverse) or recently added
+ * (e.g. g_tree_search_node, added in 2.68); neither have callers in QEMU.
+ *
+ * The reason for this import is to allow us to control the memory allocator
+ * used by the tree implementation. Until Glib 2.75.3, GTree uses Glib's
+ * slice allocator, which causes problems when forking in user-mode;
+ * see https://gitlab.com/qemu-project/qemu/-/issues/285 and glib's
+ * "45b5a6c1e gslice: Remove slice allocator and use malloc() instead".
+ *
+ * TODO: remove QTree when QEMU's minimum Glib version is >= 2.75.3.
+ */
+
+#ifndef QEMU_QTREE_H
+#define QEMU_QTREE_H
+
+#include "qemu/osdep.h"
+
+#ifdef HAVE_GLIB_WITH_SLICE_ALLOCATOR
+
+typedef struct _QTree  QTree;
+
+typedef struct _QTreeNode QTreeNode;
+
+typedef gboolean (*QTraverseNodeFunc)(QTreeNode *node,
+                                      gpointer user_data);
+
+/*
+ * Balanced binary trees
+ */
+QTree *q_tree_new(GCompareFunc key_compare_func);
+QTree *q_tree_new_with_data(GCompareDataFunc key_compare_func,
+                            gpointer key_compare_data);
+QTree *q_tree_new_full(GCompareDataFunc key_compare_func,
+                       gpointer key_compare_data,
+                       GDestroyNotify key_destroy_func,
+                       GDestroyNotify value_destroy_func);
+QTree *q_tree_ref(QTree *tree);
+void q_tree_unref(QTree *tree);
+void q_tree_destroy(QTree *tree);
+void q_tree_insert(QTree *tree,
+                   gpointer key,
+                   gpointer value);
+void q_tree_replace(QTree *tree,
+                    gpointer key,
+                    gpointer value);
+gboolean q_tree_remove(QTree *tree,
+                       gconstpointer key);
+gboolean q_tree_steal(QTree *tree,
+                      gconstpointer key);
+gpointer q_tree_lookup(QTree *tree,
+                       gconstpointer key);
+gboolean q_tree_lookup_extended(QTree *tree,
+                                gconstpointer lookup_key,
+                                gpointer *orig_key,
+                                gpointer *value);
+void q_tree_foreach(QTree *tree,
+                    GTraverseFunc func,
+                    gpointer user_data);
+gpointer q_tree_search(QTree *tree,
+                       GCompareFunc search_func,
+                       gconstpointer user_data);
+gint q_tree_height(QTree *tree);
+gint q_tree_nnodes(QTree *tree);
+
+#else /* !HAVE_GLIB_WITH_SLICE_ALLOCATOR */
+
+typedef GTree QTree;
+typedef GTreeNode QTreeNode;
+typedef GTraverseNodeFunc QTraverseNodeFunc;
+
+static inline QTree *q_tree_new(GCompareFunc key_compare_func)
+{
+    return g_tree_new(key_compare_func);
+}
+
+static inline QTree *q_tree_new_with_data(GCompareDataFunc key_compare_func,
+                                          gpointer key_compare_data)
+{
+    return g_tree_new_with_data(key_compare_func, key_compare_data);
+}
+
+static inline QTree *q_tree_new_full(GCompareDataFunc key_compare_func,
+                                     gpointer key_compare_data,
+                                     GDestroyNotify key_destroy_func,
+                                     GDestroyNotify value_destroy_func)
+{
+    return g_tree_new_full(key_compare_func, key_compare_data,
+                           key_destroy_func, value_destroy_func);
+}
+
+static inline QTree *q_tree_ref(QTree *tree)
+{
+    return g_tree_ref(tree);
+}
+
+static inline void q_tree_unref(QTree *tree)
+{
+    g_tree_unref(tree);
+}
+
+static inline void q_tree_destroy(QTree *tree)
+{
+    g_tree_destroy(tree);
+}
+
+static inline void q_tree_insert(QTree *tree,
+                                 gpointer key,
+                                 gpointer value)
+{
+    g_tree_insert(tree, key, value);
+}
+
+static inline void q_tree_replace(QTree *tree,
+                                  gpointer key,
+                                  gpointer value)
+{
+    g_tree_replace(tree, key, value);
+}
+
+static inline gboolean q_tree_remove(QTree *tree,
+                                     gconstpointer key)
+{
+    return g_tree_remove(tree, key);
+}
+
+static inline gboolean q_tree_steal(QTree *tree,
+                                    gconstpointer key)
+{
+    return g_tree_steal(tree, key);
+}
+
+static inline gpointer q_tree_lookup(QTree *tree,
+                                     gconstpointer key)
+{
+    return g_tree_lookup(tree, key);
+}
+
+static inline gboolean q_tree_lookup_extended(QTree *tree,
+                                              gconstpointer lookup_key,
+                                              gpointer *orig_key,
+                                              gpointer *value)
+{
+    return g_tree_lookup_extended(tree, lookup_key, orig_key, value);
+}
+
+static inline void q_tree_foreach(QTree *tree,
+                                  GTraverseFunc func,
+                                  gpointer user_data)
+{
+    return g_tree_foreach(tree, func, user_data);
+}
+
+static inline gpointer q_tree_search(QTree *tree,
+                                     GCompareFunc search_func,
+                                     gconstpointer user_data)
+{
+    return g_tree_search(tree, search_func, user_data);
+}
+
+static inline gint q_tree_height(QTree *tree)
+{
+    return g_tree_height(tree);
+}
+
+static inline gint q_tree_nnodes(QTree *tree)
+{
+    return g_tree_nnodes(tree);
+}
+
+#endif /* HAVE_GLIB_WITH_SLICE_ALLOCATOR */
+
+#endif /* QEMU_QTREE_H */
diff --git a/tests/bench/qtree-bench.c b/tests/bench/qtree-bench.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/tests/bench/qtree-bench.c
@@ -XXX,XX +XXX,XX @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#include "qemu/osdep.h"
+#include "qemu/qtree.h"
+#include "qemu/timer.h"
+
+enum tree_op {
+    OP_LOOKUP,
+    OP_INSERT,
+    OP_REMOVE,
+    OP_REMOVE_ALL,
+    OP_TRAVERSE,
+};
+
+struct benchmark {
+    const char * const name;
+    enum tree_op op;
+    bool fill_on_init;
+};
+
+enum impl_type {
+    IMPL_GTREE,
+    IMPL_QTREE,
+};
+
+struct tree_implementation {
+    const char * const name;
+    enum impl_type type;
+};
+
+static const struct benchmark benchmarks[] = {
+    {
+        .name = "Lookup",
+        .op = OP_LOOKUP,
+        .fill_on_init = true,
+    },
+    {
+        .name = "Insert",
+        .op = OP_INSERT,
+        .fill_on_init = false,
+    },
+    {
+        .name = "Remove",
+        .op = OP_REMOVE,
+        .fill_on_init = true,
+    },
+    {
+        .name = "RemoveAll",
+        .op = OP_REMOVE_ALL,
+        .fill_on_init = true,
+    },
+    {
+        .name = "Traverse",
+        .op = OP_TRAVERSE,
+        .fill_on_init = true,
+    },
+};
+
+static const struct tree_implementation impls[] = {
+    {
+        .name = "GTree",
+        .type = IMPL_GTREE,
+    },
+    {
+        .name = "QTree",
+        .type = IMPL_QTREE,
+    },
+};
+
+static int compare_func(const void *ap, const void *bp)
+{
+    const size_t *a = ap;
+    const size_t *b = bp;
+
+    return *a - *b;
+}
+
+static void init_empty_tree_and_keys(enum impl_type impl,
+                                     void **ret_tree, size_t **ret_keys,
+                                     size_t n_elems)
+{
+    size_t *keys = g_malloc_n(n_elems, sizeof(*keys));
+    for (size_t i = 0; i < n_elems; i++) {
+        keys[i] = i;
+    }
+
+    void *tree;
+    switch (impl) {
+    case IMPL_GTREE:
+        tree = g_tree_new(compare_func);
+        break;
+    case IMPL_QTREE:
+        tree = q_tree_new(compare_func);
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    *ret_tree = tree;
+    *ret_keys = keys;
+}
+
+static gboolean traverse_func(gpointer key, gpointer value, gpointer data)
+{
+    return FALSE;
+}
+
+static inline void remove_all(void *tree, enum impl_type impl)
+{
+    switch (impl) {
+    case IMPL_GTREE:
+        g_tree_destroy(tree);
+        break;
+    case IMPL_QTREE:
+        q_tree_destroy(tree);
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static int64_t run_benchmark(const struct benchmark *bench,
+                             enum impl_type impl,
+                             size_t n_elems)
+{
+    void *tree;
+    size_t *keys;
+
+    init_empty_tree_and_keys(impl, &tree, &keys, n_elems);
+    if (bench->fill_on_init) {
+        for (size_t i = 0; i < n_elems; i++) {
+            switch (impl) {
+            case IMPL_GTREE:
+                g_tree_insert(tree, &keys[i], &keys[i]);
+                break;
+            case IMPL_QTREE:
+                q_tree_insert(tree, &keys[i], &keys[i]);
+                break;
+            default:
+                g_assert_not_reached();
+            }
+        }
+    }
+
+    int64_t start_ns = get_clock();
+    switch (bench->op) {
+    case OP_LOOKUP:
+        for (size_t i = 0; i < n_elems; i++) {
+            void *value;
+            switch (impl) {
+            case IMPL_GTREE:
+                value = g_tree_lookup(tree, &keys[i]);
+                break;
+            case IMPL_QTREE:
+                value = q_tree_lookup(tree, &keys[i]);
+                break;
+            default:
+                g_assert_not_reached();
+            }
+            (void)value;
+        }
+        break;
+    case OP_INSERT:
+        for (size_t i = 0; i < n_elems; i++) {
+            switch (impl) {
+            case IMPL_GTREE:
+                g_tree_insert(tree, &keys[i], &keys[i]);
+                break;
+            case IMPL_QTREE:
+                q_tree_insert(tree, &keys[i], &keys[i]);
+                break;
+            default:
+                g_assert_not_reached();
+            }
+        }
+        break;
+    case OP_REMOVE:
+        for (size_t i = 0; i < n_elems; i++) {
+            switch (impl) {
+            case IMPL_GTREE:
+                g_tree_remove(tree, &keys[i]);
+                break;
+            case IMPL_QTREE:
+                q_tree_remove(tree, &keys[i]);
+                break;
+            default:
+                g_assert_not_reached();
+            }
+        }
+        break;
+    case OP_REMOVE_ALL:
+        remove_all(tree, impl);
+        break;
+    case OP_TRAVERSE:
+        switch (impl) {
+        case IMPL_GTREE:
+            g_tree_foreach(tree, traverse_func, NULL);
+            break;
+        case IMPL_QTREE:
+            q_tree_foreach(tree, traverse_func, NULL);
+            break;
+        default:
+            g_assert_not_reached();
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+    int64_t ns = get_clock() - start_ns;
+
+    if (bench->op != OP_REMOVE_ALL) {
+        remove_all(tree, impl);
+    }
+    g_free(keys);
+
+    return ns;
+}
+
+int main(int argc, char *argv[])
+{
+    size_t sizes[] = {
+        32,
+        1024,
+        1024 * 4,
+        1024 * 128,
+        1024 * 1024,
+    };
+
+    double res[ARRAY_SIZE(benchmarks)][ARRAY_SIZE(impls)][ARRAY_SIZE(sizes)];
+    for (int i = 0; i < ARRAY_SIZE(sizes); i++) {
+        size_t size = sizes[i];
+        for (int j = 0; j < ARRAY_SIZE(impls); j++) {
+            const struct tree_implementation *impl = &impls[j];
+            for (int k = 0; k < ARRAY_SIZE(benchmarks); k++) {
+                const struct benchmark *bench = &benchmarks[k];
+
+                /* warm-up run */
+                run_benchmark(bench, impl->type, size);
+
+                int64_t total_ns = 0;
+                int64_t n_runs = 0;
+                while (total_ns < 2e8 || n_runs < 5) {
+                    total_ns += run_benchmark(bench, impl->type, size);
+                    n_runs++;
+                }
+                double ns_per_run = (double)total_ns / n_runs;
+
+                /* Throughput, in Mops/s */
+                res[k][j][i] = size / ns_per_run * 1e3;
+            }
+        }
+    }
+
+    printf("# Results' breakdown: Tree, Op and #Elements. Units: Mops/s\n");
+    printf("%5s %10s ", "Tree", "Op");
+    for (int i = 0; i < ARRAY_SIZE(sizes); i++) {
+        printf("%7zu         ", sizes[i]);
+    }
+    printf("\n");
+    char separator[97];
+    for (int i = 0; i < ARRAY_SIZE(separator) - 1; i++) {
+        separator[i] = '-';
+    }
+    separator[ARRAY_SIZE(separator) - 1] = '\0';
+    printf("%s\n", separator);
+    for (int i = 0; i < ARRAY_SIZE(benchmarks); i++) {
+        for (int j = 0; j < ARRAY_SIZE(impls); j++) {
+            printf("%5s %10s ", impls[j].name, benchmarks[i].name);
+            for (int k = 0; k < ARRAY_SIZE(sizes); k++) {
+                printf("%7.2f ", res[i][j][k]);
+                if (j == 0) {
+                    printf("        ");
+                } else {
+                    if (res[i][0][k] != 0) {
+                        double speedup = res[i][j][k] / res[i][0][k];
+                        printf("(%4.2fx) ", speedup);
+                    } else {
+                        printf("(     ) ");
+                    }
+                }
+            }
+            printf("\n");
+        }
+    }
+    printf("%s\n", separator);
+    return 0;
+}
diff --git a/tests/unit/test-qtree.c b/tests/unit/test-qtree.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/tests/unit/test-qtree.c
@@ -XXX,XX +XXX,XX @@
+/*
+ * SPDX-License-Identifier: LGPL-2.1-or-later
+ *
+ * Tests for QTree.
+ * Original source: glib
+ *   https://gitlab.gnome.org/GNOME/glib/-/blob/main/glib/tests/tree.c
+ *   LGPL license.
+ *   Copyright (C) 1995-1997 Peter Mattis, Spencer Kimball and Josh MacDonald
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/qtree.h"
+
+static gint my_compare(gconstpointer a, gconstpointer b)
+{
+    const char *cha = a;
+    const char *chb = b;
+
+    return *cha - *chb;
+}
+
+static gint my_compare_with_data(gconstpointer a,
+                                 gconstpointer b,
+                                 gpointer user_data)
+{
+    const char *cha = a;
+    const char *chb = b;
+
+    /* just check that we got the right data */
+    g_assert(GPOINTER_TO_INT(user_data) == 123);
+
+    return *cha - *chb;
+}
+
+static gint my_search(gconstpointer a, gconstpointer b)
+{
+    return my_compare(b, a);
+}
+
+static gpointer destroyed_key;
+static gpointer destroyed_value;
+static guint destroyed_key_count;
+static guint destroyed_value_count;
+
+static void my_key_destroy(gpointer key)
+{
+    destroyed_key = key;
+    destroyed_key_count++;
+}
+
+static void my_value_destroy(gpointer value)
+{
+    destroyed_value = value;
+    destroyed_value_count++;
+}
+
+static gint my_traverse(gpointer key, gpointer value, gpointer data)
+{
+    char *ch = key;
+
+    g_assert((*ch) > 0);
+
+    if (*ch == 'd') {
+        return TRUE;
+    }
+
+    return FALSE;
+}
+
+char chars[] =
+    "0123456789"
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    "abcdefghijklmnopqrstuvwxyz";
+
+char chars2[] =
+    "0123456789"
+    "abcdefghijklmnopqrstuvwxyz";
+
+static gint check_order(gpointer key, gpointer value, gpointer data)
+{
+    char **p = data;
+    char *ch = key;
+
+    g_assert(**p == *ch);
+
+    (*p)++;
+
+    return FALSE;
+}
+
+static void test_tree_search(void)
+{
+    gint i;
+    QTree *tree;
+    gboolean removed;
+    gchar c;
+    gchar *p, *d;
+
+    tree = q_tree_new_with_data(my_compare_with_data, GINT_TO_POINTER(123));
+
+    for (i = 0; chars[i]; i++) {
+        q_tree_insert(tree, &chars[i], &chars[i]);
+    }
+
+    q_tree_foreach(tree, my_traverse, NULL);
+
+    g_assert(q_tree_nnodes(tree) == strlen(chars));
+    g_assert(q_tree_height(tree) == 6);
+
+    p = chars;
+    q_tree_foreach(tree, check_order, &p);
+
+    for (i = 0; i < 26; i++) {
+        removed = q_tree_remove(tree, &chars[i + 10]);
+        g_assert(removed);
+    }
+
+    c = '\0';
+    removed = q_tree_remove(tree, &c);
+    g_assert(!removed);
+
+    q_tree_foreach(tree, my_traverse, NULL);
+
+    g_assert(q_tree_nnodes(tree) == strlen(chars2));
+    g_assert(q_tree_height(tree) == 6);
+
+    p = chars2;
+    q_tree_foreach(tree, check_order, &p);
+
+    for (i = 25; i >= 0; i--) {
+        q_tree_insert(tree, &chars[i + 10], &chars[i + 10]);
+    }
+
+    p = chars;
+    q_tree_foreach(tree, check_order, &p);
+
+    c = '0';
+    p = q_tree_lookup(tree, &c);
+    g_assert(p && *p == c);
+    g_assert(q_tree_lookup_extended(tree, &c, (gpointer *)&d, (gpointer *)&p));
+    g_assert(c == *d && c == *p);
+
+    c = 'A';
+    p = q_tree_lookup(tree, &c);
+    g_assert(p && *p == c);
+
+    c = 'a';
+    p = q_tree_lookup(tree, &c);
+    g_assert(p && *p == c);
+
+    c = 'z';
+    p = q_tree_lookup(tree, &c);
+    g_assert(p && *p == c);
+
+    c = '!';
+    p = q_tree_lookup(tree, &c);
+    g_assert(p == NULL);
+
+    c = '=';
+    p = q_tree_lookup(tree, &c);
+    g_assert(p == NULL);
+
+    c = '|';
+    p = q_tree_lookup(tree, &c);
+    g_assert(p == NULL);
+
+    c = '0';
+    p = q_tree_search(tree, my_search, &c);
+    g_assert(p && *p == c);
+
+    c = 'A';
+    p = q_tree_search(tree, my_search, &c);
+    g_assert(p && *p == c);
+
+    c = 'a';
+    p = q_tree_search(tree, my_search, &c);
+    g_assert(p && *p == c);
+
+    c = 'z';
+    p = q_tree_search(tree, my_search, &c);
+    g_assert(p && *p == c);
+
+    c = '!';
+    p = q_tree_search(tree, my_search, &c);
+    g_assert(p == NULL);
+
+    c = '=';
+    p = q_tree_search(tree, my_search, &c);
+    g_assert(p == NULL);
+
+    c = '|';
+    p = q_tree_search(tree, my_search, &c);
+    g_assert(p == NULL);
+
+    q_tree_destroy(tree);
+}
+
+static void test_tree_remove(void)
+{
+    QTree *tree;
+    char c, d;
+    gint i;
+    gboolean removed;
+
+    tree = q_tree_new_full((GCompareDataFunc)my_compare, NULL,
+                           my_key_destroy,
+                           my_value_destroy);
+
+    for (i = 0; chars[i]; i++) {
+        q_tree_insert(tree, &chars[i], &chars[i]);
+    }
+
+    c = '0';
+    q_tree_insert(tree, &c, &c);
+    g_assert(destroyed_key == &c);
+    g_assert(destroyed_value == &chars[0]);
+    destroyed_key = NULL;
+    destroyed_value = NULL;
+
+    d = '1';
+    q_tree_replace(tree, &d, &d);
+    g_assert(destroyed_key == &chars[1]);
+    g_assert(destroyed_value == &chars[1]);
+    destroyed_key = NULL;
+    destroyed_value = NULL;
+
+    c = '2';
+    removed = q_tree_remove(tree, &c);
+    g_assert(removed);
+    g_assert(destroyed_key == &chars[2]);
+    g_assert(destroyed_value == &chars[2]);
+    destroyed_key = NULL;
+    destroyed_value = NULL;
+
+    c = '3';
+    removed = q_tree_steal(tree, &c);
+    g_assert(removed);
+    g_assert(destroyed_key == NULL);
+    g_assert(destroyed_value == NULL);
+
+    const gchar *remove = "omkjigfedba";
+    for (i = 0; remove[i]; i++) {
+        removed = q_tree_remove(tree, &remove[i]);
+        g_assert(removed);
+    }
+
+    q_tree_destroy(tree);
+}
+
+static void test_tree_destroy(void)
+{
+    QTree *tree;
+    gint i;
+
+    tree = q_tree_new(my_compare);
+
+    for (i = 0; chars[i]; i++) {
+        q_tree_insert(tree, &chars[i], &chars[i]);
+    }
+
+    g_assert(q_tree_nnodes(tree) == strlen(chars));
+
+    g_test_message("nnodes: %d", q_tree_nnodes(tree));
+    q_tree_ref(tree);
+    q_tree_destroy(tree);
+
+    g_test_message("nnodes: %d", q_tree_nnodes(tree));
+    g_assert(q_tree_nnodes(tree) == 0);
+
+    q_tree_unref(tree);
+}
+
+static void test_tree_insert(void)
+{
+    QTree *tree;
+    gchar *p;
+    gint i;
+    gchar *scrambled;
+
+    tree = q_tree_new(my_compare);
+
+    for (i = 0; chars[i]; i++) {
+        q_tree_insert(tree, &chars[i], &chars[i]);
+    }
+    p = chars;
+    q_tree_foreach(tree, check_order, &p);
+
+    q_tree_unref(tree);
+    tree = q_tree_new(my_compare);
+
+    for (i = strlen(chars) - 1; i >= 0; i--) {
+        q_tree_insert(tree, &chars[i], &chars[i]);
+    }
+    p = chars;
+    q_tree_foreach(tree, check_order, &p);
+
+    q_tree_unref(tree);
+    tree = q_tree_new(my_compare);
+
+    scrambled = g_strdup(chars);
+
+    for (i = 0; i < 30; i++) {
+        gchar tmp;
+        gint a, b;
+
+        a = g_random_int_range(0, strlen(scrambled));
+        b = g_random_int_range(0, strlen(scrambled));
+        tmp = scrambled[a];
+        scrambled[a] = scrambled[b];
+        scrambled[b] = tmp;
+    }
+
+    for (i = 0; scrambled[i]; i++) {
+        q_tree_insert(tree, &scrambled[i], &scrambled[i]);
+    }
+    p = chars;
+    q_tree_foreach(tree, check_order, &p);
+
+    g_free(scrambled);
+    q_tree_unref(tree);
+}
+
+int main(int argc, char *argv[])
+{
+    g_test_init(&argc, &argv, NULL);
+
+    g_test_add_func("/qtree/search", test_tree_search);
+    g_test_add_func("/qtree/remove", test_tree_remove);
+    g_test_add_func("/qtree/destroy", test_tree_destroy);
+    g_test_add_func("/qtree/insert", test_tree_insert);
+
+    return g_test_run();
+}
diff --git a/util/qtree.c b/util/qtree.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/util/qtree.c
@@ -XXX,XX +XXX,XX @@
+/*
+ * GLIB - Library of useful routines for C programming
+ * Copyright (C) 1995-1997  Peter Mattis, Spencer Kimball and Josh MacDonald
+ *
+ * SPDX-License-Identifier: LGPL-2.1-or-later
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * Modified by the GLib Team and others 1997-2000.  See the AUTHORS
+ * file for a list of people on the GLib Team.  See the ChangeLog
+ * files for a list of changes.  These files are distributed with
+ * GLib at ftp://ftp.gtk.org/pub/gtk/.
+ */
+
+/*
+ * MT safe
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/qtree.h"
+
+/**
+ * SECTION:trees-binary
+ * @title: Balanced Binary Trees
+ * @short_description: a sorted collection of key/value pairs optimized
+ *                     for searching and traversing in order
+ *
+ * The #QTree structure and its associated functions provide a sorted
+ * collection of key/value pairs optimized for searching and traversing
+ * in order. This means that most of the operations  (access, search,
+ * insertion, deletion, ...) on #QTree are O(log(n)) in average and O(n)
+ * in worst case for time complexity. But, note that maintaining a
+ * balanced sorted #QTree of n elements is done in time O(n log(n)).
+ *
+ * To create a new #QTree use q_tree_new().
+ *
+ * To insert a key/value pair into a #QTree use q_tree_insert()
+ * (O(n log(n))).
+ *
+ * To remove a key/value pair use q_tree_remove() (O(n log(n))).
+ *
+ * To look up the value corresponding to a given key, use
+ * q_tree_lookup() and q_tree_lookup_extended().
+ *
+ * To find out the number of nodes in a #QTree, use q_tree_nnodes(). To
+ * get the height of a #QTree, use q_tree_height().
+ *
+ * To traverse a #QTree, calling a function for each node visited in
+ * the traversal, use q_tree_foreach().
+ *
+ * To destroy a #QTree, use q_tree_destroy().
+ **/
+
+#define MAX_GTREE_HEIGHT 40
+
+/**
+ * QTree:
+ *
+ * The QTree struct is an opaque data structure representing a
+ * [balanced binary tree][glib-Balanced-Binary-Trees]. It should be
+ * accessed only by using the following functions.
+ */
+struct _QTree {
+    QTreeNode        *root;
+    GCompareDataFunc  key_compare;
+    GDestroyNotify    key_destroy_func;
+    GDestroyNotify    value_destroy_func;
+    gpointer          key_compare_data;
+    guint             nnodes;
+    gint              ref_count;
+};
+
+struct _QTreeNode {
+    gpointer   key;         /* key for this node */
+    gpointer   value;       /* value stored at this node */
+    QTreeNode *left;        /* left subtree */
+    QTreeNode *right;       /* right subtree */
+    gint8      balance;     /* height (right) - height (left) */
+    guint8     left_child;
+    guint8     right_child;
+};
+
+
+static QTreeNode *q_tree_node_new(gpointer       key,
+                                  gpointer       value);
+static QTreeNode *q_tree_insert_internal(QTree *tree,
+                                         gpointer key,
+                                         gpointer value,
+                                         gboolean replace);
+static gboolean   q_tree_remove_internal(QTree         *tree,
+                                         gconstpointer  key,
+                                         gboolean       steal);
+static QTreeNode *q_tree_node_balance(QTreeNode     *node);
+static QTreeNode *q_tree_find_node(QTree         *tree,
+                                   gconstpointer  key);
+static QTreeNode *q_tree_node_search(QTreeNode *node,
+                                     GCompareFunc search_func,
+                                     gconstpointer data);
+static QTreeNode *q_tree_node_rotate_left(QTreeNode     *node);
+static QTreeNode *q_tree_node_rotate_right(QTreeNode     *node);
+#ifdef Q_TREE_DEBUG
+static void       q_tree_node_check(QTreeNode     *node);
+#endif
+
+static QTreeNode*
+q_tree_node_new(gpointer key,
+                gpointer value)
+{
+    QTreeNode *node = g_new(QTreeNode, 1);
+
+    node->balance = 0;
+    node->left = NULL;
+    node->right = NULL;
+    node->left_child = FALSE;
+    node->right_child = FALSE;
+    node->key = key;
+    node->value = value;
+
+    return node;
+}
+
+/**
+ * q_tree_new:
+ * @key_compare_func: the function used to order the nodes in the #QTree.
+ *   It should return values similar to the standard strcmp() function -
+ *   0 if the two arguments are equal, a negative value if the first argument
+ *   comes before the second, or a positive value if the first argument comes
+ *   after the second.
+ *
+ * Creates a new #QTree.
+ *
+ * Returns: a newly allocated #QTree
+ */
+QTree *
+q_tree_new(GCompareFunc key_compare_func)
+{
+    g_return_val_if_fail(key_compare_func != NULL, NULL);
+
+    return q_tree_new_full((GCompareDataFunc) key_compare_func, NULL,
+                           NULL, NULL);
+}
+
+/**
+ * q_tree_new_with_data:
+ * @key_compare_func: qsort()-style comparison function
+ * @key_compare_data: data to pass to comparison function
+ *
+ * Creates a new #QTree with a comparison function that accepts user data.
+ * See q_tree_new() for more details.
+ *
+ * Returns: a newly allocated #QTree
+ */
+QTree *
+q_tree_new_with_data(GCompareDataFunc key_compare_func,
+                     gpointer         key_compare_data)
+{
+    g_return_val_if_fail(key_compare_func != NULL, NULL);
+
+    return q_tree_new_full(key_compare_func, key_compare_data,
+                           NULL, NULL);
+}
+
+/**
+ * q_tree_new_full:
+ * @key_compare_func: qsort()-style comparison function
+ * @key_compare_data: data to pass to comparison function
+ * @key_destroy_func: a function to free the memory allocated for the key
+ *   used when removing the entry from the #QTree or %NULL if you don't
+ *   want to supply such a function
+ * @value_destroy_func: a function to free the memory allocated for the
+ *   value used when removing the entry from the #QTree or %NULL if you
+ *   don't want to supply such a function
+ *
+ * Creates a new #QTree like q_tree_new() and allows to specify functions
+ * to free the memory allocated for the key and value that get called when
+ * removing the entry from the #QTree.
+ *
+ * Returns: a newly allocated #QTree
+ */
+QTree *
+q_tree_new_full(GCompareDataFunc key_compare_func,
+                gpointer         key_compare_data,
+                GDestroyNotify   key_destroy_func,
+                GDestroyNotify   value_destroy_func)
+{
+    QTree *tree;
+
+    g_return_val_if_fail(key_compare_func != NULL, NULL);
+
+    tree = g_new(QTree, 1);
+    tree->root               = NULL;
+    tree->key_compare        = key_compare_func;
+    tree->key_destroy_func   = key_destroy_func;
+    tree->value_destroy_func = value_destroy_func;
+    tree->key_compare_data   = key_compare_data;
+    tree->nnodes             = 0;
+    tree->ref_count          = 1;
+
+    return tree;
+}
+
+/**
+ * q_tree_node_first:
+ * @tree: a #QTree
+ *
+ * Returns the first in-order node of the tree, or %NULL
+ * for an empty tree.
+ *
+ * Returns: (nullable) (transfer none): the first node in the tree
+ *
+ * Since: 2.68 in GLib. Internal in Qtree, i.e. not in the public API.
+ */
+static QTreeNode *
+q_tree_node_first(QTree *tree)
+{
+    QTreeNode *tmp;
+
+    g_return_val_if_fail(tree != NULL, NULL);
+
+    if (!tree->root) {
+        return NULL;
+    }
+
+    tmp = tree->root;
+
+    while (tmp->left_child) {
+        tmp = tmp->left;
+    }
+
+    return tmp;
+}
+
+/**
+ * q_tree_node_previous
+ * @node: a #QTree node
+ *
+ * Returns the previous in-order node of the tree, or %NULL
+ * if the passed node was already the first one.
+ *
+ * Returns: (nullable) (transfer none): the previous node in the tree
+ *
+ * Since: 2.68 in GLib. Internal in Qtree, i.e. not in the public API.
+ */
+static QTreeNode *
+q_tree_node_previous(QTreeNode *node)
+{
+    QTreeNode *tmp;
+
+    g_return_val_if_fail(node != NULL, NULL);
+
+    tmp = node->left;
+
+    if (node->left_child) {
+        while (tmp->right_child) {
+            tmp = tmp->right;
+        }
+    }
+
+    return tmp;
+}
+
+/**
+ * q_tree_node_next
+ * @node: a #QTree node
+ *
+ * Returns the next in-order node of the tree, or %NULL
+ * if the passed node was already the last one.
+ *
+ * Returns: (nullable) (transfer none): the next node in the tree
+ *
+ * Since: 2.68 in GLib. Internal in Qtree, i.e. not in the public API.
+ */
+static QTreeNode *
+q_tree_node_next(QTreeNode *node)
+{
+    QTreeNode *tmp;
+
+    g_return_val_if_fail(node != NULL, NULL);
+
+    tmp = node->right;
+
+    if (node->right_child) {
+        while (tmp->left_child) {
+            tmp = tmp->left;
+        }
+    }
+
+    return tmp;
+}
+
+/**
+ * q_tree_remove_all:
+ * @tree: a #QTree
+ *
+ * Removes all nodes from a #QTree and destroys their keys and values,
+ * then resets the #QTree’s root to %NULL.
+ *
+ * Since: 2.70 in GLib. Internal in Qtree, i.e. not in the public API.
+ */
+static void
+q_tree_remove_all(QTree *tree)
+{
+    QTreeNode *node;
+    QTreeNode *next;
+
+    g_return_if_fail(tree != NULL);
+
+    node = q_tree_node_first(tree);
+
+    while (node) {
+        next = q_tree_node_next(node);
+
+        if (tree->key_destroy_func) {
+            tree->key_destroy_func(node->key);
+        }
+        if (tree->value_destroy_func) {
+            tree->value_destroy_func(node->value);
+        }
+        g_free(node);
+
+#ifdef Q_TREE_DEBUG
+        g_assert(tree->nnodes > 0);
+        tree->nnodes--;
+#endif
+
+        node = next;
+    }
+
+#ifdef Q_TREE_DEBUG
+    g_assert(tree->nnodes == 0);
+#endif
+
+    tree->root = NULL;
+#ifndef Q_TREE_DEBUG
+    tree->nnodes = 0;
+#endif
+}
+
+/**
+ * q_tree_ref:
+ * @tree: a #QTree
+ *
+ * Increments the reference count of @tree by one.
+ *
+ * It is safe to call this function from any thread.
+ *
+ * Returns: the passed in #QTree
+ *
+ * Since: 2.22
+ */
+QTree *
+q_tree_ref(QTree *tree)
+{
+    g_return_val_if_fail(tree != NULL, NULL);
+
+    g_atomic_int_inc(&tree->ref_count);
+
+    return tree;
+}
+
+/**
+ * q_tree_unref:
+ * @tree: a #QTree
+ *
+ * Decrements the reference count of @tree by one.
+ * If the reference count drops to 0, all keys and values will
+ * be destroyed (if destroy functions were specified) and all
+ * memory allocated by @tree will be released.
+ *
+ * It is safe to call this function from any thread.
+ *
+ * Since: 2.22
+ */
+void
+q_tree_unref(QTree *tree)
+{
+    g_return_if_fail(tree != NULL);
+
+    if (g_atomic_int_dec_and_test(&tree->ref_count)) {
+        q_tree_remove_all(tree);
+        g_free(tree);
+    }
+}
+
+/**
+ * q_tree_destroy:
+ * @tree: a #QTree
+ *
+ * Removes all keys and values from the #QTree and decreases its
+ * reference count by one. If keys and/or values are dynamically
+ * allocated, you should either free them first or create the #QTree
+ * using q_tree_new_full(). In the latter case the destroy functions
+ * you supplied will be called on all keys and values before destroying
+ * the #QTree.
+ */
+void
+q_tree_destroy(QTree *tree)
+{
+    g_return_if_fail(tree != NULL);
+
+    q_tree_remove_all(tree);
+    q_tree_unref(tree);
+}
+
+/**
+ * q_tree_insert_node:
+ * @tree: a #QTree
+ * @key: the key to insert
+ * @value: the value corresponding to the key
+ *
+ * Inserts a key/value pair into a #QTree.
+ *
+ * If the given key already exists in the #QTree its corresponding value
+ * is set to the new value. If you supplied a @value_destroy_func when
+ * creating the #QTree, the old value is freed using that function. If
+ * you supplied a @key_destroy_func when creating the #QTree, the passed
+ * key is freed using that function.
+ *
+ * The tree is automatically 'balanced' as new key/value pairs are added,
+ * so that the distance from the root to every leaf is as small as possible.
+ * The cost of maintaining a balanced tree while inserting new key/value
+ * result in a O(n log(n)) operation where most of the other operations
+ * are O(log(n)).
+ *
+ * Returns: (transfer none): the inserted (or set) node.
+ *
+ * Since: 2.68 in GLib. Internal in Qtree, i.e. not in the public API.
+ */
+static QTreeNode *
+q_tree_insert_node(QTree    *tree,
+                   gpointer  key,
+                   gpointer  value)
+{
+    QTreeNode *node;
+
+    g_return_val_if_fail(tree != NULL, NULL);
+
+    node = q_tree_insert_internal(tree, key, value, FALSE);
+
+#ifdef Q_TREE_DEBUG
+    q_tree_node_check(tree->root);
+#endif
+
+    return node;
+}
+
+/**
+ * q_tree_insert:
+ * @tree: a #QTree
+ * @key: the key to insert
+ * @value: the value corresponding to the key
+ *
+ * Inserts a key/value pair into a #QTree.
+ *
+ * Inserts a new key and value into a #QTree as q_tree_insert_node() does,
+ * only this function does not return the inserted or set node.
+ */
+void
+q_tree_insert(QTree    *tree,
+              gpointer  key,
+              gpointer  value)
+{
+    q_tree_insert_node(tree, key, value);
+}
+
+/**
+ * q_tree_replace_node:
+ * @tree: a #QTree
+ * @key: the key to insert
+ * @value: the value corresponding to the key
+ *
+ * Inserts a new key and value into a #QTree similar to q_tree_insert_node().
+ * The difference is that if the key already exists in the #QTree, it gets
+ * replaced by the new key. If you supplied a @value_destroy_func when
+ * creating the #QTree, the old value is freed using that function. If you
+ * supplied a @key_destroy_func when creating the #QTree, the old key is
+ * freed using that function.
+ *
+ * The tree is automatically 'balanced' as new key/value pairs are added,
+ * so that the distance from the root to every leaf is as small as possible.
+ *
+ * Returns: (transfer none): the inserted (or set) node.
+ *
+ * Since: 2.68 in GLib. Internal in Qtree, i.e. not in the public API.
+ */
+static QTreeNode *
+q_tree_replace_node(QTree    *tree,
+                    gpointer  key,
+                    gpointer  value)
+{
+    QTreeNode *node;
+
+    g_return_val_if_fail(tree != NULL, NULL);
+
+    node = q_tree_insert_internal(tree, key, value, TRUE);
+
+#ifdef Q_TREE_DEBUG
+    q_tree_node_check(tree->root);
+#endif
+
+    return node;
+}
+
+/**
+ * q_tree_replace:
+ * @tree: a #QTree
+ * @key: the key to insert
+ * @value: the value corresponding to the key
+ *
+ * Inserts a new key and value into a #QTree as q_tree_replace_node() does,
+ * only this function does not return the inserted or set node.
+ */
+void
+q_tree_replace(QTree    *tree,
+               gpointer  key,
+               gpointer  value)
+{
+    q_tree_replace_node(tree, key, value);
+}
+
+/* internal insert routine */
+static QTreeNode *
+q_tree_insert_internal(QTree    *tree,
+                       gpointer  key,
+                       gpointer  value,
+                       gboolean  replace)
+{
+    QTreeNode *node, *retnode;
+    QTreeNode *path[MAX_GTREE_HEIGHT];
+    int idx;
+
+    g_return_val_if_fail(tree != NULL, NULL);
+
+    if (!tree->root) {
+        tree->root = q_tree_node_new(key, value);
+        tree->nnodes++;
+        return tree->root;
+    }
+
+    idx = 0;
+    path[idx++] = NULL;
+    node = tree->root;
+
+    while (1) {
+        int cmp = tree->key_compare(key, node->key, tree->key_compare_data);
+
+        if (cmp == 0) {
+            if (tree->value_destroy_func) {
+                tree->value_destroy_func(node->value);
+            }
+
+            node->value = value;
+
+            if (replace) {
+                if (tree->key_destroy_func) {
+                    tree->key_destroy_func(node->key);
+                }
+
+                node->key = key;
+            } else {
+                /* free the passed key */
+                if (tree->key_destroy_func) {
+                    tree->key_destroy_func(key);
+                }
+            }
+
+            return node;
+        } else if (cmp < 0) {
+            if (node->left_child) {
+                path[idx++] = node;
+                node = node->left;
+            } else {
+                QTreeNode *child = q_tree_node_new(key, value);
+
+                child->left = node->left;
+                child->right = node;
+                node->left = child;
+                node->left_child = TRUE;
+                node->balance -= 1;
+
+                tree->nnodes++;
+
+                retnode = child;
+                break;
+            }
+        } else {
+            if (node->right_child) {
+                path[idx++] = node;
+                node = node->right;
+            } else {
+                QTreeNode *child = q_tree_node_new(key, value);
+
+                child->right = node->right;
+                child->left = node;
+                node->right = child;
+                node->right_child = TRUE;
+                node->balance += 1;
+
+                tree->nnodes++;
+
+                retnode = child;
+                break;
+            }
+        }
+    }
+
+    /*
+     * Restore balance. This is the goodness of a non-recursive
+     * implementation, when we are done with balancing we 'break'
+     * the loop and we are done.
+     */
+    while (1) {
+        QTreeNode *bparent = path[--idx];
+        gboolean left_node = (bparent && node == bparent->left);
+        g_assert(!bparent || bparent->left == node || bparent->right == node);
+
+        if (node->balance < -1 || node->balance > 1) {
+            node = q_tree_node_balance(node);
+            if (bparent == NULL) {
+                tree->root = node;
+            } else if (left_node) {
+                bparent->left = node;
+            } else {
+                bparent->right = node;
+            }
+        }
+
+        if (node->balance == 0 || bparent == NULL) {
+            break;
+        }
+
+        if (left_node) {
+            bparent->balance -= 1;
+        } else {
+            bparent->balance += 1;
+        }
+
+        node = bparent;
+    }
+
+    return retnode;
+}
+
+/**
+ * q_tree_remove:
+ * @tree: a #QTree
+ * @key: the key to remove
+ *
+ * Removes a key/value pair from a #QTree.
+ *
+ * If the #QTree was created using q_tree_new_full(), the key and value
+ * are freed using the supplied destroy functions, otherwise you have to
+ * make sure that any dynamically allocated values are freed yourself.
+ * If the key does not exist in the #QTree, the function does nothing.
+ *
+ * The cost of maintaining a balanced tree while removing a key/value
+ * result in a O(n log(n)) operation where most of the other operations
+ * are O(log(n)).
+ *
+ * Returns: %TRUE if the key was found (prior to 2.8, this function
+ *     returned nothing)
+ */
+gboolean
+q_tree_remove(QTree         *tree,
+              gconstpointer  key)
+{
+    gboolean removed;
+
+    g_return_val_if_fail(tree != NULL, FALSE);
+
+    removed = q_tree_remove_internal(tree, key, FALSE);
+
+#ifdef Q_TREE_DEBUG
+    q_tree_node_check(tree->root);
+#endif
+
+    return removed;
+}
+
+/**
+ * q_tree_steal:
+ * @tree: a #QTree
+ * @key: the key to remove
+ *
+ * Removes a key and its associated value from a #QTree without calling
+ * the key and value destroy functions.
+ *
+ * If the key does not exist in the #QTree, the function does nothing.
+ *
+ * Returns: %TRUE if the key was found (prior to 2.8, this function
+ *     returned nothing)
+ */
+gboolean
+q_tree_steal(QTree         *tree,
+             gconstpointer  key)
+{
+    gboolean removed;
+
+    g_return_val_if_fail(tree != NULL, FALSE);
+
+    removed = q_tree_remove_internal(tree, key, TRUE);
+
+#ifdef Q_TREE_DEBUG
+    q_tree_node_check(tree->root);
+#endif
+
+    return removed;
+}
+
+/* internal remove routine */
+static gboolean
+q_tree_remove_internal(QTree         *tree,
+                       gconstpointer  key,
+                       gboolean       steal)
+{
+    QTreeNode *node, *parent, *balance;
+    QTreeNode *path[MAX_GTREE_HEIGHT];
+    int idx;
+    gboolean left_node;
+
+    g_return_val_if_fail(tree != NULL, FALSE);
+
+    if (!tree->root) {
+        return FALSE;
+    }
+
+    idx = 0;
+    path[idx++] = NULL;
+    node = tree->root;
+
+    while (1) {
+        int cmp = tree->key_compare(key, node->key, tree->key_compare_data);
+
+        if (cmp == 0) {
+            break;
+        } else if (cmp < 0) {
+            if (!node->left_child) {
+                return FALSE;
+            }
+
+            path[idx++] = node;
+            node = node->left;
+        } else {
+            if (!node->right_child) {
+                return FALSE;
+            }
+
+            path[idx++] = node;
+            node = node->right;
+        }
+    }
+
+    /*
+     * The following code is almost equal to q_tree_remove_node,
+     * except that we do not have to call q_tree_node_parent.
+     */
+    balance = parent = path[--idx];
+    g_assert(!parent || parent->left == node || parent->right == node);
+    left_node = (parent && node == parent->left);
+
+    if (!node->left_child) {
+        if (!node->right_child) {
+            if (!parent) {
+                tree->root = NULL;
+            } else if (left_node) {
+                parent->left_child = FALSE;
+                parent->left = node->left;
+                parent->balance += 1;
+            } else {
+                parent->right_child = FALSE;
+                parent->right = node->right;
+                parent->balance -= 1;
+            }
+        } else {
+            /* node has a right child */
+            QTreeNode *tmp = q_tree_node_next(node);
+            tmp->left = node->left;
+
+            if (!parent) {
+                tree->root = node->right;
+            } else if (left_node) {
+                parent->left = node->right;
+                parent->balance += 1;
+            } else {
+                parent->right = node->right;
+                parent->balance -= 1;
+            }
+        }
+    } else {
+        /* node has a left child */
+        if (!node->right_child) {
+            QTreeNode *tmp = q_tree_node_previous(node);
+            tmp->right = node->right;
+
+            if (parent == NULL) {
+                tree->root = node->left;
+            } else if (left_node) {
+                parent->left = node->left;
+                parent->balance += 1;
+            } else {
+                parent->right = node->left;
+                parent->balance -= 1;
+            }
+        } else {
+            /* node has a both children (pant, pant!) */
+            QTreeNode *prev = node->left;
+            QTreeNode *next = node->right;
+            QTreeNode *nextp = node;
+            int old_idx = idx + 1;
+            idx++;
+
+            /* path[idx] == parent */
+            /* find the immediately next node (and its parent) */
+            while (next->left_child) {
+                path[++idx] = nextp = next;
+                next = next->left;
+            }
+
+            path[old_idx] = next;
+            balance = path[idx];
+
+            /* remove 'next' from the tree */
+            if (nextp != node) {
+                if (next->right_child) {
+                    nextp->left = next->right;
+                } else {
+                    nextp->left_child = FALSE;
+                }
+                nextp->balance += 1;
+
+                next->right_child = TRUE;
+                next->right = node->right;
+            } else {
+                node->balance -= 1;
+            }
+
+            /* set the prev to point to the right place */
+            while (prev->right_child) {
+                prev = prev->right;
+            }
+            prev->right = next;
+
+            /* prepare 'next' to replace 'node' */
+            next->left_child = TRUE;
+            next->left = node->left;
+            next->balance = node->balance;
+
+            if (!parent) {
+                tree->root = next;
+            } else if (left_node) {
+                parent->left = next;
+            } else {
+                parent->right = next;
+            }
+        }
+    }
+
+    /* restore balance */
+    if (balance) {
+        while (1) {
+            QTreeNode *bparent = path[--idx];
+            g_assert(!bparent ||
+                     bparent->left == balance ||
+                     bparent->right == balance);
+            left_node = (bparent && balance == bparent->left);
+
+            if (balance->balance < -1 || balance->balance > 1) {
+                balance = q_tree_node_balance(balance);
+                if (!bparent) {
+                    tree->root = balance;
+                } else if (left_node) {
+                    bparent->left = balance;
+                } else {
+                    bparent->right = balance;
+                }
+            }
+
+            if (balance->balance != 0 || !bparent) {
+                break;
+            }
+
+            if (left_node) {
+                bparent->balance += 1;
+            } else {
+                bparent->balance -= 1;
+            }
+
+            balance = bparent;
+        }
+    }
+
+    if (!steal) {
+        if (tree->key_destroy_func) {
+            tree->key_destroy_func(node->key);
+        }
+        if (tree->value_destroy_func) {
+            tree->value_destroy_func(node->value);
+        }
+    }
+
+    g_free(node);
+
+    tree->nnodes--;
+
+    return TRUE;
+}
+
+/**
+ * q_tree_lookup_node:
+ * @tree: a #QTree
+ * @key: the key to look up
+ *
+ * Gets the tree node corresponding to the given key. Since a #QTree is
+ * automatically balanced as key/value pairs are added, key lookup
+ * is O(log n) (where n is the number of key/value pairs in the tree).
+ *
+ * Returns: (nullable) (transfer none): the tree node corresponding to
+ *          the key, or %NULL if the key was not found
+ *
+ * Since: 2.68 in GLib. Internal in Qtree, i.e. not in the public API.
+ */
+static QTreeNode *
+q_tree_lookup_node(QTree         *tree,
+                   gconstpointer  key)
+{
+    g_return_val_if_fail(tree != NULL, NULL);
+
+    return q_tree_find_node(tree, key);
+}
+
+/**
+ * q_tree_lookup:
+ * @tree: a #QTree
+ * @key: the key to look up
+ *
+ * Gets the value corresponding to the given key. Since a #QTree is
+ * automatically balanced as key/value pairs are added, key lookup
+ * is O(log n) (where n is the number of key/value pairs in the tree).
+ *
+ * Returns: the value corresponding to the key, or %NULL
+ *     if the key was not found
+ */
+gpointer
+q_tree_lookup(QTree         *tree,
+              gconstpointer  key)
+{
+    QTreeNode *node;
+
+    node = q_tree_lookup_node(tree, key);
+
+    return node ? node->value : NULL;
+}
+
+/**
+ * q_tree_lookup_extended:
+ * @tree: a #QTree
+ * @lookup_key: the key to look up
+ * @orig_key: (out) (optional) (nullable): returns the original key
+ * @value: (out) (optional) (nullable): returns the value associated with
+ *         the key
+ *
+ * Looks up a key in the #QTree, returning the original key and the
+ * associated value. This is useful if you need to free the memory
+ * allocated for the original key, for example before calling
+ * q_tree_remove().
+ *
+ * Returns: %TRUE if the key was found in the #QTree
+ */
+gboolean
+q_tree_lookup_extended(QTree         *tree,
+                       gconstpointer  lookup_key,
+                       gpointer      *orig_key,
+                       gpointer      *value)
+{
+    QTreeNode *node;
+
+    g_return_val_if_fail(tree != NULL, FALSE);
+
+    node = q_tree_find_node(tree, lookup_key);
+
+    if (node) {
+        if (orig_key) {
+            *orig_key = node->key;
+        }
+        if (value) {
+            *value = node->value;
+        }
+        return TRUE;
+    } else {
+        return FALSE;
+    }
+}
+
+/**
+ * q_tree_foreach:
+ * @tree: a #QTree
+ * @func: the function to call for each node visited.
+ *     If this function returns %TRUE, the traversal is stopped.
+ * @user_data: user data to pass to the function
+ *
+ * Calls the given function for each of the key/value pairs in the #QTree.
+ * The function is passed the key and value of each pair, and the given
+ * @data parameter. The tree is traversed in sorted order.
+ *
+ * The tree may not be modified while iterating over it (you can't
+ * add/remove items). To remove all items matching a predicate, you need
+ * to add each item to a list in your #GTraverseFunc as you walk over
+ * the tree, then walk the list and remove each item.
+ */
+void
+q_tree_foreach(QTree         *tree,
+               GTraverseFunc  func,
+               gpointer       user_data)
+{
+    QTreeNode *node;
+
+    g_return_if_fail(tree != NULL);
+
+    if (!tree->root) {
+        return;
+    }
+
+    node = q_tree_node_first(tree);
+
+    while (node) {
+        if ((*func)(node->key, node->value, user_data)) {
+            break;
+        }
+
+        node = q_tree_node_next(node);
+    }
+}
+
+/**
+ * q_tree_search_node:
+ * @tree: a #QTree
+ * @search_func: a function used to search the #QTree
+ * @user_data: the data passed as the second argument to @search_func
+ *
+ * Searches a #QTree using @search_func.
+ *
+ * The @search_func is called with a pointer to the key of a key/value
+ * pair in the tree, and the passed in @user_data. If @search_func returns
+ * 0 for a key/value pair, then the corresponding node is returned as
+ * the result of q_tree_search(). If @search_func returns -1, searching
+ * will proceed among the key/value pairs that have a smaller key; if
+ * @search_func returns 1, searching will proceed among the key/value
+ * pairs that have a larger key.
+ *
+ * Returns: (nullable) (transfer none): the node corresponding to the
+ *          found key, or %NULL if the key was not found
+ *
+ * Since: 2.68 in GLib. Internal in Qtree, i.e. not in the public API.
+ */
+static QTreeNode *
+q_tree_search_node(QTree         *tree,
+                   GCompareFunc   search_func,
+                   gconstpointer  user_data)
+{
+    g_return_val_if_fail(tree != NULL, NULL);
+
+    if (!tree->root) {
+        return NULL;
+    }
+
+    return q_tree_node_search(tree->root, search_func, user_data);
+}
+
+/**
+ * q_tree_search:
+ * @tree: a #QTree
+ * @search_func: a function used to search the #QTree
+ * @user_data: the data passed as the second argument to @search_func
+ *
+ * Searches a #QTree using @search_func.
+ *
+ * The @search_func is called with a pointer to the key of a key/value
+ * pair in the tree, and the passed in @user_data. If @search_func returns
+ * 0 for a key/value pair, then the corresponding value is returned as
+ * the result of q_tree_search(). If @search_func returns -1, searching
+ * will proceed among the key/value pairs that have a smaller key; if
+ * @search_func returns 1, searching will proceed among the key/value
+ * pairs that have a larger key.
+ *
+ * Returns: the value corresponding to the found key, or %NULL
+ *     if the key was not found
+ */
+gpointer
+q_tree_search(QTree         *tree,
+              GCompareFunc   search_func,
+              gconstpointer  user_data)
+{
+    QTreeNode *node;
+
+    node = q_tree_search_node(tree, search_func, user_data);
+
+    return node ? node->value : NULL;
+}
+
+/**
+ * q_tree_height:
+ * @tree: a #QTree
+ *
+ * Gets the height of a #QTree.
+ *
+ * If the #QTree contains no nodes, the height is 0.
+ * If the #QTree contains only one root node the height is 1.
+ * If the root node has children the height is 2, etc.
+ *
+ * Returns: the height of @tree
+ */
+gint
+q_tree_height(QTree *tree)
+{
+    QTreeNode *node;
+    gint height;
+
+    g_return_val_if_fail(tree != NULL, 0);
+
+    if (!tree->root) {
+        return 0;
+    }
+
+    height = 0;
+    node = tree->root;
+
+    while (1) {
+        height += 1 + MAX(node->balance, 0);
+
+        if (!node->left_child) {
+            return height;
+        }
+
+        node = node->left;
+    }
+}
+
+/**
+ * q_tree_nnodes:
+ * @tree: a #QTree
+ *
+ * Gets the number of nodes in a #QTree.
+ *
+ * Returns: the number of nodes in @tree
+ */
+gint
+q_tree_nnodes(QTree *tree)
+{
+    g_return_val_if_fail(tree != NULL, 0);
+
+    return tree->nnodes;
+}
+
+static QTreeNode *
+q_tree_node_balance(QTreeNode *node)
+{
+    if (node->balance < -1) {
+        if (node->left->balance > 0) {
+            node->left = q_tree_node_rotate_left(node->left);
+        }
+        node = q_tree_node_rotate_right(node);
+    } else if (node->balance > 1) {
+        if (node->right->balance < 0) {
+            node->right = q_tree_node_rotate_right(node->right);
+        }
+        node = q_tree_node_rotate_left(node);
+    }
+
+    return node;
+}
+
+static QTreeNode *
+q_tree_find_node(QTree        *tree,
+                 gconstpointer key)
+{
+    QTreeNode *node;
+    gint cmp;
+
+    node = tree->root;
+    if (!node) {
+        return NULL;
+    }
+
+    while (1) {
+        cmp = tree->key_compare(key, node->key, tree->key_compare_data);
+        if (cmp == 0) {
+            return node;
+        } else if (cmp < 0) {
+            if (!node->left_child) {
+                return NULL;
+            }
+
+            node = node->left;
+        } else {
+            if (!node->right_child) {
+                return NULL;
+            }
+
+            node = node->right;
+        }
+    }
+}
+
+static QTreeNode *
+q_tree_node_search(QTreeNode     *node,
+                   GCompareFunc   search_func,
+                   gconstpointer  data)
+{
+    gint dir;
+
+    if (!node) {
+        return NULL;
+    }
+
+    while (1) {
+        dir = (*search_func)(node->key, data);
+        if (dir == 0) {
+            return node;
+        } else if (dir < 0) {
+            if (!node->left_child) {
+                return NULL;
+            }
+
+            node = node->left;
+        } else {
+            if (!node->right_child) {
+                return NULL;
+            }
+
+            node = node->right;
+        }
+    }
+}
+
+static QTreeNode *
+q_tree_node_rotate_left(QTreeNode *node)
+{
+    QTreeNode *right;
+    gint a_bal;
+    gint b_bal;
+
+    right = node->right;
+
+    if (right->left_child) {
+        node->right = right->left;
+    } else {
+        node->right_child = FALSE;
+        right->left_child = TRUE;
+    }
+    right->left = node;
+
+    a_bal = node->balance;
+    b_bal = right->balance;
+
+    if (b_bal <= 0) {
+        if (a_bal >= 1) {
+            right->balance = b_bal - 1;
+        } else {
+            right->balance = a_bal + b_bal - 2;
+        }
+        node->balance = a_bal - 1;
+    } else {
+        if (a_bal <= b_bal) {
+            right->balance = a_bal - 2;
+        } else {
+            right->balance = b_bal - 1;
+        }
+        node->balance = a_bal - b_bal - 1;
+    }
+
+    return right;
+}
+
+static QTreeNode *
+q_tree_node_rotate_right(QTreeNode *node)
+{
+    QTreeNode *left;
+    gint a_bal;
+    gint b_bal;
+
+    left = node->left;
+
+    if (left->right_child) {
+        node->left = left->right;
+    } else {
+        node->left_child = FALSE;
+        left->right_child = TRUE;
+    }
+    left->right = node;
+
+    a_bal = node->balance;
+    b_bal = left->balance;
+
+    if (b_bal <= 0) {
+        if (b_bal > a_bal) {
+            left->balance = b_bal + 1;
+        } else {
+            left->balance = a_bal + 2;
+        }
+        node->balance = a_bal - b_bal + 1;
+    } else {
+        if (a_bal <= -1) {
+            left->balance = b_bal + 1;
+        } else {
+            left->balance = a_bal + b_bal + 2;
+        }
+        node->balance = a_bal + 1;
+    }
+
+    return left;
+}
+
+#ifdef Q_TREE_DEBUG
+static gint
+q_tree_node_height(QTreeNode *node)
+{
+    gint left_height;
+    gint right_height;
+
+    if (node) {
+        left_height = 0;
+        right_height = 0;
+
+        if (node->left_child) {
+            left_height = q_tree_node_height(node->left);
+        }
+
+        if (node->right_child) {
+            right_height = q_tree_node_height(node->right);
+        }
+
+        return MAX(left_height, right_height) + 1;
+    }
+
+    return 0;
+}
+
+static void q_tree_node_check(QTreeNode *node)
+{
+    gint left_height;
+    gint right_height;
+    gint balance;
+    QTreeNode *tmp;
+
+    if (node) {
+        if (node->left_child) {
+            tmp = q_tree_node_previous(node);
+            g_assert(tmp->right == node);
+        }
+
+        if (node->right_child) {
+            tmp = q_tree_node_next(node);
+            g_assert(tmp->left == node);
+        }
+
+        left_height = 0;
+        right_height = 0;
+
+        if (node->left_child) {
+            left_height = q_tree_node_height(node->left);
+        }
+        if (node->right_child) {
+            right_height = q_tree_node_height(node->right);
+        }
+
+        balance = right_height - left_height;
+        g_assert(balance == node->balance);
+
+        if (node->left_child) {
+            q_tree_node_check(node->left);
+        }
+        if (node->right_child) {
+            q_tree_node_check(node->right);
+        }
+    }
+}
+#endif
diff --git a/tests/bench/meson.build b/tests/bench/meson.build
index XXXXXXX..XXXXXXX 100644
--- a/tests/bench/meson.build
+++ b/tests/bench/meson.build
@@ -XXX,XX +XXX,XX @@ xbzrle_bench = executable('xbzrle-bench',
                        dependencies: [qemuutil,migration])
 endif
 
+qtree_bench = executable('qtree-bench',
+                         sources: 'qtree-bench.c',
+                         dependencies: [qemuutil])
+
 executable('atomic_add-bench',
            sources: files('atomic_add-bench.c'),
            dependencies: [qemuutil],
diff --git a/tests/unit/meson.build b/tests/unit/meson.build
index XXXXXXX..XXXXXXX 100644
--- a/tests/unit/meson.build
+++ b/tests/unit/meson.build
@@ -XXX,XX +XXX,XX @@ tests = {
   'test-rcu-slist': [],
   'test-qdist': [],
   'test-qht': [],
+  'test-qtree': [],
   'test-bitops': [],
   'test-bitcnt': [],
   'test-qgraph': ['../qtest/libqos/qgraph.c'],
diff --git a/util/meson.build b/util/meson.build
index XXXXXXX..XXXXXXX 100644
--- a/util/meson.build
+++ b/util/meson.build
@@ -XXX,XX +XXX,XX @@ util_ss.add(when: 'CONFIG_WIN32', if_true: files('oslib-win32.c'))
 util_ss.add(when: 'CONFIG_WIN32', if_true: files('qemu-thread-win32.c'))
 util_ss.add(when: 'CONFIG_WIN32', if_true: winmm)
 util_ss.add(when: 'CONFIG_WIN32', if_true: pathcch)
+util_ss.add(when: 'HAVE_GLIB_WITH_SLICE_ALLOCATOR', if_true: files('qtree.c'))
 util_ss.add(files('envlist.c', 'path.c', 'module.c'))
 util_ss.add(files('host-utils.c'))
 util_ss.add(files('bitmap.c', 'bitops.c'))
-- 
2.34.1

From: Emilio Cota <cota@braap.org>

qemu-user can hang in a multi-threaded fork. One common
reason is that when creating a TB, between fork and exec
we manipulate a GTree whose memory allocator (GSlice) is
not fork-safe.

Although POSIX does not mandate it, the system's allocator
(e.g. tcmalloc, libc malloc) is probably fork-safe.

Fix some of these hangs by using QTree, which uses the system's
allocator regardless of the Glib version that we used at
configuration time.

Tested with the test program in the original bug report, i.e.:
```

void garble() {
  int pid = fork();
  if (pid == 0) {
    exit(0);
  } else {
    int wstatus;
    waitpid(pid, &wstatus, 0);
  }
}

void supragarble(unsigned depth) {
  if (depth == 0)
    return ;

std::thread a(supragarble, depth-1);
  std::thread b(supragarble, depth-1);
  garble();
  a.join();
  b.join();
}

int main() {
  supragarble(10);
}
```

Resolves: https://gitlab.com/qemu-project/qemu/-/issues/285
Reported-by: Valentin David <me@valentindavid.com>
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Emilio Cota <cota@braap.org>
Message-Id: <20230205163758.416992-3-cota@braap.org>
[rth: Add QEMU_DISABLE_CFI for all callback using functions.]
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 accel/tcg/tb-maint.c | 17 +++++++++--------
 tcg/region.c         | 19 ++++++++++---------
 util/qtree.c         |  8 ++++----
 3 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/tb-maint.c
+++ b/accel/tcg/tb-maint.c
@@ -XXX,XX +XXX,XX @@
 
 #include "qemu/osdep.h"
 #include "qemu/interval-tree.h"
+#include "qemu/qtree.h"
 #include "exec/cputlb.h"
 #include "exec/log.h"
 #include "exec/exec-all.h"
@@ -XXX,XX +XXX,XX @@ struct page_entry {
  * See also: page_collection_lock().
  */
 struct page_collection {
-    GTree *tree;
+    QTree *tree;
     struct page_entry *max;
 };
 
@@ -XXX,XX +XXX,XX @@ static bool page_trylock_add(struct page_collection *set, tb_page_addr_t addr)
     struct page_entry *pe;
     PageDesc *pd;
 
-    pe = g_tree_lookup(set->tree, &index);
+    pe = q_tree_lookup(set->tree, &index);
     if (pe) {
         return false;
     }
@@ -XXX,XX +XXX,XX @@ static bool page_trylock_add(struct page_collection *set, tb_page_addr_t addr)
     }
 
     pe = page_entry_new(pd, index);
-    g_tree_insert(set->tree, &pe->index, pe);
+    q_tree_insert(set->tree, &pe->index, pe);
 
     /*
      * If this is either (1) the first insertion or (2) a page whose index
@@ -XXX,XX +XXX,XX @@ static struct page_collection *page_collection_lock(tb_page_addr_t start,
     end   >>= TARGET_PAGE_BITS;
     g_assert(start <= end);
 
-    set->tree = g_tree_new_full(tb_page_addr_cmp, NULL, NULL,
+    set->tree = q_tree_new_full(tb_page_addr_cmp, NULL, NULL,
                                 page_entry_destroy);
     set->max = NULL;
     assert_no_pages_locked();
 
  retry:
-    g_tree_foreach(set->tree, page_entry_lock, NULL);
+    q_tree_foreach(set->tree, page_entry_lock, NULL);
 
     for (index = start; index <= end; index++) {
         TranslationBlock *tb;
@@ -XXX,XX +XXX,XX @@ static struct page_collection *page_collection_lock(tb_page_addr_t start,
             continue;
         }
         if (page_trylock_add(set, index << TARGET_PAGE_BITS)) {
-            g_tree_foreach(set->tree, page_entry_unlock, NULL);
+            q_tree_foreach(set->tree, page_entry_unlock, NULL);
             goto retry;
         }
         assert_page_locked(pd);
@@ -XXX,XX +XXX,XX @@ static struct page_collection *page_collection_lock(tb_page_addr_t start,
                 (tb_page_addr1(tb) != -1 &&
                  page_trylock_add(set, tb_page_addr1(tb)))) {
                 /* drop all locks, and reacquire in order */
-                g_tree_foreach(set->tree, page_entry_unlock, NULL);
+                q_tree_foreach(set->tree, page_entry_unlock, NULL);
                 goto retry;
             }
         }
@@ -XXX,XX +XXX,XX @@ static struct page_collection *page_collection_lock(tb_page_addr_t start,
 static void page_collection_unlock(struct page_collection *set)
 {
     /* entries are unlocked and freed via page_entry_destroy */
-    g_tree_destroy(set->tree);
+    q_tree_destroy(set->tree);
     g_free(set);
 }
 
diff --git a/tcg/region.c b/tcg/region.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/region.c
+++ b/tcg/region.c
@@ -XXX,XX +XXX,XX @@
 #include "qemu/mprotect.h"
 #include "qemu/memalign.h"
 #include "qemu/cacheinfo.h"
+#include "qemu/qtree.h"
 #include "qapi/error.h"
 #include "exec/exec-all.h"
 #include "tcg/tcg.h"
@@ -XXX,XX +XXX,XX @@
 
 struct tcg_region_tree {
     QemuMutex lock;
-    GTree *tree;
+    QTree *tree;
     /* padding to avoid false sharing is computed at run-time */
 };
 
@@ -XXX,XX +XXX,XX @@ static void tcg_region_trees_init(void)
         struct tcg_region_tree *rt = region_trees + i * tree_size;
 
         qemu_mutex_init(&rt->lock);
-        rt->tree = g_tree_new_full(tb_tc_cmp, NULL, NULL, tb_destroy);
+        rt->tree = q_tree_new_full(tb_tc_cmp, NULL, NULL, tb_destroy);
     }
 }
 
@@ -XXX,XX +XXX,XX @@ void tcg_tb_insert(TranslationBlock *tb)
 
     g_assert(rt != NULL);
     qemu_mutex_lock(&rt->lock);
-    g_tree_insert(rt->tree, &tb->tc, tb);
+    q_tree_insert(rt->tree, &tb->tc, tb);
     qemu_mutex_unlock(&rt->lock);
 }
 
@@ -XXX,XX +XXX,XX @@ void tcg_tb_remove(TranslationBlock *tb)
 
     g_assert(rt != NULL);
     qemu_mutex_lock(&rt->lock);
-    g_tree_remove(rt->tree, &tb->tc);
+    q_tree_remove(rt->tree, &tb->tc);
     qemu_mutex_unlock(&rt->lock);
 }
 
@@ -XXX,XX +XXX,XX @@ TranslationBlock *tcg_tb_lookup(uintptr_t tc_ptr)
     }
 
     qemu_mutex_lock(&rt->lock);
-    tb = g_tree_lookup(rt->tree, &s);
+    tb = q_tree_lookup(rt->tree, &s);
     qemu_mutex_unlock(&rt->lock);
     return tb;
 }
@@ -XXX,XX +XXX,XX @@ void tcg_tb_foreach(GTraverseFunc func, gpointer user_data)
     for (i = 0; i < region.n; i++) {
         struct tcg_region_tree *rt = region_trees + i * tree_size;
 
-        g_tree_foreach(rt->tree, func, user_data);
+        q_tree_foreach(rt->tree, func, user_data);
     }
     tcg_region_tree_unlock_all();
 }
@@ -XXX,XX +XXX,XX @@ size_t tcg_nb_tbs(void)
     for (i = 0; i < region.n; i++) {
         struct tcg_region_tree *rt = region_trees + i * tree_size;
 
-        nb_tbs += g_tree_nnodes(rt->tree);
+        nb_tbs += q_tree_nnodes(rt->tree);
     }
     tcg_region_tree_unlock_all();
     return nb_tbs;
@@ -XXX,XX +XXX,XX @@ static void tcg_region_tree_reset_all(void)
         struct tcg_region_tree *rt = region_trees + i * tree_size;
 
         /* Increment the refcount first so that destroy acts as a reset */
-        g_tree_ref(rt->tree);
-        g_tree_destroy(rt->tree);
+        q_tree_ref(rt->tree);
+        q_tree_destroy(rt->tree);
     }
     tcg_region_tree_unlock_all();
 }
diff --git a/util/qtree.c b/util/qtree.c
index XXXXXXX..XXXXXXX 100644
--- a/util/qtree.c
+++ b/util/qtree.c
@@ -XXX,XX +XXX,XX @@ q_tree_node_next(QTreeNode *node)
  *
  * Since: 2.70 in GLib. Internal in Qtree, i.e. not in the public API.
  */
-static void
+static void QEMU_DISABLE_CFI
 q_tree_remove_all(QTree *tree)
 {
     QTreeNode *node;
@@ -XXX,XX +XXX,XX @@ q_tree_replace(QTree    *tree,
 }
 
 /* internal insert routine */
-static QTreeNode *
+static QTreeNode * QEMU_DISABLE_CFI
 q_tree_insert_internal(QTree    *tree,
                        gpointer  key,
                        gpointer  value,
@@ -XXX,XX +XXX,XX @@ q_tree_steal(QTree         *tree,
 }
 
 /* internal remove routine */
-static gboolean
+static gboolean QEMU_DISABLE_CFI
 q_tree_remove_internal(QTree         *tree,
                        gconstpointer  key,
                        gboolean       steal)
@@ -XXX,XX +XXX,XX @@ q_tree_node_balance(QTreeNode *node)
     return node;
 }
 
-static QTreeNode *
+static QTreeNode * QEMU_DISABLE_CFI
 q_tree_find_node(QTree        *tree,
                  gconstpointer key)
 {
-- 
2.34.1

We have been enforcing host page alignment for the non-R
fallback of MAX_RESERVED_VA, but failing to enforce for -R.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 linux-user/main.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/linux-user/main.c b/linux-user/main.c
index XXXXXXX..XXXXXXX 100644
--- a/linux-user/main.c
+++ b/linux-user/main.c
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
      */
     max_reserved_va = MAX_RESERVED_VA(cpu);
     if (reserved_va != 0) {
+        if (reserved_va % qemu_host_page_size) {
+            char *s = size_to_str(qemu_host_page_size);
+            fprintf(stderr, "Reserved virtual address not aligned mod %s\n", s);
+            g_free(s);
+            exit(EXIT_FAILURE);
+        }
         if (max_reserved_va && reserved_va > max_reserved_va) {
             fprintf(stderr, "Reserved virtual address too big\n");
             exit(EXIT_FAILURE);
-- 
2.34.1

Change the semantics to be the last byte of the guest va, rather
than the following byte.  This avoids some overflow conditions.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 include/exec/cpu-all.h      | 11 ++++++++++-
 linux-user/arm/target_cpu.h |  2 +-
 bsd-user/main.c             | 10 +++-------
 bsd-user/mmap.c             |  4 ++--
 linux-user/elfload.c        | 21 +++++++++++----------
 linux-user/main.c           | 27 +++++++++++++--------------
 linux-user/mmap.c           |  4 ++--
 7 files changed, 42 insertions(+), 37 deletions(-)

diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h
index XXXXXXX..XXXXXXX 100644
--- a/include/exec/cpu-all.h
+++ b/include/exec/cpu-all.h
@@ -XXX,XX +XXX,XX @@ static inline void tswap64s(uint64_t *s)
  */
 extern uintptr_t guest_base;
 extern bool have_guest_base;
+
+/*
+ * If non-zero, the guest virtual address space is a contiguous subset
+ * of the host virtual address space, i.e. '-R reserved_va' is in effect
+ * either from the command-line or by default.  The value is the last
+ * byte of the guest address space e.g. UINT32_MAX.
+ *
+ * If zero, the host and guest virtual address spaces are intermingled.
+ */
 extern unsigned long reserved_va;
 
 /*
@@ -XXX,XX +XXX,XX @@ extern unsigned long reserved_va;
 #define GUEST_ADDR_MAX_                                                 \
     ((MIN_CONST(TARGET_VIRT_ADDR_SPACE_BITS, TARGET_ABI_BITS) <= 32) ?  \
      UINT32_MAX : ~0ul)
-#define GUEST_ADDR_MAX    (reserved_va ? reserved_va - 1 : GUEST_ADDR_MAX_)
+#define GUEST_ADDR_MAX    (reserved_va ? : GUEST_ADDR_MAX_)
 
 #else
 
diff --git a/linux-user/arm/target_cpu.h b/linux-user/arm/target_cpu.h
index XXXXXXX..XXXXXXX 100644
--- a/linux-user/arm/target_cpu.h
+++ b/linux-user/arm/target_cpu.h
@@ -XXX,XX +XXX,XX @@ static inline unsigned long arm_max_reserved_va(CPUState *cs)
          * the high addresses.  Restrict linux-user to the
          * cached write-back RAM in the system map.
          */
-        return 0x80000000ul;
+        return 0x7ffffffful;
     } else {
         /*
          * We need to be able to map the commpage.
diff --git a/bsd-user/main.c b/bsd-user/main.c
index XXXXXXX..XXXXXXX 100644
--- a/bsd-user/main.c
+++ b/bsd-user/main.c
@@ -XXX,XX +XXX,XX @@ bool have_guest_base;
 # if HOST_LONG_BITS > TARGET_VIRT_ADDR_SPACE_BITS
 #  if TARGET_VIRT_ADDR_SPACE_BITS == 32 && \
       (TARGET_LONG_BITS == 32 || defined(TARGET_ABI32))
-/*
- * There are a number of places where we assign reserved_va to a variable
- * of type abi_ulong and expect it to fit.  Avoid the last page.
- */
-#   define MAX_RESERVED_VA  (0xfffffffful & TARGET_PAGE_MASK)
+#   define MAX_RESERVED_VA  0xfffffffful
 #  else
-#   define MAX_RESERVED_VA  (1ul << TARGET_VIRT_ADDR_SPACE_BITS)
+#   define MAX_RESERVED_VA  ((1ul << TARGET_VIRT_ADDR_SPACE_BITS) - 1)
 #  endif
 # else
 #  define MAX_RESERVED_VA  0
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
     envlist_free(envlist);
 
     if (reserved_va) {
-            mmap_next_start = reserved_va;
+        mmap_next_start = reserved_va + 1;
     }
 
     {
diff --git a/bsd-user/mmap.c b/bsd-user/mmap.c
index XXXXXXX..XXXXXXX 100644
--- a/bsd-user/mmap.c
+++ b/bsd-user/mmap.c
@@ -XXX,XX +XXX,XX @@ static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size,
     size = HOST_PAGE_ALIGN(size) + alignment;
     end_addr = start + size;
     if (end_addr > reserved_va) {
-        end_addr = reserved_va;
+        end_addr = reserved_va + 1;
     }
     addr = end_addr - qemu_host_page_size;
 
@@ -XXX,XX +XXX,XX @@ static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size,
             if (looped) {
                 return (abi_ulong)-1;
             }
-            end_addr = reserved_va;
+            end_addr = reserved_va + 1;
             addr = end_addr - qemu_host_page_size;
             looped = 1;
             continue;
diff --git a/linux-user/elfload.c b/linux-user/elfload.c
index XXXXXXX..XXXXXXX 100644
--- a/linux-user/elfload.c
+++ b/linux-user/elfload.c
@@ -XXX,XX +XXX,XX @@ static bool init_guest_commpage(void)
      * has specified -R reserved_va, which would trigger an assert().
      */
     if (reserved_va != 0 &&
-        TARGET_VSYSCALL_PAGE + TARGET_PAGE_SIZE >= reserved_va) {
+        TARGET_VSYSCALL_PAGE + TARGET_PAGE_SIZE - 1 > reserved_va) {
         error_report("Cannot allocate vsyscall page");
         exit(EXIT_FAILURE);
     }
@@ -XXX,XX +XXX,XX @@ static void pgb_have_guest_base(const char *image_name, abi_ulong guest_loaddr,
 
     /* Sanity check the guest binary. */
     if (reserved_va) {
-        if (guest_hiaddr > reserved_va) {
+        if (guest_hiaddr - 1 > reserved_va) {
             error_report("%s: requires more than reserved virtual "
                          "address space (0x%" PRIx64 " > 0x%lx)",
-                         image_name, (uint64_t)guest_hiaddr, reserved_va);
+                         image_name, (uint64_t)guest_hiaddr - 1,
+                         reserved_va);
             exit(EXIT_FAILURE);
         }
     } else {
@@ -XXX,XX +XXX,XX @@ static void pgb_have_guest_base(const char *image_name, abi_ulong guest_loaddr,
     if (reserved_va) {
         guest_loaddr = (guest_base >= mmap_min_addr ? 0
                         : mmap_min_addr - guest_base);
-        guest_hiaddr = reserved_va;
+        guest_hiaddr = reserved_va + 1;
     }
 
     /* Reserve the address space for the binary, or reserved_va. */
@@ -XXX,XX +XXX,XX @@ static void pgb_reserved_va(const char *image_name, abi_ulong guest_loaddr,
     int flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE;
     void *addr, *test;
 
-    if (guest_hiaddr > reserved_va) {
+    if (guest_hiaddr - 1 > reserved_va) {
         error_report("%s: requires more than reserved virtual "
                      "address space (0x%" PRIx64 " > 0x%lx)",
-                     image_name, (uint64_t)guest_hiaddr, reserved_va);
+                     image_name, (uint64_t)guest_hiaddr - 1, reserved_va);
         exit(EXIT_FAILURE);
     }
 
     /* Widen the "image" to the entire reserved address space. */
-    pgb_static(image_name, 0, reserved_va, align);
+    pgb_static(image_name, 0, reserved_va + 1, align);
 
     /* osdep.h defines this as 0 if it's missing */
     flags |= MAP_FIXED_NOREPLACE;
@@ -XXX,XX +XXX,XX @@ static void pgb_reserved_va(const char *image_name, abi_ulong guest_loaddr,
     /* Reserve the memory on the host. */
     assert(guest_base != 0);
     test = g2h_untagged(0);
-    addr = mmap(test, reserved_va, PROT_NONE, flags, -1, 0);
+    addr = mmap(test, reserved_va + 1, PROT_NONE, flags, -1, 0);
     if (addr == MAP_FAILED || addr != test) {
         error_report("Unable to reserve 0x%lx bytes of virtual address "
                      "space at %p (%s) for use as guest address space (check your "
                      "virtual memory ulimit setting, min_mmap_addr or reserve less "
-                     "using -R option)", reserved_va, test, strerror(errno));
+                     "using -R option)", reserved_va + 1, test, strerror(errno));
         exit(EXIT_FAILURE);
     }
 
     qemu_log_mask(CPU_LOG_PAGE, "%s: base @ %p for %lu bytes\n",
-                  __func__, addr, reserved_va);
+                  __func__, addr, reserved_va + 1);
 }
 
 void probe_guest_base(const char *image_name, abi_ulong guest_loaddr,
diff --git a/linux-user/main.c b/linux-user/main.c
index XXXXXXX..XXXXXXX 100644
--- a/linux-user/main.c
+++ b/linux-user/main.c
@@ -XXX,XX +XXX,XX @@ static const char *last_log_filename;
 # if HOST_LONG_BITS > TARGET_VIRT_ADDR_SPACE_BITS
 #  if TARGET_VIRT_ADDR_SPACE_BITS == 32 && \
       (TARGET_LONG_BITS == 32 || defined(TARGET_ABI32))
-/* There are a number of places where we assign reserved_va to a variable
-   of type abi_ulong and expect it to fit.  Avoid the last page.  */
-#   define MAX_RESERVED_VA(CPU)  (0xfffffffful & TARGET_PAGE_MASK)
+#   define MAX_RESERVED_VA(CPU)  0xfffffffful
 #  else
-#   define MAX_RESERVED_VA(CPU)  (1ul << TARGET_VIRT_ADDR_SPACE_BITS)
+#   define MAX_RESERVED_VA(CPU)  ((1ul << TARGET_VIRT_ADDR_SPACE_BITS) - 1)
 #  endif
 # else
 #  define MAX_RESERVED_VA(CPU)  0
@@ -XXX,XX +XXX,XX @@ static void handle_arg_reserved_va(const char *arg)
 {
     char *p;
     int shift = 0;
-    reserved_va = strtoul(arg, &p, 0);
+    unsigned long val;
+
+    val = strtoul(arg, &p, 0);
     switch (*p) {
     case 'k':
     case 'K':
@@ -XXX,XX +XXX,XX @@ static void handle_arg_reserved_va(const char *arg)
         break;
     }
     if (shift) {
-        unsigned long unshifted = reserved_va;
+        unsigned long unshifted = val;
         p++;
-        reserved_va <<= shift;
-        if (reserved_va >> shift != unshifted) {
+        val <<= shift;
+        if (val >> shift != unshifted) {
             fprintf(stderr, "Reserved virtual address too big\n");
             exit(EXIT_FAILURE);
         }
@@ -XXX,XX +XXX,XX @@ static void handle_arg_reserved_va(const char *arg)
         fprintf(stderr, "Unrecognised -R size suffix '%s'\n", p);
         exit(EXIT_FAILURE);
     }
+    /* The representation is size - 1, with 0 remaining "default". */
+    reserved_va = val ? val - 1 : 0;
 }
 
 static void handle_arg_singlestep(const char *arg)
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
      */
     max_reserved_va = MAX_RESERVED_VA(cpu);
     if (reserved_va != 0) {
-        if (reserved_va % qemu_host_page_size) {
+        if ((reserved_va + 1) % qemu_host_page_size) {
             char *s = size_to_str(qemu_host_page_size);
             fprintf(stderr, "Reserved virtual address not aligned mod %s\n", s);
             g_free(s);
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
             exit(EXIT_FAILURE);
         }
     } else if (HOST_LONG_BITS == 64 && TARGET_VIRT_ADDR_SPACE_BITS <= 32) {
-        /*
-         * reserved_va must be aligned with the host page size
-         * as it is used with mmap()
-         */
-        reserved_va = max_reserved_va & qemu_host_page_mask;
+        /* MAX_RESERVED_VA + 1 is a large power of 2, so is aligned. */
+        reserved_va = max_reserved_va;
     }
 
     {
diff --git a/linux-user/mmap.c b/linux-user/mmap.c
index XXXXXXX..XXXXXXX 100644
--- a/linux-user/mmap.c
+++ b/linux-user/mmap.c
@@ -XXX,XX +XXX,XX @@ static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size,
     end_addr = start + size;
     if (start > reserved_va - size) {
         /* Start at the top of the address space.  */
-        end_addr = ((reserved_va - size) & -align) + size;
+        end_addr = ((reserved_va + 1 - size) & -align) + size;
         looped = true;
     }
 
@@ -XXX,XX +XXX,XX @@ static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size,
                 return (abi_ulong)-1;
             }
             /* Re-start at the top of the address space.  */
-            addr = end_addr = ((reserved_va - size) & -align) + size;
+            addr = end_addr = ((reserved_va + 1 - size) & -align) + size;
             looped = true;
         } else {
             prot = page_get_flags(addr);
-- 
2.34.1

Pass the address of the last byte to be changed, rather than
the first address past the last byte.  This avoids overflow
when the last page of the address space is involved.

Resolves: https://gitlab.com/qemu-project/qemu/-/issues/1528
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 include/exec/cpu-all.h |  2 +-
 accel/tcg/user-exec.c  | 16 +++++++---------
 bsd-user/mmap.c        |  6 +++---
 linux-user/elfload.c   | 11 ++++++-----
 linux-user/mmap.c      | 16 ++++++++--------
 linux-user/syscall.c   |  4 ++--
 6 files changed, 27 insertions(+), 28 deletions(-)

diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h
index XXXXXXX..XXXXXXX 100644
--- a/include/exec/cpu-all.h
+++ b/include/exec/cpu-all.h
@@ -XXX,XX +XXX,XX @@ typedef int (*walk_memory_regions_fn)(void *, target_ulong,
 int walk_memory_regions(void *, walk_memory_regions_fn);
 
 int page_get_flags(target_ulong address);
-void page_set_flags(target_ulong start, target_ulong end, int flags);
+void page_set_flags(target_ulong start, target_ulong last, int flags);
 void page_reset_target_data(target_ulong start, target_ulong end);
 int page_check_range(target_ulong start, target_ulong len, int flags);
 
diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/user-exec.c
+++ b/accel/tcg/user-exec.c
@@ -XXX,XX +XXX,XX @@ static bool pageflags_set_clear(target_ulong start, target_ulong last,
  * The flag PAGE_WRITE_ORG is positioned automatically depending
  * on PAGE_WRITE.  The mmap_lock should already be held.
  */
-void page_set_flags(target_ulong start, target_ulong end, int flags)
+void page_set_flags(target_ulong start, target_ulong last, int flags)
 {
-    target_ulong last;
     bool reset = false;
     bool inval_tb = false;
 
     /* This function should never be called with addresses outside the
        guest address space.  If this assert fires, it probably indicates
        a missing call to h2g_valid.  */
-    assert(start < end);
-    assert(end - 1 <= GUEST_ADDR_MAX);
+    assert(start <= last);
+    assert(last <= GUEST_ADDR_MAX);
     /* Only set PAGE_ANON with new mappings. */
     assert(!(flags & PAGE_ANON) || (flags & PAGE_RESET));
     assert_memory_lock();
 
-    start = start & TARGET_PAGE_MASK;
-    end = TARGET_PAGE_ALIGN(end);
-    last = end - 1;
+    start &= TARGET_PAGE_MASK;
+    last |= ~TARGET_PAGE_MASK;
 
     if (!(flags & PAGE_VALID)) {
         flags = 0;
@@ -XXX,XX +XXX,XX @@ void page_set_flags(target_ulong start, target_ulong end, int flags)
     }
 
     if (!flags || reset) {
-        page_reset_target_data(start, end);
+        page_reset_target_data(start, last + 1);
         inval_tb |= pageflags_unset(start, last);
     }
     if (flags) {
@@ -XXX,XX +XXX,XX @@ void page_set_flags(target_ulong start, target_ulong end, int flags)
                                         ~(reset ? 0 : PAGE_STICKY));
     }
     if (inval_tb) {
-        tb_invalidate_phys_range(start, end);
+        tb_invalidate_phys_range(start, last + 1);
     }
 }
 
diff --git a/bsd-user/mmap.c b/bsd-user/mmap.c
index XXXXXXX..XXXXXXX 100644
--- a/bsd-user/mmap.c
+++ b/bsd-user/mmap.c
@@ -XXX,XX +XXX,XX @@ int target_mprotect(abi_ulong start, abi_ulong len, int prot)
         if (ret != 0)
             goto error;
     }
-    page_set_flags(start, start + len, prot | PAGE_VALID);
+    page_set_flags(start, start + len - 1, prot | PAGE_VALID);
     mmap_unlock();
     return 0;
 error:
@@ -XXX,XX +XXX,XX @@ abi_long target_mmap(abi_ulong start, abi_ulong len, int prot,
         }
     }
  the_end1:
-    page_set_flags(start, start + len, prot | PAGE_VALID);
+    page_set_flags(start, start + len - 1, prot | PAGE_VALID);
  the_end:
 #ifdef DEBUG_MMAP
     printf("ret=0x" TARGET_ABI_FMT_lx "\n", start);
@@ -XXX,XX +XXX,XX @@ int target_munmap(abi_ulong start, abi_ulong len)
     }
 
     if (ret == 0) {
-        page_set_flags(start, start + len, 0);
+        page_set_flags(start, start + len - 1, 0);
     }
     mmap_unlock();
     return ret;
diff --git a/linux-user/elfload.c b/linux-user/elfload.c
index XXXXXXX..XXXXXXX 100644
--- a/linux-user/elfload.c
+++ b/linux-user/elfload.c
@@ -XXX,XX +XXX,XX @@ static bool init_guest_commpage(void)
         exit(EXIT_FAILURE);
     }
     page_set_flags(TARGET_VSYSCALL_PAGE,
-                   TARGET_VSYSCALL_PAGE + TARGET_PAGE_SIZE,
+                   TARGET_VSYSCALL_PAGE | ~TARGET_PAGE_MASK,
                    PAGE_EXEC | PAGE_VALID);
     return true;
 }
@@ -XXX,XX +XXX,XX @@ static bool init_guest_commpage(void)
         exit(EXIT_FAILURE);
     }
 
-    page_set_flags(commpage, commpage + qemu_host_page_size,
+    page_set_flags(commpage, commpage | ~qemu_host_page_mask,
                    PAGE_READ | PAGE_EXEC | PAGE_VALID);
     return true;
 }
@@ -XXX,XX +XXX,XX @@ static bool init_guest_commpage(void)
         exit(EXIT_FAILURE);
     }
 
-    page_set_flags(LO_COMMPAGE, LO_COMMPAGE + TARGET_PAGE_SIZE,
+    page_set_flags(LO_COMMPAGE, LO_COMMPAGE | ~TARGET_PAGE_MASK,
                    PAGE_READ | PAGE_EXEC | PAGE_VALID);
     return true;
 }
@@ -XXX,XX +XXX,XX @@ static bool init_guest_commpage(void)
      * and implement syscalls.  Here, simply mark the page executable.
      * Special case the entry points during translation (see do_page_zero).
      */
-    page_set_flags(LO_COMMPAGE, LO_COMMPAGE + TARGET_PAGE_SIZE,
+    page_set_flags(LO_COMMPAGE, LO_COMMPAGE | ~TARGET_PAGE_MASK,
                    PAGE_EXEC | PAGE_VALID);
     return true;
 }
@@ -XXX,XX +XXX,XX @@ static void zero_bss(abi_ulong elf_bss, abi_ulong last_bss, int prot)
 
     /* Ensure that the bss page(s) are valid */
     if ((page_get_flags(last_bss-1) & prot) != prot) {
-        page_set_flags(elf_bss & TARGET_PAGE_MASK, last_bss, prot | PAGE_VALID);
+        page_set_flags(elf_bss & TARGET_PAGE_MASK, last_bss - 1,
+                       prot | PAGE_VALID);
     }
 
     if (host_start < host_map_start) {
diff --git a/linux-user/mmap.c b/linux-user/mmap.c
index XXXXXXX..XXXXXXX 100644
--- a/linux-user/mmap.c
+++ b/linux-user/mmap.c
@@ -XXX,XX +XXX,XX @@ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot)
         }
     }
 
-    page_set_flags(start, start + len, page_flags);
+    page_set_flags(start, start + len - 1, page_flags);
     ret = 0;
 
 error:
@@ -XXX,XX +XXX,XX @@ abi_long target_mmap(abi_ulong start, abi_ulong len, int target_prot,
     }
     page_flags |= PAGE_RESET;
     if (passthrough_start == passthrough_end) {
-        page_set_flags(start, start + len, page_flags);
+        page_set_flags(start, start + len - 1, page_flags);
     } else {
         if (start < passthrough_start) {
-            page_set_flags(start, passthrough_start, page_flags);
+            page_set_flags(start, passthrough_start - 1, page_flags);
         }
-        page_set_flags(passthrough_start, passthrough_end,
+        page_set_flags(passthrough_start, passthrough_end - 1,
                        page_flags | PAGE_PASSTHROUGH);
         if (passthrough_end < start + len) {
-            page_set_flags(passthrough_end, start + len, page_flags);
+            page_set_flags(passthrough_end, start + len - 1, page_flags);
         }
     }
  the_end:
@@ -XXX,XX +XXX,XX @@ int target_munmap(abi_ulong start, abi_ulong len)
     }
 
     if (ret == 0) {
-        page_set_flags(start, start + len, 0);
+        page_set_flags(start, start + len - 1, 0);
     }
     mmap_unlock();
     return ret;
@@ -XXX,XX +XXX,XX @@ abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size,
     } else {
         new_addr = h2g(host_addr);
         prot = page_get_flags(old_addr);
-        page_set_flags(old_addr, old_addr + old_size, 0);
-        page_set_flags(new_addr, new_addr + new_size,
+        page_set_flags(old_addr, old_addr + old_size - 1, 0);
+        page_set_flags(new_addr, new_addr + new_size - 1,
                        prot | PAGE_VALID | PAGE_RESET);
     }
     mmap_unlock();
diff --git a/linux-user/syscall.c b/linux-user/syscall.c
index XXXXXXX..XXXXXXX 100644
--- a/linux-user/syscall.c
+++ b/linux-user/syscall.c
@@ -XXX,XX +XXX,XX @@ static inline abi_ulong do_shmat(CPUArchState *cpu_env,
     }
     raddr=h2g((unsigned long)host_raddr);
 
-    page_set_flags(raddr, raddr + shm_info.shm_segsz,
+    page_set_flags(raddr, raddr + shm_info.shm_segsz - 1,
                    PAGE_VALID | PAGE_RESET | PAGE_READ |
                    (shmflg & SHM_RDONLY ? 0 : PAGE_WRITE));
 
@@ -XXX,XX +XXX,XX @@ static inline abi_long do_shmdt(abi_ulong shmaddr)
     for (i = 0; i < N_SHM_REGIONS; ++i) {
         if (shm_regions[i].in_use && shm_regions[i].start == shmaddr) {
             shm_regions[i].in_use = false;
-            page_set_flags(shmaddr, shmaddr + shm_regions[i].size, 0);
+            page_set_flags(shmaddr, shmaddr + shm_regions[i].size - 1, 0);
             break;
         }
     }
-- 
2.34.1

Pass the address of the last byte to be changed, rather than
the first address past the last byte.  This avoids overflow
when the last page of the address space is involved.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 include/exec/cpu-all.h |  2 +-
 accel/tcg/user-exec.c  | 11 +++++------
 linux-user/mmap.c      |  2 +-
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h
index XXXXXXX..XXXXXXX 100644
--- a/include/exec/cpu-all.h
+++ b/include/exec/cpu-all.h
@@ -XXX,XX +XXX,XX @@ int walk_memory_regions(void *, walk_memory_regions_fn);
 
 int page_get_flags(target_ulong address);
 void page_set_flags(target_ulong start, target_ulong last, int flags);
-void page_reset_target_data(target_ulong start, target_ulong end);
+void page_reset_target_data(target_ulong start, target_ulong last);
 int page_check_range(target_ulong start, target_ulong len, int flags);
 
 /**
diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/user-exec.c
+++ b/accel/tcg/user-exec.c
@@ -XXX,XX +XXX,XX @@ void page_set_flags(target_ulong start, target_ulong last, int flags)
     }
 
     if (!flags || reset) {
-        page_reset_target_data(start, last + 1);
+        page_reset_target_data(start, last);
         inval_tb |= pageflags_unset(start, last);
     }
     if (flags) {
@@ -XXX,XX +XXX,XX @@ typedef struct TargetPageDataNode {
 
 static IntervalTreeRoot targetdata_root;
 
-void page_reset_target_data(target_ulong start, target_ulong end)
+void page_reset_target_data(target_ulong start, target_ulong last)
 {
     IntervalTreeNode *n, *next;
-    target_ulong last;
 
     assert_memory_lock();
 
-    start = start & TARGET_PAGE_MASK;
-    last = TARGET_PAGE_ALIGN(end) - 1;
+    start &= TARGET_PAGE_MASK;
+    last |= ~TARGET_PAGE_MASK;
 
     for (n = interval_tree_iter_first(&targetdata_root, start, last),
          next = n ? interval_tree_iter_next(n, start, last) : NULL;
@@ -XXX,XX +XXX,XX @@ void *page_get_target_data(target_ulong address)
     return t->data[(page - region) >> TARGET_PAGE_BITS];
 }
 #else
-void page_reset_target_data(target_ulong start, target_ulong end) { }
+void page_reset_target_data(target_ulong start, target_ulong last) { }
 #endif /* TARGET_PAGE_DATA_SIZE */
 
 /* The softmmu versions of these helpers are in cputlb.c.  */
diff --git a/linux-user/mmap.c b/linux-user/mmap.c
index XXXXXXX..XXXXXXX 100644
--- a/linux-user/mmap.c
+++ b/linux-user/mmap.c
@@ -XXX,XX +XXX,XX @@ abi_long target_madvise(abi_ulong start, abi_ulong len_in, int advice)
         if (can_passthrough_madvise(start, end)) {
             ret = get_errno(madvise(g2h_untagged(start), len, advice));
             if ((advice == MADV_DONTNEED) && (ret == 0)) {
-                page_reset_target_data(start, start + len);
+                page_reset_target_data(start, start + len - 1);
             }
         }
     }
-- 
2.34.1

Pass the address of the last byte to be changed, rather than
the first address past the last byte.  This avoids overflow
when the last page of the address space is involved.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 accel/tcg/tb-maint.c | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/tb-maint.c
+++ b/accel/tcg/tb-maint.c
@@ -XXX,XX +XXX,XX @@ static void tb_remove(TranslationBlock *tb)
 }
 
 /* TODO: For now, still shared with translate-all.c for system mode. */
-#define PAGE_FOR_EACH_TB(start, end, pagedesc, T, N)    \
-    for (T = foreach_tb_first(start, end),              \
-         N = foreach_tb_next(T, start, end);            \
+#define PAGE_FOR_EACH_TB(start, last, pagedesc, T, N)   \
+    for (T = foreach_tb_first(start, last),             \
+         N = foreach_tb_next(T, start, last);           \
          T != NULL;                                     \
-         T = N, N = foreach_tb_next(N, start, end))
+         T = N, N = foreach_tb_next(N, start, last))
 
 typedef TranslationBlock *PageForEachNext;
 
 static PageForEachNext foreach_tb_first(tb_page_addr_t start,
-                                        tb_page_addr_t end)
+                                        tb_page_addr_t last)
 {
-    IntervalTreeNode *n = interval_tree_iter_first(&tb_root, start, end - 1);
+    IntervalTreeNode *n = interval_tree_iter_first(&tb_root, start, last);
     return n ? container_of(n, TranslationBlock, itree) : NULL;
 }
 
 static PageForEachNext foreach_tb_next(PageForEachNext tb,
                                        tb_page_addr_t start,
-                                       tb_page_addr_t end)
+                                       tb_page_addr_t last)
 {
     IntervalTreeNode *n;
 
     if (tb) {
-        n = interval_tree_iter_next(&tb->itree, start, end - 1);
+        n = interval_tree_iter_next(&tb->itree, start, last);
         if (n) {
             return container_of(n, TranslationBlock, itree);
         }
@@ -XXX,XX +XXX,XX @@ struct page_collection {
 };
 
 typedef int PageForEachNext;
-#define PAGE_FOR_EACH_TB(start, end, pagedesc, tb, n) \
+#define PAGE_FOR_EACH_TB(start, last, pagedesc, tb, n) \
     TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next)
 
 #ifdef CONFIG_DEBUG_TCG
@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
 {
     TranslationBlock *tb;
     PageForEachNext n;
+    tb_page_addr_t last = end - 1;
 
     assert_memory_lock();
 
-    PAGE_FOR_EACH_TB(start, end, unused, tb, n) {
+    PAGE_FOR_EACH_TB(start, last, unused, tb, n) {
         tb_phys_invalidate__locked(tb);
     }
 }
@@ -XXX,XX +XXX,XX @@ bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc)
     bool current_tb_modified;
     TranslationBlock *tb;
     PageForEachNext n;
+    tb_page_addr_t last;
 
     /*
      * Without precise smc semantics, or when outside of a TB,
@@ -XXX,XX +XXX,XX @@ bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc)
     assert_memory_lock();
     current_tb = tcg_tb_lookup(pc);
 
+    last = addr | ~TARGET_PAGE_MASK;
     addr &= TARGET_PAGE_MASK;
     current_tb_modified = false;
 
-    PAGE_FOR_EACH_TB(addr, addr + TARGET_PAGE_SIZE, unused, tb, n) {
+    PAGE_FOR_EACH_TB(addr, last, unused, tb, n) {
         if (current_tb == tb &&
             (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
             /*
@@ -XXX,XX +XXX,XX @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages,
     bool current_tb_modified = false;
     TranslationBlock *current_tb = retaddr ? tcg_tb_lookup(retaddr) : NULL;
 #endif /* TARGET_HAS_PRECISE_SMC */
+    tb_page_addr_t last G_GNUC_UNUSED = end - 1;
 
     /*
      * We remove all the TBs in the range [start, end[.
      * XXX: see if in some cases it could be faster to invalidate all the code
      */
-    PAGE_FOR_EACH_TB(start, end, p, tb, n) {
+    PAGE_FOR_EACH_TB(start, last, p, tb, n) {
         /* NOTE: this is subtle as a TB may span two physical pages */
         if (n == 0) {
             /* NOTE: tb_end may be after the end of the page, but
-- 
2.34.1

Pass the address of the last byte to be changed, rather than
the first address past the last byte.  This avoids overflow
when the last page of the address space is involved.

Fixes a bug in the loop comparision where "<= end" would lock
one more page than required.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 accel/tcg/tb-maint.c | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/tb-maint.c
+++ b/accel/tcg/tb-maint.c
@@ -XXX,XX +XXX,XX @@ static gint tb_page_addr_cmp(gconstpointer ap, gconstpointer bp, gpointer udata)
 }
 
 /*
- * Lock a range of pages ([@start,@end[) as well as the pages of all
+ * Lock a range of pages ([@start,@last]) as well as the pages of all
  * intersecting TBs.
  * Locking order: acquire locks in ascending order of page index.
  */
 static struct page_collection *page_collection_lock(tb_page_addr_t start,
-                                                    tb_page_addr_t end)
+                                                    tb_page_addr_t last)
 {
     struct page_collection *set = g_malloc(sizeof(*set));
     tb_page_addr_t index;
     PageDesc *pd;
 
     start >>= TARGET_PAGE_BITS;
-    end   >>= TARGET_PAGE_BITS;
-    g_assert(start <= end);
+    last >>= TARGET_PAGE_BITS;
+    g_assert(start <= last);
 
     set->tree = q_tree_new_full(tb_page_addr_cmp, NULL, NULL,
                                 page_entry_destroy);
@@ -XXX,XX +XXX,XX @@ static struct page_collection *page_collection_lock(tb_page_addr_t start,
  retry:
     q_tree_foreach(set->tree, page_entry_lock, NULL);
 
-    for (index = start; index <= end; index++) {
+    for (index = start; index <= last; index++) {
         TranslationBlock *tb;
         PageForEachNext n;
 
@@ -XXX,XX +XXX,XX @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages,
 void tb_invalidate_phys_page(tb_page_addr_t addr)
 {
     struct page_collection *pages;
-    tb_page_addr_t start, end;
+    tb_page_addr_t start, last;
     PageDesc *p;
 
     p = page_find(addr >> TARGET_PAGE_BITS);
@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_page(tb_page_addr_t addr)
     }
 
     start = addr & TARGET_PAGE_MASK;
-    end = start + TARGET_PAGE_SIZE;
-    pages = page_collection_lock(start, end);
-    tb_invalidate_phys_page_range__locked(pages, p, start, end, 0);
+    last = addr | ~TARGET_PAGE_MASK;
+    pages = page_collection_lock(start, last);
+    tb_invalidate_phys_page_range__locked(pages, p, start, last + 1, 0);
     page_collection_unlock(pages);
 }
 
@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
     struct page_collection *pages;
     tb_page_addr_t next;
 
-    pages = page_collection_lock(start, end);
+    pages = page_collection_lock(start, end - 1);
     for (next = (start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
          start < end;
          start = next, next += TARGET_PAGE_SIZE) {
@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_range_fast(ram_addr_t ram_addr,
 {
     struct page_collection *pages;
 
-    pages = page_collection_lock(ram_addr, ram_addr + size);
+    pages = page_collection_lock(ram_addr, ram_addr + size - 1);
     tb_invalidate_phys_page_fast__locked(pages, ram_addr, size, retaddr);
     page_collection_unlock(pages);
 }
-- 
2.34.1

Pass the address of the last byte to be changed, rather than
the first address past the last byte.  This avoids overflow
when the last page of the address space is involved.

Properly truncate tb_last to the end of the page; the comment about
tb_end being past the end of the page being ok is not correct,
considering overflow.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 accel/tcg/tb-maint.c | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/tb-maint.c
+++ b/accel/tcg/tb-maint.c
@@ -XXX,XX +XXX,XX @@ bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc)
 static void
 tb_invalidate_phys_page_range__locked(struct page_collection *pages,
                                       PageDesc *p, tb_page_addr_t start,
-                                      tb_page_addr_t end,
+                                      tb_page_addr_t last,
                                       uintptr_t retaddr)
 {
     TranslationBlock *tb;
-    tb_page_addr_t tb_start, tb_end;
     PageForEachNext n;
 #ifdef TARGET_HAS_PRECISE_SMC
     bool current_tb_modified = false;
     TranslationBlock *current_tb = retaddr ? tcg_tb_lookup(retaddr) : NULL;
 #endif /* TARGET_HAS_PRECISE_SMC */
-    tb_page_addr_t last G_GNUC_UNUSED = end - 1;
 
     /*
-     * We remove all the TBs in the range [start, end[.
+     * We remove all the TBs in the range [start, last].
      * XXX: see if in some cases it could be faster to invalidate all the code
      */
     PAGE_FOR_EACH_TB(start, last, p, tb, n) {
+        tb_page_addr_t tb_start, tb_last;
+
         /* NOTE: this is subtle as a TB may span two physical pages */
+        tb_start = tb_page_addr0(tb);
+        tb_last = tb_start + tb->size - 1;
         if (n == 0) {
-            /* NOTE: tb_end may be after the end of the page, but
-               it is not a problem */
-            tb_start = tb_page_addr0(tb);
-            tb_end = tb_start + tb->size;
+            tb_last = MIN(tb_last, tb_start | ~TARGET_PAGE_MASK);
         } else {
             tb_start = tb_page_addr1(tb);
-            tb_end = tb_start + ((tb_page_addr0(tb) + tb->size)
-                                 & ~TARGET_PAGE_MASK);
+            tb_last = tb_start + (tb_last & ~TARGET_PAGE_MASK);
         }
-        if (!(tb_end <= start || tb_start >= end)) {
+        if (!(tb_last < start || tb_start > last)) {
 #ifdef TARGET_HAS_PRECISE_SMC
             if (current_tb == tb &&
                 (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_page(tb_page_addr_t addr)
     start = addr & TARGET_PAGE_MASK;
     last = addr | ~TARGET_PAGE_MASK;
     pages = page_collection_lock(start, last);
-    tb_invalidate_phys_page_range__locked(pages, p, start, last + 1, 0);
+    tb_invalidate_phys_page_range__locked(pages, p, start, last, 0);
     page_collection_unlock(pages);
 }
 
@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
             continue;
         }
         assert_page_locked(pd);
-        tb_invalidate_phys_page_range__locked(pages, pd, start, bound, 0);
+        tb_invalidate_phys_page_range__locked(pages, pd, start, bound - 1, 0);
     }
     page_collection_unlock(pages);
 }
@@ -XXX,XX +XXX,XX @@ static void tb_invalidate_phys_page_fast__locked(struct page_collection *pages,
     }
 
     assert_page_locked(p);
-    tb_invalidate_phys_page_range__locked(pages, p, start, start + len, ra);
+    tb_invalidate_phys_page_range__locked(pages, p, start, start + len - 1, ra);
 }
 
 /*
-- 
2.34.1

Pass the address of the last byte to be changed, rather than
the first address past the last byte.  This avoids overflow
when the last page of the address space is involved.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 include/exec/exec-all.h   |  2 +-
 accel/tcg/tb-maint.c      | 31 ++++++++++++++++---------------
 accel/tcg/translate-all.c |  2 +-
 accel/tcg/user-exec.c     |  2 +-
 softmmu/physmem.c         |  2 +-
 5 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
index XXXXXXX..XXXXXXX 100644
--- a/include/exec/exec-all.h
+++ b/include/exec/exec-all.h
@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_addr(target_ulong addr);
 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs);
 #endif
 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr);
-void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end);
+void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t last);
 void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr);
 
 /* GETPC is the true target of the return instruction that we'll execute.  */
diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/tb-maint.c
+++ b/accel/tcg/tb-maint.c
@@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
  * Called with mmap_lock held for user-mode emulation.
  * NOTE: this function must not be called while a TB is running.
  */
-void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
+void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t last)
 {
     TranslationBlock *tb;
     PageForEachNext n;
-    tb_page_addr_t last = end - 1;
 
     assert_memory_lock();
 
@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
  */
 void tb_invalidate_phys_page(tb_page_addr_t addr)
 {
-    tb_page_addr_t start, end;
+    tb_page_addr_t start, last;
 
     start = addr & TARGET_PAGE_MASK;
-    end = start + TARGET_PAGE_SIZE;
-    tb_invalidate_phys_range(start, end);
+    last = addr | ~TARGET_PAGE_MASK;
+    tb_invalidate_phys_range(start, last);
 }
 
 /*
@@ -XXX,XX +XXX,XX @@ void tb_invalidate_phys_page(tb_page_addr_t addr)
 
 /*
  * Invalidate all TBs which intersect with the target physical address range
- * [start;end[. NOTE: start and end may refer to *different* physical pages.
+ * [start;last]. NOTE: start and end may refer to *different* physical pages.
  * 'is_cpu_write_access' should be true if called from a real cpu write
  * access: the virtual CPU will exit the current TB if code is modified inside
  * this TB.
  */
-void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
+void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t last)
 {
     struct page_collection *pages;
-    tb_page_addr_t next;
+    tb_page_addr_t index, index_last;
 
-    pages = page_collection_lock(start, end - 1);
-    for (next = (start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-         start < end;
-         start = next, next += TARGET_PAGE_SIZE) {
-        PageDesc *pd = page_find(start >> TARGET_PAGE_BITS);
-        tb_page_addr_t bound = MIN(next, end);
+    pages = page_collection_lock(start, last);
+
+    index_last = last >> TARGET_PAGE_BITS;
+    for (index = start >> TARGET_PAGE_BITS; index <= index_last; index++) {
+        PageDesc *pd = page_find(index);
+        tb_page_addr_t bound;
 
         if (pd == NULL) {
             continue;
         }
         assert_page_locked(pd);
-        tb_invalidate_phys_page_range__locked(pages, pd, start, bound - 1, 0);
+        bound = (index << TARGET_PAGE_BITS) | ~TARGET_PAGE_MASK;
+        bound = MIN(bound, last);
+        tb_invalidate_phys_page_range__locked(pages, pd, start, bound, 0);
     }
     page_collection_unlock(pages);
 }
diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/translate-all.c
+++ b/accel/tcg/translate-all.c
@@ -XXX,XX +XXX,XX @@ void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr)
         cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
         addr = get_page_addr_code(env, pc);
         if (addr != -1) {
-            tb_invalidate_phys_range(addr, addr + 1);
+            tb_invalidate_phys_range(addr, addr);
         }
     }
 }
diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/user-exec.c
+++ b/accel/tcg/user-exec.c
@@ -XXX,XX +XXX,XX @@ void page_set_flags(target_ulong start, target_ulong last, int flags)
                                         ~(reset ? 0 : PAGE_STICKY));
     }
     if (inval_tb) {
-        tb_invalidate_phys_range(start, last + 1);
+        tb_invalidate_phys_range(start, last);
     }
 }
 
diff --git a/softmmu/physmem.c b/softmmu/physmem.c
index XXXXXXX..XXXXXXX 100644
--- a/softmmu/physmem.c
+++ b/softmmu/physmem.c
@@ -XXX,XX +XXX,XX @@ static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
     }
     if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
         assert(tcg_enabled());
-        tb_invalidate_phys_range(addr, addr + length);
+        tb_invalidate_phys_range(addr, addr + length - 1);
         dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
     }
     cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
-- 
2.34.1

User setting of -R reserved_va can lead to an assertion
failure in page_set_flags.  Sanity check the value of
reserved_va and print an error message instead.  Do not
allocate a commpage at all for m-profile cpus.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 linux-user/elfload.c | 37 +++++++++++++++++++++++++++----------
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/linux-user/elfload.c b/linux-user/elfload.c
index XXXXXXX..XXXXXXX 100644
--- a/linux-user/elfload.c
+++ b/linux-user/elfload.c
@@ -XXX,XX +XXX,XX @@ enum {
 
 static bool init_guest_commpage(void)
 {
-    abi_ptr commpage = HI_COMMPAGE & -qemu_host_page_size;
-    void *want = g2h_untagged(commpage);
-    void *addr = mmap(want, qemu_host_page_size, PROT_READ | PROT_WRITE,
-                      MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0);
+    ARMCPU *cpu = ARM_CPU(thread_cpu);
+    abi_ptr want = HI_COMMPAGE & TARGET_PAGE_MASK;
+    abi_ptr addr;
 
-    if (addr == MAP_FAILED) {
+    /*
+     * M-profile allocates maximum of 2GB address space, so can never
+     * allocate the commpage.  Skip it.
+     */
+    if (arm_feature(&cpu->env, ARM_FEATURE_M)) {
+        return true;
+    }
+
+    /*
+     * If reserved_va does not cover the commpage, we get an assert
+     * in page_set_flags.  Produce an intelligent error instead.
+     */
+    if (reserved_va != 0 && want + TARGET_PAGE_SIZE - 1 > reserved_va) {
+        error_report("Allocating guest commpage: -R 0x%" PRIx64 " too small",
+                     (uint64_t)reserved_va + 1);
+        exit(EXIT_FAILURE);
+    }
+
+    addr = target_mmap(want, TARGET_PAGE_SIZE, PROT_READ | PROT_WRITE,
+                       MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0);
+
+    if (addr == -1) {
         perror("Allocating guest commpage");
         exit(EXIT_FAILURE);
     }
@@ -XXX,XX +XXX,XX @@ static bool init_guest_commpage(void)
     }
 
     /* Set kernel helper versions; rest of page is 0.  */
-    __put_user(5, (uint32_t *)g2h_untagged(0xffff0ffcu));
+    put_user_u32(5, 0xffff0ffcu);
 
-    if (mprotect(addr, qemu_host_page_size, PROT_READ)) {
+    if (target_mprotect(addr, qemu_host_page_size, PROT_READ | PROT_EXEC)) {
         perror("Protecting guest commpage");
         exit(EXIT_FAILURE);
     }
-
-    page_set_flags(commpage, commpage | ~qemu_host_page_mask,
-                   PAGE_READ | PAGE_EXEC | PAGE_VALID);
     return true;
 }
 
-- 
2.34.1