Series comparison

-[Qemu-devel] [PULL 00/21] tcg patch queue
+[PULL 00/15] tcg patch queue
-The following changes since commit ff56877e911782dedc9a424233fd3f62369c258c:
+Merge the first set of reviewed patches from my queue.
-  Merge remote-tracking branch 'remotes/kraxel/tags/vga-20181015-pull-request' into staging (2018-10-15 15:03:45 +0100)
+r~
 The following changes since commit 6dd06214892d71cbbdd25daed7693e58afcb1093:
   Merge tag 'pull-hex-20230421' of https://github.com/quic/qemu into staging (2023-04-22 08:31:38 +0100)
 are available in the Git repository at:
-  https://github.com/rth7680/qemu.git tags/pull-tcg-20181016
+  https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20230423
-for you to fetch changes up to e3e9d1ea20c75718ce7c528c588a0a497f12f750:
+for you to fetch changes up to 3ea9be33400f14305565a9a094cb6031c07183d5:
-  cputlb: read CPUTLBEntry.addr_write atomically (2018-10-16 10:04:27 -0700)
+  tcg/riscv: Conditionalize tcg_out_exts_i32_i64 (2023-04-23 08:46:45 +0100)
 ----------------------------------------------------------------
-Queued tcg patches
+tcg cleanups:
   - Remove tcg_abort()
   - Split out extensions as known backend interfaces
   - Put the separate extensions together as tcg_out_movext
   - Introduce tcg_out_xchg as a backend interface
   - Clear TCGLabelQemuLdst on allocation
   - Avoid redundant extensions for riscv
 ----------------------------------------------------------------
-Emilio G. Cota (10):
+Richard Henderson (15):
-      tcg: access cpu->icount_decr.u16.high with atomics
+      tcg: Replace if + tcg_abort with tcg_debug_assert
-      tcg: fix use of uninitialized variable under CONFIG_PROFILER
+      tcg: Replace tcg_abort with g_assert_not_reached
-      tcg: plug holes in struct TCGProfile
+      tcg: Split out tcg_out_ext8s
-      tcg: distribute tcg_time into TCG contexts
+      tcg: Split out tcg_out_ext8u
-      target/alpha: remove tlb_flush from alpha_cpu_initfn
+      tcg: Split out tcg_out_ext16s
-      target/unicore32: remove tlb_flush from uc32_init_fn
+      tcg: Split out tcg_out_ext16u
-      exec: introduce tlb_init
+      tcg: Split out tcg_out_ext32s
-      cputlb: fix assert_cpu_is_self macro
+      tcg: Split out tcg_out_ext32u
-      cputlb: serialize tlb updates with env->tlb_lock
+      tcg: Split out tcg_out_exts_i32_i64
-      cputlb: read CPUTLBEntry.addr_write atomically
+      tcg: Split out tcg_out_extu_i32_i64
       tcg: Split out tcg_out_extrl_i64_i32
       tcg: Introduce tcg_out_movext
       tcg: Introduce tcg_out_xchg
       tcg: Clear TCGLabelQemuLdst on allocation
       tcg/riscv: Conditionalize tcg_out_exts_i32_i64
-Richard Henderson (11):
+ include/tcg/tcg.h                |   6 --
-      tcg: Implement CPU_LOG_TB_NOCHAIN during expansion
+ target/i386/tcg/translate.c      |  20 +++---
-      tcg: Add tlb_index and tlb_entry helpers
+ target/s390x/tcg/translate.c     |   4 +-
-      tcg: Split CONFIG_ATOMIC128
+ tcg/optimize.c                   |  10 ++-
-      target/i386: Convert to HAVE_CMPXCHG128
+ tcg/tcg.c                        | 135 +++++++++++++++++++++++++++++++++++----
-      target/arm: Convert to HAVE_CMPXCHG128
+ tcg/aarch64/tcg-target.c.inc     | 106 +++++++++++++++++++-----------
-      target/arm: Check HAVE_CMPXCHG128 at translate time
+ tcg/arm/tcg-target.c.inc         |  93 +++++++++++++++++----------
-      target/ppc: Convert to HAVE_CMPXCHG128 and HAVE_ATOMIC128
+ tcg/i386/tcg-target.c.inc        | 129 ++++++++++++++++++-------------------
-      target/s390x: Convert to HAVE_CMPXCHG128 and HAVE_ATOMIC128
+ tcg/loongarch64/tcg-target.c.inc | 123 +++++++++++++----------------------
-      target/s390x: Split do_cdsg, do_lpq, do_stpq
+ tcg/mips/tcg-target.c.inc        |  94 +++++++++++++++++++--------
-      target/s390x: Skip wout, cout helpers if op helper does not return
+ tcg/ppc/tcg-target.c.inc         | 119 ++++++++++++++++++----------------
-      target/s390x: Check HAVE_ATOMIC128 and HAVE_CMPXCHG128 at translate
+ tcg/riscv/tcg-target.c.inc       |  83 +++++++++++-------------
+ tcg/s390x/tcg-target.c.inc       | 128 +++++++++++++++++--------------------
- accel/tcg/atomic_template.h      |  20 +++-
+ tcg/sparc64/tcg-target.c.inc     | 117 +++++++++++++++++++++------------
- accel/tcg/softmmu_template.h     |  64 +++++-----
+ tcg/tcg-ldst.c.inc               |   1 +
- include/exec/cpu-defs.h          |   3 +
+ tcg/tci/tcg-target.c.inc         | 116 ++++++++++++++++++++++++++++++---
- include/exec/cpu_ldst.h          |  30 ++++-
+files changed, 786 insertions(+), 498 deletions(-)
  include/exec/cpu_ldst_template.h |  25 ++--
  include/exec/exec-all.h          |   8 ++
  include/qemu/atomic128.h         | 155 ++++++++++++++++++++++++
  include/qemu/timer.h             |   1 -
  target/ppc/helper.h              |   2 +-
  tcg/tcg.h                        |  20 ++--
  accel/tcg/cpu-exec.c             |   2 +-
  accel/tcg/cputlb.c               | 235 +++++++++++++++++++-----------------
  accel/tcg/tcg-all.c              |   2 +-
  accel/tcg/translate-all.c        |   2 +-
  accel/tcg/user-exec.c            |   5 +-
  cpus.c                           |   3 +-
  exec.c                           |   1 +
  monitor.c                        |  13 +-
  qom/cpu.c                        |   2 +-
  target/alpha/cpu.c               |   1 -
  target/arm/helper-a64.c          | 251 +++++++++++++++++++--------------------
  target/arm/translate-a64.c       |  38 +++---
  target/i386/mem_helper.c         |   9 +-
  target/ppc/mem_helper.c          |  33 ++++-
  target/ppc/translate.c           | 115 +++++++++---------
  target/s390x/mem_helper.c        | 202 +++++++++++++++----------------
  target/s390x/translate.c         |  45 +++++--
  target/unicore32/cpu.c           |   2 -
  tcg/tcg-op.c                     |   9 +-
  tcg/tcg.c                        |  25 +++-
  configure                        |  19 +++
 files changed, 830 insertions(+), 512 deletions(-)
  create mode 100644 include/qemu/atomic128.h

-[Qemu-devel] [PULL 17/21] target/s390x: Convert to HAVE_CMPXCHG128 and HAVE_ATOMIC128
+[PULL 01/15] tcg: Replace if + tcg_abort with tcg_debug_assert
-Reviewed-by: David Hildenbrand <david@redhat.com>
+Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- target/s390x/mem_helper.c | 92 +++++++++++++++++----------------------
+ tcg/tcg.c                 | 4 +---
-file changed, 41 insertions(+), 51 deletions(-)
+ tcg/i386/tcg-target.c.inc | 8 +++-----
 files changed, 4 insertions(+), 8 deletions(-)
-diff --git a/target/s390x/mem_helper.c b/target/s390x/mem_helper.c
+diff --git a/tcg/tcg.c b/tcg/tcg.c
 index XXXXXXX..XXXXXXX 100644
---- a/target/s390x/mem_helper.c
+--- a/tcg/tcg.c
-+++ b/target/s390x/mem_helper.c
++++ b/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@
+@@ -XXX,XX +XXX,XX @@ static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
  #include "exec/exec-all.h"
  #include "exec/cpu_ldst.h"
  #include "qemu/int128.h"
 +#include "qemu/atomic128.h"
  #if !defined(CONFIG_USER_ONLY)
  #include "hw/s390x/storage-keys.h"
@@ -XXX,XX +XXX,XX @@ static void do_cdsg(CPUS390XState *env, uint64_t addr,
      bool fail;
      if (parallel) {
 -#ifndef CONFIG_ATOMIC128
 +#if !HAVE_CMPXCHG128
          cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
  #else
          int mem_idx = cpu_mmu_index(env, false);
@@ -XXX,XX +XXX,XX @@ void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr,
  static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
                          uint64_t a2, bool parallel)
  {
--#if !defined(CONFIG_USER_ONLY) || defined(CONFIG_ATOMIC128)
+     TCGTemp *ts;
-     uint32_t mem_idx = cpu_mmu_index(env, false);
--#endif
+-    if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
-     uintptr_t ra = GETPC();
+-        tcg_abort();
-     uint32_t fc = extract32(env->regs[0], 0, 8);
+-    }
-     uint32_t sc = extract32(env->regs[0], 8, 8);
++    tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
-@@ -XXX,XX +XXX,XX @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
-     probe_write(env, a2, 0, mem_idx, ra);
+     ts = tcg_global_alloc(s);
- #endif
+     ts->base_type = type;
+diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
--    /* Note that the compare-and-swap is atomic, and the store is atomic, but
+index XXXXXXX..XXXXXXX 100644
--       the complete operation is not.  Therefore we do not need to assert serial
+--- a/tcg/i386/tcg-target.c.inc
--       context in order to implement this.  That said, restart early if we can't
++++ b/tcg/i386/tcg-target.c.inc
--       support either operation that is supposed to be atomic.  */
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
 +    /*
 +     * Note that the compare-and-swap is atomic, and the store is atomic,
 +     * but the complete operation is not.  Therefore we do not need to
 +     * assert serial context in order to implement this.  That said,
 +     * restart early if we can't support either operation that is supposed
 +     * to be atomic.
 +     */
      if (parallel) {
 -        int mask = 0;
 -#if !defined(CONFIG_ATOMIC64)
 -        mask = -8;
 -#elif !defined(CONFIG_ATOMIC128)
 -        mask = -16;
 +        uint32_t max = 2;
 +#ifdef CONFIG_ATOMIC64
 +        max = 3;
  #endif
 -        if (((4 << fc) | (1 << sc)) & mask) {
 +        if ((HAVE_CMPXCHG128 ? 0 : fc + 2 > max) ||
 +            (HAVE_ATOMIC128  ? 0 : sc > max)) {
              cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
          }
      }
@@ -XXX,XX +XXX,XX @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
              Int128 cv = int128_make128(env->regs[r3 + 1], env->regs[r3]);
              Int128 ov;
 -            if (parallel) {
 -#ifdef CONFIG_ATOMIC128
 -                TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
 -                ov = helper_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi, ra);
 -                cc = !int128_eq(ov, cv);
 -#else
 -                /* Note that we asserted !parallel above.  */
 -                g_assert_not_reached();
 -#endif
 -            } else {
 +            if (!parallel) {
                  uint64_t oh = cpu_ldq_data_ra(env, a1 + 0, ra);
                  uint64_t ol = cpu_ldq_data_ra(env, a1 + 8, ra);
@@ -XXX,XX +XXX,XX @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
                  cpu_stq_data_ra(env, a1 + 0, int128_gethi(nv), ra);
                  cpu_stq_data_ra(env, a1 + 8, int128_getlo(nv), ra);
 +            } else if (HAVE_CMPXCHG128) {
 +                TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
 +                ov = helper_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi, ra);
 +                cc = !int128_eq(ov, cv);
 +            } else {
 +                /* Note that we asserted !parallel above.  */
 +                g_assert_not_reached();
              }
              env->regs[r3 + 0] = int128_gethi(ov);
@@ -XXX,XX +XXX,XX @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
              cpu_stq_data_ra(env, a2, svh, ra);
              break;
          case 4:
 -            if (parallel) {
 -#ifdef CONFIG_ATOMIC128
 +            if (!parallel) {
 +                cpu_stq_data_ra(env, a2 + 0, svh, ra);
 +                cpu_stq_data_ra(env, a2 + 8, svl, ra);
 +            } else if (HAVE_ATOMIC128) {
                  TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
                  Int128 sv = int128_make128(svl, svh);
                  helper_atomic_sto_be_mmu(env, a2, sv, oi, ra);
 -#else
 +            } else {
                  /* Note that we asserted !parallel above.  */
                  g_assert_not_reached();
 -#endif
 -            } else {
 -                cpu_stq_data_ra(env, a2 + 0, svh, ra);
 -                cpu_stq_data_ra(env, a2 + 8, svl, ra);
              }
              break;
          default:
@@ -XXX,XX +XXX,XX @@ static uint64_t do_lpq(CPUS390XState *env, uint64_t addr, bool parallel)
      uintptr_t ra = GETPC();
      uint64_t hi, lo;
 -    if (parallel) {
 -#ifndef CONFIG_ATOMIC128
 -        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
 -#else
 +    if (!parallel) {
 +        check_alignment(env, addr, 16, ra);
 +        hi = cpu_ldq_data_ra(env, addr + 0, ra);
 +        lo = cpu_ldq_data_ra(env, addr + 8, ra);
 +    } else if (HAVE_ATOMIC128) {
          int mem_idx = cpu_mmu_index(env, false);
          TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
          Int128 v = helper_atomic_ldo_be_mmu(env, addr, oi, ra);
          hi = int128_gethi(v);
          lo = int128_getlo(v);
 -#endif
      } else {
 -        check_alignment(env, addr, 16, ra);
 -
 -        hi = cpu_ldq_data_ra(env, addr + 0, ra);
 -        lo = cpu_ldq_data_ra(env, addr + 8, ra);
 +        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
      }
      env->retxl = lo;
@@ -XXX,XX +XXX,XX @@ static void do_stpq(CPUS390XState *env, uint64_t addr,
  {
      uintptr_t ra = GETPC();
 -    if (parallel) {
 -#ifndef CONFIG_ATOMIC128
 -        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
 -#else
 -        int mem_idx = cpu_mmu_index(env, false);
 -        TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
 -
 -        Int128 v = int128_make128(low, high);
 -        helper_atomic_sto_be_mmu(env, addr, v, oi, ra);
 -#endif
 -    } else {
 +    if (!parallel) {
          check_alignment(env, addr, 16, ra);
 -
          cpu_stq_data_ra(env, addr + 0, high, ra);
          cpu_stq_data_ra(env, addr + 8, low, ra);
 +    } else if (HAVE_ATOMIC128) {
 +        int mem_idx = cpu_mmu_index(env, false);
 +        TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
 +        Int128 v = int128_make128(low, high);
 +        helper_atomic_sto_be_mmu(env, addr, v, oi, ra);
 +    } else {
 +        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
      }
  }
+-/* Use SMALL != 0 to force a short forward branch.  */
+-static void tcg_out_jxx(TCGContext *s, int opc, TCGLabel *l, int small)
++/* Set SMALL to force a short forward branch.  */
++static void tcg_out_jxx(TCGContext *s, int opc, TCGLabel *l, bool small)
+ {
+     int32_t val, val1;
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_jxx(TCGContext *s, int opc, TCGLabel *l, int small)
+             }
+             tcg_out8(s, val1);
+         } else {
+-            if (small) {
+-                tcg_abort();
+-            }
++            tcg_debug_assert(!small);
+             if (opc == -1) {
+                 tcg_out8(s, OPC_JMP_long);
+                 tcg_out32(s, val - 5);
 --
-.17.2
+.34.1

-[Qemu-devel] [PULL 05/21] tcg: distribute tcg_time into TCG contexts
+[PULL 02/15] tcg: Replace tcg_abort with g_assert_not_reached
-From: "Emilio G. Cota" <cota@braap.org>
+Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 When we implemented per-vCPU TCG contexts, we forgot to also
 distribute the tcg_time counter, which has remained as a global
 accessed without any serialization, leading to potentially missed
 counts.
 Fix it by distributing the field over the TCG contexts, embedding
 it into TCGProfile with a field called "cpu_exec_time", which is more
 descriptive than "tcg_time". Add a function to query this value
 directly, and for completeness, fill in the field in
 tcg_profile_snapshot, even though its callers do not use it.
 Signed-off-by: Emilio G. Cota <cota@braap.org>
 Message-Id: <20181010144853.13005-5-cota@braap.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- include/qemu/timer.h |  1 -
+ include/tcg/tcg.h            |  6 ------
- tcg/tcg.h            |  2 ++
+ target/i386/tcg/translate.c  | 20 ++++++++++----------
- cpus.c               |  3 ++-
+ target/s390x/tcg/translate.c |  4 ++--
- monitor.c            | 13 ++++++++++---
+ tcg/optimize.c               | 10 ++++------
- tcg/tcg.c            | 23 +++++++++++++++++++++++
+ tcg/tcg.c                    |  8 ++++----
-files changed, 37 insertions(+), 5 deletions(-)
+ tcg/aarch64/tcg-target.c.inc |  4 ++--
  tcg/arm/tcg-target.c.inc     |  2 +-
  tcg/i386/tcg-target.c.inc    | 14 +++++++-------
  tcg/mips/tcg-target.c.inc    | 14 +++++++-------
  tcg/ppc/tcg-target.c.inc     |  8 ++++----
  tcg/s390x/tcg-target.c.inc   |  8 ++++----
  tcg/sparc64/tcg-target.c.inc |  2 +-
  tcg/tci/tcg-target.c.inc     |  2 +-
 files changed, 47 insertions(+), 55 deletions(-)
-diff --git a/include/qemu/timer.h b/include/qemu/timer.h
+diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h
 index XXXXXXX..XXXXXXX 100644
---- a/include/qemu/timer.h
+--- a/include/tcg/tcg.h
-+++ b/include/qemu/timer.h
++++ b/include/tcg/tcg.h
-@@ -XXX,XX +XXX,XX @@ static inline int64_t profile_getclock(void)
+@@ -XXX,XX +XXX,XX @@ typedef struct TCGTargetOpDef {
-     return get_clock();
+     const char *args_ct_str[TCG_MAX_OP_ARGS];
- }
+ } TCGTargetOpDef;
--extern int64_t tcg_time;
+-#define tcg_abort() \
- extern int64_t dev_time;
+-do {\
 -    fprintf(stderr, "%s:%d: tcg fatal error\n", __FILE__, __LINE__);\
 -    abort();\
 -} while (0)
 -
  bool tcg_op_supported(TCGOpcode op);
  void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args);
 diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c
 index XXXXXXX..XXXXXXX 100644
 --- a/target/i386/tcg/translate.c
 +++ b/target/i386/tcg/translate.c
@@ -XXX,XX +XXX,XX @@ static TCGv gen_op_deposit_reg_v(DisasContext *s, MemOp ot, int reg, TCGv dest,
          break;
  #endif
+     default:
-diff --git a/tcg/tcg.h b/tcg/tcg.h
+-        tcg_abort();
-index XXXXXXX..XXXXXXX 100644
++        g_assert_not_reached();
---- a/tcg/tcg.h
+     }
-+++ b/tcg/tcg.h
+     return cpu_regs[reg];
-@@ -XXX,XX +XXX,XX @@ typedef struct TCGOp {
+ }
- QEMU_BUILD_BUG_ON(NB_OPS > (1 << 8));
+@@ -XXX,XX +XXX,XX @@ static void gen_lea_v_seg(DisasContext *s, MemOp aflag, TCGv a0,
+         }
- typedef struct TCGProfile {
+         break;
-+    int64_t cpu_exec_time;
+     default:
-     int64_t tb_count1;
+-        tcg_abort();
-     int64_t tb_count;
++        g_assert_not_reached();
-     int64_t op_count; /* total insn count */
+     }
-@@ -XXX,XX +XXX,XX @@ int tcg_check_temp_count(void);
- #define tcg_check_temp_count() 0
+     if (ovr_seg >= 0) {
@@ -XXX,XX +XXX,XX @@ static void gen_helper_in_func(MemOp ot, TCGv v, TCGv_i32 n)
          gen_helper_inl(v, cpu_env, n);
          break;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
@@ -XXX,XX +XXX,XX @@ static void gen_helper_out_func(MemOp ot, TCGv_i32 v, TCGv_i32 n)
          gen_helper_outl(cpu_env, v, n);
          break;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
@@ -XXX,XX +XXX,XX @@ static void gen_rotc_rm_T1(DisasContext *s, MemOp ot, int op1,
              break;
  #endif
+         default:
-+int64_t tcg_cpu_exec_time(void);
+-            tcg_abort();
- void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf);
++            g_assert_not_reached();
- void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf);
+         }
+     } else {
-diff --git a/cpus.c b/cpus.c
+         switch (ot) {
-index XXXXXXX..XXXXXXX 100644
+@@ -XXX,XX +XXX,XX @@ static void gen_rotc_rm_T1(DisasContext *s, MemOp ot, int op1,
---- a/cpus.c
+             break;
 +++ b/cpus.c
@@ -XXX,XX +XXX,XX @@ static int tcg_cpu_exec(CPUState *cpu)
      ret = cpu_exec(cpu);
      cpu_exec_end(cpu);
  #ifdef CONFIG_PROFILER
 -    tcg_time += profile_getclock() - ti;
 +    atomic_set(&tcg_ctx->prof.cpu_exec_time,
 +               tcg_ctx->prof.cpu_exec_time + profile_getclock() - ti);
  #endif
+         default:
+-            tcg_abort();
++            g_assert_not_reached();
+         }
+     }
+     /* store */
+@@ -XXX,XX +XXX,XX @@ static AddressParts gen_lea_modrm_0(CPUX86State *env, DisasContext *s,
+         break;
+     default:
+-        tcg_abort();
++        g_assert_not_reached();
+     }
+  done:
+@@ -XXX,XX +XXX,XX @@ static inline uint32_t insn_get(CPUX86State *env, DisasContext *s, MemOp ot)
+         ret = x86_ldl_code(env, s);
+         break;
+     default:
+-        tcg_abort();
++        g_assert_not_reached();
+     }
      return ret;
  }
-diff --git a/monitor.c b/monitor.c
+@@ -XXX,XX +XXX,XX @@ static bool disas_insn(DisasContext *s, CPUState *cpu)
-index XXXXXXX..XXXXXXX 100644
+             gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0);
---- a/monitor.c
+             break;
-+++ b/monitor.c
+         default:
-@@ -XXX,XX +XXX,XX @@
+-            tcg_abort();
- #include "sysemu/cpus.h"
++            g_assert_not_reached();
- #include "sysemu/iothread.h"
+         }
- #include "qemu/cutils.h"
+         break;
-+#include "tcg/tcg.h"
+     case 0x99: /* CDQ/CWD */
+@@ -XXX,XX +XXX,XX @@ static bool disas_insn(DisasContext *s, CPUState *cpu)
- #if defined(TARGET_S390X)
+             gen_op_mov_reg_v(s, MO_16, R_EDX, s->T0);
- #include "hw/s390x/storage-keys.h"
+             break;
-@@ -XXX,XX +XXX,XX @@ static void hmp_info_numa(Monitor *mon, const QDict *qdict)
+         default:
+-            tcg_abort();
- #ifdef CONFIG_PROFILER
++            g_assert_not_reached();
+         }
--int64_t tcg_time;
+         break;
- int64_t dev_time;
+     case 0x1af: /* imul Gv, Ev */
+diff --git a/target/s390x/tcg/translate.c b/target/s390x/tcg/translate.c
- static void hmp_info_profile(Monitor *mon, const QDict *qdict)
+index XXXXXXX..XXXXXXX 100644
- {
+--- a/target/s390x/tcg/translate.c
-+    static int64_t last_cpu_exec_time;
++++ b/target/s390x/tcg/translate.c
-+    int64_t cpu_exec_time;
+@@ -XXX,XX +XXX,XX @@ static int get_mem_index(DisasContext *s)
-+    int64_t delta;
+     case PSW_ASC_HOME >> FLAG_MASK_PSW_SHIFT:
-+
+         return MMU_HOME_IDX;
-+    cpu_exec_time = tcg_cpu_exec_time();
+     default:
-+    delta = cpu_exec_time - last_cpu_exec_time;
+-        tcg_abort();
-+
++        g_assert_not_reached();
-     monitor_printf(mon, "async time  %" PRId64 " (%0.3f)\n",
+         break;
-                    dev_time, dev_time / (double)NANOSECONDS_PER_SECOND);
+     }
-     monitor_printf(mon, "qemu time   %" PRId64 " (%0.3f)\n",
+ #endif
--                   tcg_time, tcg_time / (double)NANOSECONDS_PER_SECOND);
+@@ -XXX,XX +XXX,XX @@ static void gen_op_calc_cc(DisasContext *s)
--    tcg_time = 0;
+         gen_helper_calc_cc(cc_op, cpu_env, cc_op, cc_src, cc_dst, cc_vr);
-+                   delta, delta / (double)NANOSECONDS_PER_SECOND);
+         break;
-+    last_cpu_exec_time = cpu_exec_time;
+     default:
-     dev_time = 0;
+-        tcg_abort();
- }
++        g_assert_not_reached();
- #else
+     }
      /* We now have cc in cc_op as constant */
 diff --git a/tcg/optimize.c b/tcg/optimize.c
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/optimize.c
 +++ b/tcg/optimize.c
@@ -XXX,XX +XXX,XX @@ static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y)
          return (uint64_t)x % ((uint64_t)y ? : 1);
      default:
 -        fprintf(stderr,
 -                "Unrecognized operation %d in do_constant_folding.\n", op);
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
@@ -XXX,XX +XXX,XX @@ static bool do_constant_folding_cond_32(uint32_t x, uint32_t y, TCGCond c)
      case TCG_COND_GTU:
          return x > y;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
@@ -XXX,XX +XXX,XX @@ static bool do_constant_folding_cond_64(uint64_t x, uint64_t y, TCGCond c)
      case TCG_COND_GTU:
          return x > y;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
@@ -XXX,XX +XXX,XX @@ static bool do_constant_folding_cond_eq(TCGCond c)
      case TCG_COND_EQ:
          return 1;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
 diff --git a/tcg/tcg.c b/tcg/tcg.c
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/tcg.c
 +++ b/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@
+@@ -XXX,XX +XXX,XX @@ static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
- /* Define to jump the ELF file used to communicate with GDB.  */
- #undef DEBUG_JIT
+         case TEMP_VAL_DEAD:
+         default:
-+#include "qemu/error-report.h"
+-            tcg_abort();
- #include "qemu/cutils.h"
++            g_assert_not_reached();
- #include "qemu/host-utils.h"
+         }
- #include "qemu/timer.h"
+         ts->mem_coherent = 1;
-@@ -XXX,XX +XXX,XX @@ void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
+     }
-         const TCGProfile *orig = &s->prof;
+@@ -XXX,XX +XXX,XX @@ static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
+         }
-         if (counters) {
+     }
-+            PROF_ADD(prof, orig, cpu_exec_time);
-             PROF_ADD(prof, orig, tb_count1);
+-    tcg_abort();
-             PROF_ADD(prof, orig, tb_count);
++    g_assert_not_reached();
-             PROF_ADD(prof, orig, op_count);
+ }
-@@ -XXX,XX +XXX,XX @@ void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf)
-                     prof.table_op_count[i]);
+ static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs,
-     }
+@@ -XXX,XX +XXX,XX @@ static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs,
- }
+             }
-+
+         }
-+int64_t tcg_cpu_exec_time(void)
+     }
-+{
+-    tcg_abort();
-+    unsigned int n_ctxs = atomic_read(&n_tcg_ctxs);
++    g_assert_not_reached();
-+    unsigned int i;
+ }
-+    int64_t ret = 0;
-+
+ /* Make sure the temporary is in a register.  If needed, allocate the register
-+    for (i = 0; i < n_ctxs; i++) {
+@@ -XXX,XX +XXX,XX @@ static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
-+        const TCGContext *s = atomic_read(&tcg_ctxs[i]);
+         break;
-+        const TCGProfile *prof = &s->prof;
+     case TEMP_VAL_DEAD:
-+
+     default:
-+        ret += atomic_read(&prof->cpu_exec_time);
+-        tcg_abort();
-+    }
++        g_assert_not_reached();
-+    return ret;
+     }
-+}
+     set_temp_val_reg(s, ts, reg);
- #else
+ }
- void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf)
+diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
- {
+index XXXXXXX..XXXXXXX 100644
-     cpu_fprintf(f, "[TCG profiler not compiled]\n");
+--- a/tcg/aarch64/tcg-target.c.inc
- }
++++ b/tcg/aarch64/tcg-target.c.inc
-+
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp memop, TCGType ext,
-+int64_t tcg_cpu_exec_time(void)
+         tcg_out_ldst_r(s, I3312_LDRX, data_r, addr_r, otype, off_r);
-+{
+         break;
-+    error_report("%s: TCG profiler not compiled", __func__);
+     default:
-+    exit(EXIT_FAILURE);
+-        tcg_abort();
-+}
++        g_assert_not_reached();
- #endif
+     }
+ }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp memop,
          tcg_out_ldst_r(s, I3312_STRX, data_r, addr_r, otype, off_r);
          break;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
 diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/arm/tcg-target.c.inc
 +++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
 diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/i386/tcg-target.c.inc
 +++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
          tcg_patch8(code_ptr, value);
          break;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
      return true;
  }
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_pushi(TCGContext *s, tcg_target_long val)
          tcg_out_opc(s, OPC_PUSH_Iv, 0, 0, 0);
          tcg_out32(s, val);
      } else {
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
@@ -XXX,XX +XXX,XX @@ static void tgen_arithi(TCGContext *s, int c, int r0,
          return;
      }
 -    tcg_abort();
 +    g_assert_not_reached();
  }
  static void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
@@ -XXX,XX +XXX,XX @@ static void tcg_out_brcond2(TCGContext *s, const TCGArg *args,
                           label_this, small);
          break;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
      tcg_out_label(s, label_next);
  }
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
          }
          break;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
      /* Jump to the code corresponding to next IR of qemu_st */
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
              /* load bits 0..15 */
              tcg_out_modrm(s, OPC_MOVL_EvGv | P_DATA16, a2, a0);
          } else {
 -            tcg_abort();
 +            g_assert_not_reached();
          }
          break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  #undef OP_32_64
 diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/mips/tcg-target.c.inc
 +++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret,
          break;
       default:
 -         tcg_abort();
 +         g_assert_not_reached();
           break;
       }
  }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1,
          break;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
          break;
      }
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
          }
          break;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
      i = tcg_out_call_iarg_imm(s, i, oi);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi,
          }
          break;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi,
          break;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_unalign(TCGContext *s, TCGReg lo, TCGReg hi,
          break;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
  static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
 diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/ppc/tcg-target.c.inc
 +++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2,
          break;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
      op |= BF(cr) | ((type == TCG_TYPE_I64) << 21);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond,
          break;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_cmp2(TCGContext *s, const TCGArg *args,
          break;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_exit_tb:   /* Always emitted via tcg_out_exit_tb.  */
      case INDEX_op_goto_tb:   /* Always emitted via tcg_out_goto_tb.  */
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
 diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/s390x/tcg-target.c.inc
 +++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data,
          break;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg data,
          break;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
          tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
          break;
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
      tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R5, oi);
      tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R6, (uintptr_t)lb->raddr);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
 diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/sparc64/tcg-target.c.inc
 +++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
 diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/tci/tcg-target.c.inc
 +++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      default:
 -        tcg_abort();
 +        g_assert_not_reached();
      }
  }
 --
-.17.2
+.34.1

-[Qemu-devel] [PULL 21/21] cputlb: read CPUTLBEntry.addr_write atomically
+[PULL 03/15] tcg: Split out tcg_out_ext8s
-From: "Emilio G. Cota" <cota@braap.org>
+We will need a backend interface for performing 8-bit sign-extend.
 Use it in tcg_reg_alloc_op in the meantime.
-Updates can come from other threads, so readers that do not
+Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 take tlb_lock must use atomic_read to avoid undefined
 behaviour (UB).
 This completes the conversion to tlb_lock. This conversion results
 on average in no performance loss, as the following experiments
 (run on an Intel i7-6700K CPU @ 4.00GHz) show.
 . aarch64 bootup+shutdown test:
 - Before:
  Performance counter stats for 'taskset -c 0 ../img/aarch64/die.sh' (10 runs):
 .087786      task-clock (msec)         #    0.998 CPUs utilized            ( +-  0.12% )
 ,574,905,303      cycles                    #    4.217 GHz                      ( +-  0.12% )
 ,097,908,812      instructions              #    1.81  insns per cycle          ( +-  0.08% )
 ,255,415,367      branches                  # 1369.747 M/sec                    ( +-  0.08% )
 ,278,962      branch-misses             #    1.69% of all branches          ( +-  0.18% )
 .504481349 seconds time elapsed                                          ( +-  0.14% )
 - After:
  Performance counter stats for 'taskset -c 0 ../img/aarch64/die.sh' (10 runs):
 .441328      task-clock (msec)         #    0.998 CPUs utilized            ( +-  0.07% )
 ,478,476,520      cycles                    #    4.218 GHz                      ( +-  0.07% )
 ,017,330,084      instructions              #    1.81  insns per cycle          ( +-  0.05% )
 ,251,929,667      branches                  # 1373.804 M/sec                    ( +-  0.05% )
 ,023,787      branch-misses             #    1.69% of all branches          ( +-  0.11% )
 .474970463 seconds time elapsed                                          ( +-  0.07% )
 . SPEC06int:
                                               SPEC06int (test set)
                                            [Y axis: Speedup over master]
 .15 +-+----+------+------+------+------+------+-------+------+------+------+------+------+------+----+-+
        |                                                                                                  |
 .1 +-+.................................+++.............................+  tlb-lock-v2 (m+++x)       +-+
        |                                +++ |                   +++        tlb-lock-v3 (spinl|ck)         |
        |                    +++          |  |     +++    +++     |                           |            |
 .05 +-+....+++...........####.........|####.+++.|......|.....###....+++...........+++....###.........+-+
        |      ###         ++#| #         |# |# ***### +++### +++#+#     |     +++     |     #|#    ###    |
 +-+++***+#++++####+++#++#++++++++++#++#+*+*++#++++#+#+****+#++++###++++###++++###++++#+#++++#+#+++-+
        |    *+* #    #++# ***  #   #### ***  # * *++# ****+# *| * # ****|#   |# #    #|#    #+#    # #    |
 .95 +-+..*.*.#....#..#.*|*..#...#..#.*|*..#.*.*..#.*|.*.#.*++*.#.*++*+#.****.#....#+#....#.#..++#.#..+-+
        |    * * #    #  # *|*  #   #  # *|*  # * *  # *++* # *  * # *  * # * |* #  ++# #    # #  *** #    |
        |    * * #  ++#  # *+*  #   #  # *|*  # * *  # *  * # *  * # *  * # *++* # **** #  ++# #  * * #    |
 .9 +-+..*.*.#...|#..#.*.*..#.++#..#.*|*..#.*.*..#.*..*.#.*..*.#.*..*.#.*..*.#.*.|*.#...|#.#..*.*.#..+-+
        |    * * #  ***  # * *  #  |#  # *+*  # * *  # *  * # *  * # *  * # *  * # *++* #   |# #  * * #    |
 .85 +-+..*.*.#..*|*..#.*.*..#.***..#.*.*..#.*.*..#.*..*.#.*..*.#.*..*.#.*..*.#.*..*.#.****.#..*.*.#..+-+
        |    * * #  *+*  # * *  # *|*  # * *  # * *  # *  * # *  * # *  * # *  * # *  * # * |* #  * * #    |
        |    * * #  * *  # * *  # *+*  # * *  # * *  # *  * # *  * # *  * # *  * # *  * # * |* #  * * #    |
 .8 +-+..*.*.#..*.*..#.*.*..#.*.*..#.*.*..#.*.*..#.*..*.#.*..*.#.*..*.#.*..*.#.*..*.#.*++*.#..*.*.#..+-+
        |    * * #  * *  # * *  # * *  # * *  # * *  # *  * # *  * # *  * # *  * # *  * # *  * #  * * #    |
 .75 +-+--***##--***###-***###-***###-***###-***###-****##-****##-****##-****##-****##-****##--***##--+-+
 .perlben401.bzip2403.gcc429.m445.gob456.hmme45462.libqua464.h26471.omnet473483.xalancbmkgeomean
   png: https://imgur.com/a/BHzpPTW
 Notes:
 - tlb-lock-v2 corresponds to an implementation with a mutex.
 - tlb-lock-v3 corresponds to the current implementation, i.e.
   a spinlock and a single lock acquisition in tlb_set_page_with_attrs.
 Signed-off-by: Emilio G. Cota <cota@braap.org>
 Message-Id: <20181016153840.25877-1-cota@braap.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- accel/tcg/softmmu_template.h     | 12 ++++++------
+ tcg/tcg.c                        | 21 ++++++++++++++++-----
- include/exec/cpu_ldst.h          | 11 ++++++++++-
+ tcg/aarch64/tcg-target.c.inc     | 11 +++++++----
- include/exec/cpu_ldst_template.h |  2 +-
+ tcg/arm/tcg-target.c.inc         | 10 ++++------
- accel/tcg/cputlb.c               | 19 +++++++++++++------
+ tcg/i386/tcg-target.c.inc        | 10 +++++-----
-files changed, 30 insertions(+), 14 deletions(-)
+ tcg/loongarch64/tcg-target.c.inc | 11 ++++-------
  tcg/mips/tcg-target.c.inc        | 12 ++++++++----
  tcg/ppc/tcg-target.c.inc         | 10 ++++------
  tcg/riscv/tcg-target.c.inc       |  9 +++------
  tcg/s390x/tcg-target.c.inc       | 10 +++-------
  tcg/sparc64/tcg-target.c.inc     |  7 +++++++
  tcg/tci/tcg-target.c.inc         | 21 ++++++++++++++++++++-
 files changed, 81 insertions(+), 51 deletions(-)
-diff --git a/accel/tcg/softmmu_template.h b/accel/tcg/softmmu_template.h
+diff --git a/tcg/tcg.c b/tcg/tcg.c
 index XXXXXXX..XXXXXXX 100644
---- a/accel/tcg/softmmu_template.h
+--- a/tcg/tcg.c
-+++ b/accel/tcg/softmmu_template.h
++++ b/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
-     uintptr_t mmu_idx = get_mmuidx(oi);
+ static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
-     uintptr_t index = tlb_index(env, mmu_idx, addr);
+ static void tcg_out_movi(TCGContext *s, TCGType type,
-     CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
+                          TCGReg ret, tcg_target_long arg);
--    target_ulong tlb_addr = entry->addr_write;
++static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
-+    target_ulong tlb_addr = tlb_addr_write(entry);
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
-     unsigned a_bits = get_alignment_bits(get_memop(oi));
+ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
-     uintptr_t haddr;
+ static void tcg_out_goto_tb(TCGContext *s, int which);
+@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
-@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
+     }
-             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE,
-                      mmu_idx, retaddr);
+     /* emit instruction */
-         }
+-    if (def->flags & TCG_OPF_VECTOR) {
--        tlb_addr = entry->addr_write & ~TLB_INVALID_MASK;
+-        tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
-+        tlb_addr = tlb_addr_write(entry) & ~TLB_INVALID_MASK;
+-                       new_args, const_args);
-     }
+-    } else {
+-        tcg_out_op(s, op->opc, new_args, const_args);
-     /* Handle an IO access.  */
++    switch (op->opc) {
-@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
++    case INDEX_op_ext8s_i32:
-            cannot evict the first.  */
++        tcg_out_ext8s(s, TCG_TYPE_I32, new_args[0], new_args[1]);
-         page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
++        break;
-         entry2 = tlb_entry(env, mmu_idx, page2);
++    case INDEX_op_ext8s_i64:
--        if (!tlb_hit_page(entry2->addr_write, page2)
++        tcg_out_ext8s(s, TCG_TYPE_I64, new_args[0], new_args[1]);
-+        if (!tlb_hit_page(tlb_addr_write(entry2), page2)
++        break;
-             && !VICTIM_TLB_HIT(addr_write, page2)) {
++    default:
-             tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE,
++        if (def->flags & TCG_OPF_VECTOR) {
-                      mmu_idx, retaddr);
++            tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
-@@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
++                           new_args, const_args);
-     uintptr_t mmu_idx = get_mmuidx(oi);
++        } else {
-     uintptr_t index = tlb_index(env, mmu_idx, addr);
++            tcg_out_op(s, op->opc, new_args, const_args);
-     CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
++        }
--    target_ulong tlb_addr = entry->addr_write;
++        break;
-+    target_ulong tlb_addr = tlb_addr_write(entry);
+     }
-     unsigned a_bits = get_alignment_bits(get_memop(oi));
-     uintptr_t haddr;
+     /* move the outputs in the correct register if needed */
+diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
+index XXXXXXX..XXXXXXX 100644
-             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE,
+--- a/tcg/aarch64/tcg-target.c.inc
-                      mmu_idx, retaddr);
++++ b/tcg/aarch64/tcg-target.c.inc
-         }
+@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_sxt(TCGContext *s, TCGType ext, MemOp s_bits,
--        tlb_addr = entry->addr_write & ~TLB_INVALID_MASK;
+     tcg_out_sbfm(s, ext, rd, rn, 0, bits);
-+        tlb_addr = tlb_addr_write(entry) & ~TLB_INVALID_MASK;
+ }
-     }
++static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rn)
      /* Handle an IO access.  */
@@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
             cannot evict the first.  */
          page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
          entry2 = tlb_entry(env, mmu_idx, page2);
 -        if (!tlb_hit_page(entry2->addr_write, page2)
 +        if (!tlb_hit_page(tlb_addr_write(entry2), page2)
              && !VICTIM_TLB_HIT(addr_write, page2)) {
              tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE,
                       mmu_idx, retaddr);
 diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h
 index XXXXXXX..XXXXXXX 100644
 --- a/include/exec/cpu_ldst.h
 +++ b/include/exec/cpu_ldst.h
@@ -XXX,XX +XXX,XX @@ extern __thread uintptr_t helper_retaddr;
  /* The memory helpers for tcg-generated code need tcg_target_long etc.  */
  #include "tcg.h"
 +static inline target_ulong tlb_addr_write(const CPUTLBEntry *entry)
 +{
-+#if TCG_OVERSIZED_GUEST
++    tcg_out_sxt(s, type, MO_8, rd, rn);
 +    return entry->addr_write;
 +#else
 +    return atomic_read(&entry->addr_write);
 +#endif
 +}
 +
- /* Find the TLB index corresponding to the mmu_idx + address pair.  */
+ static inline void tcg_out_uxt(TCGContext *s, MemOp s_bits,
- static inline uintptr_t tlb_index(CPUArchState *env, uintptr_t mmu_idx,
+                                TCGReg rd, TCGReg rn)
-                                   target_ulong addr)
+ {
-@@ -XXX,XX +XXX,XX @@ static inline void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr,
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-         tlb_addr = tlbentry->addr_read;
+         }
          break;
-     case 1:
--        tlb_addr = tlbentry->addr_write;
+-    case INDEX_op_ext8s_i64:
-+        tlb_addr = tlb_addr_write(tlbentry);
+-    case INDEX_op_ext8s_i32:
-         break;
+-        tcg_out_sxt(s, ext, MO_8, a0, a1);
-     case 2:
+-        break;
-         tlb_addr = tlbentry->addr_code;
+     case INDEX_op_ext16s_i64:
-diff --git a/include/exec/cpu_ldst_template.h b/include/exec/cpu_ldst_template.h
+     case INDEX_op_ext16s_i32:
-index XXXXXXX..XXXXXXX 100644
+         tcg_out_sxt(s, ext, MO_16, a0, a1);
---- a/include/exec/cpu_ldst_template.h
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-+++ b/include/exec/cpu_ldst_template.h
+     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
-@@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
-     addr = ptr;
+     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
-     mmu_idx = CPU_MMU_INDEX;
++    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
-     entry = tlb_entry(env, mmu_idx, addr);
++    case INDEX_op_ext8s_i64:
--    if (unlikely(entry->addr_write !=
+     default:
-+    if (unlikely(tlb_addr_write(entry) !=
+         g_assert_not_reached();
-                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
+     }
-         oi = make_memop_idx(SHIFT, mmu_idx);
+diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
-         glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(env, addr, v, oi,
+index XXXXXXX..XXXXXXX 100644
-diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
+--- a/tcg/arm/tcg-target.c.inc
-index XXXXXXX..XXXXXXX 100644
++++ b/tcg/arm/tcg-target.c.inc
---- a/accel/tcg/cputlb.c
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_udiv(TCGContext *s, ARMCond cond,
-+++ b/accel/tcg/cputlb.c
+     tcg_out32(s, 0x0730f010 | (cond << 28) | (rd << 16) | rn | (rm << 8));
-@@ -XXX,XX +XXX,XX @@ static inline bool tlb_hit_page_anyprot(CPUTLBEntry *tlb_entry,
+ }
-                                         target_ulong page)
- {
+-static void tcg_out_ext8s(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn)
-     return tlb_hit_page(tlb_entry->addr_read, page) ||
++static void tcg_out_ext8s(TCGContext *s, TCGType t, TCGReg rd, TCGReg rn)
--           tlb_hit_page(tlb_entry->addr_write, page) ||
+ {
-+           tlb_hit_page(tlb_addr_write(tlb_entry), page) ||
+     /* sxtb */
-            tlb_hit_page(tlb_entry->addr_code, page);
+-    tcg_out32(s, 0x06af0070 | (cond << 28) | (rd << 12) | rn);
- }
++    tcg_out32(s, 0x06af0070 | (COND_AL << 28) | (rd << 12) | rn);
+ }
-@@ -XXX,XX +XXX,XX @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry,
-         tlb_fill(cpu, addr, size, MMU_DATA_STORE, mmu_idx, retaddr);
+ static void __attribute__((unused))
+@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
-         entry = tlb_entry(env, mmu_idx, addr);
+     datahi = lb->datahi_reg;
--        tlb_addr = entry->addr_write;
+     switch (opc & MO_SSIZE) {
-+        tlb_addr = tlb_addr_write(entry);
+     case MO_SB:
-         if (!(tlb_addr & ~(TARGET_PAGE_MASK | TLB_RECHECK))) {
+-        tcg_out_ext8s(s, COND_AL, datalo, TCG_REG_R0);
-             /* RAM access */
++        tcg_out_ext8s(s, TCG_TYPE_I32, datalo, TCG_REG_R0);
-             uintptr_t haddr = addr + entry->addend;
+         break;
-@@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
+     case MO_SW:
-     assert_cpu_is_self(ENV_GET_CPU(env));
+         tcg_out_ext16s(s, COND_AL, datalo, TCG_REG_R0);
-     for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) {
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-         CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx];
+         tcg_out_bswap32(s, COND_AL, args[0], args[1]);
--        target_ulong cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs);
+         break;
-+        target_ulong cmp;
 -    case INDEX_op_ext8s_i32:
 -        tcg_out_ext8s(s, COND_AL, args[0], args[1]);
 -        break;
      case INDEX_op_ext16s_i32:
          tcg_out_ext16s(s, COND_AL, args[0], args[1]);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
 +    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/i386/tcg-target.c.inc
 +++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_ext8u(TCGContext *s, int dest, int src)
      tcg_out_modrm(s, OPC_MOVZBL + P_REXB_RM, dest, src);
  }
 -static void tcg_out_ext8s(TCGContext *s, int dest, int src, int rexw)
 +static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
  {
 +    int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
      /* movsbl */
      tcg_debug_assert(src < 4 || TCG_TARGET_REG_BITS == 64);
      tcg_out_modrm(s, OPC_MOVSBL + P_REXB_RM + rexw, dest, src);
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
      data_reg = l->datalo_reg;
      switch (opc & MO_SSIZE) {
      case MO_SB:
 -        tcg_out_ext8s(s, data_reg, TCG_REG_EAX, rexw);
 +        tcg_out_ext8s(s, l->type, data_reg, TCG_REG_EAX);
          break;
      case MO_SW:
          tcg_out_ext16s(s, data_reg, TCG_REG_EAX, rexw);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, a0);
          break;
 -    OP_32_64(ext8s):
 -        tcg_out_ext8s(s, a0, a1, rexw);
 -        break;
      OP_32_64(ext16s):
          tcg_out_ext16s(s, a0, a1, rexw);
          break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
 +    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
 +    case INDEX_op_ext8s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/loongarch64/tcg-target.c.inc
 +++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg)
      tcg_out_opc_bstrpick_d(s, ret, arg, 0, 31);
  }
 -static void tcg_out_ext8s(TCGContext *s, TCGReg ret, TCGReg arg)
 +static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
  {
      tcg_out_opc_sext_b(s, ret, arg);
  }
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
      switch (opc & MO_SSIZE) {
      case MO_SB:
 -        tcg_out_ext8s(s, l->datalo_reg, TCG_REG_A0);
 +        tcg_out_ext8s(s, type, l->datalo_reg, TCG_REG_A0);
          break;
      case MO_SW:
          tcg_out_ext16s(s, l->datalo_reg, TCG_REG_A0);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_brcond(s, a2, a0, a1, arg_label(args[3]));
          break;
 -    case INDEX_op_ext8s_i32:
 -    case INDEX_op_ext8s_i64:
 -        tcg_out_ext8s(s, a0, a1);
 -        break;
 -
      case INDEX_op_ext8u_i32:
      case INDEX_op_ext8u_i64:
          tcg_out_ext8u(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
 +    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
 +    case INDEX_op_ext8s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/mips/tcg-target.c.inc
 +++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type,
      }
  }
 +static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
 +{
 +    tcg_debug_assert(TCG_TARGET_HAS_ext8s_i32);
 +    tcg_out_opc_reg(s, OPC_SEB, rd, TCG_REG_ZERO, rs);
 +}
 +
-+        /* elt_ofs might correspond to .addr_write, so use atomic_read */
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
-+#if TCG_OVERSIZED_GUEST
+                              tcg_target_long imm)
-+        cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs);
+ {
-+#else
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-+        cmp = atomic_read((target_ulong *)((uintptr_t)vtlb + elt_ofs));
+     case INDEX_op_not_i64:
          i1 = OPC_NOR;
          goto do_unary;
 -    case INDEX_op_ext8s_i32:
 -    case INDEX_op_ext8s_i64:
 -        i1 = OPC_SEB;
 -        goto do_unary;
      case INDEX_op_ext16s_i32:
      case INDEX_op_ext16s_i64:
          i1 = OPC_SEH;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
 +    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
 +    case INDEX_op_ext8s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/ppc/tcg-target.c.inc
 +++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_rlw(TCGContext *s, int op, TCGReg ra, TCGReg rs,
      tcg_out32(s, op | RA(ra) | RS(rs) | SH(sh) | MB(mb) | ME(me));
  }
 -static inline void tcg_out_ext8s(TCGContext *s, TCGReg dst, TCGReg src)
 +static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
  {
      tcg_out32(s, EXTSB | RA(dst) | RS(src));
  }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ld8s_i32:
      case INDEX_op_ld8s_i64:
          tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]);
 -        tcg_out_ext8s(s, args[0], args[0]);
 +        tcg_out_ext8s(s, TCG_TYPE_REG, args[0], args[0]);
          break;
      case INDEX_op_ld16u_i32:
      case INDEX_op_ld16u_i64:
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_qemu_st(s, args, true);
          break;
 -    case INDEX_op_ext8s_i32:
 -    case INDEX_op_ext8s_i64:
 -        tcg_out_ext8s(s, args[0], args[1]);
 -        break;
      case INDEX_op_ext16s_i32:
      case INDEX_op_ext16s_i64:
          tcg_out_ext16s(s, args[0], args[1]);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_call:      /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:   /* Always emitted via tcg_out_exit_tb.  */
      case INDEX_op_goto_tb:   /* Always emitted via tcg_out_goto_tb.  */
 +    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
 +    case INDEX_op_ext8s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/riscv/tcg-target.c.inc
 +++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg)
      tcg_out_opc_imm(s, OPC_SRLI, ret, ret, 32);
  }
 -static void tcg_out_ext8s(TCGContext *s, TCGReg ret, TCGReg arg)
 +static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
  {
      tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 24);
      tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 24);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_ext32u(s, a0, a1);
          break;
 -    case INDEX_op_ext8s_i32:
 -    case INDEX_op_ext8s_i64:
 -        tcg_out_ext8s(s, a0, a1);
 -        break;
 -
      case INDEX_op_ext16s_i32:
      case INDEX_op_ext16s_i64:
          tcg_out_ext16s(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
 +    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
 +    case INDEX_op_ext8s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/s390x/tcg-target.c.inc
 +++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_risbg(TCGContext *s, TCGReg dest, TCGReg src,
      tcg_out16(s, (ofs << 8) | (RIEf_RISBG & 0xff));
  }
 -static void tgen_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
 +static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
  {
      tcg_out_insn(s, RRE, LGBR, dest, src);
  }
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
          }
          break;
 -    case INDEX_op_ext8s_i32:
 -        tgen_ext8s(s, TCG_TYPE_I32, args[0], args[1]);
 -        break;
      case INDEX_op_ext16s_i32:
          tgen_ext16s(s, TCG_TYPE_I32, args[0], args[1]);
          break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
          }
          break;
 -    case INDEX_op_ext8s_i64:
 -        tgen_ext8s(s, TCG_TYPE_I64, args[0], args[1]);
 -        break;
      case INDEX_op_ext16s_i64:
          tgen_ext16s(s, TCG_TYPE_I64, args[0], args[1]);
          break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
 +    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
 +    case INDEX_op_ext8s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/sparc64/tcg-target.c.inc
 +++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type,
      tcg_out_movi_int(s, type, ret, arg, false, TCG_REG_T2);
  }
 +static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
 +{
 +    g_assert_not_reached();
 +}
 +
  static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
      case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
 +    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
 +    case INDEX_op_ext8s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/tci/tcg-target.c.inc
 +++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type,
      }
  }
 +static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
 +{
 +    switch (type) {
 +    case TCG_TYPE_I32:
 +        tcg_debug_assert(TCG_TARGET_HAS_ext8s_i32);
 +        tcg_out_op_rr(s, INDEX_op_ext8s_i32, rd, rs);
 +        break;
 +#if TCG_TARGET_REG_BITS == 64
 +    case TCG_TYPE_I64:
 +        tcg_debug_assert(TCG_TARGET_HAS_ext8s_i64);
 +        tcg_out_op_rr(s, INDEX_op_ext8s_i64, rd, rs);
 +        break;
 +#endif
++    default:
-         if (cmp == page) {
++        g_assert_not_reached();
-             /* Found entry in victim tlb, swap tlb and iotlb.  */
++    }
-@@ -XXX,XX +XXX,XX @@ void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx,
++}
-     uintptr_t index = tlb_index(env, mmu_idx, addr);
++
-     CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
+                              tcg_target_long imm)
--    if (!tlb_hit(entry->addr_write, addr)) {
+ {
-+    if (!tlb_hit(tlb_addr_write(entry), addr)) {
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-         /* TLB entry is for a different page */
-         if (!VICTIM_TLB_HIT(addr_write, addr)) {
+     CASE_32_64(neg)      /* Optional (TCG_TARGET_HAS_neg_*). */
-             tlb_fill(ENV_GET_CPU(env), addr, size, MMU_DATA_STORE,
+     CASE_32_64(not)      /* Optional (TCG_TARGET_HAS_not_*). */
-@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
+-    CASE_32_64(ext8s)    /* Optional (TCG_TARGET_HAS_ext8s_*). */
-     size_t mmu_idx = get_mmuidx(oi);
+     CASE_32_64(ext8u)    /* Optional (TCG_TARGET_HAS_ext8u_*). */
-     uintptr_t index = tlb_index(env, mmu_idx, addr);
+     CASE_32_64(ext16s)   /* Optional (TCG_TARGET_HAS_ext16s_*). */
-     CPUTLBEntry *tlbe = tlb_entry(env, mmu_idx, addr);
+     CASE_32_64(ext16u)   /* Optional (TCG_TARGET_HAS_ext16u_*). */
--    target_ulong tlb_addr = tlbe->addr_write;
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-+    target_ulong tlb_addr = tlb_addr_write(tlbe);
+     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
-     TCGMemOp mop = get_memop(oi);
+     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
-     int a_bits = get_alignment_bits(mop);
+     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
-     int s_bits = mop & MO_SIZE;
++    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
-@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
++    case INDEX_op_ext8s_i64:
-             tlb_fill(ENV_GET_CPU(env), addr, 1 << s_bits, MMU_DATA_STORE,
+     default:
-                      mmu_idx, retaddr);
+         g_assert_not_reached();
-         }
+     }
 -        tlb_addr = tlbe->addr_write & ~TLB_INVALID_MASK;
 +        tlb_addr = tlb_addr_write(tlbe) & ~TLB_INVALID_MASK;
      }
      /* Notice an IO access or a needs-MMU-lookup access */
 --
-.17.2
+.34.1

-[Qemu-devel] [PULL 11/21] tcg: Add tlb_index and tlb_entry helpers
+[PULL 04/15] tcg: Split out tcg_out_ext8u
-Isolate the computation of an index from an address into a
+We will need a backend interface for performing 8-bit zero-extend.
-helper before we change that function.
+Use it in tcg_reg_alloc_op in the meantime.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
+Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
-[ cota: convert tlb_vaddr_to_host; use atomic_read on addr_write ]
-Signed-off-by: Emilio G. Cota <cota@braap.org>
-Message-Id: <20181009175129.17888-2-cota@braap.org>
 ---
- accel/tcg/softmmu_template.h     | 64 +++++++++++++++++---------------
+ tcg/tcg.c                        |  5 +++++
- include/exec/cpu_ldst.h          | 19 ++++++++--
+ tcg/aarch64/tcg-target.c.inc     | 11 +++++++----
- include/exec/cpu_ldst_template.h | 25 +++++++------
+ tcg/arm/tcg-target.c.inc         | 12 +++++++++---
- accel/tcg/cputlb.c               | 60 ++++++++++++++----------------
+ tcg/i386/tcg-target.c.inc        |  7 +++----
-files changed, 90 insertions(+), 78 deletions(-)
+ tcg/loongarch64/tcg-target.c.inc |  7 ++-----
  tcg/mips/tcg-target.c.inc        |  9 ++++++++-
  tcg/ppc/tcg-target.c.inc         |  7 +++++++
  tcg/riscv/tcg-target.c.inc       |  7 ++-----
  tcg/s390x/tcg-target.c.inc       | 14 +++++---------
  tcg/sparc64/tcg-target.c.inc     |  9 ++++++++-
  tcg/tci/tcg-target.c.inc         | 14 +++++++++++++-
 files changed, 69 insertions(+), 33 deletions(-)
-diff --git a/accel/tcg/softmmu_template.h b/accel/tcg/softmmu_template.h
+diff --git a/tcg/tcg.c b/tcg/tcg.c
 index XXXXXXX..XXXXXXX 100644
---- a/accel/tcg/softmmu_template.h
+--- a/tcg/tcg.c
-+++ b/accel/tcg/softmmu_template.h
++++ b/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@ static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
+@@ -XXX,XX +XXX,XX @@ static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
- WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
+ static void tcg_out_movi(TCGContext *s, TCGType type,
-                             TCGMemOpIdx oi, uintptr_t retaddr)
+                          TCGReg ret, tcg_target_long arg);
- {
+ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
--    unsigned mmu_idx = get_mmuidx(oi);
++static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
--    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
--    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
+ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
-+    uintptr_t mmu_idx = get_mmuidx(oi);
+ static void tcg_out_goto_tb(TCGContext *s, int which);
-+    uintptr_t index = tlb_index(env, mmu_idx, addr);
+@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
-+    CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
+     case INDEX_op_ext8s_i64:
-+    target_ulong tlb_addr = entry->ADDR_READ;
+         tcg_out_ext8s(s, TCG_TYPE_I64, new_args[0], new_args[1]);
-     unsigned a_bits = get_alignment_bits(get_memop(oi));
+         break;
-     uintptr_t haddr;
++    case INDEX_op_ext8u_i32:
-     DATA_TYPE res;
++    case INDEX_op_ext8u_i64:
-@@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
++        tcg_out_ext8u(s, new_args[0], new_args[1]);
-             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, READ_ACCESS_TYPE,
++        break;
-                      mmu_idx, retaddr);
+     default:
-         }
+         if (def->flags & TCG_OPF_VECTOR) {
--        tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
+             tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
-+        tlb_addr = entry->ADDR_READ;
+diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
-     }
+index XXXXXXX..XXXXXXX 100644
+--- a/tcg/aarch64/tcg-target.c.inc
-     /* Handle an IO access.  */
++++ b/tcg/aarch64/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
+@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_uxt(TCGContext *s, MemOp s_bits,
-         return res;
+     tcg_out_ubfm(s, 0, rd, rn, 0, bits);
-     }
+ }
--    haddr = addr + env->tlb_table[mmu_idx][index].addend;
++static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rn)
-+    haddr = addr + entry->addend;
++{
- #if DATA_SIZE == 1
++    tcg_out_uxt(s, MO_8, rd, rn);
-     res = glue(glue(ld, LSUFFIX), _p)((uint8_t *)haddr);
++}
- #else
++
-@@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
+ static void tcg_out_addsubi(TCGContext *s, int ext, TCGReg rd,
- WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,
+                             TCGReg rn, int64_t aimm)
-                             TCGMemOpIdx oi, uintptr_t retaddr)
+ {
- {
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
--    unsigned mmu_idx = get_mmuidx(oi);
+     case INDEX_op_ext32s_i64:
--    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+         tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
--    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
+         break;
-+    uintptr_t mmu_idx = get_mmuidx(oi);
+-    case INDEX_op_ext8u_i64:
-+    uintptr_t index = tlb_index(env, mmu_idx, addr);
+-    case INDEX_op_ext8u_i32:
-+    CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
+-        tcg_out_uxt(s, MO_8, a0, a1);
-+    target_ulong tlb_addr = entry->ADDR_READ;
+-        break;
-     unsigned a_bits = get_alignment_bits(get_memop(oi));
+     case INDEX_op_ext16u_i64:
-     uintptr_t haddr;
+     case INDEX_op_ext16u_i32:
-     DATA_TYPE res;
+         tcg_out_uxt(s, MO_16, a0, a1);
-@@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, READ_ACCESS_TYPE,
+     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
-                      mmu_idx, retaddr);
+     case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
-         }
+     case INDEX_op_ext8s_i64:
--        tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
++    case INDEX_op_ext8u_i32:
-+        tlb_addr = entry->ADDR_READ;
++    case INDEX_op_ext8u_i64:
-     }
+     default:
+         g_assert_not_reached();
-     /* Handle an IO access.  */
+     }
-@@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,
+diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
-         return res;
+index XXXXXXX..XXXXXXX 100644
-     }
+--- a/tcg/arm/tcg-target.c.inc
++++ b/tcg/arm/tcg-target.c.inc
--    haddr = addr + env->tlb_table[mmu_idx][index].addend;
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType t, TCGReg rd, TCGReg rn)
-+    haddr = addr + entry->addend;
+     tcg_out32(s, 0x06af0070 | (COND_AL << 28) | (rd << 12) | rn);
-     res = glue(glue(ld, LSUFFIX), _be_p)((uint8_t *)haddr);
+ }
-     return res;
- }
++static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rn)
-@@ -XXX,XX +XXX,XX @@ static inline void glue(io_write, SUFFIX)(CPUArchState *env,
++{
- void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
++    tcg_out_dat_imm(s, COND_AL, ARITH_AND, rd, rn, 0xff);
-                        TCGMemOpIdx oi, uintptr_t retaddr)
++}
- {
++
--    unsigned mmu_idx = get_mmuidx(oi);
+ static void __attribute__((unused))
--    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+-tcg_out_ext8u(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn)
--    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
++tcg_out_ext8u_cond(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn)
-+    uintptr_t mmu_idx = get_mmuidx(oi);
+ {
-+    uintptr_t index = tlb_index(env, mmu_idx, addr);
+     tcg_out_dat_imm(s, cond, ARITH_AND, rd, rn, 0xff);
-+    CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
+ }
-+    target_ulong tlb_addr = entry->addr_write;
+@@ -XXX,XX +XXX,XX @@ static TCGReg NAME(TCGContext *s, TCGReg argreg, ARGTYPE arg)              \
-     unsigned a_bits = get_alignment_bits(get_memop(oi));
-     uintptr_t haddr;
+ DEFINE_TCG_OUT_ARG(tcg_out_arg_imm32, uint32_t, tcg_out_movi32,
+     (tcg_out_movi32(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
-@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
+-DEFINE_TCG_OUT_ARG(tcg_out_arg_reg8, TCGReg, tcg_out_ext8u,
-             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE,
+-    (tcg_out_ext8u(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
-                      mmu_idx, retaddr);
++DEFINE_TCG_OUT_ARG(tcg_out_arg_reg8, TCGReg, tcg_out_ext8u_cond,
-         }
++    (tcg_out_ext8u_cond(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
--        tlb_addr = env->tlb_table[mmu_idx][index].addr_write & ~TLB_INVALID_MASK;
+ DEFINE_TCG_OUT_ARG(tcg_out_arg_reg16, TCGReg, tcg_out_ext16u,
-+        tlb_addr = entry->addr_write & ~TLB_INVALID_MASK;
+     (tcg_out_ext16u(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
-     }
+ DEFINE_TCG_OUT_ARG(tcg_out_arg_reg32, TCGReg, tcg_out_mov_reg, )
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-     /* Handle an IO access.  */
+     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
-@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
+     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
-     if (DATA_SIZE > 1
+     case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
-         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
++    case INDEX_op_ext8u_i32:
-                      >= TARGET_PAGE_SIZE)) {
+     default:
--        int i, index2;
+         g_assert_not_reached();
--        target_ulong page2, tlb_addr2;
+     }
-+        int i;
+diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
-+        target_ulong page2;
+index XXXXXXX..XXXXXXX 100644
-+        CPUTLBEntry *entry2;
+--- a/tcg/i386/tcg-target.c.inc
-     do_unaligned_access:
++++ b/tcg/i386/tcg-target.c.inc
-         /* Ensure the second page is in the TLB.  Note that the first page
+@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_rolw_8(TCGContext *s, int reg)
-            is already guaranteed to be filled, and that the second page
+     tcg_out_shifti(s, SHIFT_ROL + P_DATA16, reg, 8);
-            cannot evict the first.  */
+ }
-         page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
--        index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+-static inline void tcg_out_ext8u(TCGContext *s, int dest, int src)
--        tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write;
++static void tcg_out_ext8u(TCGContext *s, TCGReg dest, TCGReg src)
--        if (!tlb_hit_page(tlb_addr2, page2)
+ {
-+        entry2 = tlb_entry(env, mmu_idx, page2);
+     /* movzbl */
-+        if (!tlb_hit_page(entry2->addr_write, page2)
+     tcg_debug_assert(src < 4 || TCG_TARGET_REG_BITS == 64);
-             && !VICTIM_TLB_HIT(addr_write, page2)) {
+@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
-             tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE,
+     OP_32_64(ext16s):
-                      mmu_idx, retaddr);
+         tcg_out_ext16s(s, a0, a1, rexw);
-@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
+         break;
 -    OP_32_64(ext8u):
 -        tcg_out_ext8u(s, a0, a1);
 -        break;
      OP_32_64(ext16u):
          tcg_out_ext16u(s, a0, a1);
          break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
      case INDEX_op_ext8s_i64:
 +    case INDEX_op_ext8u_i32:
 +    case INDEX_op_ext8u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/loongarch64/tcg-target.c.inc
 +++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_brcond(s, a2, a0, a1, arg_label(args[3]));
          break;
 -    case INDEX_op_ext8u_i32:
 -    case INDEX_op_ext8u_i64:
 -        tcg_out_ext8u(s, a0, a1);
 -        break;
 -
      case INDEX_op_ext16s_i32:
      case INDEX_op_ext16s_i64:
          tcg_out_ext16s(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
      case INDEX_op_ext8s_i64:
 +    case INDEX_op_ext8u_i32:
 +    case INDEX_op_ext8u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/mips/tcg-target.c.inc
 +++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
      tcg_out_opc_reg(s, OPC_SEB, rd, TCG_REG_ZERO, rs);
  }
 +static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs)
 +{
 +    tcg_out_opc_imm(s, OPC_ANDI, rd, rs, 0xff);
 +}
 +
  static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
@@ -XXX,XX +XXX,XX @@ static int tcg_out_call_iarg_reg8(TCGContext *s, int i, TCGReg arg)
      if (i < ARRAY_SIZE(tcg_target_call_iarg_regs)) {
          tmp = tcg_target_call_iarg_regs[i];
      }
 -    tcg_out_opc_imm(s, OPC_ANDI, tmp, arg, 0xff);
 +    tcg_out_ext8u(s, tmp, arg);
      return tcg_out_call_iarg_reg(s, i, tmp);
  }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
      case INDEX_op_ext8s_i64:
 +    case INDEX_op_ext8u_i32:
 +    case INDEX_op_ext8u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/ppc/tcg-target.c.inc
 +++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
      tcg_out32(s, EXTSB | RA(dst) | RS(src));
  }
 +static void tcg_out_ext8u(TCGContext *s, TCGReg dst, TCGReg src)
 +{
 +    tcg_out32(s, ANDI | SAI(src, dst, 0xff));
 +}
 +
  static inline void tcg_out_ext16s(TCGContext *s, TCGReg dst, TCGReg src)
  {
      tcg_out32(s, EXTSH | RA(dst) | RS(src));
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_goto_tb:   /* Always emitted via tcg_out_goto_tb.  */
      case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
      case INDEX_op_ext8s_i64:
 +    case INDEX_op_ext8u_i32:
 +    case INDEX_op_ext8u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/riscv/tcg-target.c.inc
 +++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_qemu_st(s, args, true);
          break;
 -    case INDEX_op_ext8u_i32:
 -    case INDEX_op_ext8u_i64:
 -        tcg_out_ext8u(s, a0, a1);
 -        break;
 -
      case INDEX_op_ext16u_i32:
      case INDEX_op_ext16u_i64:
          tcg_out_ext16u(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
      case INDEX_op_ext8s_i64:
 +    case INDEX_op_ext8u_i32:
 +    case INDEX_op_ext8u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/s390x/tcg-target.c.inc
 +++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
      tcg_out_insn(s, RRE, LGBR, dest, src);
  }
 -static void tgen_ext8u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
 +static void tcg_out_ext8u(TCGContext *s, TCGReg dest, TCGReg src)
  {
      tcg_out_insn(s, RRE, LLGCR, dest, src);
  }
@@ -XXX,XX +XXX,XX @@ static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
          return;
      }
+     if ((val & valid) == 0xff) {
--    haddr = addr + env->tlb_table[mmu_idx][index].addend;
+-        tgen_ext8u(s, TCG_TYPE_I64, dest, dest);
-+    haddr = addr + entry->addend;
++        tcg_out_ext8u(s, dest, dest);
  #if DATA_SIZE == 1
      glue(glue(st, SUFFIX), _p)((uint8_t *)haddr, val);
  #else
@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
  void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
                         TCGMemOpIdx oi, uintptr_t retaddr)
  {
 -    unsigned mmu_idx = get_mmuidx(oi);
 -    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 -    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
 +    uintptr_t mmu_idx = get_mmuidx(oi);
 +    uintptr_t index = tlb_index(env, mmu_idx, addr);
 +    CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
 +    target_ulong tlb_addr = entry->addr_write;
      unsigned a_bits = get_alignment_bits(get_memop(oi));
      uintptr_t haddr;
@@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
              tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE,
                       mmu_idx, retaddr);
          }
 -        tlb_addr = env->tlb_table[mmu_idx][index].addr_write & ~TLB_INVALID_MASK;
 +        tlb_addr = entry->addr_write & ~TLB_INVALID_MASK;
      }
      /* Handle an IO access.  */
@@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
      if (DATA_SIZE > 1
          && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
                       >= TARGET_PAGE_SIZE)) {
 -        int i, index2;
 -        target_ulong page2, tlb_addr2;
 +        int i;
 +        target_ulong page2;
 +        CPUTLBEntry *entry2;
      do_unaligned_access:
          /* Ensure the second page is in the TLB.  Note that the first page
             is already guaranteed to be filled, and that the second page
             cannot evict the first.  */
          page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
 -        index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 -        tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write;
 -        if (!tlb_hit_page(tlb_addr2, page2)
 +        entry2 = tlb_entry(env, mmu_idx, page2);
 +        if (!tlb_hit_page(entry2->addr_write, page2)
              && !VICTIM_TLB_HIT(addr_write, page2)) {
              tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE,
                       mmu_idx, retaddr);
@@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
          return;
      }
+     if ((val & valid) == 0xffff) {
--    haddr = addr + env->tlb_table[mmu_idx][index].addend;
+@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
-+    haddr = addr + entry->addend;
+     }
-     glue(glue(st, SUFFIX), _be_p)((uint8_t *)haddr, val);
+     switch (opc & MO_SIZE) {
- }
+     case MO_UB:
- #endif /* DATA_SIZE > 1 */
+-        tgen_ext8u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
-diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h
++        tcg_out_ext8u(s, TCG_REG_R4, data_reg);
-index XXXXXXX..XXXXXXX 100644
+         break;
---- a/include/exec/cpu_ldst.h
+     case MO_UW:
-+++ b/include/exec/cpu_ldst.h
+         tgen_ext16u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
-@@ -XXX,XX +XXX,XX @@ extern __thread uintptr_t helper_retaddr;
+@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
- /* The memory helpers for tcg-generated code need tcg_target_long etc.  */
+     case INDEX_op_ext16s_i32:
- #include "tcg.h"
+         tgen_ext16s(s, TCG_TYPE_I32, args[0], args[1]);
+         break;
-+/* Find the TLB index corresponding to the mmu_idx + address pair.  */
+-    case INDEX_op_ext8u_i32:
-+static inline uintptr_t tlb_index(CPUArchState *env, uintptr_t mmu_idx,
+-        tgen_ext8u(s, TCG_TYPE_I32, args[0], args[1]);
-+                                  target_ulong addr)
+-        break;
-+{
+     case INDEX_op_ext16u_i32:
-+    return (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+         tgen_ext16u(s, TCG_TYPE_I32, args[0], args[1]);
-+}
+         break;
-+
+@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
-+/* Find the TLB entry corresponding to the mmu_idx + address pair.  */
+     case INDEX_op_ext32s_i64:
-+static inline CPUTLBEntry *tlb_entry(CPUArchState *env, uintptr_t mmu_idx,
+         tgen_ext32s(s, args[0], args[1]);
-+                                     target_ulong addr)
+         break;
-+{
+-    case INDEX_op_ext8u_i64:
-+    return &env->tlb_table[mmu_idx][tlb_index(env, mmu_idx, addr)];
+-        tgen_ext8u(s, TCG_TYPE_I64, args[0], args[1]);
-+}
+-        break;
-+
+     case INDEX_op_ext16u_i64:
- #ifdef MMU_MODE0_SUFFIX
+         tgen_ext16u(s, TCG_TYPE_I64, args[0], args[1]);
- #define CPU_MMU_INDEX 0
+         break;
- #define MEMSUFFIX MMU_MODE0_SUFFIX
+@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
-@@ -XXX,XX +XXX,XX @@ static inline void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr,
+     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
- #if defined(CONFIG_USER_ONLY)
+     case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
-     return g2h(addr);
+     case INDEX_op_ext8s_i64:
- #else
++    case INDEX_op_ext8u_i32:
--    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
++    case INDEX_op_ext8u_i64:
--    CPUTLBEntry *tlbentry = &env->tlb_table[mmu_idx][index];
+     default:
-+    CPUTLBEntry *tlbentry = tlb_entry(env, mmu_idx, addr);
+         g_assert_not_reached();
-     abi_ptr tlb_addr;
+     }
-     uintptr_t haddr;
+diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
+index XXXXXXX..XXXXXXX 100644
-@@ -XXX,XX +XXX,XX @@ static inline void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr,
+--- a/tcg/sparc64/tcg-target.c.inc
-         return NULL;
++++ b/tcg/sparc64/tcg-target.c.inc
-     }
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
+     g_assert_not_reached();
--    haddr = addr + env->tlb_table[mmu_idx][index].addend;
+ }
-+    haddr = addr + tlbentry->addend;
-     return (void *)haddr;
++static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs)
- #endif /* defined(CONFIG_USER_ONLY) */
++{
- }
++    tcg_out_arithi(s, rd, rs, 0xff, ARITH_AND);
-diff --git a/include/exec/cpu_ldst_template.h b/include/exec/cpu_ldst_template.h
++}
-index XXXXXXX..XXXXXXX 100644
++
---- a/include/exec/cpu_ldst_template.h
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
-+++ b/include/exec/cpu_ldst_template.h
+                              tcg_target_long imm)
-@@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_ld, USUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+ {
-                                                   target_ulong ptr,
+@@ -XXX,XX +XXX,XX @@ static void emit_extend(TCGContext *s, TCGReg r, int op)
-                                                   uintptr_t retaddr)
+      */
- {
+     switch (op & MO_SIZE) {
--    int page_index;
+     case MO_8:
-+    CPUTLBEntry *entry;
+-        tcg_out_arithi(s, r, r, 0xff, ARITH_AND);
-     RES_TYPE res;
++        tcg_out_ext8u(s, r, r);
-     target_ulong addr;
+         break;
-     int mmu_idx;
+     case MO_16:
-@@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_ld, USUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+         tcg_out_arithi(s, r, r, 16, SHIFT_SLL);
- #endif
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
+     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
-     addr = ptr;
+     case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
--    page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+     case INDEX_op_ext8s_i64:
-     mmu_idx = CPU_MMU_INDEX;
++    case INDEX_op_ext8u_i32:
--    if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ !=
++    case INDEX_op_ext8u_i64:
-+    entry = tlb_entry(env, mmu_idx, addr);
+     default:
-+    if (unlikely(entry->ADDR_READ !=
+         g_assert_not_reached();
-                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
+     }
-         oi = make_memop_idx(SHIFT, mmu_idx);
+diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
-         res = glue(glue(helper_ret_ld, URETSUFFIX), MMUSUFFIX)(env, addr,
+index XXXXXXX..XXXXXXX 100644
-                                                             oi, retaddr);
+--- a/tcg/tci/tcg-target.c.inc
-     } else {
++++ b/tcg/tci/tcg-target.c.inc
--        uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
-+        uintptr_t hostaddr = addr + entry->addend;
+     }
-         res = glue(glue(ld, USUFFIX), _p)((uint8_t *)hostaddr);
+ }
-     }
-     return res;
++static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs)
-@@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_lds, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
++{
-                                                   target_ulong ptr,
++    if (TCG_TARGET_REG_BITS == 64) {
-                                                   uintptr_t retaddr)
++        tcg_debug_assert(TCG_TARGET_HAS_ext8u_i64);
- {
++        tcg_out_op_rr(s, INDEX_op_ext8u_i64, rd, rs);
--    int res, page_index;
++    } else {
-+    CPUTLBEntry *entry;
++        tcg_debug_assert(TCG_TARGET_HAS_ext8u_i32);
-+    int res;
++        tcg_out_op_rr(s, INDEX_op_ext8u_i32, rd, rs);
-     target_ulong addr;
++    }
-     int mmu_idx;
++}
-     TCGMemOpIdx oi;
++
-@@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_lds, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
- #endif
+                              tcg_target_long imm)
+ {
-     addr = ptr;
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
--    page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-     mmu_idx = CPU_MMU_INDEX;
+     CASE_32_64(neg)      /* Optional (TCG_TARGET_HAS_neg_*). */
--    if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ !=
+     CASE_32_64(not)      /* Optional (TCG_TARGET_HAS_not_*). */
-+    entry = tlb_entry(env, mmu_idx, addr);
+-    CASE_32_64(ext8u)    /* Optional (TCG_TARGET_HAS_ext8u_*). */
-+    if (unlikely(entry->ADDR_READ !=
+     CASE_32_64(ext16s)   /* Optional (TCG_TARGET_HAS_ext16s_*). */
-                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
+     CASE_32_64(ext16u)   /* Optional (TCG_TARGET_HAS_ext16u_*). */
-         oi = make_memop_idx(SHIFT, mmu_idx);
+     CASE_64(ext32s)      /* Optional (TCG_TARGET_HAS_ext32s_i64). */
-         res = (DATA_STYPE)glue(glue(helper_ret_ld, SRETSUFFIX),
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-                                MMUSUFFIX)(env, addr, oi, retaddr);
+     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
-     } else {
+     case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
--        uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
+     case INDEX_op_ext8s_i64:
-+        uintptr_t hostaddr = addr + entry->addend;
++    case INDEX_op_ext8u_i32:
-         res = glue(glue(lds, SUFFIX), _p)((uint8_t *)hostaddr);
++    case INDEX_op_ext8u_i64:
-     }
+     default:
-     return res;
+         g_assert_not_reached();
-@@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+     }
                                                   target_ulong ptr,
                                                   RES_TYPE v, uintptr_t retaddr)
  {
 -    int page_index;
 +    CPUTLBEntry *entry;
      target_ulong addr;
      int mmu_idx;
      TCGMemOpIdx oi;
@@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
  #endif
      addr = ptr;
 -    page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
      mmu_idx = CPU_MMU_INDEX;
 -    if (unlikely(env->tlb_table[mmu_idx][page_index].addr_write !=
 +    entry = tlb_entry(env, mmu_idx, addr);
 +    if (unlikely(entry->addr_write !=
                   (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
          oi = make_memop_idx(SHIFT, mmu_idx);
          glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(env, addr, v, oi,
                                                       retaddr);
      } else {
 -        uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
 +        uintptr_t hostaddr = addr + entry->addend;
          glue(glue(st, SUFFIX), _p)((uint8_t *)hostaddr, v);
      }
  }
 diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
 index XXXXXXX..XXXXXXX 100644
 --- a/accel/tcg/cputlb.c
 +++ b/accel/tcg/cputlb.c
@@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data)
  {
      CPUArchState *env = cpu->env_ptr;
      target_ulong addr = (target_ulong) data.target_ptr;
 -    int i;
      int mmu_idx;
      assert_cpu_is_self(cpu);
@@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data)
      }
      addr &= TARGET_PAGE_MASK;
 -    i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
      qemu_spin_lock(&env->tlb_lock);
      for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 -        tlb_flush_entry_locked(&env->tlb_table[mmu_idx][i], addr);
 +        tlb_flush_entry_locked(tlb_entry(env, mmu_idx, addr), addr);
          tlb_flush_vtlb_page_locked(env, mmu_idx, addr);
      }
      qemu_spin_unlock(&env->tlb_lock);
@@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_by_mmuidx_async_work(CPUState *cpu,
      target_ulong addr_and_mmuidx = (target_ulong) data.target_ptr;
      target_ulong addr = addr_and_mmuidx & TARGET_PAGE_MASK;
      unsigned long mmu_idx_bitmap = addr_and_mmuidx & ALL_MMUIDX_BITS;
 -    int page = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
      int mmu_idx;
      assert_cpu_is_self(cpu);
 -    tlb_debug("page:%d addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n",
 -              page, addr, mmu_idx_bitmap);
 +    tlb_debug("flush page addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n",
 +              addr, mmu_idx_bitmap);
      qemu_spin_lock(&env->tlb_lock);
      for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
          if (test_bit(mmu_idx, &mmu_idx_bitmap)) {
 -            tlb_flush_entry_locked(&env->tlb_table[mmu_idx][page], addr);
 +            tlb_flush_entry_locked(tlb_entry(env, mmu_idx, addr), addr);
              tlb_flush_vtlb_page_locked(env, mmu_idx, addr);
          }
      }
@@ -XXX,XX +XXX,XX @@ static inline void tlb_set_dirty1_locked(CPUTLBEntry *tlb_entry,
  void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
  {
      CPUArchState *env = cpu->env_ptr;
 -    int i;
      int mmu_idx;
      assert_cpu_is_self(cpu);
      vaddr &= TARGET_PAGE_MASK;
 -    i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
      qemu_spin_lock(&env->tlb_lock);
      for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 -        tlb_set_dirty1_locked(&env->tlb_table[mmu_idx][i], vaddr);
 +        tlb_set_dirty1_locked(tlb_entry(env, mmu_idx, vaddr), vaddr);
      }
      for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
      iotlb = memory_region_section_get_iotlb(cpu, section, vaddr_page,
                                              paddr_page, xlat, prot, &address);
 -    index = (vaddr_page >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 -    te = &env->tlb_table[mmu_idx][index];
 +    index = tlb_index(env, mmu_idx, vaddr_page);
 +    te = tlb_entry(env, mmu_idx, vaddr_page);
      /*
       * Hold the TLB lock for the rest of the function. We could acquire/release
@@ -XXX,XX +XXX,XX @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry,
           * repeat the MMU check here. This tlb_fill() call might
           * longjump out if this access should cause a guest exception.
           */
 -        int index;
 +        CPUTLBEntry *entry;
          target_ulong tlb_addr;
          tlb_fill(cpu, addr, size, MMU_DATA_LOAD, mmu_idx, retaddr);
 -        index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 -        tlb_addr = env->tlb_table[mmu_idx][index].addr_read;
 +        entry = tlb_entry(env, mmu_idx, addr);
 +        tlb_addr = entry->addr_read;
          if (!(tlb_addr & ~(TARGET_PAGE_MASK | TLB_RECHECK))) {
              /* RAM access */
 -            uintptr_t haddr = addr + env->tlb_table[mmu_idx][index].addend;
 +            uintptr_t haddr = addr + entry->addend;
              return ldn_p((void *)haddr, size);
          }
@@ -XXX,XX +XXX,XX @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry,
           * repeat the MMU check here. This tlb_fill() call might
           * longjump out if this access should cause a guest exception.
           */
 -        int index;
 +        CPUTLBEntry *entry;
          target_ulong tlb_addr;
          tlb_fill(cpu, addr, size, MMU_DATA_STORE, mmu_idx, retaddr);
 -        index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 -        tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
 +        entry = tlb_entry(env, mmu_idx, addr);
 +        tlb_addr = entry->addr_write;
          if (!(tlb_addr & ~(TARGET_PAGE_MASK | TLB_RECHECK))) {
              /* RAM access */
 -            uintptr_t haddr = addr + env->tlb_table[mmu_idx][index].addend;
 +            uintptr_t haddr = addr + entry->addend;
              stn_p((void *)haddr, size, val);
              return;
@@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
   */
  tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr)
  {
 -    int mmu_idx, index;
 +    uintptr_t mmu_idx = cpu_mmu_index(env, true);
 +    uintptr_t index = tlb_index(env, mmu_idx, addr);
 +    CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
      void *p;
 -    index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 -    mmu_idx = cpu_mmu_index(env, true);
 -    if (unlikely(!tlb_hit(env->tlb_table[mmu_idx][index].addr_code, addr))) {
 +    if (unlikely(!tlb_hit(entry->addr_code, addr))) {
          if (!VICTIM_TLB_HIT(addr_code, addr)) {
              tlb_fill(ENV_GET_CPU(env), addr, 0, MMU_INST_FETCH, mmu_idx, 0);
          }
 -        assert(tlb_hit(env->tlb_table[mmu_idx][index].addr_code, addr));
 +        assert(tlb_hit(entry->addr_code, addr));
      }
 -    if (unlikely(env->tlb_table[mmu_idx][index].addr_code &
 -                 (TLB_RECHECK | TLB_MMIO))) {
 +    if (unlikely(entry->addr_code & (TLB_RECHECK | TLB_MMIO))) {
          /*
           * Return -1 if we can't translate and execute from an entire
           * page of RAM here, which will cause us to execute by loading
@@ -XXX,XX +XXX,XX @@ tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr)
          return -1;
      }
 -    p = (void *)((uintptr_t)addr + env->tlb_table[mmu_idx][index].addend);
 +    p = (void *)((uintptr_t)addr + entry->addend);
      return qemu_ram_addr_from_host_nofail(p);
  }
@@ -XXX,XX +XXX,XX @@ tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr)
  void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx,
                   uintptr_t retaddr)
  {
 -    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 -    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
 +    uintptr_t index = tlb_index(env, mmu_idx, addr);
 +    CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
 -    if (!tlb_hit(tlb_addr, addr)) {
 +    if (!tlb_hit(entry->addr_write, addr)) {
          /* TLB entry is for a different page */
          if (!VICTIM_TLB_HIT(addr_write, addr)) {
              tlb_fill(ENV_GET_CPU(env), addr, size, MMU_DATA_STORE,
@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
                                 NotDirtyInfo *ndi)
  {
      size_t mmu_idx = get_mmuidx(oi);
 -    size_t index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 -    CPUTLBEntry *tlbe = &env->tlb_table[mmu_idx][index];
 +    uintptr_t index = tlb_index(env, mmu_idx, addr);
 +    CPUTLBEntry *tlbe = tlb_entry(env, mmu_idx, addr);
      target_ulong tlb_addr = tlbe->addr_write;
      TCGMemOp mop = get_memop(oi);
      int a_bits = get_alignment_bits(mop);
 --
-.17.2
+.34.1

-[Qemu-devel] [PULL 08/21] exec: introduce tlb_init
+[PULL 05/15] tcg: Split out tcg_out_ext16s
-From: "Emilio G. Cota" <cota@braap.org>
+We will need a backend interface for performing 16-bit sign-extend.
 Use it in tcg_reg_alloc_op in the meantime.
-Paves the way for the addition of a per-TLB lock.
+Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
 Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
 Signed-off-by: Emilio G. Cota <cota@braap.org>
 Message-Id: <20181009174557.16125-4-cota@braap.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- include/exec/exec-all.h | 8 ++++++++
+ tcg/tcg.c                        |  7 +++++++
- accel/tcg/cputlb.c      | 4 ++++
+ tcg/aarch64/tcg-target.c.inc     | 13 ++++++++-----
- exec.c                  | 1 +
+ tcg/arm/tcg-target.c.inc         | 10 ++++------
-files changed, 13 insertions(+)
+ tcg/i386/tcg-target.c.inc        | 16 ++++++++--------
  tcg/loongarch64/tcg-target.c.inc | 13 +++++--------
  tcg/mips/tcg-target.c.inc        | 11 ++++++++---
  tcg/ppc/tcg-target.c.inc         | 12 +++++-------
  tcg/riscv/tcg-target.c.inc       |  9 +++------
  tcg/s390x/tcg-target.c.inc       | 12 ++++--------
  tcg/sparc64/tcg-target.c.inc     |  7 +++++++
  tcg/tci/tcg-target.c.inc         | 21 ++++++++++++++++++++-
 files changed, 79 insertions(+), 52 deletions(-)
-diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
+diff --git a/tcg/tcg.c b/tcg/tcg.c
 index XXXXXXX..XXXXXXX 100644
---- a/include/exec/exec-all.h
+--- a/tcg/tcg.c
-+++ b/include/exec/exec-all.h
++++ b/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@ void cpu_address_space_init(CPUState *cpu, int asidx,
+@@ -XXX,XX +XXX,XX @@ static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
+ static void tcg_out_movi(TCGContext *s, TCGType type,
- #if !defined(CONFIG_USER_ONLY) && defined(CONFIG_TCG)
+                          TCGReg ret, tcg_target_long arg);
- /* cputlb.c */
+ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
-+/**
++static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
-+ * tlb_init - initialize a CPU's TLB
+ static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
-+ * @cpu: CPU whose TLB should be initialized
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
-+ */
+ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
-+void tlb_init(CPUState *cpu);
+@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
- /**
+     case INDEX_op_ext8u_i64:
-  * tlb_flush_page:
+         tcg_out_ext8u(s, new_args[0], new_args[1]);
-  * @cpu: CPU whose TLB should be flushed
+         break;
-@@ -XXX,XX +XXX,XX @@ void tlb_set_page(CPUState *cpu, target_ulong vaddr,
++    case INDEX_op_ext16s_i32:
- void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx,
++        tcg_out_ext16s(s, TCG_TYPE_I32, new_args[0], new_args[1]);
-                  uintptr_t retaddr);
++        break;
- #else
++    case INDEX_op_ext16s_i64:
-+static inline void tlb_init(CPUState *cpu)
++        tcg_out_ext16s(s, TCG_TYPE_I64, new_args[0], new_args[1]);
 +        break;
      default:
          if (def->flags & TCG_OPF_VECTOR) {
              tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
 diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/aarch64/tcg-target.c.inc
 +++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rn)
      tcg_out_sxt(s, type, MO_8, rd, rn);
  }
 +static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rn)
 +{
-+}
++    tcg_out_sxt(s, type, MO_16, rd, rn);
  static inline void tlb_flush_page(CPUState *cpu, target_ulong addr)
  {
  }
 diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
 index XXXXXXX..XXXXXXX 100644
 --- a/accel/tcg/cputlb.c
 +++ b/accel/tcg/cputlb.c
@@ -XXX,XX +XXX,XX @@ QEMU_BUILD_BUG_ON(sizeof(target_ulong) > sizeof(run_on_cpu_data));
  QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16);
  #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)
 +void tlb_init(CPUState *cpu)
 +{
 +}
 +
- /* flush_all_helper: run fn across all cpus
+ static inline void tcg_out_uxt(TCGContext *s, MemOp s_bits,
-  *
+                                TCGReg rd, TCGReg rn)
-  * If the wait flag is set then the src cpu's helper will be queued as
+ {
-diff --git a/exec.c b/exec.c
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-index XXXXXXX..XXXXXXX 100644
+         tcg_out_rev(s, TCG_TYPE_I32, MO_16, a0, a1);
---- a/exec.c
+         if (a2 & TCG_BSWAP_OS) {
-+++ b/exec.c
+             /* Output must be sign-extended. */
-@@ -XXX,XX +XXX,XX @@ void cpu_exec_realizefn(CPUState *cpu, Error **errp)
+-            tcg_out_sxt(s, ext, MO_16, a0, a0);
-         tcg_target_initialized = true;
++            tcg_out_ext16s(s, ext, a0, a0);
-         cc->tcg_initialize();
+         } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
-     }
+             /* Output must be zero-extended, but input isn't. */
-+    tlb_init(cpu);
+             tcg_out_uxt(s, MO_16, a0, a0);
+         }
- #ifndef CONFIG_USER_ONLY
+         break;
-     if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
 -    case INDEX_op_ext16s_i64:
 -    case INDEX_op_ext16s_i32:
 -        tcg_out_sxt(s, ext, MO_16, a0, a1);
 -        break;
      case INDEX_op_ext_i32_i64:
      case INDEX_op_ext32s_i64:
          tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext8s_i64:
      case INDEX_op_ext8u_i32:
      case INDEX_op_ext8u_i64:
 +    case INDEX_op_ext16s_i64:
 +    case INDEX_op_ext16s_i32:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/arm/tcg-target.c.inc
 +++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ tcg_out_ext8u_cond(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn)
      tcg_out_dat_imm(s, cond, ARITH_AND, rd, rn, 0xff);
  }
 -static void tcg_out_ext16s(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn)
 +static void tcg_out_ext16s(TCGContext *s, TCGType t, TCGReg rd, TCGReg rn)
  {
      /* sxth */
 -    tcg_out32(s, 0x06bf0070 | (cond << 28) | (rd << 12) | rn);
 +    tcg_out32(s, 0x06bf0070 | (COND_AL << 28) | (rd << 12) | rn);
  }
  static void tcg_out_ext16u(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn)
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
          tcg_out_ext8s(s, TCG_TYPE_I32, datalo, TCG_REG_R0);
          break;
      case MO_SW:
 -        tcg_out_ext16s(s, COND_AL, datalo, TCG_REG_R0);
 +        tcg_out_ext16s(s, TCG_TYPE_I32, datalo, TCG_REG_R0);
          break;
      default:
          tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_R0);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_bswap32(s, COND_AL, args[0], args[1]);
          break;
 -    case INDEX_op_ext16s_i32:
 -        tcg_out_ext16s(s, COND_AL, args[0], args[1]);
 -        break;
      case INDEX_op_ext16u_i32:
          tcg_out_ext16u(s, COND_AL, args[0], args[1]);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
      case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
      case INDEX_op_ext8u_i32:
 +    case INDEX_op_ext16s_i32:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/i386/tcg-target.c.inc
 +++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_ext16u(TCGContext *s, int dest, int src)
      tcg_out_modrm(s, OPC_MOVZWL, dest, src);
  }
 -static inline void tcg_out_ext16s(TCGContext *s, int dest, int src, int rexw)
 +static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
  {
 +    int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
      /* movsw[lq] */
      tcg_out_modrm(s, OPC_MOVSWL + rexw, dest, src);
  }
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
      MemOp opc = get_memop(oi);
      TCGReg data_reg;
      tcg_insn_unit **label_ptr = &l->label_ptr[0];
 -    int rexw = (l->type == TCG_TYPE_I64 ? P_REXW : 0);
      /* resolve label address */
      tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4);
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
          tcg_out_ext8s(s, l->type, data_reg, TCG_REG_EAX);
          break;
      case MO_SW:
 -        tcg_out_ext16s(s, data_reg, TCG_REG_EAX, rexw);
 +        tcg_out_ext16s(s, l->type, data_reg, TCG_REG_EAX);
          break;
  #if TCG_TARGET_REG_BITS == 64
      case MO_SL:
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
                                     TCGReg base, int index, intptr_t ofs,
                                     int seg, bool is64, MemOp memop)
  {
 +    TCGType type = is64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
      bool use_movbe = false;
      int rexw = is64 * P_REXW;
      int movop = OPC_MOVL_GvEv;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
          if (use_movbe) {
              tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg,
                                       datalo, base, index, 0, ofs);
 -            tcg_out_ext16s(s, datalo, datalo, rexw);
 +            tcg_out_ext16s(s, type, datalo, datalo);
          } else {
              tcg_out_modrm_sib_offset(s, OPC_MOVSWL + rexw + seg,
                                       datalo, base, index, 0, ofs);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, a0);
          break;
 -    OP_32_64(ext16s):
 -        tcg_out_ext16s(s, a0, a1, rexw);
 -        break;
      OP_32_64(ext16u):
          tcg_out_ext16u(s, a0, a1);
          break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
          if (a1 < 4 && a0 < 8) {
              tcg_out_modrm(s, OPC_MOVSBL, a0, a1 + 4);
          } else {
 -            tcg_out_ext16s(s, a0, a1, 0);
 +            tcg_out_ext16s(s, TCG_TYPE_I32, a0, a1);
              tcg_out_shifti(s, SHIFT_SAR, a0, 8);
          }
          break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext8s_i64:
      case INDEX_op_ext8u_i32:
      case INDEX_op_ext8u_i64:
 +    case INDEX_op_ext16s_i32:
 +    case INDEX_op_ext16s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/loongarch64/tcg-target.c.inc
 +++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
      tcg_out_opc_sext_b(s, ret, arg);
  }
 -static void tcg_out_ext16s(TCGContext *s, TCGReg ret, TCGReg arg)
 +static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
  {
      tcg_out_opc_sext_h(s, ret, arg);
  }
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
          tcg_out_ext8s(s, type, l->datalo_reg, TCG_REG_A0);
          break;
      case MO_SW:
 -        tcg_out_ext16s(s, l->datalo_reg, TCG_REG_A0);
 +        tcg_out_ext16s(s, type, l->datalo_reg, TCG_REG_A0);
          break;
      case MO_SL:
          tcg_out_ext32s(s, l->datalo_reg, TCG_REG_A0);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_brcond(s, a2, a0, a1, arg_label(args[3]));
          break;
 -    case INDEX_op_ext16s_i32:
 -    case INDEX_op_ext16s_i64:
 -        tcg_out_ext16s(s, a0, a1);
 -        break;
 -
      case INDEX_op_ext16u_i32:
      case INDEX_op_ext16u_i64:
          tcg_out_ext16u(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_bswap16_i64:
          tcg_out_opc_revb_2h(s, a0, a1);
          if (a2 & TCG_BSWAP_OS) {
 -            tcg_out_ext16s(s, a0, a0);
 +            tcg_out_ext16s(s, TCG_TYPE_REG, a0, a0);
          } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
              tcg_out_ext16u(s, a0, a0);
          }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext8s_i64:
      case INDEX_op_ext8u_i32:
      case INDEX_op_ext8u_i64:
 +    case INDEX_op_ext16s_i32:
 +    case INDEX_op_ext16s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/mips/tcg-target.c.inc
 +++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs)
      tcg_out_opc_imm(s, OPC_ANDI, rd, rs, 0xff);
  }
 +static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
 +{
 +    tcg_debug_assert(TCG_TARGET_HAS_ext16s_i32);
 +    tcg_out_opc_reg(s, OPC_SEH, rd, TCG_REG_ZERO, rs);
 +}
 +
  static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_not_i64:
          i1 = OPC_NOR;
          goto do_unary;
 -    case INDEX_op_ext16s_i32:
 -    case INDEX_op_ext16s_i64:
 -        i1 = OPC_SEH;
      do_unary:
          tcg_out_opc_reg(s, i1, a0, TCG_REG_ZERO, a1);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext8s_i64:
      case INDEX_op_ext8u_i32:
      case INDEX_op_ext8u_i64:
 +    case INDEX_op_ext16s_i32:
 +    case INDEX_op_ext16s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/ppc/tcg-target.c.inc
 +++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8u(TCGContext *s, TCGReg dst, TCGReg src)
      tcg_out32(s, ANDI | SAI(src, dst, 0xff));
  }
 -static inline void tcg_out_ext16s(TCGContext *s, TCGReg dst, TCGReg src)
 +static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
  {
      tcg_out32(s, EXTSH | RA(dst) | RS(src));
  }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_bswap16(TCGContext *s, TCGReg dst, TCGReg src, int flags)
      if (have_isa_3_10) {
          tcg_out32(s, BRH | RA(dst) | RS(src));
          if (flags & TCG_BSWAP_OS) {
 -            tcg_out_ext16s(s, dst, dst);
 +            tcg_out_ext16s(s, TCG_TYPE_REG, dst, dst);
          } else if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
              tcg_out_ext16u(s, dst, dst);
          }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_bswap16(TCGContext *s, TCGReg dst, TCGReg src, int flags)
      tcg_out_rlw(s, RLWIMI, tmp, src, 8, 16, 23);
      if (flags & TCG_BSWAP_OS) {
 -        tcg_out_ext16s(s, dst, tmp);
 +        tcg_out_ext16s(s, TCG_TYPE_REG, dst, tmp);
      } else {
          tcg_out_mov(s, TCG_TYPE_REG, dst, tmp);
      }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_qemu_st(s, args, true);
          break;
 -    case INDEX_op_ext16s_i32:
 -    case INDEX_op_ext16s_i64:
 -        tcg_out_ext16s(s, args[0], args[1]);
 -        break;
      case INDEX_op_ext_i32_i64:
      case INDEX_op_ext32s_i64:
          tcg_out_ext32s(s, args[0], args[1]);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext8s_i64:
      case INDEX_op_ext8u_i32:
      case INDEX_op_ext8u_i64:
 +    case INDEX_op_ext16s_i32:
 +    case INDEX_op_ext16s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/riscv/tcg-target.c.inc
 +++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
      tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 24);
  }
 -static void tcg_out_ext16s(TCGContext *s, TCGReg ret, TCGReg arg)
 +static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
  {
      tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16);
      tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 16);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_ext32u(s, a0, a1);
          break;
 -    case INDEX_op_ext16s_i32:
 -    case INDEX_op_ext16s_i64:
 -        tcg_out_ext16s(s, a0, a1);
 -        break;
 -
      case INDEX_op_ext32s_i64:
      case INDEX_op_extrl_i64_i32:
      case INDEX_op_ext_i32_i64:
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext8s_i64:
      case INDEX_op_ext8u_i32:
      case INDEX_op_ext8u_i64:
 +    case INDEX_op_ext16s_i32:
 +    case INDEX_op_ext16s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/s390x/tcg-target.c.inc
 +++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8u(TCGContext *s, TCGReg dest, TCGReg src)
      tcg_out_insn(s, RRE, LLGCR, dest, src);
  }
 -static void tgen_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
 +static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
  {
      tcg_out_insn(s, RRE, LGHR, dest, src);
  }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data,
      case MO_SW | MO_BSWAP:
          /* swapped sign-extended halfword load */
          tcg_out_insn(s, RXY, LRVH, data, base, index, disp);
 -        tgen_ext16s(s, TCG_TYPE_I64, data, data);
 +        tcg_out_ext16s(s, TCG_TYPE_REG, data, data);
          break;
      case MO_SW:
          tcg_out_insn(s, RXY, LGH, data, base, index, disp);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
          }
          break;
 -    case INDEX_op_ext16s_i32:
 -        tgen_ext16s(s, TCG_TYPE_I32, args[0], args[1]);
 -        break;
      case INDEX_op_ext16u_i32:
          tgen_ext16u(s, TCG_TYPE_I32, args[0], args[1]);
          break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
          }
          break;
 -    case INDEX_op_ext16s_i64:
 -        tgen_ext16s(s, TCG_TYPE_I64, args[0], args[1]);
 -        break;
      case INDEX_op_ext_i32_i64:
      case INDEX_op_ext32s_i64:
          tgen_ext32s(s, args[0], args[1]);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext8s_i64:
      case INDEX_op_ext8u_i32:
      case INDEX_op_ext8u_i64:
 +    case INDEX_op_ext16s_i32:
 +    case INDEX_op_ext16s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/sparc64/tcg-target.c.inc
 +++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
      g_assert_not_reached();
  }
 +static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
 +{
 +    g_assert_not_reached();
 +}
 +
  static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs)
  {
      tcg_out_arithi(s, rd, rs, 0xff, ARITH_AND);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext8s_i64:
      case INDEX_op_ext8u_i32:
      case INDEX_op_ext8u_i64:
 +    case INDEX_op_ext16s_i32:
 +    case INDEX_op_ext16s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/tci/tcg-target.c.inc
 +++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs)
      }
  }
 +static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
 +{
 +    switch (type) {
 +    case TCG_TYPE_I32:
 +        tcg_debug_assert(TCG_TARGET_HAS_ext16s_i32);
 +        tcg_out_op_rr(s, INDEX_op_ext16s_i32, rd, rs);
 +        break;
 +#if TCG_TARGET_REG_BITS == 64
 +    case TCG_TYPE_I64:
 +        tcg_debug_assert(TCG_TARGET_HAS_ext16s_i64);
 +        tcg_out_op_rr(s, INDEX_op_ext16s_i64, rd, rs);
 +        break;
 +#endif
 +    default:
 +        g_assert_not_reached();
 +    }
 +}
 +
  static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      CASE_32_64(neg)      /* Optional (TCG_TARGET_HAS_neg_*). */
      CASE_32_64(not)      /* Optional (TCG_TARGET_HAS_not_*). */
 -    CASE_32_64(ext16s)   /* Optional (TCG_TARGET_HAS_ext16s_*). */
      CASE_32_64(ext16u)   /* Optional (TCG_TARGET_HAS_ext16u_*). */
      CASE_64(ext32s)      /* Optional (TCG_TARGET_HAS_ext32s_i64). */
      CASE_64(ext32u)      /* Optional (TCG_TARGET_HAS_ext32u_i64). */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext8s_i64:
      case INDEX_op_ext8u_i32:
      case INDEX_op_ext8u_i64:
 +    case INDEX_op_ext16s_i32:
 +    case INDEX_op_ext16s_i64:
      default:
          g_assert_not_reached();
      }
 --
-.17.2
+.34.1

-[Qemu-devel] [PULL 12/21] tcg: Split CONFIG_ATOMIC128
+[PULL 06/15] tcg: Split out tcg_out_ext16u
-GCC7+ will no longer advertise support for 16-byte __atomic operations
+We will need a backend interface for performing 16-bit zero-extend.
-if only cmpxchg is supported, as for x86_64.  Fortunately, x86_64 still
+Use it in tcg_reg_alloc_op in the meantime.
 has support for __sync_compare_and_swap_16 and we can make use of that.
 AArch64 does not have, nor ever has had such support, so open-code it.
-Reviewed-by: Emilio G. Cota <cota@braap.org>
+Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- accel/tcg/atomic_template.h |  20 ++++-
+ tcg/tcg.c                        |  5 +++++
- include/qemu/atomic128.h    | 155 ++++++++++++++++++++++++++++++++++++
+ tcg/aarch64/tcg-target.c.inc     | 13 ++++++++-----
- tcg/tcg.h                   |  16 ++--
+ tcg/arm/tcg-target.c.inc         | 17 ++++++++++-------
- accel/tcg/cputlb.c          |   3 +-
+ tcg/i386/tcg-target.c.inc        |  8 +++-----
- accel/tcg/user-exec.c       |   5 +-
+ tcg/loongarch64/tcg-target.c.inc |  7 ++-----
- configure                   |  19 +++++
+ tcg/mips/tcg-target.c.inc        |  5 +++++
-files changed, 204 insertions(+), 14 deletions(-)
+ tcg/ppc/tcg-target.c.inc         |  4 +++-
- create mode 100644 include/qemu/atomic128.h
+ tcg/riscv/tcg-target.c.inc       |  7 ++-----
  tcg/s390x/tcg-target.c.inc       | 17 ++++++-----------
  tcg/sparc64/tcg-target.c.inc     | 11 +++++++++--
  tcg/tci/tcg-target.c.inc         | 14 +++++++++++++-
 files changed, 66 insertions(+), 42 deletions(-)
-diff --git a/accel/tcg/atomic_template.h b/accel/tcg/atomic_template.h
+diff --git a/tcg/tcg.c b/tcg/tcg.c
 index XXXXXXX..XXXXXXX 100644
---- a/accel/tcg/atomic_template.h
+--- a/tcg/tcg.c
-+++ b/accel/tcg/atomic_template.h
++++ b/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type,
-     DATA_TYPE ret;
+ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
+ static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
-     ATOMIC_TRACE_RMW;
+ static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
-+#if DATA_SIZE == 16
++static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
-+    ret = atomic16_cmpxchg(haddr, cmpv, newv);
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
-+#else
+ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
-     ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
+ static void tcg_out_goto_tb(TCGContext *s, int which);
-+#endif
+@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
-     ATOMIC_MMU_CLEANUP;
+     case INDEX_op_ext16s_i64:
-     return ret;
+         tcg_out_ext16s(s, TCG_TYPE_I64, new_args[0], new_args[1]);
- }
+         break;
++    case INDEX_op_ext16u_i32:
- #if DATA_SIZE >= 16
++    case INDEX_op_ext16u_i64:
-+#if HAVE_ATOMIC128
++        tcg_out_ext16u(s, new_args[0], new_args[1]);
- ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
++        break;
- {
+     default:
-     ATOMIC_MMU_DECLS;
+         if (def->flags & TCG_OPF_VECTOR) {
-     DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
+             tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
+diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
-     ATOMIC_TRACE_LD;
+index XXXXXXX..XXXXXXX 100644
--    __atomic_load(haddr, &val, __ATOMIC_RELAXED);
+--- a/tcg/aarch64/tcg-target.c.inc
-+    val = atomic16_read(haddr);
++++ b/tcg/aarch64/tcg-target.c.inc
-     ATOMIC_MMU_CLEANUP;
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rn)
-     return val;
+     tcg_out_uxt(s, MO_8, rd, rn);
  }
-@@ -XXX,XX +XXX,XX @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
-     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
++static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rn)
      ATOMIC_TRACE_ST;
 -    __atomic_store(haddr, &val, __ATOMIC_RELAXED);
 +    atomic16_set(haddr, val);
      ATOMIC_MMU_CLEANUP;
  }
 +#endif
  #else
  ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
                             ABI_TYPE val EXTRA_ARGS)
@@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
      DATA_TYPE ret;
      ATOMIC_TRACE_RMW;
 +#if DATA_SIZE == 16
 +    ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
 +#else
      ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
 +#endif
      ATOMIC_MMU_CLEANUP;
      return BSWAP(ret);
  }
  #if DATA_SIZE >= 16
 +#if HAVE_ATOMIC128
  ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
  {
      ATOMIC_MMU_DECLS;
      DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
      ATOMIC_TRACE_LD;
 -    __atomic_load(haddr, &val, __ATOMIC_RELAXED);
 +    val = atomic16_read(haddr);
      ATOMIC_MMU_CLEANUP;
      return BSWAP(val);
  }
@@ -XXX,XX +XXX,XX @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
      ATOMIC_TRACE_ST;
      val = BSWAP(val);
 -    __atomic_store(haddr, &val, __ATOMIC_RELAXED);
 +    atomic16_set(haddr, val);
      ATOMIC_MMU_CLEANUP;
  }
 +#endif
  #else
  ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
                             ABI_TYPE val EXTRA_ARGS)
 diff --git a/include/qemu/atomic128.h b/include/qemu/atomic128.h
 new file mode 100644
 index XXXXXXX..XXXXXXX
 --- /dev/null
 +++ b/include/qemu/atomic128.h
@@ -XXX,XX +XXX,XX @@
 +/*
 + * Simple interface for 128-bit atomic operations.
 + *
 + * Copyright (C) 2018 Linaro, Ltd.
 + *
 + * This work is licensed under the terms of the GNU GPL, version 2 or later.
 + * See the COPYING file in the top-level directory.
 + *
 + * See docs/devel/atomics.txt for discussion about the guarantees each
 + * atomic primitive is meant to provide.
 + */
 +
 +#ifndef QEMU_ATOMIC128_H
 +#define QEMU_ATOMIC128_H
 +
 +/*
 + * GCC is a house divided about supporting large atomic operations.
 + *
 + * For hosts that only have large compare-and-swap, a legalistic reading
 + * of the C++ standard means that one cannot implement __atomic_read on
 + * read-only memory, and thus all atomic operations must synchronize
 + * through libatomic.
 + *
 + * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80878
 + *
 + * This interpretation is not especially helpful for QEMU.
 + * For softmmu, all RAM is always read/write from the hypervisor.
 + * For user-only, if the guest doesn't implement such an __atomic_read
 + * then the host need not worry about it either.
 + *
 + * Moreover, using libatomic is not an option, because its interface is
 + * built for std::atomic<T>, and requires that *all* accesses to such an
 + * object go through the library.  In our case we do not have an object
 + * in the C/C++ sense, but a view of memory as seen by the guest.
 + * The guest may issue a large atomic operation and then access those
 + * pieces using word-sized accesses.  From the hypervisor, we have no
 + * way to connect those two actions.
 + *
 + * Therefore, special case each platform.
 + */
 +
 +#if defined(CONFIG_ATOMIC128)
 +static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
 +{
-+    return atomic_cmpxchg__nocheck(ptr, cmp, new);
++    tcg_out_uxt(s, MO_16, rd, rn);
 +}
 +# define HAVE_CMPXCHG128 1
 +#elif defined(CONFIG_CMPXCHG128)
 +static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
 +{
 +    return __sync_val_compare_and_swap_16(ptr, cmp, new);
 +}
 +# define HAVE_CMPXCHG128 1
 +#elif defined(__aarch64__)
 +/* Through gcc 8, aarch64 has no support for 128-bit at all.  */
 +static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
 +{
 +    uint64_t cmpl = int128_getlo(cmp), cmph = int128_gethi(cmp);
 +    uint64_t newl = int128_getlo(new), newh = int128_gethi(new);
 +    uint64_t oldl, oldh;
 +    uint32_t tmp;
 +
 +    asm("0: ldaxp %[oldl], %[oldh], %[mem]\n\t"
 +        "cmp %[oldl], %[cmpl]\n\t"
 +        "ccmp %[oldh], %[cmph], #0, eq\n\t"
 +        "b.ne 1f\n\t"
 +        "stlxp %w[tmp], %[newl], %[newh], %[mem]\n\t"
 +        "cbnz %w[tmp], 0b\n"
 +        "1:"
 +        : [mem] "+m"(*ptr), [tmp] "=&r"(tmp),
 +          [oldl] "=&r"(oldl), [oldh] "=r"(oldh)
 +        : [cmpl] "r"(cmpl), [cmph] "r"(cmph),
 +          [newl] "r"(newl), [newh] "r"(newh)
 +        : "memory", "cc");
 +
 +    return int128_make128(oldl, oldh);
 +}
 +# define HAVE_CMPXCHG128 1
 +#else
 +/* Fallback definition that must be optimized away, or error.  */
 +Int128 __attribute__((error("unsupported atomic")))
 +    atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new);
 +# define HAVE_CMPXCHG128 0
 +#endif /* Some definition for HAVE_CMPXCHG128 */
 +
 +
 +#if defined(CONFIG_ATOMIC128)
 +static inline Int128 atomic16_read(Int128 *ptr)
 +{
 +    return atomic_read__nocheck(ptr);
 +}
 +
-+static inline void atomic16_set(Int128 *ptr, Int128 val)
+ static void tcg_out_addsubi(TCGContext *s, int ext, TCGReg rd,
                              TCGReg rn, int64_t aimm)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
              tcg_out_ext16s(s, ext, a0, a0);
          } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
              /* Output must be zero-extended, but input isn't. */
 -            tcg_out_uxt(s, MO_16, a0, a0);
 +            tcg_out_ext16u(s, a0, a0);
          }
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext32s_i64:
          tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
          break;
 -    case INDEX_op_ext16u_i64:
 -    case INDEX_op_ext16u_i32:
 -        tcg_out_uxt(s, MO_16, a0, a1);
 -        break;
      case INDEX_op_extu_i32_i64:
      case INDEX_op_ext32u_i64:
          tcg_out_movr(s, TCG_TYPE_I32, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext8u_i64:
      case INDEX_op_ext16s_i64:
      case INDEX_op_ext16s_i32:
 +    case INDEX_op_ext16u_i64:
 +    case INDEX_op_ext16u_i32:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/arm/tcg-target.c.inc
 +++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16s(TCGContext *s, TCGType t, TCGReg rd, TCGReg rn)
      tcg_out32(s, 0x06bf0070 | (COND_AL << 28) | (rd << 12) | rn);
  }
 -static void tcg_out_ext16u(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn)
 +static void tcg_out_ext16u_cond(TCGContext *s, ARMCond cond,
 +                                TCGReg rd, TCGReg rn)
  {
      /* uxth */
      tcg_out32(s, 0x06ff0070 | (cond << 28) | (rd << 12) | rn);
  }
 +static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rn)
 +{
-+    atomic_set__nocheck(ptr, val);
++    tcg_out_ext16u_cond(s, COND_AL, rd, rn);
 +}
 +
-+# define HAVE_ATOMIC128 1
+ static void tcg_out_bswap16(TCGContext *s, ARMCond cond,
-+#elif !defined(CONFIG_USER_ONLY) && defined(__aarch64__)
+                             TCGReg rd, TCGReg rn, int flags)
-+/* We can do better than cmpxchg for AArch64.  */
+ {
-+static inline Int128 atomic16_read(Int128 *ptr)
+@@ -XXX,XX +XXX,XX @@ DEFINE_TCG_OUT_ARG(tcg_out_arg_imm32, uint32_t, tcg_out_movi32,
      (tcg_out_movi32(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
  DEFINE_TCG_OUT_ARG(tcg_out_arg_reg8, TCGReg, tcg_out_ext8u_cond,
      (tcg_out_ext8u_cond(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
 -DEFINE_TCG_OUT_ARG(tcg_out_arg_reg16, TCGReg, tcg_out_ext16u,
 -    (tcg_out_ext16u(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
 +DEFINE_TCG_OUT_ARG(tcg_out_arg_reg16, TCGReg, tcg_out_ext16u_cond,
 +    (tcg_out_ext16u_cond(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
  DEFINE_TCG_OUT_ARG(tcg_out_arg_reg32, TCGReg, tcg_out_mov_reg, )
  static TCGReg tcg_out_arg_reg64(TCGContext *s, TCGReg argreg,
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_bswap32(s, COND_AL, args[0], args[1]);
          break;
 -    case INDEX_op_ext16u_i32:
 -        tcg_out_ext16u(s, COND_AL, args[0], args[1]);
 -        break;
 -
      case INDEX_op_deposit_i32:
          tcg_out_deposit(s, COND_AL, args[0], args[2],
                          args[3], args[4], const_args[2]);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
      case INDEX_op_ext8u_i32:
      case INDEX_op_ext16s_i32:
 +    case INDEX_op_ext16u_i32:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/i386/tcg-target.c.inc
 +++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
      tcg_out_modrm(s, OPC_MOVSBL + P_REXB_RM + rexw, dest, src);
  }
 -static inline void tcg_out_ext16u(TCGContext *s, int dest, int src)
 +static void tcg_out_ext16u(TCGContext *s, TCGReg dest, TCGReg src)
  {
      /* movzwl */
      tcg_out_modrm(s, OPC_MOVZWL, dest, src);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, a0);
          break;
 -    OP_32_64(ext16u):
 -        tcg_out_ext16u(s, a0, a1);
 -        break;
 -
      case INDEX_op_qemu_ld_i32:
          tcg_out_qemu_ld(s, args, 0);
          break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext8u_i64:
      case INDEX_op_ext16s_i32:
      case INDEX_op_ext16s_i64:
 +    case INDEX_op_ext16u_i32:
 +    case INDEX_op_ext16u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/loongarch64/tcg-target.c.inc
 +++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_brcond(s, a2, a0, a1, arg_label(args[3]));
          break;
 -    case INDEX_op_ext16u_i32:
 -    case INDEX_op_ext16u_i64:
 -        tcg_out_ext16u(s, a0, a1);
 -        break;
 -
      case INDEX_op_ext32u_i64:
      case INDEX_op_extu_i32_i64:
          tcg_out_ext32u(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext8u_i64:
      case INDEX_op_ext16s_i32:
      case INDEX_op_ext16s_i64:
 +    case INDEX_op_ext16u_i32:
 +    case INDEX_op_ext16u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/mips/tcg-target.c.inc
 +++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
      tcg_out_opc_reg(s, OPC_SEH, rd, TCG_REG_ZERO, rs);
  }
 +static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rs)
 +{
-+    uint64_t l, h;
++    tcg_out_opc_imm(s, OPC_ANDI, rd, rs, 0xffff);
 +    uint32_t tmp;
 +
 +    /* The load must be paired with the store to guarantee not tearing.  */
 +    asm("0: ldxp %[l], %[h], %[mem]\n\t"
 +        "stxp %w[tmp], %[l], %[h], %[mem]\n\t"
 +        "cbnz %w[tmp], 0b"
 +        : [mem] "+m"(*ptr), [tmp] "=r"(tmp), [l] "=r"(l), [h] "=r"(h));
 +
 +    return int128_make128(l, h);
 +}
 +
-+static inline void atomic16_set(Int128 *ptr, Int128 val)
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
 diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/ppc/tcg-target.c.inc
 +++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
      tcg_out32(s, EXTSH | RA(dst) | RS(src));
  }
 -static inline void tcg_out_ext16u(TCGContext *s, TCGReg dst, TCGReg src)
 +static void tcg_out_ext16u(TCGContext *s, TCGReg dst, TCGReg src)
  {
      tcg_out32(s, ANDI | SAI(src, dst, 0xffff));
  }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext8u_i64:
      case INDEX_op_ext16s_i32:
      case INDEX_op_ext16s_i64:
 +    case INDEX_op_ext16u_i32:
 +    case INDEX_op_ext16u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/riscv/tcg-target.c.inc
 +++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_qemu_st(s, args, true);
          break;
 -    case INDEX_op_ext16u_i32:
 -    case INDEX_op_ext16u_i64:
 -        tcg_out_ext16u(s, a0, a1);
 -        break;
 -
      case INDEX_op_ext32u_i64:
      case INDEX_op_extu_i32_i64:
          tcg_out_ext32u(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext8u_i64:
      case INDEX_op_ext16s_i32:
      case INDEX_op_ext16s_i64:
 +    case INDEX_op_ext16u_i32:
 +    case INDEX_op_ext16u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/s390x/tcg-target.c.inc
 +++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
      tcg_out_insn(s, RRE, LGHR, dest, src);
  }
 -static void tgen_ext16u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
 +static void tcg_out_ext16u(TCGContext *s, TCGReg dest, TCGReg src)
  {
      tcg_out_insn(s, RRE, LLGHR, dest, src);
  }
@@ -XXX,XX +XXX,XX @@ static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
          return;
      }
      if ((val & valid) == 0xffff) {
 -        tgen_ext16u(s, TCG_TYPE_I64, dest, dest);
 +        tcg_out_ext16u(s, dest, dest);
          return;
      }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data,
      case MO_UW | MO_BSWAP:
          /* swapped unsigned halfword load with upper bits zeroed */
          tcg_out_insn(s, RXY, LRVH, data, base, index, disp);
 -        tgen_ext16u(s, TCG_TYPE_I64, data, data);
 +        tcg_out_ext16u(s, data, data);
          break;
      case MO_UW:
          tcg_out_insn(s, RXY, LLGH, data, base, index, disp);
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
          tcg_out_ext8u(s, TCG_REG_R4, data_reg);
          break;
      case MO_UW:
 -        tgen_ext16u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
 +        tcg_out_ext16u(s, TCG_REG_R4, data_reg);
          break;
      case MO_UL:
          tgen_ext32u(s, TCG_REG_R4, data_reg);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
          }
          break;
 -    case INDEX_op_ext16u_i32:
 -        tgen_ext16u(s, TCG_TYPE_I32, args[0], args[1]);
 -        break;
 -
      case INDEX_op_bswap16_i32:
          a0 = args[0], a1 = args[1], a2 = args[2];
          tcg_out_insn(s, RRE, LRVR, a0, a1);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext32s_i64:
          tgen_ext32s(s, args[0], args[1]);
          break;
 -    case INDEX_op_ext16u_i64:
 -        tgen_ext16u(s, TCG_TYPE_I64, args[0], args[1]);
 -        break;
      case INDEX_op_extu_i32_i64:
      case INDEX_op_ext32u_i64:
          tgen_ext32u(s, args[0], args[1]);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext8u_i64:
      case INDEX_op_ext16s_i32:
      case INDEX_op_ext16s_i64:
 +    case INDEX_op_ext16u_i32:
 +    case INDEX_op_ext16u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/sparc64/tcg-target.c.inc
 +++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs)
      tcg_out_arithi(s, rd, rs, 0xff, ARITH_AND);
  }
 +static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rs)
 +{
-+    uint64_t l = int128_getlo(val), h = int128_gethi(val);
++    tcg_out_arithi(s, rd, rs, 16, SHIFT_SLL);
-+    uint64_t t1, t2;
++    tcg_out_arithi(s, rd, rd, 16, SHIFT_SRL);
 +
 +    /* Load into temporaries to acquire the exclusive access lock.  */
 +    asm("0: ldxp %[t1], %[t2], %[mem]\n\t"
 +        "stxp %w[t1], %[l], %[h], %[mem]\n\t"
 +        "cbnz %w[t1], 0b"
 +        : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2)
 +        : [l] "r"(l), [h] "r"(h));
 +}
 +
-+# define HAVE_ATOMIC128 1
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
-+#elif !defined(CONFIG_USER_ONLY) && HAVE_CMPXCHG128
+                              tcg_target_long imm)
-+static inline Int128 atomic16_read(Int128 *ptr)
+ {
@@ -XXX,XX +XXX,XX @@ static void emit_extend(TCGContext *s, TCGReg r, int op)
          tcg_out_ext8u(s, r, r);
          break;
      case MO_16:
 -        tcg_out_arithi(s, r, r, 16, SHIFT_SLL);
 -        tcg_out_arithi(s, r, r, 16, SHIFT_SRL);
 +        tcg_out_ext16u(s, r, r);
          break;
      case MO_32:
          tcg_out_arith(s, r, r, 0, SHIFT_SRL);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext8u_i64:
      case INDEX_op_ext16s_i32:
      case INDEX_op_ext16s_i64:
 +    case INDEX_op_ext16u_i32:
 +    case INDEX_op_ext16u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/tci/tcg-target.c.inc
 +++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
      }
  }
 +static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rs)
 +{
-+    /* Maybe replace 0 with 0, returning the old value.  */
++    if (TCG_TARGET_REG_BITS == 64) {
-+    return atomic16_cmpxchg(ptr, 0, 0);
++        tcg_debug_assert(TCG_TARGET_HAS_ext16u_i64);
 +        tcg_out_op_rr(s, INDEX_op_ext16u_i64, rd, rs);
 +    } else {
 +        tcg_debug_assert(TCG_TARGET_HAS_ext16u_i32);
 +        tcg_out_op_rr(s, INDEX_op_ext16u_i32, rd, rs);
 +    }
 +}
 +
-+static inline void atomic16_set(Int128 *ptr, Int128 val)
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
-+{
+                              tcg_target_long imm)
-+    Int128 old = *ptr, cmp;
+ {
-+    do {
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-+        cmp = old;
-+        old = atomic16_cmpxchg(ptr, cmp, val);
+     CASE_32_64(neg)      /* Optional (TCG_TARGET_HAS_neg_*). */
-+    } while (old != cmp);
+     CASE_32_64(not)      /* Optional (TCG_TARGET_HAS_not_*). */
-+}
+-    CASE_32_64(ext16u)   /* Optional (TCG_TARGET_HAS_ext16u_*). */
-+
+     CASE_64(ext32s)      /* Optional (TCG_TARGET_HAS_ext32s_i64). */
-+# define HAVE_ATOMIC128 1
+     CASE_64(ext32u)      /* Optional (TCG_TARGET_HAS_ext32u_i64). */
-+#else
+     CASE_64(ext_i32)
-+/* Fallback definitions that must be optimized away, or error.  */
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-+Int128 __attribute__((error("unsupported atomic")))
+     case INDEX_op_ext8u_i64:
-+    atomic16_read(Int128 *ptr);
+     case INDEX_op_ext16s_i32:
-+void __attribute__((error("unsupported atomic")))
+     case INDEX_op_ext16s_i64:
-+    atomic16_set(Int128 *ptr, Int128 val);
++    case INDEX_op_ext16u_i32:
-+# define HAVE_ATOMIC128 0
++    case INDEX_op_ext16u_i64:
-+#endif /* Some definition for HAVE_ATOMIC128 */
+     default:
-+
+         g_assert_not_reached();
-+#endif /* QEMU_ATOMIC128_H */
+     }
 diff --git a/tcg/tcg.h b/tcg/tcg.h
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/tcg.h
 +++ b/tcg/tcg.h
@@ -XXX,XX +XXX,XX @@
  #include "qemu/queue.h"
  #include "tcg-mo.h"
  #include "tcg-target.h"
 +#include "qemu/int128.h"
  /* XXX: make safe guess about sizes */
  #define MAX_OP_PER_INSTR 266
@@ -XXX,XX +XXX,XX @@ GEN_ATOMIC_HELPER_ALL(xchg)
  #undef GEN_ATOMIC_HELPER
  #endif /* CONFIG_SOFTMMU */
 -#ifdef CONFIG_ATOMIC128
 -#include "qemu/int128.h"
 -
 -/* These aren't really a "proper" helpers because TCG cannot manage Int128.
 -   However, use the same format as the others, for use by the backends. */
 +/*
 + * These aren't really a "proper" helpers because TCG cannot manage Int128.
 + * However, use the same format as the others, for use by the backends.
 + *
 + * The cmpxchg functions are only defined if HAVE_CMPXCHG128;
 + * the ld/st functions are only defined if HAVE_ATOMIC128,
 + * as defined by <qemu/atomic128.h>.
 + */
  Int128 helper_atomic_cmpxchgo_le_mmu(CPUArchState *env, target_ulong addr,
                                       Int128 cmpv, Int128 newv,
                                       TCGMemOpIdx oi, uintptr_t retaddr);
@@ -XXX,XX +XXX,XX @@ void helper_atomic_sto_le_mmu(CPUArchState *env, target_ulong addr, Int128 val,
  void helper_atomic_sto_be_mmu(CPUArchState *env, target_ulong addr, Int128 val,
                                TCGMemOpIdx oi, uintptr_t retaddr);
 -#endif /* CONFIG_ATOMIC128 */
 -
  #endif /* TCG_H */
 diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
 index XXXXXXX..XXXXXXX 100644
 --- a/accel/tcg/cputlb.c
 +++ b/accel/tcg/cputlb.c
@@ -XXX,XX +XXX,XX @@
  #include "exec/log.h"
  #include "exec/helper-proto.h"
  #include "qemu/atomic.h"
 +#include "qemu/atomic128.h"
  /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
  /* #define DEBUG_TLB */
@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
  #include "atomic_template.h"
  #endif
 -#ifdef CONFIG_ATOMIC128
 +#if HAVE_CMPXCHG128 || HAVE_ATOMIC128
  #define DATA_SIZE 16
  #include "atomic_template.h"
  #endif
 diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c
 index XXXXXXX..XXXXXXX 100644
 --- a/accel/tcg/user-exec.c
 +++ b/accel/tcg/user-exec.c
@@ -XXX,XX +XXX,XX @@
  #include "exec/cpu_ldst.h"
  #include "translate-all.h"
  #include "exec/helper-proto.h"
 +#include "qemu/atomic128.h"
  #undef EAX
  #undef ECX
@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
  /* The following is only callable from other helpers, and matches up
     with the softmmu version.  */
 -#ifdef CONFIG_ATOMIC128
 +#if HAVE_ATOMIC128 || HAVE_CMPXCHG128
  #undef EXTRA_ARGS
  #undef ATOMIC_NAME
@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
  #define DATA_SIZE 16
  #include "atomic_template.h"
 -#endif /* CONFIG_ATOMIC128 */
 +#endif
 diff --git a/configure b/configure
 index XXXXXXX..XXXXXXX 100755
 --- a/configure
 +++ b/configure
@@ -XXX,XX +XXX,XX @@ EOF
    fi
  fi
 +cmpxchg128=no
 +if test "$int128" = yes -a "$atomic128" = no; then
 +  cat > $TMPC << EOF
 +int main(void)
 +{
 +  unsigned __int128 x = 0, y = 0;
 +  __sync_val_compare_and_swap_16(&x, y, x);
 +  return 0;
 +}
 +EOF
 +  if compile_prog "" "" ; then
 +    cmpxchg128=yes
 +  fi
 +fi
 +
  #########################################
  # See if 64-bit atomic operations are supported.
  # Note that without __atomic builtins, we can only
@@ -XXX,XX +XXX,XX @@ if test "$atomic128" = "yes" ; then
    echo "CONFIG_ATOMIC128=y" >> $config_host_mak
  fi
 +if test "$cmpxchg128" = "yes" ; then
 +  echo "CONFIG_CMPXCHG128=y" >> $config_host_mak
 +fi
 +
  if test "$atomic64" = "yes" ; then
    echo "CONFIG_ATOMIC64=y" >> $config_host_mak
  fi
 --
-.17.2
+.34.1

-[Qemu-devel] [PULL 20/21] target/s390x: Check HAVE_ATOMIC128 and HAVE_CMPXCHG128 at translate
+[PULL 07/15] tcg: Split out tcg_out_ext32s
-Reviewed-by: David Hildenbrand <david@redhat.com>
+We will need a backend interface for performing 32-bit sign-extend.
 Use it in tcg_reg_alloc_op in the meantime.
 Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- target/s390x/mem_helper.c | 40 +++++++++++++++++++--------------------
+ tcg/tcg.c                        |  4 ++++
- target/s390x/translate.c  | 25 +++++++++++++++++-------
+ tcg/aarch64/tcg-target.c.inc     |  9 +++++++--
-files changed, 38 insertions(+), 27 deletions(-)
+ tcg/arm/tcg-target.c.inc         |  5 +++++
  tcg/i386/tcg-target.c.inc        |  5 +++--
  tcg/loongarch64/tcg-target.c.inc |  2 +-
  tcg/mips/tcg-target.c.inc        | 12 +++++++++---
  tcg/ppc/tcg-target.c.inc         |  5 +++--
  tcg/riscv/tcg-target.c.inc       |  2 +-
  tcg/s390x/tcg-target.c.inc       | 10 +++++-----
  tcg/sparc64/tcg-target.c.inc     | 11 ++++++++---
  tcg/tci/tcg-target.c.inc         |  9 ++++++++-
 files changed, 54 insertions(+), 20 deletions(-)
-diff --git a/target/s390x/mem_helper.c b/target/s390x/mem_helper.c
+diff --git a/tcg/tcg.c b/tcg/tcg.c
 index XXXXXXX..XXXXXXX 100644
---- a/target/s390x/mem_helper.c
+--- a/tcg/tcg.c
-+++ b/target/s390x/mem_helper.c
++++ b/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@ void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr,
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
-     Int128 oldv;
+ static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
-     bool fail;
+ static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
+ static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
--    if (!HAVE_CMPXCHG128) {
++static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
--        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
--    }
+ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
-+    assert(HAVE_CMPXCHG128);
+ static void tcg_out_goto_tb(TCGContext *s, int which);
+@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
-     mem_idx = cpu_mmu_index(env, false);
+     case INDEX_op_ext16u_i64:
-     oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
+         tcg_out_ext16u(s, new_args[0], new_args[1]);
-@@ -XXX,XX +XXX,XX @@ uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr)
+         break;
- {
++    case INDEX_op_ext32s_i64:
-     uintptr_t ra = GETPC();
++        tcg_out_ext32s(s, new_args[0], new_args[1]);
-     uint64_t hi, lo;
++        break;
-+    int mem_idx;
+     default:
-+    TCGMemOpIdx oi;
+         if (def->flags & TCG_OPF_VECTOR) {
-+    Int128 v;
+             tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
+diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
--    if (HAVE_ATOMIC128) {
+index XXXXXXX..XXXXXXX 100644
--        int mem_idx = cpu_mmu_index(env, false);
+--- a/tcg/aarch64/tcg-target.c.inc
--        TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
++++ b/tcg/aarch64/tcg-target.c.inc
--        Int128 v = helper_atomic_ldo_be_mmu(env, addr, oi, ra);
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rn)
--        hi = int128_gethi(v);
+     tcg_out_sxt(s, type, MO_16, rd, rn);
--        lo = int128_getlo(v);
+ }
--    } else {
--        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
++static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rn)
--    }
++{
-+    assert(HAVE_ATOMIC128);
++    tcg_out_sxt(s, TCG_TYPE_I64, MO_32, rd, rn);
-+
++}
-+    mem_idx = cpu_mmu_index(env, false);
++
-+    oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
+ static inline void tcg_out_uxt(TCGContext *s, MemOp s_bits,
-+    v = helper_atomic_ldo_be_mmu(env, addr, oi, ra);
+                                TCGReg rd, TCGReg rn)
-+    hi = int128_gethi(v);
+ {
-+    lo = int128_getlo(v);
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
+     case INDEX_op_bswap32_i64:
-     env->retxl = lo;
+         tcg_out_rev(s, TCG_TYPE_I32, MO_32, a0, a1);
-     return hi;
+         if (a2 & TCG_BSWAP_OS) {
-@@ -XXX,XX +XXX,XX @@ void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr,
+-            tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a0);
-                            uint64_t low, uint64_t high)
++            tcg_out_ext32s(s, a0, a0);
- {
+         }
-     uintptr_t ra = GETPC();
+         break;
-+    int mem_idx;
+     case INDEX_op_bswap32_i32:
-+    TCGMemOpIdx oi;
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-+    Int128 v;
+         break;
--    if (HAVE_ATOMIC128) {
+     case INDEX_op_ext_i32_i64:
--        int mem_idx = cpu_mmu_index(env, false);
+-    case INDEX_op_ext32s_i64:
--        TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
+         tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
--        Int128 v = int128_make128(low, high);
+         break;
--        helper_atomic_sto_be_mmu(env, addr, v, oi, ra);
+     case INDEX_op_extu_i32_i64:
--    } else {
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
--        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
+     case INDEX_op_ext16s_i32:
--    }
+     case INDEX_op_ext16u_i64:
-+    assert(HAVE_ATOMIC128);
+     case INDEX_op_ext16u_i32:
-+
++    case INDEX_op_ext32s_i64:
-+    mem_idx = cpu_mmu_index(env, false);
+     default:
-+    oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
+         g_assert_not_reached();
-+    v = int128_make128(low, high);
+     }
-+    helper_atomic_sto_be_mmu(env, addr, v, oi, ra);
+diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
- }
+index XXXXXXX..XXXXXXX 100644
+--- a/tcg/arm/tcg-target.c.inc
- /* Execute instruction.  This instruction executes an insn modified with
++++ b/tcg/arm/tcg-target.c.inc
-diff --git a/target/s390x/translate.c b/target/s390x/translate.c
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rn)
-index XXXXXXX..XXXXXXX 100644
+     tcg_out_ext16u_cond(s, COND_AL, rd, rn);
---- a/target/s390x/translate.c
+ }
-+++ b/target/s390x/translate.c
-@@ -XXX,XX +XXX,XX @@
++static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rn)
- #include "trace-tcg.h"
++{
- #include "exec/translator.h"
++    g_assert_not_reached();
- #include "exec/log.h"
++}
-+#include "qemu/atomic128.h"
++
+ static void tcg_out_bswap16(TCGContext *s, ARMCond cond,
+                             TCGReg rd, TCGReg rn, int flags)
- /* Information that (most) every instruction needs to manipulate.  */
+ {
-@@ -XXX,XX +XXX,XX @@ static DisasJumpType op_cdsg(DisasContext *s, DisasOps *o)
+diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
-     int r3 = get_field(s->fields, r3);
+index XXXXXXX..XXXXXXX 100644
-     int d2 = get_field(s->fields, d2);
+--- a/tcg/i386/tcg-target.c.inc
-     int b2 = get_field(s->fields, b2);
++++ b/tcg/i386/tcg-target.c.inc
-+    DisasJumpType ret = DISAS_NEXT;
+@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_ext32u(TCGContext *s, int dest, int src)
-     TCGv_i64 addr;
+     tcg_out_modrm(s, OPC_MOVL_GvEv, dest, src);
-     TCGv_i32 t_r1, t_r3;
+ }
-@@ -XXX,XX +XXX,XX @@ static DisasJumpType op_cdsg(DisasContext *s, DisasOps *o)
+-static inline void tcg_out_ext32s(TCGContext *s, int dest, int src)
-     addr = get_address(s, 0, b2, d2);
++static void tcg_out_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
-     t_r1 = tcg_const_i32(r1);
+ {
-     t_r3 = tcg_const_i32(r3);
++    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
--    if (tb_cflags(s->base.tb) & CF_PARALLEL) {
+     tcg_out_modrm(s, OPC_MOVSLQ, dest, src);
-+    if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
+ }
-+        gen_helper_cdsg(cpu_env, addr, t_r1, t_r3);
-+    } else if (HAVE_CMPXCHG128) {
+@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
-         gen_helper_cdsg_parallel(cpu_env, addr, t_r1, t_r3);
+         tcg_out_ext32u(s, a0, a1);
          break;
      case INDEX_op_ext_i32_i64:
 -    case INDEX_op_ext32s_i64:
          tcg_out_ext32s(s, a0, a1);
          break;
      case INDEX_op_extrh_i64_i32:
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16s_i64:
      case INDEX_op_ext16u_i32:
      case INDEX_op_ext16u_i64:
 +    case INDEX_op_ext32s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/loongarch64/tcg-target.c.inc
 +++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_ext32u(s, a0, a1);
          break;
 -    case INDEX_op_ext32s_i64:
      case INDEX_op_extrl_i64_i32:
      case INDEX_op_ext_i32_i64:
          tcg_out_ext32s(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16s_i64:
      case INDEX_op_ext16u_i32:
      case INDEX_op_ext16u_i64:
 +    case INDEX_op_ext32s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/mips/tcg-target.c.inc
 +++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rs)
      tcg_out_opc_imm(s, OPC_ANDI, rd, rs, 0xffff);
  }
 +static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rs)
 +{
 +    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
 +    tcg_out_opc_sa(s, OPC_SLL, rd, rs, 0);
 +}
 +
  static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
      /* delay slot */
      if (TCG_TARGET_REG_BITS == 64 && l->type == TCG_TYPE_I32) {
          /* we always sign-extend 32-bit loads */
 -        tcg_out_opc_sa(s, OPC_SLL, v0, TCG_REG_V0, 0);
 +        tcg_out_ext32s(s, v0, TCG_REG_V0);
      } else {
--        gen_helper_cdsg(cpu_env, addr, t_r1, t_r3);
+         tcg_out_opc_reg(s, OPC_OR, v0, TCG_REG_V0, TCG_REG_ZERO);
-+        gen_helper_exit_atomic(cpu_env);
+     }
-+        ret = DISAS_NORETURN;
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-     }
+     case INDEX_op_extrh_i64_i32:
-     tcg_temp_free_i64(addr);
+         tcg_out_dsra(s, a0, a1, 32);
-     tcg_temp_free_i32(t_r1);
+         break;
-     tcg_temp_free_i32(t_r3);
+-    case INDEX_op_ext32s_i64:
+     case INDEX_op_ext_i32_i64:
-     set_cc_static(s);
+     case INDEX_op_extrl_i64_i32:
--    return DISAS_NEXT;
+-        tcg_out_opc_sa(s, OPC_SLL, a0, a1, 0);
-+    return ret;
++        tcg_out_ext32s(s, a0, a1);
- }
+         break;
+     case INDEX_op_ext32u_i64:
- static DisasJumpType op_csst(DisasContext *s, DisasOps *o)
+     case INDEX_op_extu_i32_i64:
-@@ -XXX,XX +XXX,XX @@ static DisasJumpType op_lpd(DisasContext *s, DisasOps *o)
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
+     case INDEX_op_ext8u_i64:
- static DisasJumpType op_lpq(DisasContext *s, DisasOps *o)
+     case INDEX_op_ext16s_i32:
- {
+     case INDEX_op_ext16s_i64:
--    if (tb_cflags(s->base.tb) & CF_PARALLEL) {
++    case INDEX_op_ext32s_i64:
-+    if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
+     default:
-+        gen_helper_lpq(o->out, cpu_env, o->in2);
+         g_assert_not_reached();
-+    } else if (HAVE_ATOMIC128) {
+     }
-         gen_helper_lpq_parallel(o->out, cpu_env, o->in2);
+diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/ppc/tcg-target.c.inc
 +++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16u(TCGContext *s, TCGReg dst, TCGReg src)
      tcg_out32(s, ANDI | SAI(src, dst, 0xffff));
  }
 -static inline void tcg_out_ext32s(TCGContext *s, TCGReg dst, TCGReg src)
 +static void tcg_out_ext32s(TCGContext *s, TCGReg dst, TCGReg src)
  {
 +    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
      tcg_out32(s, EXTSW | RA(dst) | RS(src));
  }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          break;
      case INDEX_op_ext_i32_i64:
 -    case INDEX_op_ext32s_i64:
          tcg_out_ext32s(s, args[0], args[1]);
          break;
      case INDEX_op_extu_i32_i64:
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16s_i64:
      case INDEX_op_ext16u_i32:
      case INDEX_op_ext16u_i64:
 +    case INDEX_op_ext32s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/riscv/tcg-target.c.inc
 +++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_ext32u(s, a0, a1);
          break;
 -    case INDEX_op_ext32s_i64:
      case INDEX_op_extrl_i64_i32:
      case INDEX_op_ext_i32_i64:
          tcg_out_ext32s(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16s_i64:
      case INDEX_op_ext16u_i32:
      case INDEX_op_ext16u_i64:
 +    case INDEX_op_ext32s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/s390x/tcg-target.c.inc
 +++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16u(TCGContext *s, TCGReg dest, TCGReg src)
      tcg_out_insn(s, RRE, LLGHR, dest, src);
  }
 -static inline void tgen_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
 +static void tcg_out_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
  {
      tcg_out_insn(s, RRE, LGFR, dest, src);
  }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data,
      case MO_SL | MO_BSWAP:
          /* swapped sign-extended int load */
          tcg_out_insn(s, RXY, LRV, data, base, index, disp);
 -        tgen_ext32s(s, data, data);
 +        tcg_out_ext32s(s, data, data);
          break;
      case MO_SL:
          tcg_out_insn(s, RXY, LGF, data, base, index, disp);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
          a0 = args[0], a1 = args[1], a2 = args[2];
          tcg_out_insn(s, RRE, LRVR, a0, a1);
          if (a2 & TCG_BSWAP_OS) {
 -            tgen_ext32s(s, a0, a0);
 +            tcg_out_ext32s(s, a0, a0);
          } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
              tgen_ext32u(s, a0, a0);
          }
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
          break;
      case INDEX_op_ext_i32_i64:
 -    case INDEX_op_ext32s_i64:
 -        tgen_ext32s(s, args[0], args[1]);
 +        tcg_out_ext32s(s, args[0], args[1]);
          break;
      case INDEX_op_extu_i32_i64:
      case INDEX_op_ext32u_i64:
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16s_i64:
      case INDEX_op_ext16u_i32:
      case INDEX_op_ext16u_i64:
 +    case INDEX_op_ext32s_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/sparc64/tcg-target.c.inc
 +++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rs)
      tcg_out_arithi(s, rd, rd, 16, SHIFT_SRL);
  }
 +static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rs)
 +{
 +    tcg_out_arithi(s, rd, rs, 0, SHIFT_SRA);
 +}
 +
  static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr,
      /* We let the helper sign-extend SB and SW, but leave SL for here.  */
      if (is_64 && (memop & MO_SSIZE) == MO_SL) {
 -        tcg_out_arithi(s, data, TCG_REG_O0, 0, SHIFT_SRA);
 +        tcg_out_ext32s(s, data, TCG_REG_O0);
      } else {
--        gen_helper_lpq(o->out, cpu_env, o->in2);
+         tcg_out_mov(s, TCG_TYPE_REG, data, TCG_REG_O0);
-+        gen_helper_exit_atomic(cpu_env);
+     }
-+        return DISAS_NORETURN;
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-     }
+         c = ARITH_UDIVX;
-     return_low128(o->out2);
+         goto gen_arith;
-     return DISAS_NEXT;
+     case INDEX_op_ext_i32_i64:
-@@ -XXX,XX +XXX,XX @@ static DisasJumpType op_stmh(DisasContext *s, DisasOps *o)
+-    case INDEX_op_ext32s_i64:
+-        tcg_out_arithi(s, a0, a1, 0, SHIFT_SRA);
- static DisasJumpType op_stpq(DisasContext *s, DisasOps *o)
++        tcg_out_ext32s(s, a0, a1);
- {
+         break;
--    if (tb_cflags(s->base.tb) & CF_PARALLEL) {
+     case INDEX_op_extu_i32_i64:
-+    if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
+     case INDEX_op_ext32u_i64:
-+        gen_helper_stpq(cpu_env, o->in2, o->out2, o->out);
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-+    } else if (HAVE_ATOMIC128) {
+     case INDEX_op_ext16s_i64:
-         gen_helper_stpq_parallel(cpu_env, o->in2, o->out2, o->out);
+     case INDEX_op_ext16u_i32:
-     } else {
+     case INDEX_op_ext16u_i64:
--        gen_helper_stpq(cpu_env, o->in2, o->out2, o->out);
++    case INDEX_op_ext32s_i64:
-+        gen_helper_exit_atomic(cpu_env);
+     default:
-+        return DISAS_NORETURN;
+         g_assert_not_reached();
      }
-     return DISAS_NEXT;
+diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
- }
+index XXXXXXX..XXXXXXX 100644
 --- a/tcg/tci/tcg-target.c.inc
 +++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rs)
      }
  }
 +static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rs)
 +{
 +    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
 +    tcg_debug_assert(TCG_TARGET_HAS_ext32s_i64);
 +    tcg_out_op_rr(s, INDEX_op_ext32s_i64, rd, rs);
 +}
 +
  static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      CASE_32_64(neg)      /* Optional (TCG_TARGET_HAS_neg_*). */
      CASE_32_64(not)      /* Optional (TCG_TARGET_HAS_not_*). */
 -    CASE_64(ext32s)      /* Optional (TCG_TARGET_HAS_ext32s_i64). */
      CASE_64(ext32u)      /* Optional (TCG_TARGET_HAS_ext32u_i64). */
      CASE_64(ext_i32)
      CASE_64(extu_i32)
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16s_i64:
      case INDEX_op_ext16u_i32:
      case INDEX_op_ext16u_i64:
 +    case INDEX_op_ext32s_i64:
      default:
          g_assert_not_reached();
      }
 --
-.17.2
+.34.1

-[Qemu-devel] [PULL 18/21] target/s390x: Split do_cdsg, do_lpq, do_stpq
+[PULL 08/15] tcg: Split out tcg_out_ext32u
-Reviewed-by: David Hildenbrand <david@redhat.com>
+We will need a backend interface for performing 32-bit zero-extend.
 Use it in tcg_reg_alloc_op in the meantime.
 Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- target/s390x/mem_helper.c | 128 ++++++++++++++++++--------------------
+ tcg/tcg.c                        |  4 ++++
-file changed, 61 insertions(+), 67 deletions(-)
+ tcg/aarch64/tcg-target.c.inc     |  9 +++++++--
  tcg/arm/tcg-target.c.inc         |  5 +++++
  tcg/i386/tcg-target.c.inc        |  4 ++--
  tcg/loongarch64/tcg-target.c.inc |  2 +-
  tcg/mips/tcg-target.c.inc        |  3 ++-
  tcg/ppc/tcg-target.c.inc         |  4 +++-
  tcg/riscv/tcg-target.c.inc       |  2 +-
  tcg/s390x/tcg-target.c.inc       | 20 ++++++++++----------
  tcg/sparc64/tcg-target.c.inc     | 17 +++++++++++------
  tcg/tci/tcg-target.c.inc         |  9 ++++++++-
 files changed, 54 insertions(+), 25 deletions(-)
-diff --git a/target/s390x/mem_helper.c b/target/s390x/mem_helper.c
+diff --git a/tcg/tcg.c b/tcg/tcg.c
 index XXXXXXX..XXXXXXX 100644
---- a/target/s390x/mem_helper.c
+--- a/tcg/tcg.c
-+++ b/target/s390x/mem_helper.c
++++ b/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@ uint32_t HELPER(trXX)(CPUS390XState *env, uint32_t r1, uint32_t r2,
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
-     return cc;
+ static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
- }
+ static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
+ static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
--static void do_cdsg(CPUS390XState *env, uint64_t addr,
++static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg);
--                    uint32_t r1, uint32_t r3, bool parallel)
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
-+void HELPER(cdsg)(CPUS390XState *env, uint64_t addr,
+ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
-+                  uint32_t r1, uint32_t r3)
+ static void tcg_out_goto_tb(TCGContext *s, int which);
- {
+@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
-     uintptr_t ra = GETPC();
+     case INDEX_op_ext32s_i64:
-     Int128 cmpv = int128_make128(env->regs[r1 + 1], env->regs[r1]);
+         tcg_out_ext32s(s, new_args[0], new_args[1]);
-     Int128 newv = int128_make128(env->regs[r3 + 1], env->regs[r3]);
+         break;
-     Int128 oldv;
++    case INDEX_op_ext32u_i64:
-+    uint64_t oldh, oldl;
++        tcg_out_ext32u(s, new_args[0], new_args[1]);
-     bool fail;
++        break;
+     default:
--    if (parallel) {
+         if (def->flags & TCG_OPF_VECTOR) {
--#if !HAVE_CMPXCHG128
+             tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
--        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
+diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
--#else
+index XXXXXXX..XXXXXXX 100644
--        int mem_idx = cpu_mmu_index(env, false);
+--- a/tcg/aarch64/tcg-target.c.inc
--        TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
++++ b/tcg/aarch64/tcg-target.c.inc
--        oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rn)
--        fail = !int128_eq(oldv, cmpv);
+     tcg_out_uxt(s, MO_16, rd, rn);
--#endif
+ }
--    } else {
--        uint64_t oldh, oldl;
++static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rn)
-+    check_alignment(env, addr, 16, ra);
++{
++    tcg_out_movr(s, TCG_TYPE_I32, rd, rn);
 -        check_alignment(env, addr, 16, ra);
 +    oldh = cpu_ldq_data_ra(env, addr + 0, ra);
 +    oldl = cpu_ldq_data_ra(env, addr + 8, ra);
 -        oldh = cpu_ldq_data_ra(env, addr + 0, ra);
 -        oldl = cpu_ldq_data_ra(env, addr + 8, ra);
 -
 -        oldv = int128_make128(oldl, oldh);
 -        fail = !int128_eq(oldv, cmpv);
 -        if (fail) {
 -            newv = oldv;
 -        }
 -
 -        cpu_stq_data_ra(env, addr + 0, int128_gethi(newv), ra);
 -        cpu_stq_data_ra(env, addr + 8, int128_getlo(newv), ra);
 +    oldv = int128_make128(oldl, oldh);
 +    fail = !int128_eq(oldv, cmpv);
 +    if (fail) {
 +        newv = oldv;
      }
 +    cpu_stq_data_ra(env, addr + 0, int128_gethi(newv), ra);
 +    cpu_stq_data_ra(env, addr + 8, int128_getlo(newv), ra);
 +
      env->cc_op = fail;
      env->regs[r1] = int128_gethi(oldv);
      env->regs[r1 + 1] = int128_getlo(oldv);
  }
 -void HELPER(cdsg)(CPUS390XState *env, uint64_t addr,
 -                  uint32_t r1, uint32_t r3)
 -{
 -    do_cdsg(env, addr, r1, r3, false);
 -}
 -
  void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr,
                             uint32_t r1, uint32_t r3)
  {
 -    do_cdsg(env, addr, r1, r3, true);
 +    uintptr_t ra = GETPC();
 +    Int128 cmpv = int128_make128(env->regs[r1 + 1], env->regs[r1]);
 +    Int128 newv = int128_make128(env->regs[r3 + 1], env->regs[r3]);
 +    int mem_idx;
 +    TCGMemOpIdx oi;
 +    Int128 oldv;
 +    bool fail;
 +
 +    if (!HAVE_CMPXCHG128) {
 +        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
 +    }
 +
 +    mem_idx = cpu_mmu_index(env, false);
 +    oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
 +    oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
 +    fail = !int128_eq(oldv, cmpv);
 +
 +    env->cc_op = fail;
 +    env->regs[r1] = int128_gethi(oldv);
 +    env->regs[r1 + 1] = int128_getlo(oldv);
  }
  static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
@@ -XXX,XX +XXX,XX @@ uint64_t HELPER(lra)(CPUS390XState *env, uint64_t addr)
  #endif
  /* load pair from quadword */
 -static uint64_t do_lpq(CPUS390XState *env, uint64_t addr, bool parallel)
 +uint64_t HELPER(lpq)(CPUS390XState *env, uint64_t addr)
  {
      uintptr_t ra = GETPC();
      uint64_t hi, lo;
 -    if (!parallel) {
 -        check_alignment(env, addr, 16, ra);
 -        hi = cpu_ldq_data_ra(env, addr + 0, ra);
 -        lo = cpu_ldq_data_ra(env, addr + 8, ra);
 -    } else if (HAVE_ATOMIC128) {
 +    check_alignment(env, addr, 16, ra);
 +    hi = cpu_ldq_data_ra(env, addr + 0, ra);
 +    lo = cpu_ldq_data_ra(env, addr + 8, ra);
 +
 +    env->retxl = lo;
 +    return hi;
 +}
 +
-+uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr)
+ static void tcg_out_addsubi(TCGContext *s, int ext, TCGReg rd,
                              TCGReg rn, int64_t aimm)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
          break;
      case INDEX_op_extu_i32_i64:
 -    case INDEX_op_ext32u_i64:
 -        tcg_out_movr(s, TCG_TYPE_I32, a0, a1);
 +        tcg_out_ext32u(s, a0, a1);
          break;
      case INDEX_op_deposit_i64:
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16u_i64:
      case INDEX_op_ext16u_i32:
      case INDEX_op_ext32s_i64:
 +    case INDEX_op_ext32u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/arm/tcg-target.c.inc
 +++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rn)
      g_assert_not_reached();
  }
 +static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rn)
 +{
-+    uintptr_t ra = GETPC();
++    g_assert_not_reached();
 +    uint64_t hi, lo;
 +
 +    if (HAVE_ATOMIC128) {
          int mem_idx = cpu_mmu_index(env, false);
          TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
          Int128 v = helper_atomic_ldo_be_mmu(env, addr, oi, ra);
@@ -XXX,XX +XXX,XX @@ static uint64_t do_lpq(CPUS390XState *env, uint64_t addr, bool parallel)
      return hi;
  }
 -uint64_t HELPER(lpq)(CPUS390XState *env, uint64_t addr)
 -{
 -    return do_lpq(env, addr, false);
 -}
 -
 -uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr)
 -{
 -    return do_lpq(env, addr, true);
 -}
 -
  /* store pair to quadword */
 -static void do_stpq(CPUS390XState *env, uint64_t addr,
 -                    uint64_t low, uint64_t high, bool parallel)
 +void HELPER(stpq)(CPUS390XState *env, uint64_t addr,
 +                  uint64_t low, uint64_t high)
  {
      uintptr_t ra = GETPC();
 -    if (!parallel) {
 -        check_alignment(env, addr, 16, ra);
 -        cpu_stq_data_ra(env, addr + 0, high, ra);
 -        cpu_stq_data_ra(env, addr + 8, low, ra);
 -    } else if (HAVE_ATOMIC128) {
 +    check_alignment(env, addr, 16, ra);
 +    cpu_stq_data_ra(env, addr + 0, high, ra);
 +    cpu_stq_data_ra(env, addr + 8, low, ra);
 +}
 +
-+void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr,
+ static void tcg_out_bswap16(TCGContext *s, ARMCond cond,
-+                           uint64_t low, uint64_t high)
+                             TCGReg rd, TCGReg rn, int flags)
  {
 diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/i386/tcg-target.c.inc
 +++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
      tcg_out_modrm(s, OPC_MOVSWL + rexw, dest, src);
  }
 -static inline void tcg_out_ext32u(TCGContext *s, int dest, int src)
 +static void tcg_out_ext32u(TCGContext *s, TCGReg dest, TCGReg src)
  {
      /* 32-bit mov zero extends.  */
      tcg_out_modrm(s, OPC_MOVL_GvEv, dest, src);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_bswap64(s, a0);
          break;
      case INDEX_op_extu_i32_i64:
 -    case INDEX_op_ext32u_i64:
      case INDEX_op_extrl_i64_i32:
          tcg_out_ext32u(s, a0, a1);
          break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16u_i32:
      case INDEX_op_ext16u_i64:
      case INDEX_op_ext32s_i64:
 +    case INDEX_op_ext32u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/loongarch64/tcg-target.c.inc
 +++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_brcond(s, a2, a0, a1, arg_label(args[3]));
          break;
 -    case INDEX_op_ext32u_i64:
      case INDEX_op_extu_i32_i64:
          tcg_out_ext32u(s, a0, a1);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16u_i32:
      case INDEX_op_ext16u_i64:
      case INDEX_op_ext32s_i64:
 +    case INDEX_op_ext32u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/mips/tcg-target.c.inc
 +++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_bswap64(TCGContext *s, TCGReg ret, TCGReg arg)
  static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg)
  {
 +    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
      if (use_mips32r2_instructions) {
          tcg_out_opc_bf(s, OPC_DEXT, ret, arg, 31, 0);
      } else {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_extrl_i64_i32:
          tcg_out_ext32s(s, a0, a1);
          break;
 -    case INDEX_op_ext32u_i64:
      case INDEX_op_extu_i32_i64:
          tcg_out_ext32u(s, a0, a1);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16s_i32:
      case INDEX_op_ext16s_i64:
      case INDEX_op_ext32s_i64:
 +    case INDEX_op_ext32u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/ppc/tcg-target.c.inc
 +++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg dst, TCGReg src)
      tcg_out32(s, EXTSW | RA(dst) | RS(src));
  }
 -static inline void tcg_out_ext32u(TCGContext *s, TCGReg dst, TCGReg src)
 +static void tcg_out_ext32u(TCGContext *s, TCGReg dst, TCGReg src)
  {
 +    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
      tcg_out_rld(s, RLDICL, dst, src, 0, 32);
  }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16u_i32:
      case INDEX_op_ext16u_i64:
      case INDEX_op_ext32s_i64:
 +    case INDEX_op_ext32u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/riscv/tcg-target.c.inc
 +++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_qemu_st(s, args, true);
          break;
 -    case INDEX_op_ext32u_i64:
      case INDEX_op_extu_i32_i64:
          tcg_out_ext32u(s, a0, a1);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16u_i32:
      case INDEX_op_ext16u_i64:
      case INDEX_op_ext32s_i64:
 +    case INDEX_op_ext32u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/s390x/tcg-target.c.inc
 +++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
      tcg_out_insn(s, RRE, LGFR, dest, src);
  }
 -static inline void tgen_ext32u(TCGContext *s, TCGReg dest, TCGReg src)
 +static void tcg_out_ext32u(TCGContext *s, TCGReg dest, TCGReg src)
  {
      tcg_out_insn(s, RRE, LLGFR, dest, src);
  }
@@ -XXX,XX +XXX,XX @@ static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
      /* Look for the zero-extensions.  */
      if ((val & valid) == 0xffffffff) {
 -        tgen_ext32u(s, dest, dest);
 +        tcg_out_ext32u(s, dest, dest);
          return;
      }
      if ((val & valid) == 0xff) {
@@ -XXX,XX +XXX,XX @@ static void tgen_ctpop(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
      /* With MIE3, and bit 0 of m4 set, we get the complete result. */
      if (HAVE_FACILITY(MISC_INSN_EXT3)) {
          if (type == TCG_TYPE_I32) {
 -            tgen_ext32u(s, dest, src);
 +            tcg_out_ext32u(s, dest, src);
              src = dest;
          }
          tcg_out_insn(s, RRFc, POPCNT, dest, src, 8);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data,
      case MO_UL | MO_BSWAP:
          /* swapped unsigned int load with upper bits zeroed */
          tcg_out_insn(s, RXY, LRV, data, base, index, disp);
 -        tgen_ext32u(s, data, data);
 +        tcg_out_ext32u(s, data, data);
          break;
      case MO_UL:
          tcg_out_insn(s, RXY, LLGF, data, base, index, disp);
@@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, MemOp opc,
                   offsetof(CPUTLBEntry, addend));
      if (TARGET_LONG_BITS == 32) {
 -        tgen_ext32u(s, TCG_REG_R3, addr_reg);
 +        tcg_out_ext32u(s, TCG_REG_R3, addr_reg);
          return TCG_REG_R3;
      }
      return addr_reg;
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
          tcg_out_ext16u(s, TCG_REG_R4, data_reg);
          break;
      case MO_UL:
 -        tgen_ext32u(s, TCG_REG_R4, data_reg);
 +        tcg_out_ext32u(s, TCG_REG_R4, data_reg);
          break;
      case MO_UQ:
          tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
@@ -XXX,XX +XXX,XX @@ static void tcg_prepare_user_ldst(TCGContext *s, TCGReg *addr_reg,
                                    TCGReg *index_reg, tcg_target_long *disp)
  {
      if (TARGET_LONG_BITS == 32) {
 -        tgen_ext32u(s, TCG_TMP0, *addr_reg);
 +        tcg_out_ext32u(s, TCG_TMP0, *addr_reg);
          *addr_reg = TCG_TMP0;
      }
      if (guest_base < 0x80000) {
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
          if (a2 & TCG_BSWAP_OS) {
              tcg_out_ext32s(s, a0, a0);
          } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
 -            tgen_ext32u(s, a0, a0);
 +            tcg_out_ext32u(s, a0, a0);
          }
          break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_ext32s(s, args[0], args[1]);
          break;
      case INDEX_op_extu_i32_i64:
 -    case INDEX_op_ext32u_i64:
 -        tgen_ext32u(s, args[0], args[1]);
 +        tcg_out_ext32u(s, args[0], args[1]);
          break;
      case INDEX_op_add2_i64:
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16u_i32:
      case INDEX_op_ext16u_i64:
      case INDEX_op_ext32s_i64:
 +    case INDEX_op_ext32u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/sparc64/tcg-target.c.inc
 +++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rs)
      tcg_out_arithi(s, rd, rs, 0, SHIFT_SRA);
  }
 +static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rs)
 +{
-+    uintptr_t ra = GETPC();
++    tcg_out_arithi(s, rd, rs, 0, SHIFT_SRL);
 +}
 +
-+    if (HAVE_ATOMIC128) {
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
-         int mem_idx = cpu_mmu_index(env, false);
+                              tcg_target_long imm)
-         TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
+ {
-         Int128 v = int128_make128(low, high);
+@@ -XXX,XX +XXX,XX @@ static void emit_extend(TCGContext *s, TCGReg r, int op)
-@@ -XXX,XX +XXX,XX @@ static void do_stpq(CPUS390XState *env, uint64_t addr,
+         tcg_out_ext16u(s, r, r);
-     }
+         break;
- }
+     case MO_32:
+-        tcg_out_arith(s, r, r, 0, SHIFT_SRL);
--void HELPER(stpq)(CPUS390XState *env, uint64_t addr,
++        tcg_out_ext32u(s, r, r);
--                  uint64_t low, uint64_t high)
+         break;
--{
+     case MO_64:
--    do_stpq(env, addr, low, high, false);
+         break;
--}
+@@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addr, int mem_index,
--
--void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr,
+     /* If the guest address must be zero-extended, do so now.  */
--                           uint64_t low, uint64_t high)
+     if (TARGET_LONG_BITS == 32) {
--{
+-        tcg_out_arithi(s, r0, addr, 0, SHIFT_SRL);
--    do_stpq(env, addr, low, high, true);
++        tcg_out_ext32u(s, r0, addr);
--}
+         return r0;
--
+     }
- /* Execute instruction.  This instruction executes an insn modified with
+     return addr;
-    the contents of r1.  It does not change the executed instruction in memory;
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr,
-    it does not change the program counter.
+     unsigned t_bits;
      if (TARGET_LONG_BITS == 32) {
 -        tcg_out_arithi(s, TCG_REG_T1, addr, 0, SHIFT_SRL);
 +        tcg_out_ext32u(s, TCG_REG_T1, addr);
          addr = TCG_REG_T1;
      }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr,
      unsigned t_bits;
      if (TARGET_LONG_BITS == 32) {
 -        tcg_out_arithi(s, TCG_REG_T1, addr, 0, SHIFT_SRL);
 +        tcg_out_ext32u(s, TCG_REG_T1, addr);
          addr = TCG_REG_T1;
      }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_ext32s(s, a0, a1);
          break;
      case INDEX_op_extu_i32_i64:
 -    case INDEX_op_ext32u_i64:
 -        tcg_out_arithi(s, a0, a1, 0, SHIFT_SRL);
 +        tcg_out_ext32u(s, a0, a1);
          break;
      case INDEX_op_extrl_i64_i32:
          tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16u_i32:
      case INDEX_op_ext16u_i64:
      case INDEX_op_ext32s_i64:
 +    case INDEX_op_ext32u_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/tci/tcg-target.c.inc
 +++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rs)
      tcg_out_op_rr(s, INDEX_op_ext32s_i64, rd, rs);
  }
 +static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rs)
 +{
 +    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
 +    tcg_debug_assert(TCG_TARGET_HAS_ext32u_i64);
 +    tcg_out_op_rr(s, INDEX_op_ext32u_i64, rd, rs);
 +}
 +
  static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      CASE_32_64(neg)      /* Optional (TCG_TARGET_HAS_neg_*). */
      CASE_32_64(not)      /* Optional (TCG_TARGET_HAS_not_*). */
 -    CASE_64(ext32u)      /* Optional (TCG_TARGET_HAS_ext32u_i64). */
      CASE_64(ext_i32)
      CASE_64(extu_i32)
      CASE_32_64(ctpop)    /* Optional (TCG_TARGET_HAS_ctpop_*). */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16u_i32:
      case INDEX_op_ext16u_i64:
      case INDEX_op_ext32s_i64:
 +    case INDEX_op_ext32u_i64:
      default:
          g_assert_not_reached();
      }
 --
-.17.2
+.34.1

-[Qemu-devel] [PULL 19/21] target/s390x: Skip wout, cout helpers if op helper does not return
+[PULL 09/15] tcg: Split out tcg_out_exts_i32_i64
-When op raises an exception, it may not have initialized the output
+We will need a backend interface for type extension with sign.
-temps that would be written back by wout or cout.
+Use it in tcg_reg_alloc_op in the meantime.
-Reviewed-by: David Hildenbrand <david@redhat.com>
+Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- target/s390x/translate.c | 20 +++++++++++++++-----
+ tcg/tcg.c                        | 4 ++++
-file changed, 15 insertions(+), 5 deletions(-)
+ tcg/aarch64/tcg-target.c.inc     | 9 ++++++---
  tcg/arm/tcg-target.c.inc         | 5 +++++
  tcg/i386/tcg-target.c.inc        | 9 ++++++---
  tcg/loongarch64/tcg-target.c.inc | 7 ++++++-
  tcg/mips/tcg-target.c.inc        | 7 ++++++-
  tcg/ppc/tcg-target.c.inc         | 9 ++++++---
  tcg/riscv/tcg-target.c.inc       | 7 ++++++-
  tcg/s390x/tcg-target.c.inc       | 9 ++++++---
  tcg/sparc64/tcg-target.c.inc     | 9 ++++++---
  tcg/tci/tcg-target.c.inc         | 7 ++++++-
 files changed, 63 insertions(+), 19 deletions(-)
-diff --git a/target/s390x/translate.c b/target/s390x/translate.c
+diff --git a/tcg/tcg.c b/tcg/tcg.c
 index XXXXXXX..XXXXXXX 100644
---- a/target/s390x/translate.c
+--- a/tcg/tcg.c
-+++ b/target/s390x/translate.c
++++ b/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@ struct DisasInsn {
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
+ static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
-     const char *name;
+ static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
+ static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg);
-+    /* Pre-process arguments before HELP_OP.  */
++static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
-     void (*help_in1)(DisasContext *, DisasFields *, DisasOps *);
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
-     void (*help_in2)(DisasContext *, DisasFields *, DisasOps *);
+ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
-     void (*help_prep)(DisasContext *, DisasFields *, DisasOps *);
+ static void tcg_out_goto_tb(TCGContext *s, int which);
-+
+@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
-+    /*
+     case INDEX_op_ext32u_i64:
-+     * Post-process output after HELP_OP.
+         tcg_out_ext32u(s, new_args[0], new_args[1]);
-+     * Note that these are not called if HELP_OP returns DISAS_NORETURN.
+         break;
-+     */
++    case INDEX_op_ext_i32_i64:
-     void (*help_wout)(DisasContext *, DisasFields *, DisasOps *);
++        tcg_out_exts_i32_i64(s, new_args[0], new_args[1]);
-     void (*help_cout)(DisasContext *, DisasOps *);
++        break;
-+
+     default:
-+    /* Implement the operation itself.  */
+         if (def->flags & TCG_OPF_VECTOR) {
-     DisasJumpType (*help_op)(DisasContext *, DisasOps *);
+             tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
+diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
-     uint64_t data;
+index XXXXXXX..XXXXXXX 100644
-@@ -XXX,XX +XXX,XX @@ static DisasJumpType translate_one(CPUS390XState *env, DisasContext *s)
+--- a/tcg/aarch64/tcg-target.c.inc
-     if (insn->help_op) {
++++ b/tcg/aarch64/tcg-target.c.inc
-         ret = insn->help_op(s, &o);
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rn)
-     }
+     tcg_out_sxt(s, TCG_TYPE_I64, MO_32, rd, rn);
--    if (insn->help_wout) {
+ }
--        insn->help_wout(s, &f, &o);
--    }
++static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn)
--    if (insn->help_cout) {
++{
--        insn->help_cout(s, &o);
++    tcg_out_ext32s(s, rd, rn);
-+    if (ret != DISAS_NORETURN) {
++}
-+        if (insn->help_wout) {
++
-+            insn->help_wout(s, &f, &o);
+ static inline void tcg_out_uxt(TCGContext *s, MemOp s_bits,
-+        }
+                                TCGReg rd, TCGReg rn)
-+        if (insn->help_cout) {
+ {
-+            insn->help_cout(s, &o);
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-+        }
+         }
-     }
+         break;
-     /* Free any temporaries created by the helpers.  */
+-    case INDEX_op_ext_i32_i64:
 -        tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
 -        break;
      case INDEX_op_extu_i32_i64:
          tcg_out_ext32u(s, a0, a1);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16u_i32:
      case INDEX_op_ext32s_i64:
      case INDEX_op_ext32u_i64:
 +    case INDEX_op_ext_i32_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/arm/tcg-target.c.inc
 +++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rn)
      g_assert_not_reached();
  }
 +static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn)
 +{
 +    g_assert_not_reached();
 +}
 +
  static void tcg_out_bswap16(TCGContext *s, ARMCond cond,
                              TCGReg rd, TCGReg rn, int flags)
  {
 diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/i386/tcg-target.c.inc
 +++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
      tcg_out_modrm(s, OPC_MOVSLQ, dest, src);
  }
 +static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
 +{
 +    tcg_out_ext32s(s, dest, src);
 +}
 +
  static inline void tcg_out_bswap64(TCGContext *s, int reg)
  {
      tcg_out_opc(s, OPC_BSWAP + P_REXW + LOWREGMASK(reg), 0, reg, 0);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_extrl_i64_i32:
          tcg_out_ext32u(s, a0, a1);
          break;
 -    case INDEX_op_ext_i32_i64:
 -        tcg_out_ext32s(s, a0, a1);
 -        break;
      case INDEX_op_extrh_i64_i32:
          tcg_out_shifti(s, SHIFT_SHR + P_REXW, a0, 32);
          break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16u_i64:
      case INDEX_op_ext32s_i64:
      case INDEX_op_ext32u_i64:
 +    case INDEX_op_ext_i32_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/loongarch64/tcg-target.c.inc
 +++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg)
      tcg_out_opc_addi_w(s, ret, arg, 0);
  }
 +static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
 +{
 +    tcg_out_ext32s(s, ret, arg);
 +}
 +
  static void tcg_out_clzctz(TCGContext *s, LoongArchInsn opc,
                             TCGReg a0, TCGReg a1, TCGReg a2,
                             bool c2, bool is_32bit)
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          break;
      case INDEX_op_extrl_i64_i32:
 -    case INDEX_op_ext_i32_i64:
          tcg_out_ext32s(s, a0, a1);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16u_i64:
      case INDEX_op_ext32s_i64:
      case INDEX_op_ext32u_i64:
 +    case INDEX_op_ext_i32_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/mips/tcg-target.c.inc
 +++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rs)
      tcg_out_opc_sa(s, OPC_SLL, rd, rs, 0);
  }
 +static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
 +{
 +    tcg_out_ext32s(s, rd, rs);
 +}
 +
  static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_extrh_i64_i32:
          tcg_out_dsra(s, a0, a1, 32);
          break;
 -    case INDEX_op_ext_i32_i64:
      case INDEX_op_extrl_i64_i32:
          tcg_out_ext32s(s, a0, a1);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16s_i64:
      case INDEX_op_ext32s_i64:
      case INDEX_op_ext32u_i64:
 +    case INDEX_op_ext_i32_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/ppc/tcg-target.c.inc
 +++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32u(TCGContext *s, TCGReg dst, TCGReg src)
      tcg_out_rld(s, RLDICL, dst, src, 0, 32);
  }
 +static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg dst, TCGReg src)
 +{
 +    tcg_out_ext32s(s, dst, src);
 +}
 +
  static inline void tcg_out_shli32(TCGContext *s, TCGReg dst, TCGReg src, int c)
  {
      tcg_out_rlw(s, RLWINM, dst, src, c, 0, 31 - c);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_qemu_st(s, args, true);
          break;
 -    case INDEX_op_ext_i32_i64:
 -        tcg_out_ext32s(s, args[0], args[1]);
 -        break;
      case INDEX_op_extu_i32_i64:
          tcg_out_ext32u(s, args[0], args[1]);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16u_i64:
      case INDEX_op_ext32s_i64:
      case INDEX_op_ext32u_i64:
 +    case INDEX_op_ext_i32_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/riscv/tcg-target.c.inc
 +++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg)
      tcg_out_opc_imm(s, OPC_ADDIW, ret, arg, 0);
  }
 +static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
 +{
 +    tcg_out_ext32s(s, ret, arg);
 +}
 +
  static void tcg_out_ldst(TCGContext *s, RISCVInsn opc, TCGReg data,
                           TCGReg addr, intptr_t offset)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          break;
      case INDEX_op_extrl_i64_i32:
 -    case INDEX_op_ext_i32_i64:
          tcg_out_ext32s(s, a0, a1);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16u_i64:
      case INDEX_op_ext32s_i64:
      case INDEX_op_ext32u_i64:
 +    case INDEX_op_ext_i32_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/s390x/tcg-target.c.inc
 +++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32u(TCGContext *s, TCGReg dest, TCGReg src)
      tcg_out_insn(s, RRE, LLGFR, dest, src);
  }
 +static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
 +{
 +    tcg_out_ext32s(s, dest, src);
 +}
 +
  static void tgen_andi_risbg(TCGContext *s, TCGReg out, TCGReg in, uint64_t val)
  {
      int msb, lsb;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
          }
          break;
 -    case INDEX_op_ext_i32_i64:
 -        tcg_out_ext32s(s, args[0], args[1]);
 -        break;
      case INDEX_op_extu_i32_i64:
          tcg_out_ext32u(s, args[0], args[1]);
          break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16u_i64:
      case INDEX_op_ext32s_i64:
      case INDEX_op_ext32u_i64:
 +    case INDEX_op_ext_i32_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/sparc64/tcg-target.c.inc
 +++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rs)
      tcg_out_arithi(s, rd, rs, 0, SHIFT_SRL);
  }
 +static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
 +{
 +    tcg_out_ext32s(s, rd, rs);
 +}
 +
  static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_divu_i64:
          c = ARITH_UDIVX;
          goto gen_arith;
 -    case INDEX_op_ext_i32_i64:
 -        tcg_out_ext32s(s, a0, a1);
 -        break;
      case INDEX_op_extu_i32_i64:
          tcg_out_ext32u(s, a0, a1);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16u_i64:
      case INDEX_op_ext32s_i64:
      case INDEX_op_ext32u_i64:
 +    case INDEX_op_ext_i32_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/tci/tcg-target.c.inc
 +++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rs)
      tcg_out_op_rr(s, INDEX_op_ext32u_i64, rd, rs);
  }
 +static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
 +{
 +    tcg_out_ext32s(s, rd, rs);
 +}
 +
  static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      CASE_32_64(neg)      /* Optional (TCG_TARGET_HAS_neg_*). */
      CASE_32_64(not)      /* Optional (TCG_TARGET_HAS_not_*). */
 -    CASE_64(ext_i32)
      CASE_64(extu_i32)
      CASE_32_64(ctpop)    /* Optional (TCG_TARGET_HAS_ctpop_*). */
      case INDEX_op_bswap32_i32: /* Optional (TCG_TARGET_HAS_bswap32_i32). */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext16u_i64:
      case INDEX_op_ext32s_i64:
      case INDEX_op_ext32u_i64:
 +    case INDEX_op_ext_i32_i64:
      default:
          g_assert_not_reached();
      }
 --
-.17.2
+.34.1

-[Qemu-devel] [PULL 16/21] target/ppc: Convert to HAVE_CMPXCHG128 and HAVE_ATOMIC128
+[PULL 10/15] tcg: Split out tcg_out_extu_i32_i64
-Reviewed-by: Emilio G. Cota <cota@braap.org>
+We will need a backend interface for type extension with zero.
 Use it in tcg_reg_alloc_op in the meantime.
 Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- target/ppc/helper.h     |   2 +-
+ tcg/tcg.c                        |  4 ++++
- target/ppc/mem_helper.c |  33 ++++++++++--
+ tcg/aarch64/tcg-target.c.inc     | 10 ++++++----
- target/ppc/translate.c  | 115 +++++++++++++++++++++-------------------
+ tcg/arm/tcg-target.c.inc         |  5 +++++
-files changed, 88 insertions(+), 62 deletions(-)
+ tcg/i386/tcg-target.c.inc        |  7 ++++++-
  tcg/loongarch64/tcg-target.c.inc | 10 ++++++----
  tcg/mips/tcg-target.c.inc        |  9 ++++++---
  tcg/ppc/tcg-target.c.inc         | 10 ++++++----
  tcg/riscv/tcg-target.c.inc       | 10 ++++++----
  tcg/s390x/tcg-target.c.inc       | 10 ++++++----
  tcg/sparc64/tcg-target.c.inc     |  9 ++++++---
  tcg/tci/tcg-target.c.inc         |  7 ++++++-
 files changed, 63 insertions(+), 28 deletions(-)
-diff --git a/target/ppc/helper.h b/target/ppc/helper.h
+diff --git a/tcg/tcg.c b/tcg/tcg.c
 index XXXXXXX..XXXXXXX 100644
---- a/target/ppc/helper.h
+--- a/tcg/tcg.c
-+++ b/target/ppc/helper.h
++++ b/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@ DEF_HELPER_4(dscliq, void, env, fprp, fprp, i32)
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
- DEF_HELPER_1(tbegin, void, env)
+ static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
- DEF_HELPER_FLAGS_1(fixup_thrm, TCG_CALL_NO_RWG, void, env)
+ static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg);
+ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
--#if defined(TARGET_PPC64) && defined(CONFIG_ATOMIC128)
++static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
-+#ifdef TARGET_PPC64
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
- DEF_HELPER_FLAGS_3(lq_le_parallel, TCG_CALL_NO_WG, i64, env, tl, i32)
+ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
- DEF_HELPER_FLAGS_3(lq_be_parallel, TCG_CALL_NO_WG, i64, env, tl, i32)
+ static void tcg_out_goto_tb(TCGContext *s, int which);
- DEF_HELPER_FLAGS_5(stq_le_parallel, TCG_CALL_NO_WG,
+@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
-diff --git a/target/ppc/mem_helper.c b/target/ppc/mem_helper.c
+     case INDEX_op_ext_i32_i64:
-index XXXXXXX..XXXXXXX 100644
+         tcg_out_exts_i32_i64(s, new_args[0], new_args[1]);
---- a/target/ppc/mem_helper.c
+         break;
-+++ b/target/ppc/mem_helper.c
++    case INDEX_op_extu_i32_i64:
-@@ -XXX,XX +XXX,XX @@
++        tcg_out_extu_i32_i64(s, new_args[0], new_args[1]);
- #include "exec/cpu_ldst.h"
++        break;
- #include "tcg.h"
+     default:
- #include "internal.h"
+         if (def->flags & TCG_OPF_VECTOR) {
-+#include "qemu/atomic128.h"
+             tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
+diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
- //#define DEBUG_OP
+index XXXXXXX..XXXXXXX 100644
+--- a/tcg/aarch64/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ target_ulong helper_lscbx(CPUPPCState *env, target_ulong addr, uint32_t reg,
++++ b/tcg/aarch64/tcg-target.c.inc
-     return i;
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rn)
- }
+     tcg_out_movr(s, TCG_TYPE_I32, rd, rn);
+ }
--#if defined(TARGET_PPC64) && defined(CONFIG_ATOMIC128)
-+#ifdef TARGET_PPC64
++static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn)
- uint64_t helper_lq_le_parallel(CPUPPCState *env, target_ulong addr,
++{
-                                uint32_t opidx)
++    tcg_out_ext32u(s, rd, rn);
- {
++}
--    Int128 ret = helper_atomic_ldo_le_mmu(env, addr, opidx, GETPC());
++
-+    Int128 ret;
+ static void tcg_out_addsubi(TCGContext *s, int ext, TCGReg rd,
-+
+                             TCGReg rn, int64_t aimm)
-+    /* We will have raised EXCP_ATOMIC from the translator.  */
+ {
-+    assert(HAVE_ATOMIC128);
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 +    ret = helper_atomic_ldo_le_mmu(env, addr, opidx, GETPC());
      env->retxh = int128_gethi(ret);
      return int128_getlo(ret);
  }
@@ -XXX,XX +XXX,XX @@ uint64_t helper_lq_le_parallel(CPUPPCState *env, target_ulong addr,
  uint64_t helper_lq_be_parallel(CPUPPCState *env, target_ulong addr,
                                 uint32_t opidx)
  {
 -    Int128 ret = helper_atomic_ldo_be_mmu(env, addr, opidx, GETPC());
 +    Int128 ret;
 +
 +    /* We will have raised EXCP_ATOMIC from the translator.  */
 +    assert(HAVE_ATOMIC128);
 +    ret = helper_atomic_ldo_be_mmu(env, addr, opidx, GETPC());
      env->retxh = int128_gethi(ret);
      return int128_getlo(ret);
  }
@@ -XXX,XX +XXX,XX @@ uint64_t helper_lq_be_parallel(CPUPPCState *env, target_ulong addr,
  void helper_stq_le_parallel(CPUPPCState *env, target_ulong addr,
                              uint64_t lo, uint64_t hi, uint32_t opidx)
  {
 -    Int128 val = int128_make128(lo, hi);
 +    Int128 val;
 +
 +    /* We will have raised EXCP_ATOMIC from the translator.  */
 +    assert(HAVE_ATOMIC128);
 +    val = int128_make128(lo, hi);
      helper_atomic_sto_le_mmu(env, addr, val, opidx, GETPC());
  }
  void helper_stq_be_parallel(CPUPPCState *env, target_ulong addr,
                              uint64_t lo, uint64_t hi, uint32_t opidx)
  {
 -    Int128 val = int128_make128(lo, hi);
 +    Int128 val;
 +
 +    /* We will have raised EXCP_ATOMIC from the translator.  */
 +    assert(HAVE_ATOMIC128);
 +    val = int128_make128(lo, hi);
      helper_atomic_sto_be_mmu(env, addr, val, opidx, GETPC());
  }
@@ -XXX,XX +XXX,XX @@ uint32_t helper_stqcx_le_parallel(CPUPPCState *env, target_ulong addr,
  {
      bool success = false;
 +    /* We will have raised EXCP_ATOMIC from the translator.  */
 +    assert(HAVE_CMPXCHG128);
 +
      if (likely(addr == env->reserve_addr)) {
          Int128 oldv, cmpv, newv;
@@ -XXX,XX +XXX,XX @@ uint32_t helper_stqcx_be_parallel(CPUPPCState *env, target_ulong addr,
  {
      bool success = false;
 +    /* We will have raised EXCP_ATOMIC from the translator.  */
 +    assert(HAVE_CMPXCHG128);
 +
      if (likely(addr == env->reserve_addr)) {
          Int128 oldv, cmpv, newv;
 diff --git a/target/ppc/translate.c b/target/ppc/translate.c
 index XXXXXXX..XXXXXXX 100644
 --- a/target/ppc/translate.c
 +++ b/target/ppc/translate.c
@@ -XXX,XX +XXX,XX @@
  #include "trace-tcg.h"
  #include "exec/translator.h"
  #include "exec/log.h"
 +#include "qemu/atomic128.h"
  #define CPU_SINGLE_STEP 0x1
@@ -XXX,XX +XXX,XX @@ static void gen_lq(DisasContext *ctx)
      hi = cpu_gpr[rd];
      if (tb_cflags(ctx->base.tb) & CF_PARALLEL) {
 -#ifdef CONFIG_ATOMIC128
 -        TCGv_i32 oi = tcg_temp_new_i32();
 -        if (ctx->le_mode) {
 -            tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ, ctx->mem_idx));
 -            gen_helper_lq_le_parallel(lo, cpu_env, EA, oi);
 +        if (HAVE_ATOMIC128) {
 +            TCGv_i32 oi = tcg_temp_new_i32();
 +            if (ctx->le_mode) {
 +                tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ, ctx->mem_idx));
 +                gen_helper_lq_le_parallel(lo, cpu_env, EA, oi);
 +            } else {
 +                tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ, ctx->mem_idx));
 +                gen_helper_lq_be_parallel(lo, cpu_env, EA, oi);
 +            }
 +            tcg_temp_free_i32(oi);
 +            tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh));
          } else {
 -            tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ, ctx->mem_idx));
 -            gen_helper_lq_be_parallel(lo, cpu_env, EA, oi);
 +            /* Restart with exclusive lock.  */
 +            gen_helper_exit_atomic(cpu_env);
 +            ctx->base.is_jmp = DISAS_NORETURN;
          }
--        tcg_temp_free_i32(oi);
+         break;
--        tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh));
--#else
+-    case INDEX_op_extu_i32_i64:
--        /* Restart with exclusive lock.  */
+-        tcg_out_ext32u(s, a0, a1);
--        gen_helper_exit_atomic(cpu_env);
+-        break;
--        ctx->base.is_jmp = DISAS_NORETURN;
+-
--#endif
+     case INDEX_op_deposit_i64:
-     } else if (ctx->le_mode) {
+     case INDEX_op_deposit_i32:
-         tcg_gen_qemu_ld_i64(lo, EA, ctx->mem_idx, MO_LEQ);
+         tcg_out_dep(s, ext, a0, REG0(2), args[3], args[4]);
-         gen_addr_add(ctx, EA, EA, 8);
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-@@ -XXX,XX +XXX,XX @@ static void gen_std(DisasContext *ctx)
+     case INDEX_op_ext32s_i64:
-         hi = cpu_gpr[rs];
+     case INDEX_op_ext32u_i64:
+     case INDEX_op_ext_i32_i64:
-         if (tb_cflags(ctx->base.tb) & CF_PARALLEL) {
++    case INDEX_op_extu_i32_i64:
--#ifdef CONFIG_ATOMIC128
+     default:
--            TCGv_i32 oi = tcg_temp_new_i32();
+         g_assert_not_reached();
--            if (ctx->le_mode) {
+     }
--                tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ, ctx->mem_idx));
+diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
--                gen_helper_stq_le_parallel(cpu_env, EA, lo, hi, oi);
+index XXXXXXX..XXXXXXX 100644
-+            if (HAVE_ATOMIC128) {
+--- a/tcg/arm/tcg-target.c.inc
-+                TCGv_i32 oi = tcg_temp_new_i32();
++++ b/tcg/arm/tcg-target.c.inc
-+                if (ctx->le_mode) {
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn)
-+                    tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ, ctx->mem_idx));
+     g_assert_not_reached();
-+                    gen_helper_stq_le_parallel(cpu_env, EA, lo, hi, oi);
+ }
-+                } else {
-+                    tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ, ctx->mem_idx));
++static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn)
-+                    gen_helper_stq_be_parallel(cpu_env, EA, lo, hi, oi);
++{
-+                }
++    g_assert_not_reached();
-+                tcg_temp_free_i32(oi);
++}
-             } else {
++
--                tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ, ctx->mem_idx));
+ static void tcg_out_bswap16(TCGContext *s, ARMCond cond,
--                gen_helper_stq_be_parallel(cpu_env, EA, lo, hi, oi);
+                             TCGReg rd, TCGReg rn, int flags)
-+                /* Restart with exclusive lock.  */
+ {
-+                gen_helper_exit_atomic(cpu_env);
+diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
-+                ctx->base.is_jmp = DISAS_NORETURN;
+index XXXXXXX..XXXXXXX 100644
-             }
+--- a/tcg/i386/tcg-target.c.inc
--            tcg_temp_free_i32(oi);
++++ b/tcg/i386/tcg-target.c.inc
--#else
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
--            /* Restart with exclusive lock.  */
+     tcg_out_ext32s(s, dest, src);
--            gen_helper_exit_atomic(cpu_env);
+ }
--            ctx->base.is_jmp = DISAS_NORETURN;
--#endif
++static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
-         } else if (ctx->le_mode) {
++{
-             tcg_gen_qemu_st_i64(lo, EA, ctx->mem_idx, MO_LEQ);
++    tcg_out_ext32u(s, dest, src);
-             gen_addr_add(ctx, EA, EA, 8);
++}
-@@ -XXX,XX +XXX,XX @@ static void gen_lqarx(DisasContext *ctx)
++
-     hi = cpu_gpr[rd];
+ static inline void tcg_out_bswap64(TCGContext *s, int reg)
+ {
-     if (tb_cflags(ctx->base.tb) & CF_PARALLEL) {
+     tcg_out_opc(s, OPC_BSWAP + P_REXW + LOWREGMASK(reg), 0, reg, 0);
--#ifdef CONFIG_ATOMIC128
+@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
--        TCGv_i32 oi = tcg_temp_new_i32();
+     case INDEX_op_bswap64_i64:
--        if (ctx->le_mode) {
+         tcg_out_bswap64(s, a0);
--            tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ | MO_ALIGN_16,
+         break;
--                                                ctx->mem_idx));
+-    case INDEX_op_extu_i32_i64:
--            gen_helper_lq_le_parallel(lo, cpu_env, EA, oi);
+     case INDEX_op_extrl_i64_i32:
-+        if (HAVE_ATOMIC128) {
+         tcg_out_ext32u(s, a0, a1);
-+            TCGv_i32 oi = tcg_temp_new_i32();
+         break;
-+            if (ctx->le_mode) {
+@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
-+                tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ | MO_ALIGN_16,
+     case INDEX_op_ext32s_i64:
-+                                                    ctx->mem_idx));
+     case INDEX_op_ext32u_i64:
-+                gen_helper_lq_le_parallel(lo, cpu_env, EA, oi);
+     case INDEX_op_ext_i32_i64:
-+            } else {
++    case INDEX_op_extu_i32_i64:
-+                tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ | MO_ALIGN_16,
+     default:
-+                                                    ctx->mem_idx));
+         g_assert_not_reached();
-+                gen_helper_lq_be_parallel(lo, cpu_env, EA, oi);
+     }
-+            }
+diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
-+            tcg_temp_free_i32(oi);
+index XXXXXXX..XXXXXXX 100644
-+            tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh));
+--- a/tcg/loongarch64/tcg-target.c.inc
-         } else {
++++ b/tcg/loongarch64/tcg-target.c.inc
--            tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ | MO_ALIGN_16,
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
--                                                ctx->mem_idx));
+     tcg_out_ext32s(s, ret, arg);
--            gen_helper_lq_be_parallel(lo, cpu_env, EA, oi);
+ }
-+            /* Restart with exclusive lock.  */
-+            gen_helper_exit_atomic(cpu_env);
++static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
-+            ctx->base.is_jmp = DISAS_NORETURN;
++{
-+            tcg_temp_free(EA);
++    tcg_out_ext32u(s, ret, arg);
-+            return;
++}
 +
  static void tcg_out_clzctz(TCGContext *s, LoongArchInsn opc,
                             TCGReg a0, TCGReg a1, TCGReg a2,
                             bool c2, bool is_32bit)
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_brcond(s, a2, a0, a1, arg_label(args[3]));
          break;
 -    case INDEX_op_extu_i32_i64:
 -        tcg_out_ext32u(s, a0, a1);
 -        break;
 -
      case INDEX_op_extrl_i64_i32:
          tcg_out_ext32s(s, a0, a1);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext32s_i64:
      case INDEX_op_ext32u_i64:
      case INDEX_op_ext_i32_i64:
 +    case INDEX_op_extu_i32_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/mips/tcg-target.c.inc
 +++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
      tcg_out_ext32s(s, rd, rs);
  }
 +static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
 +{
 +    tcg_out_ext32u(s, rd, rs);
 +}
 +
  static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_extrl_i64_i32:
          tcg_out_ext32s(s, a0, a1);
          break;
 -    case INDEX_op_extu_i32_i64:
 -        tcg_out_ext32u(s, a0, a1);
 -        break;
      case INDEX_op_sar_i32:
          i1 = OPC_SRAV, i2 = OPC_SRA;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext32s_i64:
      case INDEX_op_ext32u_i64:
      case INDEX_op_ext_i32_i64:
 +    case INDEX_op_extu_i32_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/ppc/tcg-target.c.inc
 +++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg dst, TCGReg src)
      tcg_out_ext32s(s, dst, src);
  }
 +static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg dst, TCGReg src)
 +{
 +    tcg_out_ext32u(s, dst, src);
 +}
 +
  static inline void tcg_out_shli32(TCGContext *s, TCGReg dst, TCGReg src, int c)
  {
      tcg_out_rlw(s, RLWINM, dst, src, c, 0, 31 - c);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_qemu_st(s, args, true);
          break;
 -    case INDEX_op_extu_i32_i64:
 -        tcg_out_ext32u(s, args[0], args[1]);
 -        break;
 -
      case INDEX_op_setcond_i32:
          tcg_out_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1], args[2],
                          const_args[2]);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext32s_i64:
      case INDEX_op_ext32u_i64:
      case INDEX_op_ext_i32_i64:
 +    case INDEX_op_extu_i32_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/riscv/tcg-target.c.inc
 +++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
      tcg_out_ext32s(s, ret, arg);
  }
 +static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
 +{
 +    tcg_out_ext32u(s, ret, arg);
 +}
 +
  static void tcg_out_ldst(TCGContext *s, RISCVInsn opc, TCGReg data,
                           TCGReg addr, intptr_t offset)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_qemu_st(s, args, true);
          break;
 -    case INDEX_op_extu_i32_i64:
 -        tcg_out_ext32u(s, a0, a1);
 -        break;
 -
      case INDEX_op_extrl_i64_i32:
          tcg_out_ext32s(s, a0, a1);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext32s_i64:
      case INDEX_op_ext32u_i64:
      case INDEX_op_ext_i32_i64:
 +    case INDEX_op_extu_i32_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/s390x/tcg-target.c.inc
 +++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
      tcg_out_ext32s(s, dest, src);
  }
 +static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
 +{
 +    tcg_out_ext32u(s, dest, src);
 +}
 +
  static void tgen_andi_risbg(TCGContext *s, TCGReg out, TCGReg in, uint64_t val)
  {
      int msb, lsb;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
          }
--        tcg_temp_free_i32(oi);
+         break;
--        tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh));
--#else
+-    case INDEX_op_extu_i32_i64:
--        /* Restart with exclusive lock.  */
+-        tcg_out_ext32u(s, args[0], args[1]);
--        gen_helper_exit_atomic(cpu_env);
+-        break;
--        ctx->base.is_jmp = DISAS_NORETURN;
+-
--        tcg_temp_free(EA);
+     case INDEX_op_add2_i64:
--        return;
+         if (const_args[4]) {
--#endif
+             if ((int64_t)args[4] >= 0) {
-     } else if (ctx->le_mode) {
+@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
-         tcg_gen_qemu_ld_i64(lo, EA, ctx->mem_idx, MO_LEQ | MO_ALIGN_16);
+     case INDEX_op_ext32s_i64:
-         tcg_gen_mov_tl(cpu_reserve, EA);
+     case INDEX_op_ext32u_i64:
-@@ -XXX,XX +XXX,XX @@ static void gen_stqcx_(DisasContext *ctx)
+     case INDEX_op_ext_i32_i64:
-     hi = cpu_gpr[rs];
++    case INDEX_op_extu_i32_i64:
+     default:
-     if (tb_cflags(ctx->base.tb) & CF_PARALLEL) {
+         g_assert_not_reached();
--        TCGv_i32 oi = tcg_const_i32(DEF_MEMOP(MO_Q) | MO_ALIGN_16);
+     }
--#ifdef CONFIG_ATOMIC128
+diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
--        if (ctx->le_mode) {
+index XXXXXXX..XXXXXXX 100644
--            gen_helper_stqcx_le_parallel(cpu_crf[0], cpu_env, EA, lo, hi, oi);
+--- a/tcg/sparc64/tcg-target.c.inc
-+        if (HAVE_CMPXCHG128) {
++++ b/tcg/sparc64/tcg-target.c.inc
-+            TCGv_i32 oi = tcg_const_i32(DEF_MEMOP(MO_Q) | MO_ALIGN_16);
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
-+            if (ctx->le_mode) {
+     tcg_out_ext32s(s, rd, rs);
-+                gen_helper_stqcx_le_parallel(cpu_crf[0], cpu_env,
+ }
-+                                             EA, lo, hi, oi);
-+            } else {
++static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
-+                gen_helper_stqcx_be_parallel(cpu_crf[0], cpu_env,
++{
-+                                             EA, lo, hi, oi);
++    tcg_out_ext32u(s, rd, rs);
-+            }
++}
-+            tcg_temp_free_i32(oi);
++
-         } else {
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
--            gen_helper_stqcx_le_parallel(cpu_crf[0], cpu_env, EA, lo, hi, oi);
+                              tcg_target_long imm)
-+            /* Restart with exclusive lock.  */
+ {
-+            gen_helper_exit_atomic(cpu_env);
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-+            ctx->base.is_jmp = DISAS_NORETURN;
+     case INDEX_op_divu_i64:
-         }
+         c = ARITH_UDIVX;
--#else
+         goto gen_arith;
--        /* Restart with exclusive lock.  */
+-    case INDEX_op_extu_i32_i64:
--        gen_helper_exit_atomic(cpu_env);
+-        tcg_out_ext32u(s, a0, a1);
--        ctx->base.is_jmp = DISAS_NORETURN;
+-        break;
--#endif
+     case INDEX_op_extrl_i64_i32:
-         tcg_temp_free(EA);
+         tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
--        tcg_temp_free_i32(oi);
+         break;
-     } else {
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-         TCGLabel *lab_fail = gen_new_label();
+     case INDEX_op_ext32s_i64:
-         TCGLabel *lab_over = gen_new_label();
+     case INDEX_op_ext32u_i64:
      case INDEX_op_ext_i32_i64:
 +    case INDEX_op_extu_i32_i64:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/tci/tcg-target.c.inc
 +++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
      tcg_out_ext32s(s, rd, rs);
  }
 +static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
 +{
 +    tcg_out_ext32u(s, rd, rs);
 +}
 +
  static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      CASE_32_64(neg)      /* Optional (TCG_TARGET_HAS_neg_*). */
      CASE_32_64(not)      /* Optional (TCG_TARGET_HAS_not_*). */
 -    CASE_64(extu_i32)
      CASE_32_64(ctpop)    /* Optional (TCG_TARGET_HAS_ctpop_*). */
      case INDEX_op_bswap32_i32: /* Optional (TCG_TARGET_HAS_bswap32_i32). */
      case INDEX_op_bswap64_i64: /* Optional (TCG_TARGET_HAS_bswap64_i64). */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext32s_i64:
      case INDEX_op_ext32u_i64:
      case INDEX_op_ext_i32_i64:
 +    case INDEX_op_extu_i32_i64:
      default:
          g_assert_not_reached();
      }
 --
-.17.2
+.34.1

-[Qemu-devel] [PULL 03/21] tcg: fix use of uninitialized variable under CONFIG_PROFILER
+[PULL 11/15] tcg: Split out tcg_out_extrl_i64_i32
-From: "Emilio G. Cota" <cota@braap.org>
+We will need a backend interface for type truncation.  For those backends
 that did not enable TCG_TARGET_HAS_extrl_i64_i32, use tcg_out_mov.
 Use it in tcg_reg_alloc_op in the meantime.
-We forgot to initialize n in commit 15fa08f845 ("tcg: Dynamically
+Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 allocate TCGOps", 2017-12-29).
 Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
 Signed-off-by: Emilio G. Cota <cota@braap.org>
 Message-Id: <20181010144853.13005-3-cota@braap.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- tcg/tcg.c | 2 +-
+ tcg/tcg.c                        |  4 ++++
-file changed, 1 insertion(+), 1 deletion(-)
+ tcg/aarch64/tcg-target.c.inc     |  6 ++++++
  tcg/arm/tcg-target.c.inc         |  5 +++++
  tcg/i386/tcg-target.c.inc        |  9 ++++++---
  tcg/loongarch64/tcg-target.c.inc | 10 ++++++----
  tcg/mips/tcg-target.c.inc        |  9 ++++++---
  tcg/ppc/tcg-target.c.inc         |  7 +++++++
  tcg/riscv/tcg-target.c.inc       | 10 ++++++----
  tcg/s390x/tcg-target.c.inc       |  6 ++++++
  tcg/sparc64/tcg-target.c.inc     |  9 ++++++---
  tcg/tci/tcg-target.c.inc         |  7 +++++++
 files changed, 65 insertions(+), 17 deletions(-)
 diff --git a/tcg/tcg.c b/tcg/tcg.c
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/tcg.c
 +++ b/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
+ static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg);
- #ifdef CONFIG_PROFILER
+ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
-     {
+ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
--        int n;
++static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg);
-+        int n = 0;
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
+ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
-         QTAILQ_FOREACH(op, &s->ops, link) {
+ static void tcg_out_goto_tb(TCGContext *s, int which);
-             n++;
+@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
      case INDEX_op_extu_i32_i64:
          tcg_out_extu_i32_i64(s, new_args[0], new_args[1]);
          break;
 +    case INDEX_op_extrl_i64_i32:
 +        tcg_out_extrl_i64_i32(s, new_args[0], new_args[1]);
 +        break;
      default:
          if (def->flags & TCG_OPF_VECTOR) {
              tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
 diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/aarch64/tcg-target.c.inc
 +++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn)
      tcg_out_ext32u(s, rd, rn);
  }
 +static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rn)
 +{
 +    tcg_out_mov(s, TCG_TYPE_I32, rd, rn);
 +}
 +
  static void tcg_out_addsubi(TCGContext *s, int ext, TCGReg rd,
                              TCGReg rn, int64_t aimm)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext32u_i64:
      case INDEX_op_ext_i32_i64:
      case INDEX_op_extu_i32_i64:
 +    case INDEX_op_extrl_i64_i32:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/arm/tcg-target.c.inc
 +++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn)
      g_assert_not_reached();
  }
 +static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rn)
 +{
 +    g_assert_not_reached();
 +}
 +
  static void tcg_out_bswap16(TCGContext *s, ARMCond cond,
                              TCGReg rd, TCGReg rn, int flags)
  {
 diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/i386/tcg-target.c.inc
 +++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
      tcg_out_ext32u(s, dest, src);
  }
 +static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg dest, TCGReg src)
 +{
 +    tcg_out_ext32u(s, dest, src);
 +}
 +
  static inline void tcg_out_bswap64(TCGContext *s, int reg)
  {
      tcg_out_opc(s, OPC_BSWAP + P_REXW + LOWREGMASK(reg), 0, reg, 0);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_bswap64_i64:
          tcg_out_bswap64(s, a0);
          break;
 -    case INDEX_op_extrl_i64_i32:
 -        tcg_out_ext32u(s, a0, a1);
 -        break;
      case INDEX_op_extrh_i64_i32:
          tcg_out_shifti(s, SHIFT_SHR + P_REXW, a0, 32);
          break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext32u_i64:
      case INDEX_op_ext_i32_i64:
      case INDEX_op_extu_i32_i64:
 +    case INDEX_op_extrl_i64_i32:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/loongarch64/tcg-target.c.inc
 +++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
      tcg_out_ext32u(s, ret, arg);
  }
 +static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg)
 +{
 +    tcg_out_ext32s(s, ret, arg);
 +}
 +
  static void tcg_out_clzctz(TCGContext *s, LoongArchInsn opc,
                             TCGReg a0, TCGReg a1, TCGReg a2,
                             bool c2, bool is_32bit)
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_brcond(s, a2, a0, a1, arg_label(args[3]));
          break;
 -    case INDEX_op_extrl_i64_i32:
 -        tcg_out_ext32s(s, a0, a1);
 -        break;
 -
      case INDEX_op_extrh_i64_i32:
          tcg_out_opc_srai_d(s, a0, a1, 32);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext32u_i64:
      case INDEX_op_ext_i32_i64:
      case INDEX_op_extu_i32_i64:
 +    case INDEX_op_extrl_i64_i32:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/mips/tcg-target.c.inc
 +++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
      tcg_out_ext32u(s, rd, rs);
  }
 +static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs)
 +{
 +    tcg_out_ext32s(s, rd, rs);
 +}
 +
  static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_extrh_i64_i32:
          tcg_out_dsra(s, a0, a1, 32);
          break;
 -    case INDEX_op_extrl_i64_i32:
 -        tcg_out_ext32s(s, a0, a1);
 -        break;
      case INDEX_op_sar_i32:
          i1 = OPC_SRAV, i2 = OPC_SRA;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext32u_i64:
      case INDEX_op_ext_i32_i64:
      case INDEX_op_extu_i32_i64:
 +    case INDEX_op_extrl_i64_i32:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/ppc/tcg-target.c.inc
 +++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg dst, TCGReg src)
      tcg_out_ext32u(s, dst, src);
  }
 +static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rn)
 +{
 +    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
 +    tcg_out_mov(s, TCG_TYPE_I32, rd, rn);
 +}
 +
  static inline void tcg_out_shli32(TCGContext *s, TCGReg dst, TCGReg src, int c)
  {
      tcg_out_rlw(s, RLWINM, dst, src, c, 0, 31 - c);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext32u_i64:
      case INDEX_op_ext_i32_i64:
      case INDEX_op_extu_i32_i64:
 +    case INDEX_op_extrl_i64_i32:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/riscv/tcg-target.c.inc
 +++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
      tcg_out_ext32u(s, ret, arg);
  }
 +static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg)
 +{
 +    tcg_out_ext32s(s, ret, arg);
 +}
 +
  static void tcg_out_ldst(TCGContext *s, RISCVInsn opc, TCGReg data,
                           TCGReg addr, intptr_t offset)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_qemu_st(s, args, true);
          break;
 -    case INDEX_op_extrl_i64_i32:
 -        tcg_out_ext32s(s, a0, a1);
 -        break;
 -
      case INDEX_op_extrh_i64_i32:
          tcg_out_opc_imm(s, OPC_SRAI, a0, a1, 32);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext32u_i64:
      case INDEX_op_ext_i32_i64:
      case INDEX_op_extu_i32_i64:
 +    case INDEX_op_extrl_i64_i32:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/s390x/tcg-target.c.inc
 +++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
      tcg_out_ext32u(s, dest, src);
  }
 +static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg dest, TCGReg src)
 +{
 +    tcg_out_mov(s, TCG_TYPE_I32, dest, src);
 +}
 +
  static void tgen_andi_risbg(TCGContext *s, TCGReg out, TCGReg in, uint64_t val)
  {
      int msb, lsb;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext32u_i64:
      case INDEX_op_ext_i32_i64:
      case INDEX_op_extu_i32_i64:
 +    case INDEX_op_extrl_i64_i32:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/sparc64/tcg-target.c.inc
 +++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
      tcg_out_ext32u(s, rd, rs);
  }
 +static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs)
 +{
 +    tcg_out_mov(s, TCG_TYPE_I32, rd, rs);
 +}
 +
  static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_divu_i64:
          c = ARITH_UDIVX;
          goto gen_arith;
 -    case INDEX_op_extrl_i64_i32:
 -        tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
 -        break;
      case INDEX_op_extrh_i64_i32:
          tcg_out_arithi(s, a0, a1, 32, SHIFT_SRLX);
          break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext32u_i64:
      case INDEX_op_ext_i32_i64:
      case INDEX_op_extu_i32_i64:
 +    case INDEX_op_extrl_i64_i32:
      default:
          g_assert_not_reached();
      }
 diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/tci/tcg-target.c.inc
 +++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
      tcg_out_ext32u(s, rd, rs);
  }
 +static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs)
 +{
 +    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
 +    tcg_out_mov(s, TCG_TYPE_I32, rd, rs);
 +}
 +
  static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
      case INDEX_op_ext32u_i64:
      case INDEX_op_ext_i32_i64:
      case INDEX_op_extu_i32_i64:
 +    case INDEX_op_extrl_i64_i32:
      default:
          g_assert_not_reached();
      }
 --
-.17.2
+.34.1

-[Qemu-devel] [PULL 14/21] target/arm: Convert to HAVE_CMPXCHG128
+[PULL 12/15] tcg: Introduce tcg_out_movext
-Reviewed-by: Emilio G. Cota <cota@braap.org>
+This is common code in most qemu_{ld,st} slow paths, extending the
 input value for the store helper data argument or extending the
 return value from the load helper.
 Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- target/arm/helper-a64.c | 259 +++++++++++++++++++++-------------------
+ tcg/tcg.c                        | 63 ++++++++++++++++++++++++++++++++
-file changed, 133 insertions(+), 126 deletions(-)
+ tcg/aarch64/tcg-target.c.inc     |  8 +---
  tcg/arm/tcg-target.c.inc         | 16 ++------
  tcg/i386/tcg-target.c.inc        | 30 +++------------
  tcg/loongarch64/tcg-target.c.inc | 53 ++++-----------------------
  tcg/ppc/tcg-target.c.inc         | 38 +++++--------------
  tcg/riscv/tcg-target.c.inc       | 13 +------
  tcg/s390x/tcg-target.c.inc       | 19 ++--------
  tcg/sparc64/tcg-target.c.inc     | 31 +++-------------
 files changed, 103 insertions(+), 168 deletions(-)
-diff --git a/target/arm/helper-a64.c b/target/arm/helper-a64.c
+diff --git a/tcg/tcg.c b/tcg/tcg.c
 index XXXXXXX..XXXXXXX 100644
---- a/target/arm/helper-a64.c
+--- a/tcg/tcg.c
-+++ b/target/arm/helper-a64.c
++++ b/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@
+@@ -XXX,XX +XXX,XX @@ void tcg_raise_tb_overflow(TCGContext *s)
- #include "exec/exec-all.h"
+     siglongjmp(s->jmp_trans, -2);
  #include "exec/cpu_ldst.h"
  #include "qemu/int128.h"
 +#include "qemu/atomic128.h"
  #include "tcg.h"
  #include "fpu/softfloat.h"
  #include <zlib.h> /* For crc32 */
@@ -XXX,XX +XXX,XX @@ uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes)
      return crc32c(acc, buf, bytes) ^ 0xffffffff;
  }
--/* Returns 0 on success; 1 otherwise.  */
++/**
--static uint64_t do_paired_cmpxchg64_le(CPUARMState *env, uint64_t addr,
++ * tcg_out_movext -- move and extend
--                                       uint64_t new_lo, uint64_t new_hi,
++ * @s: tcg context
--                                       bool parallel, uintptr_t ra)
++ * @dst_type: integral type for destination
-+uint64_t HELPER(paired_cmpxchg64_le)(CPUARMState *env, uint64_t addr,
++ * @dst: destination register
-+                                     uint64_t new_lo, uint64_t new_hi)
++ * @src_type: integral type for source
 + * @src_ext: extension to apply to source
 + * @src: source register
 + *
 + * Move or extend @src into @dst, depending on @src_ext and the types.
 + */
 +static void __attribute__((unused))
 +tcg_out_movext(TCGContext *s, TCGType dst_type, TCGReg dst,
 +               TCGType src_type, MemOp src_ext, TCGReg src)
 +{
 +    switch (src_ext) {
 +    case MO_UB:
 +        tcg_out_ext8u(s, dst, src);
 +        break;
 +    case MO_SB:
 +        tcg_out_ext8s(s, dst_type, dst, src);
 +        break;
 +    case MO_UW:
 +        tcg_out_ext16u(s, dst, src);
 +        break;
 +    case MO_SW:
 +        tcg_out_ext16s(s, dst_type, dst, src);
 +        break;
 +    case MO_UL:
 +    case MO_SL:
 +        if (dst_type == TCG_TYPE_I32) {
 +            if (src_type == TCG_TYPE_I32) {
 +                tcg_out_mov(s, TCG_TYPE_I32, dst, src);
 +            } else {
 +                tcg_out_extrl_i64_i32(s, dst, src);
 +            }
 +        } else if (src_type == TCG_TYPE_I32) {
 +            if (src_ext & MO_SIGN) {
 +                tcg_out_exts_i32_i64(s, dst, src);
 +            } else {
 +                tcg_out_extu_i32_i64(s, dst, src);
 +            }
 +        } else {
 +            if (src_ext & MO_SIGN) {
 +                tcg_out_ext32s(s, dst, src);
 +            } else {
 +                tcg_out_ext32u(s, dst, src);
 +            }
 +        }
 +        break;
 +    case MO_UQ:
 +        tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
 +        if (dst_type == TCG_TYPE_I32) {
 +            tcg_out_extrl_i64_i32(s, dst, src);
 +        } else {
 +            tcg_out_mov(s, TCG_TYPE_I64, dst, src);
 +        }
 +        break;
 +    default:
 +        g_assert_not_reached();
 +    }
 +}
 +
  #define C_PFX1(P, A)                    P##A
  #define C_PFX2(P, A, B)                 P##A##_##B
  #define C_PFX3(P, A, B, C)              P##A##_##B##_##C
 diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/aarch64/tcg-target.c.inc
 +++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
  {
--    Int128 oldv, cmpv, newv;
+     MemOpIdx oi = lb->oi;
-+    Int128 cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
+     MemOp opc = get_memop(oi);
-+    Int128 newv = int128_make128(new_lo, new_hi);
+-    MemOp size = opc & MO_SIZE;
-+    Int128 oldv;
-+    uintptr_t ra = GETPC();
+     if (!reloc_pc19(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
-+    uint64_t o0, o1;
+         return false;
-     bool success;
+@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
+     tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X2, oi);
--    cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
+     tcg_out_adr(s, TCG_REG_X3, lb->raddr);
--    newv = int128_make128(new_lo, new_hi);
+     tcg_out_call_int(s, qemu_ld_helpers[opc & MO_SIZE]);
 -    if (opc & MO_SIGN) {
 -        tcg_out_sxt(s, lb->type, size, lb->datalo_reg, TCG_REG_X0);
 -    } else {
 -        tcg_out_mov(s, size == MO_64, lb->datalo_reg, TCG_REG_X0);
 -    }
 +    tcg_out_movext(s, lb->type, lb->datalo_reg,
 +                   TCG_TYPE_REG, opc & MO_SSIZE, TCG_REG_X0);
      tcg_out_goto(s, lb->raddr);
      return true;
  }
 diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/arm/tcg-target.c.inc
 +++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
      datalo = lb->datalo_reg;
      datahi = lb->datahi_reg;
 -    switch (opc & MO_SSIZE) {
 -    case MO_SB:
 -        tcg_out_ext8s(s, TCG_TYPE_I32, datalo, TCG_REG_R0);
 -        break;
 -    case MO_SW:
 -        tcg_out_ext16s(s, TCG_TYPE_I32, datalo, TCG_REG_R0);
 -        break;
 -    default:
 -        tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_R0);
 -        break;
 -    case MO_UQ:
 +    if ((opc & MO_SIZE) == MO_64) {
          if (datalo != TCG_REG_R1) {
              tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_R0);
              tcg_out_mov_reg(s, COND_AL, datahi, TCG_REG_R1);
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
              tcg_out_mov_reg(s, COND_AL, datahi, TCG_REG_R1);
              tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_TMP);
          }
 -        break;
 +    } else {
 +        tcg_out_movext(s, TCG_TYPE_I32, datalo,
 +                       TCG_TYPE_I32, opc & MO_SSIZE, TCG_REG_R0);
      }
      tcg_out_goto(s, COND_AL, lb->raddr);
 diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/i386/tcg-target.c.inc
 +++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
      tcg_out_branch(s, 1, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]);
      data_reg = l->datalo_reg;
 -    switch (opc & MO_SSIZE) {
 -    case MO_SB:
 -        tcg_out_ext8s(s, l->type, data_reg, TCG_REG_EAX);
 -        break;
 -    case MO_SW:
 -        tcg_out_ext16s(s, l->type, data_reg, TCG_REG_EAX);
 -        break;
 -#if TCG_TARGET_REG_BITS == 64
 -    case MO_SL:
 -        tcg_out_ext32s(s, data_reg, TCG_REG_EAX);
 -        break;
 -#endif
 -    case MO_UB:
 -    case MO_UW:
 -        /* Note that the helpers have zero-extended to tcg_target_long.  */
 -    case MO_UL:
 -        tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
 -        break;
 -    case MO_UQ:
 -        if (TCG_TARGET_REG_BITS == 64) {
 -            tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_RAX);
 -        } else if (data_reg == TCG_REG_EDX) {
 +    if (TCG_TARGET_REG_BITS == 32 && (opc & MO_SIZE) == MO_64) {
 +        if (data_reg == TCG_REG_EDX) {
              /* xchg %edx, %eax */
              tcg_out_opc(s, OPC_XCHG_ax_r32 + TCG_REG_EDX, 0, 0, 0);
              tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EAX);
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
              tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
              tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EDX);
          }
 -        break;
 -    default:
 -        g_assert_not_reached();
 +    } else {
 +        tcg_out_movext(s, l->type, data_reg,
 +                       TCG_TYPE_REG, opc & MO_SSIZE, TCG_REG_EAX);
      }
      /* Jump to the code corresponding to next IR of qemu_st */
 diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/loongarch64/tcg-target.c.inc
 +++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
      MemOpIdx oi = l->oi;
      MemOp opc = get_memop(oi);
      MemOp size = opc & MO_SIZE;
 -    TCGType type = l->type;
      /* resolve label address */
      if (!reloc_br_sk16(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
      tcg_out_call_int(s, qemu_ld_helpers[size], false);
 -    switch (opc & MO_SSIZE) {
 -    case MO_SB:
 -        tcg_out_ext8s(s, type, l->datalo_reg, TCG_REG_A0);
 -        break;
 -    case MO_SW:
 -        tcg_out_ext16s(s, type, l->datalo_reg, TCG_REG_A0);
 -        break;
 -    case MO_SL:
 -        tcg_out_ext32s(s, l->datalo_reg, TCG_REG_A0);
 -        break;
 -    case MO_UL:
 -        if (type == TCG_TYPE_I32) {
 -            /* MO_UL loads of i32 should be sign-extended too */
 -            tcg_out_ext32s(s, l->datalo_reg, TCG_REG_A0);
 -            break;
 -        }
 -        /* fallthrough */
 -    default:
 -        tcg_out_mov(s, type, l->datalo_reg, TCG_REG_A0);
 -        break;
 -    }
 -
--    if (parallel) {
++    tcg_out_movext(s, l->type, l->datalo_reg,
--#ifndef CONFIG_ATOMIC128
++                   TCG_TYPE_REG, opc & MO_SSIZE, TCG_REG_A0);
--        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
+     return tcg_out_goto(s, l->raddr);
--#else
+ }
--        int mem_idx = cpu_mmu_index(env, false);
--        TCGMemOpIdx oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
+@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
--        oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra);
+     /* call store helper */
--        success = int128_eq(oldv, cmpv);
+     tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_A0, TCG_AREG0);
--#endif
+     tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_A1, l->addrlo_reg);
--    } else {
+-    switch (size) {
--        uint64_t o0, o1;
+-    case MO_8:
 -        tcg_out_ext8u(s, TCG_REG_A2, l->datalo_reg);
 -        break;
 -    case MO_16:
 -        tcg_out_ext16u(s, TCG_REG_A2, l->datalo_reg);
 -        break;
 -    case MO_32:
 -        tcg_out_ext32u(s, TCG_REG_A2, l->datalo_reg);
 -        break;
 -    case MO_64:
 -        tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_A2, l->datalo_reg);
 -        break;
 -    default:
 -        g_assert_not_reached();
 -        break;
 -    }
 +    tcg_out_movext(s, size == MO_64 ? TCG_TYPE_I32 : TCG_TYPE_I32, TCG_REG_A2,
 +                   l->type, size, l->datalo_reg);
      tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A3, oi);
      tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A4, (tcg_target_long)l->raddr);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_indexed(TCGContext *s, TCGReg data,
      }
  }
 -static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args)
 +static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, TCGType type)
  {
      TCGReg addr_regl;
      TCGReg data_regl;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args)
      tcg_out_tlb_load(s, addr_regl, oi, label_ptr, 0);
      base = tcg_out_zext_addr_if_32_bit(s, addr_regl, TCG_REG_TMP0);
      tcg_out_qemu_st_indexed(s, data_regl, base, TCG_REG_TMP2, opc);
 -    add_qemu_ldst_label(s, 0, oi,
 -                        0, /* type param is unused for stores */
 +    add_qemu_ldst_label(s, 0, oi, type,
                          data_regl, addr_regl,
                          s->code_ptr, label_ptr);
  #else
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
          tcg_out_qemu_ld(s, args, TCG_TYPE_I64);
          break;
      case INDEX_op_qemu_st_i32:
 -        tcg_out_qemu_st(s, args);
 +        tcg_out_qemu_st(s, args, TCG_TYPE_I32);
          break;
      case INDEX_op_qemu_st_i64:
 -        tcg_out_qemu_st(s, args);
 +        tcg_out_qemu_st(s, args, TCG_TYPE_I64);
          break;
      case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
 diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/ppc/tcg-target.c.inc
 +++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static const uint32_t qemu_stx_opc[(MO_SIZE + MO_BSWAP) + 1] = {
      [MO_BSWAP | MO_UQ] = STDBRX,
  };
 -static const uint32_t qemu_exts_opc[4] = {
 -    EXTSB, EXTSH, EXTSW, 0
 -};
 -
- #ifdef CONFIG_USER_ONLY
+ #if defined (CONFIG_SOFTMMU)
--        /* ??? Enforce alignment.  */
+ /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
--        uint64_t *haddr = g2h(addr);
+  *                                 int mmu_idx, uintptr_t ra)
-+    /* ??? Enforce alignment.  */
+@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
-+    uint64_t *haddr = g2h(addr);
+     if (TCG_TARGET_REG_BITS == 32 && (opc & MO_SIZE) == MO_64) {
+         tcg_out_mov(s, TCG_TYPE_I32, lo, TCG_REG_R4);
--        helper_retaddr = ra;
+         tcg_out_mov(s, TCG_TYPE_I32, hi, TCG_REG_R3);
--        o0 = ldq_le_p(haddr + 0);
+-    } else if (opc & MO_SIGN) {
--        o1 = ldq_le_p(haddr + 1);
+-        uint32_t insn = qemu_exts_opc[opc & MO_SIZE];
--        oldv = int128_make128(o0, o1);
+-        tcg_out32(s, insn | RA(lo) | RS(TCG_REG_R3));
-+    helper_retaddr = ra;
+     } else {
-+    o0 = ldq_le_p(haddr + 0);
+-        tcg_out_mov(s, TCG_TYPE_REG, lo, TCG_REG_R3);
-+    o1 = ldq_le_p(haddr + 1);
++        tcg_out_movext(s, lb->type, lo,
-+    oldv = int128_make128(o0, o1);
++                       TCG_TYPE_REG, opc & MO_SSIZE, TCG_REG_R3);
+     }
--        success = int128_eq(oldv, cmpv);
--        if (success) {
+     tcg_out_b(s, 0, lb->raddr);
--            stq_le_p(haddr + 0, int128_getlo(newv));
+@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
--            stq_le_p(haddr + 1, int128_gethi(newv));
      lo = lb->datalo_reg;
      hi = lb->datahi_reg;
 -    if (TCG_TARGET_REG_BITS == 32) {
 -        switch (s_bits) {
 -        case MO_64:
 -            arg |= (TCG_TARGET_CALL_ARG_I64 == TCG_CALL_ARG_EVEN);
 -            tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
 -            /* FALLTHRU */
 -        case MO_32:
 -            tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
 -            break;
 -        default:
 -            tcg_out_rlw(s, RLWINM, arg++, lo, 0, 32 - (8 << s_bits), 31);
 -            break;
 -        }
--        helper_retaddr = 0;
++    if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) {
--#else
++        arg |= (TCG_TARGET_CALL_ARG_I64 == TCG_CALL_ARG_EVEN);
--        int mem_idx = cpu_mmu_index(env, false);
++        tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
--        TCGMemOpIdx oi0 = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
++        tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
--        TCGMemOpIdx oi1 = make_memop_idx(MO_LEQ, mem_idx);
+     } else {
--
+-        if (s_bits == MO_64) {
--        o0 = helper_le_ldq_mmu(env, addr + 0, oi0, ra);
+-            tcg_out_mov(s, TCG_TYPE_I64, arg++, lo);
--        o1 = helper_le_ldq_mmu(env, addr + 8, oi1, ra);
+-        } else {
--        oldv = int128_make128(o0, o1);
+-            tcg_out_rld(s, RLDICL, arg++, lo, 0, 64 - (8 << s_bits));
 -
 -        success = int128_eq(oldv, cmpv);
 -        if (success) {
 -            helper_le_stq_mmu(env, addr + 0, int128_getlo(newv), oi1, ra);
 -            helper_le_stq_mmu(env, addr + 8, int128_gethi(newv), oi1, ra);
 -        }
--#endif
++        tcg_out_movext(s, s_bits == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32,
-+    success = int128_eq(oldv, cmpv);
++                       arg++, lb->type, s_bits, lo);
-+    if (success) {
+     }
-+        stq_le_p(haddr + 0, int128_getlo(newv));
-+        stq_le_p(haddr + 1, int128_gethi(newv));
+     tcg_out_movi(s, TCG_TYPE_I32, arg++, oi);
-     }
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
-+    helper_retaddr = 0;
+         } else {
-+#else
+             insn = qemu_ldx_opc[opc & (MO_SIZE | MO_BSWAP)];
-+    int mem_idx = cpu_mmu_index(env, false);
+             tcg_out32(s, insn | TAB(datalo, rbase, addrlo));
-+    TCGMemOpIdx oi0 = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
+-            insn = qemu_exts_opc[s_bits];
-+    TCGMemOpIdx oi1 = make_memop_idx(MO_LEQ, mem_idx);
+-            tcg_out32(s, insn | RA(datalo) | RS(datalo));
-+
++            tcg_out_movext(s, TCG_TYPE_REG, datalo,
-+    o0 = helper_le_ldq_mmu(env, addr + 0, oi0, ra);
++                           TCG_TYPE_REG, opc & MO_SSIZE, datalo);
-+    o1 = helper_le_ldq_mmu(env, addr + 8, oi1, ra);
+         }
-+    oldv = int128_make128(o0, o1);
+     }
-+
-+    success = int128_eq(oldv, cmpv);
+diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
-+    if (success) {
+index XXXXXXX..XXXXXXX 100644
-+        helper_le_stq_mmu(env, addr + 0, int128_getlo(newv), oi1, ra);
+--- a/tcg/riscv/tcg-target.c.inc
-+        helper_le_stq_mmu(env, addr + 8, int128_gethi(newv), oi1, ra);
++++ b/tcg/riscv/tcg-target.c.inc
-+    }
+@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
-+#endif
+     /* call store helper */
+     tcg_out_mov(s, TCG_TYPE_PTR, a0, TCG_AREG0);
-     return !success;
+     tcg_out_mov(s, TCG_TYPE_PTR, a1, l->addrlo_reg);
- }
+-    tcg_out_mov(s, TCG_TYPE_PTR, a2, l->datalo_reg);
+-    switch (s_bits) {
--uint64_t HELPER(paired_cmpxchg64_le)(CPUARMState *env, uint64_t addr,
+-    case MO_8:
--                                              uint64_t new_lo, uint64_t new_hi)
+-        tcg_out_ext8u(s, a2, a2);
 -        break;
 -    case MO_16:
 -        tcg_out_ext16u(s, a2, a2);
 -        break;
 -    default:
 -        break;
 -    }
 +    tcg_out_movext(s, s_bits == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32, a2,
 +                   l->type, s_bits, l->datalo_reg);
      tcg_out_movi(s, TCG_TYPE_PTR, a3, oi);
      tcg_out_movi(s, TCG_TYPE_PTR, a4, (tcg_target_long)l->raddr);
 diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/s390x/tcg-target.c.inc
 +++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
      TCGReg data_reg = lb->datalo_reg;
      MemOpIdx oi = lb->oi;
      MemOp opc = get_memop(oi);
 +    MemOp size = opc & MO_SIZE;
      if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL,
                       (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) {
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
      if (TARGET_LONG_BITS == 64) {
          tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R3, addr_reg);
      }
 -    switch (opc & MO_SIZE) {
 -    case MO_UB:
 -        tcg_out_ext8u(s, TCG_REG_R4, data_reg);
 -        break;
 -    case MO_UW:
 -        tcg_out_ext16u(s, TCG_REG_R4, data_reg);
 -        break;
 -    case MO_UL:
 -        tcg_out_ext32u(s, TCG_REG_R4, data_reg);
 -        break;
 -    case MO_UQ:
 -        tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
 -        break;
 -    default:
 -        g_assert_not_reached();
 -    }
 +    tcg_out_movext(s, size == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32,
 +                   TCG_REG_R4, lb->type, size, data_reg);
      tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R5, oi);
      tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R6, (uintptr_t)lb->raddr);
      tcg_out_call_int(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
 diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/sparc64/tcg-target.c.inc
 +++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_mb(TCGContext *s, TCGArg a0)
  static const tcg_insn_unit *qemu_ld_trampoline[(MO_SSIZE | MO_BSWAP) + 1];
  static const tcg_insn_unit *qemu_st_trampoline[(MO_SIZE | MO_BSWAP) + 1];
 -static void emit_extend(TCGContext *s, TCGReg r, int op)
 -{
--    return do_paired_cmpxchg64_le(env, addr, new_lo, new_hi, false, GETPC());
+-    /* Emit zero extend of 8, 16 or 32 bit data as
 -     * required by the MO_* value op; do nothing for 64 bit.
 -     */
 -    switch (op & MO_SIZE) {
 -    case MO_8:
 -        tcg_out_ext8u(s, r, r);
 -        break;
 -    case MO_16:
 -        tcg_out_ext16u(s, r, r);
 -        break;
 -    case MO_32:
 -        tcg_out_ext32u(s, r, r);
 -        break;
 -    case MO_64:
 -        break;
 -    }
 -}
 -
- uint64_t HELPER(paired_cmpxchg64_le_parallel)(CPUARMState *env, uint64_t addr,
+ static void build_trampolines(TCGContext *s)
-                                               uint64_t new_lo, uint64_t new_hi)
+ {
--{
+     static void * const qemu_ld_helpers[] = {
--    return do_paired_cmpxchg64_le(env, addr, new_lo, new_hi, true, GETPC());
+@@ -XXX,XX +XXX,XX @@ static void build_trampolines(TCGContext *s)
--}
+         }
          qemu_st_trampoline[i] = tcg_splitwx_to_rx(s->code_ptr);
 -        emit_extend(s, TCG_REG_O2, i);
 -
--static uint64_t do_paired_cmpxchg64_be(CPUARMState *env, uint64_t addr,
+         /* Set the retaddr operand.  */
--                                       uint64_t new_lo, uint64_t new_hi,
+         tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O4, TCG_REG_O7);
--                                       bool parallel, uintptr_t ra)
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr,
  }
  static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr,
 -                            MemOpIdx oi)
 +                            MemOpIdx oi, TCGType data_type)
  {
-     Int128 oldv, cmpv, newv;
+     MemOp memop = get_memop(oi);
-+    uintptr_t ra = GETPC();
+     tcg_insn_unit *label_ptr;
-     bool success;
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr,
-+    int mem_idx;
+     /* TLB Miss.  */
-+    TCGMemOpIdx oi;
+     tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_O1, addrz);
--    /* high and low need to be switched here because this is not actually a
+-    tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_O2, data);
--     * 128bit store but two doublewords stored consecutively
++    tcg_out_movext(s, (memop & MO_SIZE) == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32,
--     */
++                   TCG_REG_O2, data_type, memop & MO_SIZE, data);
--    cmpv = int128_make128(env->exclusive_high, env->exclusive_val);
--    newv = int128_make128(new_hi, new_lo);
+     func = qemu_st_trampoline[memop & (MO_BSWAP | MO_SIZE)];
--
+     tcg_debug_assert(func != NULL);
--    if (parallel) {
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
--#ifndef CONFIG_ATOMIC128
+         tcg_out_qemu_ld(s, a0, a1, a2, true);
-+    if (!HAVE_CMPXCHG128) {
+         break;
-         cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
+     case INDEX_op_qemu_st_i32:
--#else
++        tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32);
--        int mem_idx = cpu_mmu_index(env, false);
++        break;
--        TCGMemOpIdx oi = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
+     case INDEX_op_qemu_st_i64:
--        oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
+-        tcg_out_qemu_st(s, a0, a1, a2);
--        success = int128_eq(oldv, cmpv);
++        tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I64);
--#endif
+         break;
--    } else {
--        uint64_t o0, o1;
+     case INDEX_op_ld32s_i64:
 -
 -#ifdef CONFIG_USER_ONLY
 -        /* ??? Enforce alignment.  */
 -        uint64_t *haddr = g2h(addr);
 -
 -        helper_retaddr = ra;
 -        o1 = ldq_be_p(haddr + 0);
 -        o0 = ldq_be_p(haddr + 1);
 -        oldv = int128_make128(o0, o1);
 -
 -        success = int128_eq(oldv, cmpv);
 -        if (success) {
 -            stq_be_p(haddr + 0, int128_gethi(newv));
 -            stq_be_p(haddr + 1, int128_getlo(newv));
 -        }
 -        helper_retaddr = 0;
 -#else
 -        int mem_idx = cpu_mmu_index(env, false);
 -        TCGMemOpIdx oi0 = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
 -        TCGMemOpIdx oi1 = make_memop_idx(MO_BEQ, mem_idx);
 -
 -        o1 = helper_be_ldq_mmu(env, addr + 0, oi0, ra);
 -        o0 = helper_be_ldq_mmu(env, addr + 8, oi1, ra);
 -        oldv = int128_make128(o0, o1);
 -
 -        success = int128_eq(oldv, cmpv);
 -        if (success) {
 -            helper_be_stq_mmu(env, addr + 0, int128_gethi(newv), oi1, ra);
 -            helper_be_stq_mmu(env, addr + 8, int128_getlo(newv), oi1, ra);
 -        }
 -#endif
      }
 +    mem_idx = cpu_mmu_index(env, false);
 +    oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
 +
 +    cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
 +    newv = int128_make128(new_lo, new_hi);
 +    oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra);
 +
 +    success = int128_eq(oldv, cmpv);
      return !success;
  }
  uint64_t HELPER(paired_cmpxchg64_be)(CPUARMState *env, uint64_t addr,
                                       uint64_t new_lo, uint64_t new_hi)
  {
 -    return do_paired_cmpxchg64_be(env, addr, new_lo, new_hi, false, GETPC());
 +    /*
 +     * High and low need to be switched here because this is not actually a
 +     * 128bit store but two doublewords stored consecutively
 +     */
 +    Int128 cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
 +    Int128 newv = int128_make128(new_lo, new_hi);
 +    Int128 oldv;
 +    uintptr_t ra = GETPC();
 +    uint64_t o0, o1;
 +    bool success;
 +
 +#ifdef CONFIG_USER_ONLY
 +    /* ??? Enforce alignment.  */
 +    uint64_t *haddr = g2h(addr);
 +
 +    helper_retaddr = ra;
 +    o1 = ldq_be_p(haddr + 0);
 +    o0 = ldq_be_p(haddr + 1);
 +    oldv = int128_make128(o0, o1);
 +
 +    success = int128_eq(oldv, cmpv);
 +    if (success) {
 +        stq_be_p(haddr + 0, int128_gethi(newv));
 +        stq_be_p(haddr + 1, int128_getlo(newv));
 +    }
 +    helper_retaddr = 0;
 +#else
 +    int mem_idx = cpu_mmu_index(env, false);
 +    TCGMemOpIdx oi0 = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
 +    TCGMemOpIdx oi1 = make_memop_idx(MO_BEQ, mem_idx);
 +
 +    o1 = helper_be_ldq_mmu(env, addr + 0, oi0, ra);
 +    o0 = helper_be_ldq_mmu(env, addr + 8, oi1, ra);
 +    oldv = int128_make128(o0, o1);
 +
 +    success = int128_eq(oldv, cmpv);
 +    if (success) {
 +        helper_be_stq_mmu(env, addr + 0, int128_gethi(newv), oi1, ra);
 +        helper_be_stq_mmu(env, addr + 8, int128_getlo(newv), oi1, ra);
 +    }
 +#endif
 +
 +    return !success;
  }
  uint64_t HELPER(paired_cmpxchg64_be_parallel)(CPUARMState *env, uint64_t addr,
 -                                     uint64_t new_lo, uint64_t new_hi)
 +                                              uint64_t new_lo, uint64_t new_hi)
  {
 -    return do_paired_cmpxchg64_be(env, addr, new_lo, new_hi, true, GETPC());
 +    Int128 oldv, cmpv, newv;
 +    uintptr_t ra = GETPC();
 +    bool success;
 +    int mem_idx;
 +    TCGMemOpIdx oi;
 +
 +    if (!HAVE_CMPXCHG128) {
 +        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
 +    }
 +
 +    mem_idx = cpu_mmu_index(env, false);
 +    oi = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
 +
 +    /*
 +     * High and low need to be switched here because this is not actually a
 +     * 128bit store but two doublewords stored consecutively
 +     */
 +    cmpv = int128_make128(env->exclusive_high, env->exclusive_val);
 +    newv = int128_make128(new_hi, new_lo);
 +    oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
 +
 +    success = int128_eq(oldv, cmpv);
 +    return !success;
  }
  /* Writes back the old data into Rs.  */
  void HELPER(casp_le_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
                                uint64_t new_lo, uint64_t new_hi)
  {
 -    uintptr_t ra = GETPC();
 -#ifndef CONFIG_ATOMIC128
 -    cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
 -#else
      Int128 oldv, cmpv, newv;
 +    uintptr_t ra = GETPC();
 +    int mem_idx;
 +    TCGMemOpIdx oi;
 +
 +    if (!HAVE_CMPXCHG128) {
 +        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
 +    }
 +
 +    mem_idx = cpu_mmu_index(env, false);
 +    oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
      cmpv = int128_make128(env->xregs[rs], env->xregs[rs + 1]);
      newv = int128_make128(new_lo, new_hi);
 -
 -    int mem_idx = cpu_mmu_index(env, false);
 -    TCGMemOpIdx oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
      oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra);
      env->xregs[rs] = int128_getlo(oldv);
      env->xregs[rs + 1] = int128_gethi(oldv);
 -#endif
  }
  void HELPER(casp_be_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
                                uint64_t new_hi, uint64_t new_lo)
  {
 -    uintptr_t ra = GETPC();
 -#ifndef CONFIG_ATOMIC128
 -    cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
 -#else
      Int128 oldv, cmpv, newv;
 +    uintptr_t ra = GETPC();
 +    int mem_idx;
 +    TCGMemOpIdx oi;
 +
 +    if (!HAVE_CMPXCHG128) {
 +        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
 +    }
 +
 +    mem_idx = cpu_mmu_index(env, false);
 +    oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
      cmpv = int128_make128(env->xregs[rs + 1], env->xregs[rs]);
      newv = int128_make128(new_lo, new_hi);
 -
 -    int mem_idx = cpu_mmu_index(env, false);
 -    TCGMemOpIdx oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
      oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
      env->xregs[rs + 1] = int128_getlo(oldv);
      env->xregs[rs] = int128_gethi(oldv);
 -#endif
  }
  /*
 --
-.17.2
+.34.1

-[Qemu-devel] [PULL 10/21] cputlb: serialize tlb updates with env->tlb_lock
+[PULL 13/15] tcg: Introduce tcg_out_xchg
-From: "Emilio G. Cota" <cota@braap.org>
+We will want a backend interface for register swapping.
+This is only properly defined for x86; all others get a
-Currently we rely on atomic operations for cross-CPU invalidations.
+stub version that always indicates failure.
-There are two cases that these atomics miss: cross-CPU invalidations
-can race with either (1) vCPU threads flushing their TLB, which
+Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 happens via memset, or (2) vCPUs calling tlb_reset_dirty on their TLB,
 which updates .addr_write with a regular store. This results in
 undefined behaviour, since we're mixing regular and atomic ops
 on concurrent accesses.
 Fix it by using tlb_lock, a per-vCPU lock. All updaters of tlb_table
 and the corresponding victim cache now hold the lock.
 The readers that do not hold tlb_lock must use atomic reads when
 reading .addr_write, since this field can be updated by other threads;
 the conversion to atomic reads is done in the next patch.
 Note that an alternative fix would be to expand the use of atomic ops.
 However, in the case of TLB flushes this would have a huge performance
 impact, since (1) TLB flushes can happen very frequently and (2) we
 currently use a full memory barrier to flush each TLB entry, and a TLB
 has many entries. Instead, acquiring the lock is barely slower than a
 full memory barrier since it is uncontended, and with a single lock
 acquisition we can flush the entire TLB.
 Tested-by: Alex Bennée <alex.bennee@linaro.org>
 Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
 Signed-off-by: Emilio G. Cota <cota@braap.org>
 Message-Id: <20181009174557.16125-6-cota@braap.org>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- include/exec/cpu-defs.h |   3 +
+ tcg/tcg.c                        | 2 ++
- accel/tcg/cputlb.c      | 155 ++++++++++++++++++++++------------------
+ tcg/aarch64/tcg-target.c.inc     | 5 +++++
-files changed, 87 insertions(+), 71 deletions(-)
+ tcg/arm/tcg-target.c.inc         | 5 +++++
+ tcg/i386/tcg-target.c.inc        | 8 ++++++++
-diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h
+ tcg/loongarch64/tcg-target.c.inc | 5 +++++
-index XXXXXXX..XXXXXXX 100644
+ tcg/mips/tcg-target.c.inc        | 5 +++++
---- a/include/exec/cpu-defs.h
+ tcg/ppc/tcg-target.c.inc         | 5 +++++
-+++ b/include/exec/cpu-defs.h
+ tcg/riscv/tcg-target.c.inc       | 5 +++++
-@@ -XXX,XX +XXX,XX @@
+ tcg/s390x/tcg-target.c.inc       | 5 +++++
- #endif
+ tcg/sparc64/tcg-target.c.inc     | 5 +++++
+ tcg/tci/tcg-target.c.inc         | 5 +++++
- #include "qemu/host-utils.h"
+files changed, 55 insertions(+)
-+#include "qemu/thread.h"
- #include "qemu/queue.h"
+diff --git a/tcg/tcg.c b/tcg/tcg.c
- #ifdef CONFIG_TCG
+index XXXXXXX..XXXXXXX 100644
- #include "tcg-target.h"
+--- a/tcg/tcg.c
-@@ -XXX,XX +XXX,XX @@ typedef struct CPUIOTLBEntry {
++++ b/tcg/tcg.c
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
- #define CPU_COMMON_TLB \
+ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
-     /* The meaning of the MMU modes is defined in the target code. */   \
+ static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg);
-+    /* tlb_lock serializes updates to tlb_table and tlb_v_table */      \
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
-+    QemuSpin tlb_lock;                                                  \
++static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
-     CPUTLBEntry tlb_table[NB_MMU_MODES][CPU_TLB_SIZE];                  \
++    __attribute__((unused));
-     CPUTLBEntry tlb_v_table[NB_MMU_MODES][CPU_VTLB_SIZE];               \
+ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
-     CPUIOTLBEntry iotlb[NB_MMU_MODES][CPU_TLB_SIZE];                    \
+ static void tcg_out_goto_tb(TCGContext *s, int which);
-diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
+ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-index XXXXXXX..XXXXXXX 100644
+diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
---- a/accel/tcg/cputlb.c
+index XXXXXXX..XXXXXXX 100644
-+++ b/accel/tcg/cputlb.c
+--- a/tcg/aarch64/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16);
++++ b/tcg/aarch64/tcg-target.c.inc
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
- void tlb_init(CPUState *cpu)
+     tcg_out_insn(s, 3305, LDR, 0, rd);
- {
+ }
-+    CPUArchState *env = cpu->env_ptr;
-+
++static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
-+    qemu_spin_init(&env->tlb_lock);
++{
- }
++    return false;
++}
- /* flush_all_helper: run fn across all cpus
++
-@@ -XXX,XX +XXX,XX @@ static void tlb_flush_nocheck(CPUState *cpu)
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
-     atomic_set(&env->tlb_flush_count, env->tlb_flush_count + 1);
+                              tcg_target_long imm)
-     tlb_debug("(count: %zu)\n", tlb_flush_count());
+ {
+diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
-+    /*
+index XXXXXXX..XXXXXXX 100644
-+     * tlb_table/tlb_v_table updates from any thread must hold tlb_lock.
+--- a/tcg/arm/tcg-target.c.inc
-+     * However, updates from the owner thread (as is the case here; see the
++++ b/tcg/arm/tcg-target.c.inc
-+     * above assert_cpu_is_self) do not need atomic_set because all reads
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type,
-+     * that do not hold the lock are performed by the same owner thread.
+     tcg_out_movi32(s, COND_AL, ret, arg);
-+     */
+ }
-+    qemu_spin_lock(&env->tlb_lock);
-     memset(env->tlb_table, -1, sizeof(env->tlb_table));
++static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
-     memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table));
++{
-+    qemu_spin_unlock(&env->tlb_lock);
++    return false;
-+
++}
-     cpu_tb_jmp_cache_clear(cpu);
++
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
-     env->vtlb_index = 0;
+                              tcg_target_long imm)
-@@ -XXX,XX +XXX,XX @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)
+ {
+diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
-     tlb_debug("start: mmu_idx:0x%04lx\n", mmu_idx_bitmask);
+index XXXXXXX..XXXXXXX 100644
+--- a/tcg/i386/tcg-target.c.inc
-+    qemu_spin_lock(&env->tlb_lock);
++++ b/tcg/i386/tcg-target.c.inc
-     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
+@@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct)
+ #define OPC_VPTERNLOGQ  (0x25 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX)
-         if (test_bit(mmu_idx, &mmu_idx_bitmask)) {
+ #define OPC_VZEROUPPER  (0x77 | P_EXT)
-@@ -XXX,XX +XXX,XX @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)
+ #define OPC_XCHG_ax_r32    (0x90)
-             memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0]));
++#define OPC_XCHG_EvGv   (0x87)
-         }
  #define OPC_GRP3_Eb     (0xf6)
  #define OPC_GRP3_Ev     (0xf7)
@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type,
      }
-+    qemu_spin_unlock(&env->tlb_lock);
+ }
-     cpu_tb_jmp_cache_clear(cpu);
++static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
++{
-@@ -XXX,XX +XXX,XX @@ static inline bool tlb_hit_page_anyprot(CPUTLBEntry *tlb_entry,
++    int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
-            tlb_hit_page(tlb_entry->addr_code, page);
++    tcg_out_modrm(s, OPC_XCHG_EvGv + rexw, r1, r2);
- }
++    return true;
++}
--static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong page)
++
-+/* Called with tlb_lock held */
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
-+static inline void tlb_flush_entry_locked(CPUTLBEntry *tlb_entry,
+                              tcg_target_long imm)
-+                                          target_ulong page)
+ {
- {
+diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
-     if (tlb_hit_page_anyprot(tlb_entry, page)) {
+index XXXXXXX..XXXXXXX 100644
-         memset(tlb_entry, -1, sizeof(*tlb_entry));
+--- a/tcg/loongarch64/tcg-target.c.inc
 +++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_addi(TCGContext *s, TCGType type, TCGReg rd,
      }
  }
--static inline void tlb_flush_vtlb_page(CPUArchState *env, int mmu_idx,
++static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
--                                       target_ulong page)
++{
-+/* Called with tlb_lock held */
++    return false;
-+static inline void tlb_flush_vtlb_page_locked(CPUArchState *env, int mmu_idx,
++}
-+                                              target_ulong page)
++
- {
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
-     int k;
+                              tcg_target_long imm)
-+
+ {
-+    assert_cpu_is_self(ENV_GET_CPU(env));
+diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
-     for (k = 0; k < CPU_VTLB_SIZE; k++) {
+index XXXXXXX..XXXXXXX 100644
--        tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], page);
+--- a/tcg/mips/tcg-target.c.inc
-+        tlb_flush_entry_locked(&env->tlb_v_table[mmu_idx][k], page);
++++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs)
      tcg_out_ext32s(s, rd, rs);
  }
 +static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
 +{
 +    return false;
 +}
 +
  static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                               tcg_target_long imm)
  {
 diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/ppc/tcg-target.c.inc
 +++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg ret,
      }
  }
-@@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data)
++static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
++{
-     addr &= TARGET_PAGE_MASK;
++    return false;
-     i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
++}
-+    qemu_spin_lock(&env->tlb_lock);
++
-     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
--        tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
+                              tcg_target_long imm)
--        tlb_flush_vtlb_page(env, mmu_idx, addr);
+ {
-+        tlb_flush_entry_locked(&env->tlb_table[mmu_idx][i], addr);
+diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
-+        tlb_flush_vtlb_page_locked(env, mmu_idx, addr);
+index XXXXXXX..XXXXXXX 100644
-     }
+--- a/tcg/riscv/tcg-target.c.inc
-+    qemu_spin_unlock(&env->tlb_lock);
++++ b/tcg/riscv/tcg-target.c.inc
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
-     tb_flush_jmp_cache(cpu, addr);
+     tcg_out_opc_imm(s, OPC_LD, rd, rd, 0);
  }
-@@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_by_mmuidx_async_work(CPUState *cpu,
-     tlb_debug("page:%d addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n",
++static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
-               page, addr, mmu_idx_bitmap);
++{
++    return false;
-+    qemu_spin_lock(&env->tlb_lock);
++}
-     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
++
-         if (test_bit(mmu_idx, &mmu_idx_bitmap)) {
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
--            tlb_flush_entry(&env->tlb_table[mmu_idx][page], addr);
+                              tcg_target_long imm)
--            tlb_flush_vtlb_page(env, mmu_idx, addr);
+ {
-+            tlb_flush_entry_locked(&env->tlb_table[mmu_idx][page], addr);
+diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
-+            tlb_flush_vtlb_page_locked(env, mmu_idx, addr);
+index XXXXXXX..XXXXXXX 100644
-         }
+--- a/tcg/s390x/tcg-target.c.inc
-     }
++++ b/tcg/s390x/tcg-target.c.inc
-+    qemu_spin_unlock(&env->tlb_lock);
+@@ -XXX,XX +XXX,XX @@ static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+     return false;
-     tb_flush_jmp_cache(cpu, addr);
+ }
- }
-@@ -XXX,XX +XXX,XX @@ void tlb_unprotect_code(ram_addr_t ram_addr)
++static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
-  * most usual is detecting writes to code regions which may invalidate
++{
-  * generated code.
++    return false;
-  *
++}
-- * Because we want other vCPUs to respond to changes straight away we
++
-- * update the te->addr_write field atomically. If the TLB entry has
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
-- * been changed by the vCPU in the mean time we skip the update.
+                              tcg_target_long imm)
-+ * Other vCPUs might be reading their TLBs during guest execution, so we update
+ {
-+ * te->addr_write with atomic_set. We don't need to worry about this for
+diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
-+ * oversized guests as MTTCG is disabled for them.
+index XXXXXXX..XXXXXXX 100644
-  *
+--- a/tcg/sparc64/tcg-target.c.inc
-- * As this function uses atomic accesses we also need to ensure
++++ b/tcg/sparc64/tcg-target.c.inc
-- * updates to tlb_entries follow the same access rules. We don't need
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs)
-- * to worry about this for oversized guests as MTTCG is disabled for
+     tcg_out_mov(s, TCG_TYPE_I32, rd, rs);
-- * them.
+ }
-+ * Called with tlb_lock held.
-  */
++static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
--
++{
--static void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,
++    return false;
--                           uintptr_t length)
++}
-+static void tlb_reset_dirty_range_locked(CPUTLBEntry *tlb_entry,
++
-+                                         uintptr_t start, uintptr_t length)
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
- {
+                              tcg_target_long imm)
--#if TCG_OVERSIZED_GUEST
+ {
-     uintptr_t addr = tlb_entry->addr_write;
+diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
+index XXXXXXX..XXXXXXX 100644
-     if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) {
+--- a/tcg/tci/tcg-target.c.inc
-         addr &= TARGET_PAGE_MASK;
++++ b/tcg/tci/tcg-target.c.inc
-         addr += tlb_entry->addend;
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs)
-         if ((addr - start) < length) {
+     tcg_out_mov(s, TCG_TYPE_I32, rd, rs);
-+#if TCG_OVERSIZED_GUEST
+ }
-             tlb_entry->addr_write |= TLB_NOTDIRTY;
--        }
++static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
--    }
++{
- #else
++    return false;
--    /* paired with atomic_mb_set in tlb_set_page_with_attrs */
++}
--    uintptr_t orig_addr = atomic_mb_read(&tlb_entry->addr_write);
++
--    uintptr_t addr = orig_addr;
+ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
--
+                              tcg_target_long imm)
--    if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) {
+ {
 -        addr &= TARGET_PAGE_MASK;
 -        addr += atomic_read(&tlb_entry->addend);
 -        if ((addr - start) < length) {
 -            uintptr_t notdirty_addr = orig_addr | TLB_NOTDIRTY;
 -            atomic_cmpxchg(&tlb_entry->addr_write, orig_addr, notdirty_addr);
 +            atomic_set(&tlb_entry->addr_write,
 +                       tlb_entry->addr_write | TLB_NOTDIRTY);
 +#endif
          }
      }
 -#endif
  }
 -/* For atomic correctness when running MTTCG we need to use the right
 - * primitives when copying entries */
 -static inline void copy_tlb_helper(CPUTLBEntry *d, CPUTLBEntry *s,
 -                                   bool atomic_set)
 +/*
 + * Called with tlb_lock held.
 + * Called only from the vCPU context, i.e. the TLB's owner thread.
 + */
 +static inline void copy_tlb_helper_locked(CPUTLBEntry *d, const CPUTLBEntry *s)
  {
 -#if TCG_OVERSIZED_GUEST
      *d = *s;
 -#else
 -    if (atomic_set) {
 -        d->addr_read = s->addr_read;
 -        d->addr_code = s->addr_code;
 -        atomic_set(&d->addend, atomic_read(&s->addend));
 -        /* Pairs with flag setting in tlb_reset_dirty_range */
 -        atomic_mb_set(&d->addr_write, atomic_read(&s->addr_write));
 -    } else {
 -        d->addr_read = s->addr_read;
 -        d->addr_write = atomic_read(&s->addr_write);
 -        d->addr_code = s->addr_code;
 -        d->addend = atomic_read(&s->addend);
 -    }
 -#endif
  }
  /* This is a cross vCPU call (i.e. another vCPU resetting the flags of
 - * the target vCPU). As such care needs to be taken that we don't
 - * dangerously race with another vCPU update. The only thing actually
 - * updated is the target TLB entry ->addr_write flags.
 + * the target vCPU).
 + * We must take tlb_lock to avoid racing with another vCPU update. The only
 + * thing actually updated is the target TLB entry ->addr_write flags.
   */
  void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
  {
@@ -XXX,XX +XXX,XX @@ void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
      int mmu_idx;
      env = cpu->env_ptr;
 +    qemu_spin_lock(&env->tlb_lock);
      for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
          unsigned int i;
          for (i = 0; i < CPU_TLB_SIZE; i++) {
 -            tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
 -                                  start1, length);
 +            tlb_reset_dirty_range_locked(&env->tlb_table[mmu_idx][i], start1,
 +                                         length);
          }
          for (i = 0; i < CPU_VTLB_SIZE; i++) {
 -            tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],
 -                                  start1, length);
 +            tlb_reset_dirty_range_locked(&env->tlb_v_table[mmu_idx][i], start1,
 +                                         length);
          }
      }
 +    qemu_spin_unlock(&env->tlb_lock);
  }
 -static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
 +/* Called with tlb_lock held */
 +static inline void tlb_set_dirty1_locked(CPUTLBEntry *tlb_entry,
 +                                         target_ulong vaddr)
  {
      if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY)) {
          tlb_entry->addr_write = vaddr;
@@ -XXX,XX +XXX,XX @@ void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
      vaddr &= TARGET_PAGE_MASK;
      i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 +    qemu_spin_lock(&env->tlb_lock);
      for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 -        tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
 +        tlb_set_dirty1_locked(&env->tlb_table[mmu_idx][i], vaddr);
      }
      for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
          int k;
          for (k = 0; k < CPU_VTLB_SIZE; k++) {
 -            tlb_set_dirty1(&env->tlb_v_table[mmu_idx][k], vaddr);
 +            tlb_set_dirty1_locked(&env->tlb_v_table[mmu_idx][k], vaddr);
          }
      }
 +    qemu_spin_unlock(&env->tlb_lock);
  }
  /* Our TLB does not support large pages, so remember the area covered by
@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
          addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat;
      }
 -    /* Make sure there's no cached translation for the new page.  */
 -    tlb_flush_vtlb_page(env, mmu_idx, vaddr_page);
 -
      code_address = address;
      iotlb = memory_region_section_get_iotlb(cpu, section, vaddr_page,
                                              paddr_page, xlat, prot, &address);
@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
      index = (vaddr_page >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
      te = &env->tlb_table[mmu_idx][index];
 +    /*
 +     * Hold the TLB lock for the rest of the function. We could acquire/release
 +     * the lock several times in the function, but it is faster to amortize the
 +     * acquisition cost by acquiring it just once. Note that this leads to
 +     * a longer critical section, but this is not a concern since the TLB lock
 +     * is unlikely to be contended.
 +     */
 +    qemu_spin_lock(&env->tlb_lock);
 +
 +    /* Make sure there's no cached translation for the new page.  */
 +    tlb_flush_vtlb_page_locked(env, mmu_idx, vaddr_page);
 +
      /*
       * Only evict the old entry to the victim tlb if it's for a
       * different page; otherwise just overwrite the stale data.
@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
          CPUTLBEntry *tv = &env->tlb_v_table[mmu_idx][vidx];
          /* Evict the old entry into the victim tlb.  */
 -        copy_tlb_helper(tv, te, true);
 +        copy_tlb_helper_locked(tv, te);
          env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index];
      }
@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
          }
      }
 -    /* Pairs with flag setting in tlb_reset_dirty_range */
 -    copy_tlb_helper(te, &tn, true);
 -    /* atomic_mb_set(&te->addr_write, write_address); */
 +    copy_tlb_helper_locked(te, &tn);
 +    qemu_spin_unlock(&env->tlb_lock);
  }
  /* Add a new TLB entry, but without specifying the memory
@@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
                             size_t elt_ofs, target_ulong page)
  {
      size_t vidx;
 +
 +    assert_cpu_is_self(ENV_GET_CPU(env));
      for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) {
          CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx];
          target_ulong cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs);
@@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
              /* Found entry in victim tlb, swap tlb and iotlb.  */
              CPUTLBEntry tmptlb, *tlb = &env->tlb_table[mmu_idx][index];
 -            copy_tlb_helper(&tmptlb, tlb, false);
 -            copy_tlb_helper(tlb, vtlb, true);
 -            copy_tlb_helper(vtlb, &tmptlb, true);
 +            qemu_spin_lock(&env->tlb_lock);
 +            copy_tlb_helper_locked(&tmptlb, tlb);
 +            copy_tlb_helper_locked(tlb, vtlb);
 +            copy_tlb_helper_locked(vtlb, &tmptlb);
 +            qemu_spin_unlock(&env->tlb_lock);
              CPUIOTLBEntry tmpio, *io = &env->iotlb[mmu_idx][index];
              CPUIOTLBEntry *vio = &env->iotlb_v[mmu_idx][vidx];
 --
-.17.2
+.34.1

-[Qemu-devel] [PULL 15/21] target/arm: Check HAVE_CMPXCHG128 at translate time
+[PULL 14/15] tcg: Clear TCGLabelQemuLdst on allocation
-Reviewed-by: Emilio G. Cota <cota@braap.org>
+Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
 Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- target/arm/helper-a64.c    | 16 ++++------------
+ tcg/tcg-ldst.c.inc | 1 +
- target/arm/translate-a64.c | 38 ++++++++++++++++++++++----------------
+file changed, 1 insertion(+)
 files changed, 26 insertions(+), 28 deletions(-)
-diff --git a/target/arm/helper-a64.c b/target/arm/helper-a64.c
+diff --git a/tcg/tcg-ldst.c.inc b/tcg/tcg-ldst.c.inc
 index XXXXXXX..XXXXXXX 100644
---- a/target/arm/helper-a64.c
+--- a/tcg/tcg-ldst.c.inc
-+++ b/target/arm/helper-a64.c
++++ b/tcg/tcg-ldst.c.inc
-@@ -XXX,XX +XXX,XX @@ uint64_t HELPER(paired_cmpxchg64_le_parallel)(CPUARMState *env, uint64_t addr,
+@@ -XXX,XX +XXX,XX @@ static inline TCGLabelQemuLdst *new_ldst_label(TCGContext *s)
-     int mem_idx;
+ {
-     TCGMemOpIdx oi;
+     TCGLabelQemuLdst *l = tcg_malloc(sizeof(*l));
--    if (!HAVE_CMPXCHG128) {
++    memset(l, 0, sizeof(*l));
--        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
+     QSIMPLEQ_INSERT_TAIL(&s->ldst_labels, l, next);
--    }
-+    assert(HAVE_CMPXCHG128);
+     return l;
      mem_idx = cpu_mmu_index(env, false);
      oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
@@ -XXX,XX +XXX,XX @@ uint64_t HELPER(paired_cmpxchg64_be_parallel)(CPUARMState *env, uint64_t addr,
      int mem_idx;
      TCGMemOpIdx oi;
 -    if (!HAVE_CMPXCHG128) {
 -        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
 -    }
 +    assert(HAVE_CMPXCHG128);
      mem_idx = cpu_mmu_index(env, false);
      oi = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
@@ -XXX,XX +XXX,XX @@ void HELPER(casp_le_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
      int mem_idx;
      TCGMemOpIdx oi;
 -    if (!HAVE_CMPXCHG128) {
 -        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
 -    }
 +    assert(HAVE_CMPXCHG128);
      mem_idx = cpu_mmu_index(env, false);
      oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
@@ -XXX,XX +XXX,XX @@ void HELPER(casp_be_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
      int mem_idx;
      TCGMemOpIdx oi;
 -    if (!HAVE_CMPXCHG128) {
 -        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
 -    }
 +    assert(HAVE_CMPXCHG128);
      mem_idx = cpu_mmu_index(env, false);
      oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
 diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c
 index XXXXXXX..XXXXXXX 100644
 --- a/target/arm/translate-a64.c
 +++ b/target/arm/translate-a64.c
@@ -XXX,XX +XXX,XX @@
  #include "trace-tcg.h"
  #include "translate-a64.h"
 +#include "qemu/atomic128.h"
  static TCGv_i64 cpu_X[32];
  static TCGv_i64 cpu_pc;
@@ -XXX,XX +XXX,XX @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
                                         get_mem_index(s),
                                         MO_64 | MO_ALIGN | s->be_data);
              tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
 -        } else if (s->be_data == MO_LE) {
 -            if (tb_cflags(s->base.tb) & CF_PARALLEL) {
 +        } else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
 +            if (!HAVE_CMPXCHG128) {
 +                gen_helper_exit_atomic(cpu_env);
 +                s->base.is_jmp = DISAS_NORETURN;
 +            } else if (s->be_data == MO_LE) {
                  gen_helper_paired_cmpxchg64_le_parallel(tmp, cpu_env,
                                                          cpu_exclusive_addr,
                                                          cpu_reg(s, rt),
                                                          cpu_reg(s, rt2));
              } else {
 -                gen_helper_paired_cmpxchg64_le(tmp, cpu_env, cpu_exclusive_addr,
 -                                               cpu_reg(s, rt), cpu_reg(s, rt2));
 -            }
 -        } else {
 -            if (tb_cflags(s->base.tb) & CF_PARALLEL) {
                  gen_helper_paired_cmpxchg64_be_parallel(tmp, cpu_env,
                                                          cpu_exclusive_addr,
                                                          cpu_reg(s, rt),
                                                          cpu_reg(s, rt2));
 -            } else {
 -                gen_helper_paired_cmpxchg64_be(tmp, cpu_env, cpu_exclusive_addr,
 -                                               cpu_reg(s, rt), cpu_reg(s, rt2));
              }
 +        } else if (s->be_data == MO_LE) {
 +            gen_helper_paired_cmpxchg64_le(tmp, cpu_env, cpu_exclusive_addr,
 +                                           cpu_reg(s, rt), cpu_reg(s, rt2));
 +        } else {
 +            gen_helper_paired_cmpxchg64_be(tmp, cpu_env, cpu_exclusive_addr,
 +                                           cpu_reg(s, rt), cpu_reg(s, rt2));
          }
      } else {
          tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr, cpu_exclusive_val,
@@ -XXX,XX +XXX,XX @@ static void gen_compare_and_swap_pair(DisasContext *s, int rs, int rt,
          }
          tcg_temp_free_i64(cmp);
      } else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
 -        TCGv_i32 tcg_rs = tcg_const_i32(rs);
 -
 -        if (s->be_data == MO_LE) {
 -            gen_helper_casp_le_parallel(cpu_env, tcg_rs, addr, t1, t2);
 +        if (HAVE_CMPXCHG128) {
 +            TCGv_i32 tcg_rs = tcg_const_i32(rs);
 +            if (s->be_data == MO_LE) {
 +                gen_helper_casp_le_parallel(cpu_env, tcg_rs, addr, t1, t2);
 +            } else {
 +                gen_helper_casp_be_parallel(cpu_env, tcg_rs, addr, t1, t2);
 +            }
 +            tcg_temp_free_i32(tcg_rs);
          } else {
 -            gen_helper_casp_be_parallel(cpu_env, tcg_rs, addr, t1, t2);
 +            gen_helper_exit_atomic(cpu_env);
 +            s->base.is_jmp = DISAS_NORETURN;
          }
 -        tcg_temp_free_i32(tcg_rs);
      } else {
          TCGv_i64 d1 = tcg_temp_new_i64();
          TCGv_i64 d2 = tcg_temp_new_i64();
 --
-.17.2
+.34.1

-[Qemu-devel] [PULL 01/21] tcg: Implement CPU_LOG_TB_NOCHAIN during expansion
+[PULL 15/15] tcg/riscv: Conditionalize tcg_out_exts_i32_i64
-Rather than test NOCHAIN before linking, do not emit the
+Since TCG_TYPE_I32 values are kept sign-extended in registers, via "w"
-goto_tb opcode at all.  We already do this for goto_ptr.
+instructions, we don't need to extend if the register matches.
 This is already relied upon by comparisons.
+Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
 Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
 ---
- accel/tcg/cpu-exec.c | 2 +-
+ tcg/riscv/tcg-target.c.inc | 4 +++-
- tcg/tcg-op.c         | 9 ++++++++-
+file changed, 3 insertions(+), 1 deletion(-)
 files changed, 9 insertions(+), 2 deletions(-)
-diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c
+diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
 index XXXXXXX..XXXXXXX 100644
---- a/accel/tcg/cpu-exec.c
+--- a/tcg/riscv/tcg-target.c.inc
-+++ b/accel/tcg/cpu-exec.c
++++ b/tcg/riscv/tcg-target.c.inc
-@@ -XXX,XX +XXX,XX @@ static inline TranslationBlock *tb_find(CPUState *cpu,
+@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg)
-     }
- #endif
+ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
-     /* See if we can patch the calling TB. */
+ {
--    if (last_tb && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
+-    tcg_out_ext32s(s, ret, arg);
-+    if (last_tb) {
++    if (ret != arg) {
-         tb_add_jump(last_tb, tb_exit, tb);
++        tcg_out_ext32s(s, ret, arg);
      }
      return tb;
 diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c
 index XXXXXXX..XXXXXXX 100644
 --- a/tcg/tcg-op.c
 +++ b/tcg/tcg-op.c
@@ -XXX,XX +XXX,XX @@ void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx)
             seen this numbered exit before, via tcg_gen_goto_tb.  */
          tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
  #endif
 +        /* When not chaining, exit without indicating a link.  */
 +        if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
 +            val = 0;
 +        }
      } else {
          /* This is an exit via the exitreq label.  */
          tcg_debug_assert(idx == TB_EXIT_REQUESTED);
@@ -XXX,XX +XXX,XX @@ void tcg_gen_goto_tb(unsigned idx)
      tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
      tcg_ctx->goto_tb_issue_mask |= 1 << idx;
  #endif
 -    tcg_gen_op1i(INDEX_op_goto_tb, idx);
 +    /* When not chaining, we simply fall through to the "fallback" exit.  */
 +    if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
 +        tcg_gen_op1i(INDEX_op_goto_tb, idx);
 +    }
  }
- void tcg_gen_lookup_and_goto_ptr(void)
+ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
 --
-.17.2
+.34.1

-[Qemu-devel] [PULL 02/21] tcg: access cpu->icount_decr.u16.high with atomics
+Deleted patch
-From: "Emilio G. Cota" <cota@braap.org>
-Consistently access u16.high with atomics to avoid
-undefined behaviour in MTTCG.
-Note that icount_decr.u16.low is only used in icount mode,
-so regular accesses to it are OK.
-Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
-Signed-off-by: Emilio G. Cota <cota@braap.org>
-Message-Id: <20181010144853.13005-2-cota@braap.org>
-Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
----
- accel/tcg/tcg-all.c       | 2 +-
- accel/tcg/translate-all.c | 2 +-
- qom/cpu.c                 | 2 +-
-files changed, 3 insertions(+), 3 deletions(-)
-diff --git a/accel/tcg/tcg-all.c b/accel/tcg/tcg-all.c
-index XXXXXXX..XXXXXXX 100644
---- a/accel/tcg/tcg-all.c
-+++ b/accel/tcg/tcg-all.c
-@@ -XXX,XX +XXX,XX @@ static void tcg_handle_interrupt(CPUState *cpu, int mask)
-     if (!qemu_cpu_is_self(cpu)) {
-         qemu_cpu_kick(cpu);
-     } else {
--        cpu->icount_decr.u16.high = -1;
-+        atomic_set(&cpu->icount_decr.u16.high, -1);
-         if (use_icount &&
-             !cpu->can_do_io
-             && (mask & ~old_mask) != 0) {
-diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c
-index XXXXXXX..XXXXXXX 100644
---- a/accel/tcg/translate-all.c
-+++ b/accel/tcg/translate-all.c
-@@ -XXX,XX +XXX,XX @@ void cpu_interrupt(CPUState *cpu, int mask)
- {
-     g_assert(qemu_mutex_iothread_locked());
-     cpu->interrupt_request |= mask;
--    cpu->icount_decr.u16.high = -1;
-+    atomic_set(&cpu->icount_decr.u16.high, -1);
- }
- /*
-diff --git a/qom/cpu.c b/qom/cpu.c
-index XXXXXXX..XXXXXXX 100644
---- a/qom/cpu.c
-+++ b/qom/cpu.c
-@@ -XXX,XX +XXX,XX @@ static void cpu_common_reset(CPUState *cpu)
-     cpu->mem_io_pc = 0;
-     cpu->mem_io_vaddr = 0;
-     cpu->icount_extra = 0;
--    cpu->icount_decr.u32 = 0;
-+    atomic_set(&cpu->icount_decr.u32, 0);
-     cpu->can_do_io = 1;
-     cpu->exception_index = -1;
-     cpu->crash_occurred = false;
---
-.17.2

-[Qemu-devel] [PULL 04/21] tcg: plug holes in struct TCGProfile
+Deleted patch
-From: "Emilio G. Cota" <cota@braap.org>
-This plugs two 4-byte holes in 64-bit.
-Signed-off-by: Emilio G. Cota <cota@braap.org>
-Message-Id: <20181010144853.13005-4-cota@braap.org>
-Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
----
- tcg/tcg.h | 2 +-
-file changed, 1 insertion(+), 1 deletion(-)
-diff --git a/tcg/tcg.h b/tcg/tcg.h
-index XXXXXXX..XXXXXXX 100644
---- a/tcg/tcg.h
-+++ b/tcg/tcg.h
-@@ -XXX,XX +XXX,XX @@ typedef struct TCGProfile {
-     int64_t tb_count;
-     int64_t op_count; /* total insn count */
-     int op_count_max; /* max insn per TB */
--    int64_t temp_count;
-     int temp_count_max;
-+    int64_t temp_count;
-     int64_t del_op_count;
-     int64_t code_in_len;
-     int64_t code_out_len;
---
-.17.2

-[Qemu-devel] [PULL 06/21] target/alpha: remove tlb_flush from alpha_cpu_initfn
+Deleted patch
-From: "Emilio G. Cota" <cota@braap.org>
-As far as I can tell tlb_flush does not need to be called
-this early. tlb_flush is eventually called after the CPU
-has been realized.
-This change paves the way to the introduction of tlb_init,
-which will be called from cpu_exec_realizefn.
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
-Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
-Signed-off-by: Emilio G. Cota <cota@braap.org>
-Message-Id: <20181009174557.16125-2-cota@braap.org>
-Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
----
- target/alpha/cpu.c | 1 -
-file changed, 1 deletion(-)
-diff --git a/target/alpha/cpu.c b/target/alpha/cpu.c
-index XXXXXXX..XXXXXXX 100644
---- a/target/alpha/cpu.c
-+++ b/target/alpha/cpu.c
-@@ -XXX,XX +XXX,XX @@ static void alpha_cpu_initfn(Object *obj)
-     CPUAlphaState *env = &cpu->env;
-     cs->env_ptr = env;
--    tlb_flush(cs);
-     env->lock_addr = -1;
- #if defined(CONFIG_USER_ONLY)
---
-.17.2

-[Qemu-devel] [PULL 07/21] target/unicore32: remove tlb_flush from uc32_init_fn
+Deleted patch
-From: "Emilio G. Cota" <cota@braap.org>
-As far as I can tell tlb_flush does not need to be called
-this early. tlb_flush is eventually called after the CPU
-has been realized.
-This change paves the way to the introduction of tlb_init,
-which will be called from cpu_exec_realizefn.
-Cc: Guan Xuetao <gxt@mprc.pku.edu.cn>
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
-Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
-Signed-off-by: Emilio G. Cota <cota@braap.org>
-Message-Id: <20181009174557.16125-3-cota@braap.org>
-Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
----
- target/unicore32/cpu.c | 2 --
-file changed, 2 deletions(-)
-diff --git a/target/unicore32/cpu.c b/target/unicore32/cpu.c
-index XXXXXXX..XXXXXXX 100644
---- a/target/unicore32/cpu.c
-+++ b/target/unicore32/cpu.c
-@@ -XXX,XX +XXX,XX @@ static void uc32_cpu_initfn(Object *obj)
-     env->uncached_asr = ASR_MODE_PRIV;
-     env->regs[31] = 0x03000000;
- #endif
--
--    tlb_flush(cs);
- }
- static const VMStateDescription vmstate_uc32_cpu = {
---
-.17.2

-[Qemu-devel] [PULL 09/21] cputlb: fix assert_cpu_is_self macro
+Deleted patch
-From: "Emilio G. Cota" <cota@braap.org>
-Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
-Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
-Signed-off-by: Emilio G. Cota <cota@braap.org>
-Message-Id: <20181009174557.16125-5-cota@braap.org>
-Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
----
- accel/tcg/cputlb.c | 4 ++--
-file changed, 2 insertions(+), 2 deletions(-)
-diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
-index XXXXXXX..XXXXXXX 100644
---- a/accel/tcg/cputlb.c
-+++ b/accel/tcg/cputlb.c
-@@ -XXX,XX +XXX,XX @@
-     } \
- } while (0)
--#define assert_cpu_is_self(this_cpu) do {                         \
-+#define assert_cpu_is_self(cpu) do {                              \
-         if (DEBUG_TLB_GATE) {                                     \
--            g_assert(!cpu->created || qemu_cpu_is_self(cpu));     \
-+            g_assert(!(cpu)->created || qemu_cpu_is_self(cpu));   \
-         }                                                         \
-     } while (0)
---
-.17.2

-[Qemu-devel] [PULL 13/21] target/i386: Convert to HAVE_CMPXCHG128
+Deleted patch
-Reviewed-by: Emilio G. Cota <cota@braap.org>
-Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
-Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
----
- target/i386/mem_helper.c | 9 ++++-----
-file changed, 4 insertions(+), 5 deletions(-)
-diff --git a/target/i386/mem_helper.c b/target/i386/mem_helper.c
-index XXXXXXX..XXXXXXX 100644
---- a/target/i386/mem_helper.c
-+++ b/target/i386/mem_helper.c
-@@ -XXX,XX +XXX,XX @@
- #include "exec/exec-all.h"
- #include "exec/cpu_ldst.h"
- #include "qemu/int128.h"
-+#include "qemu/atomic128.h"
- #include "tcg.h"
- void helper_cmpxchg8b_unlocked(CPUX86State *env, target_ulong a0)
-@@ -XXX,XX +XXX,XX @@ void helper_cmpxchg16b(CPUX86State *env, target_ulong a0)
-     if ((a0 & 0xf) != 0) {
-         raise_exception_ra(env, EXCP0D_GPF, ra);
--    } else {
--#ifndef CONFIG_ATOMIC128
--        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
--#else
-+    } else if (HAVE_CMPXCHG128) {
-         int eflags = cpu_cc_compute_all(env, CC_OP);
-         Int128 cmpv = int128_make128(env->regs[R_EAX], env->regs[R_EDX]);
-@@ -XXX,XX +XXX,XX @@ void helper_cmpxchg16b(CPUX86State *env, target_ulong a0)
-             eflags &= ~CC_Z;
-         }
-         CC_SRC = eflags;
--#endif
-+    } else {
-+        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
-     }
- }
- #endif
---
-.17.2

The following changes since commit ff56877e911782dedc9a424233fd3f62369c258c:

Merge remote-tracking branch 'remotes/kraxel/tags/vga-20181015-pull-request' into staging (2018-10-15 15:03:45 +0100)

are available in the Git repository at:

https://github.com/rth7680/qemu.git tags/pull-tcg-20181016

for you to fetch changes up to e3e9d1ea20c75718ce7c528c588a0a497f12f750:

cputlb: read CPUTLBEntry.addr_write atomically (2018-10-16 10:04:27 -0700)

----------------------------------------------------------------
Queued tcg patches

----------------------------------------------------------------
Emilio G. Cota (10):
      tcg: access cpu->icount_decr.u16.high with atomics
      tcg: fix use of uninitialized variable under CONFIG_PROFILER
      tcg: plug holes in struct TCGProfile
      tcg: distribute tcg_time into TCG contexts
      target/alpha: remove tlb_flush from alpha_cpu_initfn
      target/unicore32: remove tlb_flush from uc32_init_fn
      exec: introduce tlb_init
      cputlb: fix assert_cpu_is_self macro
      cputlb: serialize tlb updates with env->tlb_lock
      cputlb: read CPUTLBEntry.addr_write atomically

Richard Henderson (11):
      tcg: Implement CPU_LOG_TB_NOCHAIN during expansion
      tcg: Add tlb_index and tlb_entry helpers
      tcg: Split CONFIG_ATOMIC128
      target/i386: Convert to HAVE_CMPXCHG128
      target/arm: Convert to HAVE_CMPXCHG128
      target/arm: Check HAVE_CMPXCHG128 at translate time
      target/ppc: Convert to HAVE_CMPXCHG128 and HAVE_ATOMIC128
      target/s390x: Convert to HAVE_CMPXCHG128 and HAVE_ATOMIC128
      target/s390x: Split do_cdsg, do_lpq, do_stpq
      target/s390x: Skip wout, cout helpers if op helper does not return
      target/s390x: Check HAVE_ATOMIC128 and HAVE_CMPXCHG128 at translate

Rather than test NOCHAIN before linking, do not emit the
goto_tb opcode at all.  We already do this for goto_ptr.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 accel/tcg/cpu-exec.c | 2 +-
 tcg/tcg-op.c         | 9 ++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/cpu-exec.c
+++ b/accel/tcg/cpu-exec.c
@@ -XXX,XX +XXX,XX @@ static inline TranslationBlock *tb_find(CPUState *cpu,
     }
 #endif
     /* See if we can patch the calling TB. */
-    if (last_tb && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
+    if (last_tb) {
         tb_add_jump(last_tb, tb_exit, tb);
     }
     return tb;
diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg-op.c
+++ b/tcg/tcg-op.c
@@ -XXX,XX +XXX,XX @@ void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx)
            seen this numbered exit before, via tcg_gen_goto_tb.  */
         tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
 #endif
+        /* When not chaining, exit without indicating a link.  */
+        if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
+            val = 0;
+        }
     } else {
         /* This is an exit via the exitreq label.  */
         tcg_debug_assert(idx == TB_EXIT_REQUESTED);
@@ -XXX,XX +XXX,XX @@ void tcg_gen_goto_tb(unsigned idx)
     tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
     tcg_ctx->goto_tb_issue_mask |= 1 << idx;
 #endif
-    tcg_gen_op1i(INDEX_op_goto_tb, idx);
+    /* When not chaining, we simply fall through to the "fallback" exit.  */
+    if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
+        tcg_gen_op1i(INDEX_op_goto_tb, idx);
+    }
 }
 
 void tcg_gen_lookup_and_goto_ptr(void)
-- 
2.17.2

From: "Emilio G. Cota" <cota@braap.org>

Consistently access u16.high with atomics to avoid
undefined behaviour in MTTCG.

Note that icount_decr.u16.low is only used in icount mode,
so regular accesses to it are OK.

Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
Signed-off-by: Emilio G. Cota <cota@braap.org>
Message-Id: <20181010144853.13005-2-cota@braap.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 accel/tcg/tcg-all.c       | 2 +-
 accel/tcg/translate-all.c | 2 +-
 qom/cpu.c                 | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/accel/tcg/tcg-all.c b/accel/tcg/tcg-all.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/tcg-all.c
+++ b/accel/tcg/tcg-all.c
@@ -XXX,XX +XXX,XX @@ static void tcg_handle_interrupt(CPUState *cpu, int mask)
     if (!qemu_cpu_is_self(cpu)) {
         qemu_cpu_kick(cpu);
     } else {
-        cpu->icount_decr.u16.high = -1;
+        atomic_set(&cpu->icount_decr.u16.high, -1);
         if (use_icount &&
             !cpu->can_do_io
             && (mask & ~old_mask) != 0) {
diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/translate-all.c
+++ b/accel/tcg/translate-all.c
@@ -XXX,XX +XXX,XX @@ void cpu_interrupt(CPUState *cpu, int mask)
 {
     g_assert(qemu_mutex_iothread_locked());
     cpu->interrupt_request |= mask;
-    cpu->icount_decr.u16.high = -1;
+    atomic_set(&cpu->icount_decr.u16.high, -1);
 }
 
 /*
diff --git a/qom/cpu.c b/qom/cpu.c
index XXXXXXX..XXXXXXX 100644
--- a/qom/cpu.c
+++ b/qom/cpu.c
@@ -XXX,XX +XXX,XX @@ static void cpu_common_reset(CPUState *cpu)
     cpu->mem_io_pc = 0;
     cpu->mem_io_vaddr = 0;
     cpu->icount_extra = 0;
-    cpu->icount_decr.u32 = 0;
+    atomic_set(&cpu->icount_decr.u32, 0);
     cpu->can_do_io = 1;
     cpu->exception_index = -1;
     cpu->crash_occurred = false;
-- 
2.17.2

From: "Emilio G. Cota" <cota@braap.org>

When we implemented per-vCPU TCG contexts, we forgot to also
distribute the tcg_time counter, which has remained as a global
accessed without any serialization, leading to potentially missed
counts.

Fix it by distributing the field over the TCG contexts, embedding
it into TCGProfile with a field called "cpu_exec_time", which is more
descriptive than "tcg_time". Add a function to query this value
directly, and for completeness, fill in the field in
tcg_profile_snapshot, even though its callers do not use it.

Signed-off-by: Emilio G. Cota <cota@braap.org>
Message-Id: <20181010144853.13005-5-cota@braap.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 include/qemu/timer.h |  1 -
 tcg/tcg.h            |  2 ++
 cpus.c               |  3 ++-
 monitor.c            | 13 ++++++++++---
 tcg/tcg.c            | 23 +++++++++++++++++++++++
 5 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/include/qemu/timer.h b/include/qemu/timer.h
index XXXXXXX..XXXXXXX 100644
--- a/include/qemu/timer.h
+++ b/include/qemu/timer.h
@@ -XXX,XX +XXX,XX @@ static inline int64_t profile_getclock(void)
     return get_clock();
 }
 
-extern int64_t tcg_time;
 extern int64_t dev_time;
 #endif
 
diff --git a/tcg/tcg.h b/tcg/tcg.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.h
+++ b/tcg/tcg.h
@@ -XXX,XX +XXX,XX @@ typedef struct TCGOp {
 QEMU_BUILD_BUG_ON(NB_OPS > (1 << 8));
 
 typedef struct TCGProfile {
+    int64_t cpu_exec_time;
     int64_t tb_count1;
     int64_t tb_count;
     int64_t op_count; /* total insn count */
@@ -XXX,XX +XXX,XX @@ int tcg_check_temp_count(void);
 #define tcg_check_temp_count() 0
 #endif
 
+int64_t tcg_cpu_exec_time(void);
 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf);
 void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf);
 
diff --git a/cpus.c b/cpus.c
index XXXXXXX..XXXXXXX 100644
--- a/cpus.c
+++ b/cpus.c
@@ -XXX,XX +XXX,XX @@ static int tcg_cpu_exec(CPUState *cpu)
     ret = cpu_exec(cpu);
     cpu_exec_end(cpu);
 #ifdef CONFIG_PROFILER
-    tcg_time += profile_getclock() - ti;
+    atomic_set(&tcg_ctx->prof.cpu_exec_time,
+               tcg_ctx->prof.cpu_exec_time + profile_getclock() - ti);
 #endif
     return ret;
 }
diff --git a/monitor.c b/monitor.c
index XXXXXXX..XXXXXXX 100644
--- a/monitor.c
+++ b/monitor.c
@@ -XXX,XX +XXX,XX @@
 #include "sysemu/cpus.h"
 #include "sysemu/iothread.h"
 #include "qemu/cutils.h"
+#include "tcg/tcg.h"
 
 #if defined(TARGET_S390X)
 #include "hw/s390x/storage-keys.h"
@@ -XXX,XX +XXX,XX @@ static void hmp_info_numa(Monitor *mon, const QDict *qdict)
 
 #ifdef CONFIG_PROFILER
 
-int64_t tcg_time;
 int64_t dev_time;
 
 static void hmp_info_profile(Monitor *mon, const QDict *qdict)
 {
+    static int64_t last_cpu_exec_time;
+    int64_t cpu_exec_time;
+    int64_t delta;
+
+    cpu_exec_time = tcg_cpu_exec_time();
+    delta = cpu_exec_time - last_cpu_exec_time;
+
     monitor_printf(mon, "async time  %" PRId64 " (%0.3f)\n",
                    dev_time, dev_time / (double)NANOSECONDS_PER_SECOND);
     monitor_printf(mon, "qemu time   %" PRId64 " (%0.3f)\n",
-                   tcg_time, tcg_time / (double)NANOSECONDS_PER_SECOND);
-    tcg_time = 0;
+                   delta, delta / (double)NANOSECONDS_PER_SECOND);
+    last_cpu_exec_time = cpu_exec_time;
     dev_time = 0;
 }
 #else
diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@
 /* Define to jump the ELF file used to communicate with GDB.  */
 #undef DEBUG_JIT
 
+#include "qemu/error-report.h"
 #include "qemu/cutils.h"
 #include "qemu/host-utils.h"
 #include "qemu/timer.h"
@@ -XXX,XX +XXX,XX @@ void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
         const TCGProfile *orig = &s->prof;
 
         if (counters) {
+            PROF_ADD(prof, orig, cpu_exec_time);
             PROF_ADD(prof, orig, tb_count1);
             PROF_ADD(prof, orig, tb_count);
             PROF_ADD(prof, orig, op_count);
@@ -XXX,XX +XXX,XX @@ void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf)
                     prof.table_op_count[i]);
     }
 }
+
+int64_t tcg_cpu_exec_time(void)
+{
+    unsigned int n_ctxs = atomic_read(&n_tcg_ctxs);
+    unsigned int i;
+    int64_t ret = 0;
+
+    for (i = 0; i < n_ctxs; i++) {
+        const TCGContext *s = atomic_read(&tcg_ctxs[i]);
+        const TCGProfile *prof = &s->prof;
+
+        ret += atomic_read(&prof->cpu_exec_time);
+    }
+    return ret;
+}
 #else
 void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf)
 {
     cpu_fprintf(f, "[TCG profiler not compiled]\n");
 }
+
+int64_t tcg_cpu_exec_time(void)
+{
+    error_report("%s: TCG profiler not compiled", __func__);
+    exit(EXIT_FAILURE);
+}
 #endif
 
 
-- 
2.17.2

From: "Emilio G. Cota" <cota@braap.org>

As far as I can tell tlb_flush does not need to be called
this early. tlb_flush is eventually called after the CPU
has been realized.

This change paves the way to the introduction of tlb_init,
which will be called from cpu_exec_realizefn.

Cc: Guan Xuetao <gxt@mprc.pku.edu.cn>
Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
Signed-off-by: Emilio G. Cota <cota@braap.org>
Message-Id: <20181009174557.16125-3-cota@braap.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/unicore32/cpu.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/target/unicore32/cpu.c b/target/unicore32/cpu.c
index XXXXXXX..XXXXXXX 100644
--- a/target/unicore32/cpu.c
+++ b/target/unicore32/cpu.c
@@ -XXX,XX +XXX,XX @@ static void uc32_cpu_initfn(Object *obj)
     env->uncached_asr = ASR_MODE_PRIV;
     env->regs[31] = 0x03000000;
 #endif
-
-    tlb_flush(cs);
 }
 
 static const VMStateDescription vmstate_uc32_cpu = {
-- 
2.17.2

From: "Emilio G. Cota" <cota@braap.org>

Paves the way for the addition of a per-TLB lock.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
Signed-off-by: Emilio G. Cota <cota@braap.org>
Message-Id: <20181009174557.16125-4-cota@braap.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 include/exec/exec-all.h | 8 ++++++++
 accel/tcg/cputlb.c      | 4 ++++
 exec.c                  | 1 +
 3 files changed, 13 insertions(+)

diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
index XXXXXXX..XXXXXXX 100644
--- a/include/exec/exec-all.h
+++ b/include/exec/exec-all.h
@@ -XXX,XX +XXX,XX @@ void cpu_address_space_init(CPUState *cpu, int asidx,
 
 #if !defined(CONFIG_USER_ONLY) && defined(CONFIG_TCG)
 /* cputlb.c */
+/**
+ * tlb_init - initialize a CPU's TLB
+ * @cpu: CPU whose TLB should be initialized
+ */
+void tlb_init(CPUState *cpu);
 /**
  * tlb_flush_page:
  * @cpu: CPU whose TLB should be flushed
@@ -XXX,XX +XXX,XX @@ void tlb_set_page(CPUState *cpu, target_ulong vaddr,
 void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx,
                  uintptr_t retaddr);
 #else
+static inline void tlb_init(CPUState *cpu)
+{
+}
 static inline void tlb_flush_page(CPUState *cpu, target_ulong addr)
 {
 }
diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/cputlb.c
+++ b/accel/tcg/cputlb.c
@@ -XXX,XX +XXX,XX @@ QEMU_BUILD_BUG_ON(sizeof(target_ulong) > sizeof(run_on_cpu_data));
 QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16);
 #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)
 
+void tlb_init(CPUState *cpu)
+{
+}
+
 /* flush_all_helper: run fn across all cpus
  *
  * If the wait flag is set then the src cpu's helper will be queued as
diff --git a/exec.c b/exec.c
index XXXXXXX..XXXXXXX 100644
--- a/exec.c
+++ b/exec.c
@@ -XXX,XX +XXX,XX @@ void cpu_exec_realizefn(CPUState *cpu, Error **errp)
         tcg_target_initialized = true;
         cc->tcg_initialize();
     }
+    tlb_init(cpu);
 
 #ifndef CONFIG_USER_ONLY
     if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
-- 
2.17.2

From: "Emilio G. Cota" <cota@braap.org>

Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Emilio G. Cota <cota@braap.org>
Message-Id: <20181009174557.16125-5-cota@braap.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 accel/tcg/cputlb.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/cputlb.c
+++ b/accel/tcg/cputlb.c
@@ -XXX,XX +XXX,XX @@
     } \
 } while (0)
 
-#define assert_cpu_is_self(this_cpu) do {                         \
+#define assert_cpu_is_self(cpu) do {                              \
         if (DEBUG_TLB_GATE) {                                     \
-            g_assert(!cpu->created || qemu_cpu_is_self(cpu));     \
+            g_assert(!(cpu)->created || qemu_cpu_is_self(cpu));   \
         }                                                         \
     } while (0)
 
-- 
2.17.2

From: "Emilio G. Cota" <cota@braap.org>

Currently we rely on atomic operations for cross-CPU invalidations.
There are two cases that these atomics miss: cross-CPU invalidations
can race with either (1) vCPU threads flushing their TLB, which
happens via memset, or (2) vCPUs calling tlb_reset_dirty on their TLB,
which updates .addr_write with a regular store. This results in
undefined behaviour, since we're mixing regular and atomic ops
on concurrent accesses.

Fix it by using tlb_lock, a per-vCPU lock. All updaters of tlb_table
and the corresponding victim cache now hold the lock.
The readers that do not hold tlb_lock must use atomic reads when
reading .addr_write, since this field can be updated by other threads;
the conversion to atomic reads is done in the next patch.

Note that an alternative fix would be to expand the use of atomic ops.
However, in the case of TLB flushes this would have a huge performance
impact, since (1) TLB flushes can happen very frequently and (2) we
currently use a full memory barrier to flush each TLB entry, and a TLB
has many entries. Instead, acquiring the lock is barely slower than a
full memory barrier since it is uncontended, and with a single lock
acquisition we can flush the entire TLB.

Tested-by: Alex Bennée <alex.bennee@linaro.org>
Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Emilio G. Cota <cota@braap.org>
Message-Id: <20181009174557.16125-6-cota@braap.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 include/exec/cpu-defs.h |   3 +
 accel/tcg/cputlb.c      | 155 ++++++++++++++++++++++------------------
 2 files changed, 87 insertions(+), 71 deletions(-)

diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h
index XXXXXXX..XXXXXXX 100644
--- a/include/exec/cpu-defs.h
+++ b/include/exec/cpu-defs.h
@@ -XXX,XX +XXX,XX @@
 #endif
 
 #include "qemu/host-utils.h"
+#include "qemu/thread.h"
 #include "qemu/queue.h"
 #ifdef CONFIG_TCG
 #include "tcg-target.h"
@@ -XXX,XX +XXX,XX @@ typedef struct CPUIOTLBEntry {
 
 #define CPU_COMMON_TLB \
     /* The meaning of the MMU modes is defined in the target code. */   \
+    /* tlb_lock serializes updates to tlb_table and tlb_v_table */      \
+    QemuSpin tlb_lock;                                                  \
     CPUTLBEntry tlb_table[NB_MMU_MODES][CPU_TLB_SIZE];                  \
     CPUTLBEntry tlb_v_table[NB_MMU_MODES][CPU_VTLB_SIZE];               \
     CPUIOTLBEntry iotlb[NB_MMU_MODES][CPU_TLB_SIZE];                    \
diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/cputlb.c
+++ b/accel/tcg/cputlb.c
@@ -XXX,XX +XXX,XX @@ QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16);
 
 void tlb_init(CPUState *cpu)
 {
+    CPUArchState *env = cpu->env_ptr;
+
+    qemu_spin_init(&env->tlb_lock);
 }
 
 /* flush_all_helper: run fn across all cpus
@@ -XXX,XX +XXX,XX @@ static void tlb_flush_nocheck(CPUState *cpu)
     atomic_set(&env->tlb_flush_count, env->tlb_flush_count + 1);
     tlb_debug("(count: %zu)\n", tlb_flush_count());
 
+    /*
+     * tlb_table/tlb_v_table updates from any thread must hold tlb_lock.
+     * However, updates from the owner thread (as is the case here; see the
+     * above assert_cpu_is_self) do not need atomic_set because all reads
+     * that do not hold the lock are performed by the same owner thread.
+     */
+    qemu_spin_lock(&env->tlb_lock);
     memset(env->tlb_table, -1, sizeof(env->tlb_table));
     memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table));
+    qemu_spin_unlock(&env->tlb_lock);
+
     cpu_tb_jmp_cache_clear(cpu);
 
     env->vtlb_index = 0;
@@ -XXX,XX +XXX,XX @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)
 
     tlb_debug("start: mmu_idx:0x%04lx\n", mmu_idx_bitmask);
 
+    qemu_spin_lock(&env->tlb_lock);
     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 
         if (test_bit(mmu_idx, &mmu_idx_bitmask)) {
@@ -XXX,XX +XXX,XX @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)
             memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0]));
         }
     }
+    qemu_spin_unlock(&env->tlb_lock);
 
     cpu_tb_jmp_cache_clear(cpu);
 
@@ -XXX,XX +XXX,XX @@ static inline bool tlb_hit_page_anyprot(CPUTLBEntry *tlb_entry,
            tlb_hit_page(tlb_entry->addr_code, page);
 }
 
-static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong page)
+/* Called with tlb_lock held */
+static inline void tlb_flush_entry_locked(CPUTLBEntry *tlb_entry,
+                                          target_ulong page)
 {
     if (tlb_hit_page_anyprot(tlb_entry, page)) {
         memset(tlb_entry, -1, sizeof(*tlb_entry));
     }
 }
 
-static inline void tlb_flush_vtlb_page(CPUArchState *env, int mmu_idx,
-                                       target_ulong page)
+/* Called with tlb_lock held */
+static inline void tlb_flush_vtlb_page_locked(CPUArchState *env, int mmu_idx,
+                                              target_ulong page)
 {
     int k;
+
+    assert_cpu_is_self(ENV_GET_CPU(env));
     for (k = 0; k < CPU_VTLB_SIZE; k++) {
-        tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], page);
+        tlb_flush_entry_locked(&env->tlb_v_table[mmu_idx][k], page);
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data)
 
     addr &= TARGET_PAGE_MASK;
     i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+    qemu_spin_lock(&env->tlb_lock);
     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
-        tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
-        tlb_flush_vtlb_page(env, mmu_idx, addr);
+        tlb_flush_entry_locked(&env->tlb_table[mmu_idx][i], addr);
+        tlb_flush_vtlb_page_locked(env, mmu_idx, addr);
     }
+    qemu_spin_unlock(&env->tlb_lock);
 
     tb_flush_jmp_cache(cpu, addr);
 }
@@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_by_mmuidx_async_work(CPUState *cpu,
     tlb_debug("page:%d addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n",
               page, addr, mmu_idx_bitmap);
 
+    qemu_spin_lock(&env->tlb_lock);
     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
         if (test_bit(mmu_idx, &mmu_idx_bitmap)) {
-            tlb_flush_entry(&env->tlb_table[mmu_idx][page], addr);
-            tlb_flush_vtlb_page(env, mmu_idx, addr);
+            tlb_flush_entry_locked(&env->tlb_table[mmu_idx][page], addr);
+            tlb_flush_vtlb_page_locked(env, mmu_idx, addr);
         }
     }
+    qemu_spin_unlock(&env->tlb_lock);
 
     tb_flush_jmp_cache(cpu, addr);
 }
@@ -XXX,XX +XXX,XX @@ void tlb_unprotect_code(ram_addr_t ram_addr)
  * most usual is detecting writes to code regions which may invalidate
  * generated code.
  *
- * Because we want other vCPUs to respond to changes straight away we
- * update the te->addr_write field atomically. If the TLB entry has
- * been changed by the vCPU in the mean time we skip the update.
+ * Other vCPUs might be reading their TLBs during guest execution, so we update
+ * te->addr_write with atomic_set. We don't need to worry about this for
+ * oversized guests as MTTCG is disabled for them.
  *
- * As this function uses atomic accesses we also need to ensure
- * updates to tlb_entries follow the same access rules. We don't need
- * to worry about this for oversized guests as MTTCG is disabled for
- * them.
+ * Called with tlb_lock held.
  */
-
-static void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,
-                           uintptr_t length)
+static void tlb_reset_dirty_range_locked(CPUTLBEntry *tlb_entry,
+                                         uintptr_t start, uintptr_t length)
 {
-#if TCG_OVERSIZED_GUEST
     uintptr_t addr = tlb_entry->addr_write;
 
     if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) {
         addr &= TARGET_PAGE_MASK;
         addr += tlb_entry->addend;
         if ((addr - start) < length) {
+#if TCG_OVERSIZED_GUEST
             tlb_entry->addr_write |= TLB_NOTDIRTY;
-        }
-    }
 #else
-    /* paired with atomic_mb_set in tlb_set_page_with_attrs */
-    uintptr_t orig_addr = atomic_mb_read(&tlb_entry->addr_write);
-    uintptr_t addr = orig_addr;
-
-    if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) {
-        addr &= TARGET_PAGE_MASK;
-        addr += atomic_read(&tlb_entry->addend);
-        if ((addr - start) < length) {
-            uintptr_t notdirty_addr = orig_addr | TLB_NOTDIRTY;
-            atomic_cmpxchg(&tlb_entry->addr_write, orig_addr, notdirty_addr);
+            atomic_set(&tlb_entry->addr_write,
+                       tlb_entry->addr_write | TLB_NOTDIRTY);
+#endif
         }
     }
-#endif
 }
 
-/* For atomic correctness when running MTTCG we need to use the right
- * primitives when copying entries */
-static inline void copy_tlb_helper(CPUTLBEntry *d, CPUTLBEntry *s,
-                                   bool atomic_set)
+/*
+ * Called with tlb_lock held.
+ * Called only from the vCPU context, i.e. the TLB's owner thread.
+ */
+static inline void copy_tlb_helper_locked(CPUTLBEntry *d, const CPUTLBEntry *s)
 {
-#if TCG_OVERSIZED_GUEST
     *d = *s;
-#else
-    if (atomic_set) {
-        d->addr_read = s->addr_read;
-        d->addr_code = s->addr_code;
-        atomic_set(&d->addend, atomic_read(&s->addend));
-        /* Pairs with flag setting in tlb_reset_dirty_range */
-        atomic_mb_set(&d->addr_write, atomic_read(&s->addr_write));
-    } else {
-        d->addr_read = s->addr_read;
-        d->addr_write = atomic_read(&s->addr_write);
-        d->addr_code = s->addr_code;
-        d->addend = atomic_read(&s->addend);
-    }
-#endif
 }
 
 /* This is a cross vCPU call (i.e. another vCPU resetting the flags of
- * the target vCPU). As such care needs to be taken that we don't
- * dangerously race with another vCPU update. The only thing actually
- * updated is the target TLB entry ->addr_write flags.
+ * the target vCPU).
+ * We must take tlb_lock to avoid racing with another vCPU update. The only
+ * thing actually updated is the target TLB entry ->addr_write flags.
  */
 void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
 {
@@ -XXX,XX +XXX,XX @@ void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
     int mmu_idx;
 
     env = cpu->env_ptr;
+    qemu_spin_lock(&env->tlb_lock);
     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
         unsigned int i;
 
         for (i = 0; i < CPU_TLB_SIZE; i++) {
-            tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
-                                  start1, length);
+            tlb_reset_dirty_range_locked(&env->tlb_table[mmu_idx][i], start1,
+                                         length);
         }
 
         for (i = 0; i < CPU_VTLB_SIZE; i++) {
-            tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],
-                                  start1, length);
+            tlb_reset_dirty_range_locked(&env->tlb_v_table[mmu_idx][i], start1,
+                                         length);
         }
     }
+    qemu_spin_unlock(&env->tlb_lock);
 }
 
-static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
+/* Called with tlb_lock held */
+static inline void tlb_set_dirty1_locked(CPUTLBEntry *tlb_entry,
+                                         target_ulong vaddr)
 {
     if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY)) {
         tlb_entry->addr_write = vaddr;
@@ -XXX,XX +XXX,XX @@ void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
 
     vaddr &= TARGET_PAGE_MASK;
     i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+    qemu_spin_lock(&env->tlb_lock);
     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
-        tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
+        tlb_set_dirty1_locked(&env->tlb_table[mmu_idx][i], vaddr);
     }
 
     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
         int k;
         for (k = 0; k < CPU_VTLB_SIZE; k++) {
-            tlb_set_dirty1(&env->tlb_v_table[mmu_idx][k], vaddr);
+            tlb_set_dirty1_locked(&env->tlb_v_table[mmu_idx][k], vaddr);
         }
     }
+    qemu_spin_unlock(&env->tlb_lock);
 }
 
 /* Our TLB does not support large pages, so remember the area covered by
@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
         addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat;
     }
 
-    /* Make sure there's no cached translation for the new page.  */
-    tlb_flush_vtlb_page(env, mmu_idx, vaddr_page);
-
     code_address = address;
     iotlb = memory_region_section_get_iotlb(cpu, section, vaddr_page,
                                             paddr_page, xlat, prot, &address);
@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
     index = (vaddr_page >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     te = &env->tlb_table[mmu_idx][index];
 
+    /*
+     * Hold the TLB lock for the rest of the function. We could acquire/release
+     * the lock several times in the function, but it is faster to amortize the
+     * acquisition cost by acquiring it just once. Note that this leads to
+     * a longer critical section, but this is not a concern since the TLB lock
+     * is unlikely to be contended.
+     */
+    qemu_spin_lock(&env->tlb_lock);
+
+    /* Make sure there's no cached translation for the new page.  */
+    tlb_flush_vtlb_page_locked(env, mmu_idx, vaddr_page);
+
     /*
      * Only evict the old entry to the victim tlb if it's for a
      * different page; otherwise just overwrite the stale data.
@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
         CPUTLBEntry *tv = &env->tlb_v_table[mmu_idx][vidx];
 
         /* Evict the old entry into the victim tlb.  */
-        copy_tlb_helper(tv, te, true);
+        copy_tlb_helper_locked(tv, te);
         env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index];
     }
 
@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
         }
     }
 
-    /* Pairs with flag setting in tlb_reset_dirty_range */
-    copy_tlb_helper(te, &tn, true);
-    /* atomic_mb_set(&te->addr_write, write_address); */
+    copy_tlb_helper_locked(te, &tn);
+    qemu_spin_unlock(&env->tlb_lock);
 }
 
 /* Add a new TLB entry, but without specifying the memory
@@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
                            size_t elt_ofs, target_ulong page)
 {
     size_t vidx;
+
+    assert_cpu_is_self(ENV_GET_CPU(env));
     for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) {
         CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx];
         target_ulong cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs);
@@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
             /* Found entry in victim tlb, swap tlb and iotlb.  */
             CPUTLBEntry tmptlb, *tlb = &env->tlb_table[mmu_idx][index];
 
-            copy_tlb_helper(&tmptlb, tlb, false);
-            copy_tlb_helper(tlb, vtlb, true);
-            copy_tlb_helper(vtlb, &tmptlb, true);
+            qemu_spin_lock(&env->tlb_lock);
+            copy_tlb_helper_locked(&tmptlb, tlb);
+            copy_tlb_helper_locked(tlb, vtlb);
+            copy_tlb_helper_locked(vtlb, &tmptlb);
+            qemu_spin_unlock(&env->tlb_lock);
 
             CPUIOTLBEntry tmpio, *io = &env->iotlb[mmu_idx][index];
             CPUIOTLBEntry *vio = &env->iotlb_v[mmu_idx][vidx];
-- 
2.17.2

Isolate the computation of an index from an address into a
helper before we change that function.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
[ cota: convert tlb_vaddr_to_host; use atomic_read on addr_write ]
Signed-off-by: Emilio G. Cota <cota@braap.org>
Message-Id: <20181009175129.17888-2-cota@braap.org>
---
 accel/tcg/softmmu_template.h     | 64 +++++++++++++++++---------------
 include/exec/cpu_ldst.h          | 19 ++++++++--
 include/exec/cpu_ldst_template.h | 25 +++++++------
 accel/tcg/cputlb.c               | 60 ++++++++++++++----------------
 4 files changed, 90 insertions(+), 78 deletions(-)

diff --git a/accel/tcg/softmmu_template.h b/accel/tcg/softmmu_template.h
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/softmmu_template.h
+++ b/accel/tcg/softmmu_template.h
@@ -XXX,XX +XXX,XX @@ static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
 WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
                             TCGMemOpIdx oi, uintptr_t retaddr)
 {
-    unsigned mmu_idx = get_mmuidx(oi);
-    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
+    uintptr_t mmu_idx = get_mmuidx(oi);
+    uintptr_t index = tlb_index(env, mmu_idx, addr);
+    CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
+    target_ulong tlb_addr = entry->ADDR_READ;
     unsigned a_bits = get_alignment_bits(get_memop(oi));
     uintptr_t haddr;
     DATA_TYPE res;
@@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, READ_ACCESS_TYPE,
                      mmu_idx, retaddr);
         }
-        tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
+        tlb_addr = entry->ADDR_READ;
     }
 
     /* Handle an IO access.  */
@@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
         return res;
     }
 
-    haddr = addr + env->tlb_table[mmu_idx][index].addend;
+    haddr = addr + entry->addend;
 #if DATA_SIZE == 1
     res = glue(glue(ld, LSUFFIX), _p)((uint8_t *)haddr);
 #else
@@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
 WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,
                             TCGMemOpIdx oi, uintptr_t retaddr)
 {
-    unsigned mmu_idx = get_mmuidx(oi);
-    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
+    uintptr_t mmu_idx = get_mmuidx(oi);
+    uintptr_t index = tlb_index(env, mmu_idx, addr);
+    CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
+    target_ulong tlb_addr = entry->ADDR_READ;
     unsigned a_bits = get_alignment_bits(get_memop(oi));
     uintptr_t haddr;
     DATA_TYPE res;
@@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,
             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, READ_ACCESS_TYPE,
                      mmu_idx, retaddr);
         }
-        tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
+        tlb_addr = entry->ADDR_READ;
     }
 
     /* Handle an IO access.  */
@@ -XXX,XX +XXX,XX @@ WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,
         return res;
     }
 
-    haddr = addr + env->tlb_table[mmu_idx][index].addend;
+    haddr = addr + entry->addend;
     res = glue(glue(ld, LSUFFIX), _be_p)((uint8_t *)haddr);
     return res;
 }
@@ -XXX,XX +XXX,XX @@ static inline void glue(io_write, SUFFIX)(CPUArchState *env,
 void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
                        TCGMemOpIdx oi, uintptr_t retaddr)
 {
-    unsigned mmu_idx = get_mmuidx(oi);
-    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
+    uintptr_t mmu_idx = get_mmuidx(oi);
+    uintptr_t index = tlb_index(env, mmu_idx, addr);
+    CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
+    target_ulong tlb_addr = entry->addr_write;
     unsigned a_bits = get_alignment_bits(get_memop(oi));
     uintptr_t haddr;
 
@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE,
                      mmu_idx, retaddr);
         }
-        tlb_addr = env->tlb_table[mmu_idx][index].addr_write & ~TLB_INVALID_MASK;
+        tlb_addr = entry->addr_write & ~TLB_INVALID_MASK;
     }
 
     /* Handle an IO access.  */
@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
     if (DATA_SIZE > 1
         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
                      >= TARGET_PAGE_SIZE)) {
-        int i, index2;
-        target_ulong page2, tlb_addr2;
+        int i;
+        target_ulong page2;
+        CPUTLBEntry *entry2;
     do_unaligned_access:
         /* Ensure the second page is in the TLB.  Note that the first page
            is already guaranteed to be filled, and that the second page
            cannot evict the first.  */
         page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
-        index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-        tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write;
-        if (!tlb_hit_page(tlb_addr2, page2)
+        entry2 = tlb_entry(env, mmu_idx, page2);
+        if (!tlb_hit_page(entry2->addr_write, page2)
             && !VICTIM_TLB_HIT(addr_write, page2)) {
             tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE,
                      mmu_idx, retaddr);
@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
         return;
     }
 
-    haddr = addr + env->tlb_table[mmu_idx][index].addend;
+    haddr = addr + entry->addend;
 #if DATA_SIZE == 1
     glue(glue(st, SUFFIX), _p)((uint8_t *)haddr, val);
 #else
@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
 void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
                        TCGMemOpIdx oi, uintptr_t retaddr)
 {
-    unsigned mmu_idx = get_mmuidx(oi);
-    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
+    uintptr_t mmu_idx = get_mmuidx(oi);
+    uintptr_t index = tlb_index(env, mmu_idx, addr);
+    CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
+    target_ulong tlb_addr = entry->addr_write;
     unsigned a_bits = get_alignment_bits(get_memop(oi));
     uintptr_t haddr;
 
@@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE,
                      mmu_idx, retaddr);
         }
-        tlb_addr = env->tlb_table[mmu_idx][index].addr_write & ~TLB_INVALID_MASK;
+        tlb_addr = entry->addr_write & ~TLB_INVALID_MASK;
     }
 
     /* Handle an IO access.  */
@@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
     if (DATA_SIZE > 1
         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
                      >= TARGET_PAGE_SIZE)) {
-        int i, index2;
-        target_ulong page2, tlb_addr2;
+        int i;
+        target_ulong page2;
+        CPUTLBEntry *entry2;
     do_unaligned_access:
         /* Ensure the second page is in the TLB.  Note that the first page
            is already guaranteed to be filled, and that the second page
            cannot evict the first.  */
         page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
-        index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-        tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write;
-        if (!tlb_hit_page(tlb_addr2, page2)
+        entry2 = tlb_entry(env, mmu_idx, page2);
+        if (!tlb_hit_page(entry2->addr_write, page2)
             && !VICTIM_TLB_HIT(addr_write, page2)) {
             tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE,
                      mmu_idx, retaddr);
@@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
         return;
     }
 
-    haddr = addr + env->tlb_table[mmu_idx][index].addend;
+    haddr = addr + entry->addend;
     glue(glue(st, SUFFIX), _be_p)((uint8_t *)haddr, val);
 }
 #endif /* DATA_SIZE > 1 */
diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h
index XXXXXXX..XXXXXXX 100644
--- a/include/exec/cpu_ldst.h
+++ b/include/exec/cpu_ldst.h
@@ -XXX,XX +XXX,XX @@ extern __thread uintptr_t helper_retaddr;
 /* The memory helpers for tcg-generated code need tcg_target_long etc.  */
 #include "tcg.h"
 
+/* Find the TLB index corresponding to the mmu_idx + address pair.  */
+static inline uintptr_t tlb_index(CPUArchState *env, uintptr_t mmu_idx,
+                                  target_ulong addr)
+{
+    return (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+}
+
+/* Find the TLB entry corresponding to the mmu_idx + address pair.  */
+static inline CPUTLBEntry *tlb_entry(CPUArchState *env, uintptr_t mmu_idx,
+                                     target_ulong addr)
+{
+    return &env->tlb_table[mmu_idx][tlb_index(env, mmu_idx, addr)];
+}
+
 #ifdef MMU_MODE0_SUFFIX
 #define CPU_MMU_INDEX 0
 #define MEMSUFFIX MMU_MODE0_SUFFIX
@@ -XXX,XX +XXX,XX @@ static inline void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr,
 #if defined(CONFIG_USER_ONLY)
     return g2h(addr);
 #else
-    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    CPUTLBEntry *tlbentry = &env->tlb_table[mmu_idx][index];
+    CPUTLBEntry *tlbentry = tlb_entry(env, mmu_idx, addr);
     abi_ptr tlb_addr;
     uintptr_t haddr;
 
@@ -XXX,XX +XXX,XX @@ static inline void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr,
         return NULL;
     }
 
-    haddr = addr + env->tlb_table[mmu_idx][index].addend;
+    haddr = addr + tlbentry->addend;
     return (void *)haddr;
 #endif /* defined(CONFIG_USER_ONLY) */
 }
diff --git a/include/exec/cpu_ldst_template.h b/include/exec/cpu_ldst_template.h
index XXXXXXX..XXXXXXX 100644
--- a/include/exec/cpu_ldst_template.h
+++ b/include/exec/cpu_ldst_template.h
@@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_ld, USUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
                                                   target_ulong ptr,
                                                   uintptr_t retaddr)
 {
-    int page_index;
+    CPUTLBEntry *entry;
     RES_TYPE res;
     target_ulong addr;
     int mmu_idx;
@@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_ld, USUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
 #endif
 
     addr = ptr;
-    page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     mmu_idx = CPU_MMU_INDEX;
-    if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ !=
+    entry = tlb_entry(env, mmu_idx, addr);
+    if (unlikely(entry->ADDR_READ !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
         oi = make_memop_idx(SHIFT, mmu_idx);
         res = glue(glue(helper_ret_ld, URETSUFFIX), MMUSUFFIX)(env, addr,
                                                             oi, retaddr);
     } else {
-        uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
+        uintptr_t hostaddr = addr + entry->addend;
         res = glue(glue(ld, USUFFIX), _p)((uint8_t *)hostaddr);
     }
     return res;
@@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_lds, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
                                                   target_ulong ptr,
                                                   uintptr_t retaddr)
 {
-    int res, page_index;
+    CPUTLBEntry *entry;
+    int res;
     target_ulong addr;
     int mmu_idx;
     TCGMemOpIdx oi;
@@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_lds, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
 #endif
 
     addr = ptr;
-    page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     mmu_idx = CPU_MMU_INDEX;
-    if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ !=
+    entry = tlb_entry(env, mmu_idx, addr);
+    if (unlikely(entry->ADDR_READ !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
         oi = make_memop_idx(SHIFT, mmu_idx);
         res = (DATA_STYPE)glue(glue(helper_ret_ld, SRETSUFFIX),
                                MMUSUFFIX)(env, addr, oi, retaddr);
     } else {
-        uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
+        uintptr_t hostaddr = addr + entry->addend;
         res = glue(glue(lds, SUFFIX), _p)((uint8_t *)hostaddr);
     }
     return res;
@@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
                                                  target_ulong ptr,
                                                  RES_TYPE v, uintptr_t retaddr)
 {
-    int page_index;
+    CPUTLBEntry *entry;
     target_ulong addr;
     int mmu_idx;
     TCGMemOpIdx oi;
@@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
 #endif
 
     addr = ptr;
-    page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     mmu_idx = CPU_MMU_INDEX;
-    if (unlikely(env->tlb_table[mmu_idx][page_index].addr_write !=
+    entry = tlb_entry(env, mmu_idx, addr);
+    if (unlikely(entry->addr_write !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
         oi = make_memop_idx(SHIFT, mmu_idx);
         glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(env, addr, v, oi,
                                                      retaddr);
     } else {
-        uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
+        uintptr_t hostaddr = addr + entry->addend;
         glue(glue(st, SUFFIX), _p)((uint8_t *)hostaddr, v);
     }
 }
diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/cputlb.c
+++ b/accel/tcg/cputlb.c
@@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data)
 {
     CPUArchState *env = cpu->env_ptr;
     target_ulong addr = (target_ulong) data.target_ptr;
-    int i;
     int mmu_idx;
 
     assert_cpu_is_self(cpu);
@@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data)
     }
 
     addr &= TARGET_PAGE_MASK;
-    i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     qemu_spin_lock(&env->tlb_lock);
     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
-        tlb_flush_entry_locked(&env->tlb_table[mmu_idx][i], addr);
+        tlb_flush_entry_locked(tlb_entry(env, mmu_idx, addr), addr);
         tlb_flush_vtlb_page_locked(env, mmu_idx, addr);
     }
     qemu_spin_unlock(&env->tlb_lock);
@@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_by_mmuidx_async_work(CPUState *cpu,
     target_ulong addr_and_mmuidx = (target_ulong) data.target_ptr;
     target_ulong addr = addr_and_mmuidx & TARGET_PAGE_MASK;
     unsigned long mmu_idx_bitmap = addr_and_mmuidx & ALL_MMUIDX_BITS;
-    int page = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     int mmu_idx;
 
     assert_cpu_is_self(cpu);
 
-    tlb_debug("page:%d addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n",
-              page, addr, mmu_idx_bitmap);
+    tlb_debug("flush page addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n",
+              addr, mmu_idx_bitmap);
 
     qemu_spin_lock(&env->tlb_lock);
     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
         if (test_bit(mmu_idx, &mmu_idx_bitmap)) {
-            tlb_flush_entry_locked(&env->tlb_table[mmu_idx][page], addr);
+            tlb_flush_entry_locked(tlb_entry(env, mmu_idx, addr), addr);
             tlb_flush_vtlb_page_locked(env, mmu_idx, addr);
         }
     }
@@ -XXX,XX +XXX,XX @@ static inline void tlb_set_dirty1_locked(CPUTLBEntry *tlb_entry,
 void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
 {
     CPUArchState *env = cpu->env_ptr;
-    int i;
     int mmu_idx;
 
     assert_cpu_is_self(cpu);
 
     vaddr &= TARGET_PAGE_MASK;
-    i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     qemu_spin_lock(&env->tlb_lock);
     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
-        tlb_set_dirty1_locked(&env->tlb_table[mmu_idx][i], vaddr);
+        tlb_set_dirty1_locked(tlb_entry(env, mmu_idx, vaddr), vaddr);
     }
 
     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
@@ -XXX,XX +XXX,XX @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
     iotlb = memory_region_section_get_iotlb(cpu, section, vaddr_page,
                                             paddr_page, xlat, prot, &address);
 
-    index = (vaddr_page >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    te = &env->tlb_table[mmu_idx][index];
+    index = tlb_index(env, mmu_idx, vaddr_page);
+    te = tlb_entry(env, mmu_idx, vaddr_page);
 
     /*
      * Hold the TLB lock for the rest of the function. We could acquire/release
@@ -XXX,XX +XXX,XX @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry,
          * repeat the MMU check here. This tlb_fill() call might
          * longjump out if this access should cause a guest exception.
          */
-        int index;
+        CPUTLBEntry *entry;
         target_ulong tlb_addr;
 
         tlb_fill(cpu, addr, size, MMU_DATA_LOAD, mmu_idx, retaddr);
 
-        index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-        tlb_addr = env->tlb_table[mmu_idx][index].addr_read;
+        entry = tlb_entry(env, mmu_idx, addr);
+        tlb_addr = entry->addr_read;
         if (!(tlb_addr & ~(TARGET_PAGE_MASK | TLB_RECHECK))) {
             /* RAM access */
-            uintptr_t haddr = addr + env->tlb_table[mmu_idx][index].addend;
+            uintptr_t haddr = addr + entry->addend;
 
             return ldn_p((void *)haddr, size);
         }
@@ -XXX,XX +XXX,XX @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry,
          * repeat the MMU check here. This tlb_fill() call might
          * longjump out if this access should cause a guest exception.
          */
-        int index;
+        CPUTLBEntry *entry;
         target_ulong tlb_addr;
 
         tlb_fill(cpu, addr, size, MMU_DATA_STORE, mmu_idx, retaddr);
 
-        index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-        tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
+        entry = tlb_entry(env, mmu_idx, addr);
+        tlb_addr = entry->addr_write;
         if (!(tlb_addr & ~(TARGET_PAGE_MASK | TLB_RECHECK))) {
             /* RAM access */
-            uintptr_t haddr = addr + env->tlb_table[mmu_idx][index].addend;
+            uintptr_t haddr = addr + entry->addend;
 
             stn_p((void *)haddr, size, val);
             return;
@@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
  */
 tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr)
 {
-    int mmu_idx, index;
+    uintptr_t mmu_idx = cpu_mmu_index(env, true);
+    uintptr_t index = tlb_index(env, mmu_idx, addr);
+    CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
     void *p;
 
-    index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    mmu_idx = cpu_mmu_index(env, true);
-    if (unlikely(!tlb_hit(env->tlb_table[mmu_idx][index].addr_code, addr))) {
+    if (unlikely(!tlb_hit(entry->addr_code, addr))) {
         if (!VICTIM_TLB_HIT(addr_code, addr)) {
             tlb_fill(ENV_GET_CPU(env), addr, 0, MMU_INST_FETCH, mmu_idx, 0);
         }
-        assert(tlb_hit(env->tlb_table[mmu_idx][index].addr_code, addr));
+        assert(tlb_hit(entry->addr_code, addr));
     }
 
-    if (unlikely(env->tlb_table[mmu_idx][index].addr_code &
-                 (TLB_RECHECK | TLB_MMIO))) {
+    if (unlikely(entry->addr_code & (TLB_RECHECK | TLB_MMIO))) {
         /*
          * Return -1 if we can't translate and execute from an entire
          * page of RAM here, which will cause us to execute by loading
@@ -XXX,XX +XXX,XX @@ tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr)
         return -1;
     }
 
-    p = (void *)((uintptr_t)addr + env->tlb_table[mmu_idx][index].addend);
+    p = (void *)((uintptr_t)addr + entry->addend);
     return qemu_ram_addr_from_host_nofail(p);
 }
 
@@ -XXX,XX +XXX,XX @@ tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr)
 void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx,
                  uintptr_t retaddr)
 {
-    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
+    uintptr_t index = tlb_index(env, mmu_idx, addr);
+    CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
 
-    if (!tlb_hit(tlb_addr, addr)) {
+    if (!tlb_hit(entry->addr_write, addr)) {
         /* TLB entry is for a different page */
         if (!VICTIM_TLB_HIT(addr_write, addr)) {
             tlb_fill(ENV_GET_CPU(env), addr, size, MMU_DATA_STORE,
@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
                                NotDirtyInfo *ndi)
 {
     size_t mmu_idx = get_mmuidx(oi);
-    size_t index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    CPUTLBEntry *tlbe = &env->tlb_table[mmu_idx][index];
+    uintptr_t index = tlb_index(env, mmu_idx, addr);
+    CPUTLBEntry *tlbe = tlb_entry(env, mmu_idx, addr);
     target_ulong tlb_addr = tlbe->addr_write;
     TCGMemOp mop = get_memop(oi);
     int a_bits = get_alignment_bits(mop);
-- 
2.17.2

GCC7+ will no longer advertise support for 16-byte __atomic operations
if only cmpxchg is supported, as for x86_64.  Fortunately, x86_64 still
has support for __sync_compare_and_swap_16 and we can make use of that.
AArch64 does not have, nor ever has had such support, so open-code it.

Reviewed-by: Emilio G. Cota <cota@braap.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 accel/tcg/atomic_template.h |  20 ++++-
 include/qemu/atomic128.h    | 155 ++++++++++++++++++++++++++++++++++++
 tcg/tcg.h                   |  16 ++--
 accel/tcg/cputlb.c          |   3 +-
 accel/tcg/user-exec.c       |   5 +-
 configure                   |  19 +++++
 6 files changed, 204 insertions(+), 14 deletions(-)
 create mode 100644 include/qemu/atomic128.h

diff --git a/accel/tcg/atomic_template.h b/accel/tcg/atomic_template.h
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/atomic_template.h
+++ b/accel/tcg/atomic_template.h
@@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
     DATA_TYPE ret;
 
     ATOMIC_TRACE_RMW;
+#if DATA_SIZE == 16
+    ret = atomic16_cmpxchg(haddr, cmpv, newv);
+#else
     ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
+#endif
     ATOMIC_MMU_CLEANUP;
     return ret;
 }
 
 #if DATA_SIZE >= 16
+#if HAVE_ATOMIC128
 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
 {
     ATOMIC_MMU_DECLS;
     DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
 
     ATOMIC_TRACE_LD;
-    __atomic_load(haddr, &val, __ATOMIC_RELAXED);
+    val = atomic16_read(haddr);
     ATOMIC_MMU_CLEANUP;
     return val;
 }
@@ -XXX,XX +XXX,XX @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 
     ATOMIC_TRACE_ST;
-    __atomic_store(haddr, &val, __ATOMIC_RELAXED);
+    atomic16_set(haddr, val);
     ATOMIC_MMU_CLEANUP;
 }
+#endif
 #else
 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
                            ABI_TYPE val EXTRA_ARGS)
@@ -XXX,XX +XXX,XX @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
     DATA_TYPE ret;
 
     ATOMIC_TRACE_RMW;
+#if DATA_SIZE == 16
+    ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
+#else
     ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
+#endif
     ATOMIC_MMU_CLEANUP;
     return BSWAP(ret);
 }
 
 #if DATA_SIZE >= 16
+#if HAVE_ATOMIC128
 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
 {
     ATOMIC_MMU_DECLS;
     DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
 
     ATOMIC_TRACE_LD;
-    __atomic_load(haddr, &val, __ATOMIC_RELAXED);
+    val = atomic16_read(haddr);
     ATOMIC_MMU_CLEANUP;
     return BSWAP(val);
 }
@@ -XXX,XX +XXX,XX @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
 
     ATOMIC_TRACE_ST;
     val = BSWAP(val);
-    __atomic_store(haddr, &val, __ATOMIC_RELAXED);
+    atomic16_set(haddr, val);
     ATOMIC_MMU_CLEANUP;
 }
+#endif
 #else
 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
                            ABI_TYPE val EXTRA_ARGS)
diff --git a/include/qemu/atomic128.h b/include/qemu/atomic128.h
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/include/qemu/atomic128.h
@@ -XXX,XX +XXX,XX @@
+/*
+ * Simple interface for 128-bit atomic operations.
+ *
+ * Copyright (C) 2018 Linaro, Ltd.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ * See docs/devel/atomics.txt for discussion about the guarantees each
+ * atomic primitive is meant to provide.
+ */
+
+#ifndef QEMU_ATOMIC128_H
+#define QEMU_ATOMIC128_H
+
+/*
+ * GCC is a house divided about supporting large atomic operations.
+ *
+ * For hosts that only have large compare-and-swap, a legalistic reading
+ * of the C++ standard means that one cannot implement __atomic_read on
+ * read-only memory, and thus all atomic operations must synchronize
+ * through libatomic.
+ *
+ * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80878
+ *
+ * This interpretation is not especially helpful for QEMU.
+ * For softmmu, all RAM is always read/write from the hypervisor.
+ * For user-only, if the guest doesn't implement such an __atomic_read
+ * then the host need not worry about it either.
+ *
+ * Moreover, using libatomic is not an option, because its interface is
+ * built for std::atomic<T>, and requires that *all* accesses to such an
+ * object go through the library.  In our case we do not have an object
+ * in the C/C++ sense, but a view of memory as seen by the guest.
+ * The guest may issue a large atomic operation and then access those
+ * pieces using word-sized accesses.  From the hypervisor, we have no
+ * way to connect those two actions.
+ *
+ * Therefore, special case each platform.
+ */
+
+#if defined(CONFIG_ATOMIC128)
+static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
+{
+    return atomic_cmpxchg__nocheck(ptr, cmp, new);
+}
+# define HAVE_CMPXCHG128 1
+#elif defined(CONFIG_CMPXCHG128)
+static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
+{
+    return __sync_val_compare_and_swap_16(ptr, cmp, new);
+}
+# define HAVE_CMPXCHG128 1
+#elif defined(__aarch64__)
+/* Through gcc 8, aarch64 has no support for 128-bit at all.  */
+static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
+{
+    uint64_t cmpl = int128_getlo(cmp), cmph = int128_gethi(cmp);
+    uint64_t newl = int128_getlo(new), newh = int128_gethi(new);
+    uint64_t oldl, oldh;
+    uint32_t tmp;
+
+    asm("0: ldaxp %[oldl], %[oldh], %[mem]\n\t"
+        "cmp %[oldl], %[cmpl]\n\t"
+        "ccmp %[oldh], %[cmph], #0, eq\n\t"
+        "b.ne 1f\n\t"
+        "stlxp %w[tmp], %[newl], %[newh], %[mem]\n\t"
+        "cbnz %w[tmp], 0b\n"
+        "1:"
+        : [mem] "+m"(*ptr), [tmp] "=&r"(tmp),
+          [oldl] "=&r"(oldl), [oldh] "=r"(oldh)
+        : [cmpl] "r"(cmpl), [cmph] "r"(cmph),
+          [newl] "r"(newl), [newh] "r"(newh)
+        : "memory", "cc");
+
+    return int128_make128(oldl, oldh);
+}
+# define HAVE_CMPXCHG128 1
+#else
+/* Fallback definition that must be optimized away, or error.  */
+Int128 __attribute__((error("unsupported atomic")))
+    atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new);
+# define HAVE_CMPXCHG128 0
+#endif /* Some definition for HAVE_CMPXCHG128 */
+
+
+#if defined(CONFIG_ATOMIC128)
+static inline Int128 atomic16_read(Int128 *ptr)
+{
+    return atomic_read__nocheck(ptr);
+}
+
+static inline void atomic16_set(Int128 *ptr, Int128 val)
+{
+    atomic_set__nocheck(ptr, val);
+}
+
+# define HAVE_ATOMIC128 1
+#elif !defined(CONFIG_USER_ONLY) && defined(__aarch64__)
+/* We can do better than cmpxchg for AArch64.  */
+static inline Int128 atomic16_read(Int128 *ptr)
+{
+    uint64_t l, h;
+    uint32_t tmp;
+
+    /* The load must be paired with the store to guarantee not tearing.  */
+    asm("0: ldxp %[l], %[h], %[mem]\n\t"
+        "stxp %w[tmp], %[l], %[h], %[mem]\n\t"
+        "cbnz %w[tmp], 0b"
+        : [mem] "+m"(*ptr), [tmp] "=r"(tmp), [l] "=r"(l), [h] "=r"(h));
+
+    return int128_make128(l, h);
+}
+
+static inline void atomic16_set(Int128 *ptr, Int128 val)
+{
+    uint64_t l = int128_getlo(val), h = int128_gethi(val);
+    uint64_t t1, t2;
+
+    /* Load into temporaries to acquire the exclusive access lock.  */
+    asm("0: ldxp %[t1], %[t2], %[mem]\n\t"
+        "stxp %w[t1], %[l], %[h], %[mem]\n\t"
+        "cbnz %w[t1], 0b"
+        : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2)
+        : [l] "r"(l), [h] "r"(h));
+}
+
+# define HAVE_ATOMIC128 1
+#elif !defined(CONFIG_USER_ONLY) && HAVE_CMPXCHG128
+static inline Int128 atomic16_read(Int128 *ptr)
+{
+    /* Maybe replace 0 with 0, returning the old value.  */
+    return atomic16_cmpxchg(ptr, 0, 0);
+}
+
+static inline void atomic16_set(Int128 *ptr, Int128 val)
+{
+    Int128 old = *ptr, cmp;
+    do {
+        cmp = old;
+        old = atomic16_cmpxchg(ptr, cmp, val);
+    } while (old != cmp);
+}
+
+# define HAVE_ATOMIC128 1
+#else
+/* Fallback definitions that must be optimized away, or error.  */
+Int128 __attribute__((error("unsupported atomic")))
+    atomic16_read(Int128 *ptr);
+void __attribute__((error("unsupported atomic")))
+    atomic16_set(Int128 *ptr, Int128 val);
+# define HAVE_ATOMIC128 0
+#endif /* Some definition for HAVE_ATOMIC128 */
+
+#endif /* QEMU_ATOMIC128_H */
diff --git a/tcg/tcg.h b/tcg/tcg.h
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.h
+++ b/tcg/tcg.h
@@ -XXX,XX +XXX,XX @@
 #include "qemu/queue.h"
 #include "tcg-mo.h"
 #include "tcg-target.h"
+#include "qemu/int128.h"
 
 /* XXX: make safe guess about sizes */
 #define MAX_OP_PER_INSTR 266
@@ -XXX,XX +XXX,XX @@ GEN_ATOMIC_HELPER_ALL(xchg)
 #undef GEN_ATOMIC_HELPER
 #endif /* CONFIG_SOFTMMU */
 
-#ifdef CONFIG_ATOMIC128
-#include "qemu/int128.h"
-
-/* These aren't really a "proper" helpers because TCG cannot manage Int128.
-   However, use the same format as the others, for use by the backends. */
+/*
+ * These aren't really a "proper" helpers because TCG cannot manage Int128.
+ * However, use the same format as the others, for use by the backends.
+ *
+ * The cmpxchg functions are only defined if HAVE_CMPXCHG128;
+ * the ld/st functions are only defined if HAVE_ATOMIC128,
+ * as defined by <qemu/atomic128.h>.
+ */
 Int128 helper_atomic_cmpxchgo_le_mmu(CPUArchState *env, target_ulong addr,
                                      Int128 cmpv, Int128 newv,
                                      TCGMemOpIdx oi, uintptr_t retaddr);
@@ -XXX,XX +XXX,XX @@ void helper_atomic_sto_le_mmu(CPUArchState *env, target_ulong addr, Int128 val,
 void helper_atomic_sto_be_mmu(CPUArchState *env, target_ulong addr, Int128 val,
                               TCGMemOpIdx oi, uintptr_t retaddr);
 
-#endif /* CONFIG_ATOMIC128 */
-
 #endif /* TCG_H */
diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/cputlb.c
+++ b/accel/tcg/cputlb.c
@@ -XXX,XX +XXX,XX @@
 #include "exec/log.h"
 #include "exec/helper-proto.h"
 #include "qemu/atomic.h"
+#include "qemu/atomic128.h"
 
 /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
 /* #define DEBUG_TLB */
@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
 #include "atomic_template.h"
 #endif
 
-#ifdef CONFIG_ATOMIC128
+#if HAVE_CMPXCHG128 || HAVE_ATOMIC128
 #define DATA_SIZE 16
 #include "atomic_template.h"
 #endif
diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/user-exec.c
+++ b/accel/tcg/user-exec.c
@@ -XXX,XX +XXX,XX @@
 #include "exec/cpu_ldst.h"
 #include "translate-all.h"
 #include "exec/helper-proto.h"
+#include "qemu/atomic128.h"
 
 #undef EAX
 #undef ECX
@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
 /* The following is only callable from other helpers, and matches up
    with the softmmu version.  */
 
-#ifdef CONFIG_ATOMIC128
+#if HAVE_ATOMIC128 || HAVE_CMPXCHG128
 
 #undef EXTRA_ARGS
 #undef ATOMIC_NAME
@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
 
 #define DATA_SIZE 16
 #include "atomic_template.h"
-#endif /* CONFIG_ATOMIC128 */
+#endif
diff --git a/configure b/configure
index XXXXXXX..XXXXXXX 100755
--- a/configure
+++ b/configure
@@ -XXX,XX +XXX,XX @@ EOF
   fi
 fi
 
+cmpxchg128=no
+if test "$int128" = yes -a "$atomic128" = no; then
+  cat > $TMPC << EOF
+int main(void)
+{
+  unsigned __int128 x = 0, y = 0;
+  __sync_val_compare_and_swap_16(&x, y, x);
+  return 0;
+}
+EOF
+  if compile_prog "" "" ; then
+    cmpxchg128=yes
+  fi
+fi
+
 #########################################
 # See if 64-bit atomic operations are supported.
 # Note that without __atomic builtins, we can only
@@ -XXX,XX +XXX,XX @@ if test "$atomic128" = "yes" ; then
   echo "CONFIG_ATOMIC128=y" >> $config_host_mak
 fi
 
+if test "$cmpxchg128" = "yes" ; then
+  echo "CONFIG_CMPXCHG128=y" >> $config_host_mak
+fi
+
 if test "$atomic64" = "yes" ; then
   echo "CONFIG_ATOMIC64=y" >> $config_host_mak
 fi
-- 
2.17.2

Reviewed-by: Emilio G. Cota <cota@braap.org>
Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/i386/mem_helper.c | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/target/i386/mem_helper.c b/target/i386/mem_helper.c
index XXXXXXX..XXXXXXX 100644
--- a/target/i386/mem_helper.c
+++ b/target/i386/mem_helper.c
@@ -XXX,XX +XXX,XX @@
 #include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 #include "qemu/int128.h"
+#include "qemu/atomic128.h"
 #include "tcg.h"
 
 void helper_cmpxchg8b_unlocked(CPUX86State *env, target_ulong a0)
@@ -XXX,XX +XXX,XX @@ void helper_cmpxchg16b(CPUX86State *env, target_ulong a0)
 
     if ((a0 & 0xf) != 0) {
         raise_exception_ra(env, EXCP0D_GPF, ra);
-    } else {
-#ifndef CONFIG_ATOMIC128
-        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
-#else
+    } else if (HAVE_CMPXCHG128) {
         int eflags = cpu_cc_compute_all(env, CC_OP);
 
         Int128 cmpv = int128_make128(env->regs[R_EAX], env->regs[R_EDX]);
@@ -XXX,XX +XXX,XX @@ void helper_cmpxchg16b(CPUX86State *env, target_ulong a0)
             eflags &= ~CC_Z;
         }
         CC_SRC = eflags;
-#endif
+    } else {
+        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
     }
 }
 #endif
-- 
2.17.2

Reviewed-by: Emilio G. Cota <cota@braap.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/helper-a64.c | 259 +++++++++++++++++++++-------------------
 1 file changed, 133 insertions(+), 126 deletions(-)

diff --git a/target/arm/helper-a64.c b/target/arm/helper-a64.c
index XXXXXXX..XXXXXXX 100644
--- a/target/arm/helper-a64.c
+++ b/target/arm/helper-a64.c
@@ -XXX,XX +XXX,XX @@
 #include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 #include "qemu/int128.h"
+#include "qemu/atomic128.h"
 #include "tcg.h"
 #include "fpu/softfloat.h"
 #include <zlib.h> /* For crc32 */
@@ -XXX,XX +XXX,XX @@ uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes)
     return crc32c(acc, buf, bytes) ^ 0xffffffff;
 }
 
-/* Returns 0 on success; 1 otherwise.  */
-static uint64_t do_paired_cmpxchg64_le(CPUARMState *env, uint64_t addr,
-                                       uint64_t new_lo, uint64_t new_hi,
-                                       bool parallel, uintptr_t ra)
+uint64_t HELPER(paired_cmpxchg64_le)(CPUARMState *env, uint64_t addr,
+                                     uint64_t new_lo, uint64_t new_hi)
 {
-    Int128 oldv, cmpv, newv;
+    Int128 cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
+    Int128 newv = int128_make128(new_lo, new_hi);
+    Int128 oldv;
+    uintptr_t ra = GETPC();
+    uint64_t o0, o1;
     bool success;
 
-    cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
-    newv = int128_make128(new_lo, new_hi);
-
-    if (parallel) {
-#ifndef CONFIG_ATOMIC128
-        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
-#else
-        int mem_idx = cpu_mmu_index(env, false);
-        TCGMemOpIdx oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
-        oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra);
-        success = int128_eq(oldv, cmpv);
-#endif
-    } else {
-        uint64_t o0, o1;
-
 #ifdef CONFIG_USER_ONLY
-        /* ??? Enforce alignment.  */
-        uint64_t *haddr = g2h(addr);
+    /* ??? Enforce alignment.  */
+    uint64_t *haddr = g2h(addr);
 
-        helper_retaddr = ra;
-        o0 = ldq_le_p(haddr + 0);
-        o1 = ldq_le_p(haddr + 1);
-        oldv = int128_make128(o0, o1);
+    helper_retaddr = ra;
+    o0 = ldq_le_p(haddr + 0);
+    o1 = ldq_le_p(haddr + 1);
+    oldv = int128_make128(o0, o1);
 
-        success = int128_eq(oldv, cmpv);
-        if (success) {
-            stq_le_p(haddr + 0, int128_getlo(newv));
-            stq_le_p(haddr + 1, int128_gethi(newv));
-        }
-        helper_retaddr = 0;
-#else
-        int mem_idx = cpu_mmu_index(env, false);
-        TCGMemOpIdx oi0 = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
-        TCGMemOpIdx oi1 = make_memop_idx(MO_LEQ, mem_idx);
-
-        o0 = helper_le_ldq_mmu(env, addr + 0, oi0, ra);
-        o1 = helper_le_ldq_mmu(env, addr + 8, oi1, ra);
-        oldv = int128_make128(o0, o1);
-
-        success = int128_eq(oldv, cmpv);
-        if (success) {
-            helper_le_stq_mmu(env, addr + 0, int128_getlo(newv), oi1, ra);
-            helper_le_stq_mmu(env, addr + 8, int128_gethi(newv), oi1, ra);
-        }
-#endif
+    success = int128_eq(oldv, cmpv);
+    if (success) {
+        stq_le_p(haddr + 0, int128_getlo(newv));
+        stq_le_p(haddr + 1, int128_gethi(newv));
     }
+    helper_retaddr = 0;
+#else
+    int mem_idx = cpu_mmu_index(env, false);
+    TCGMemOpIdx oi0 = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
+    TCGMemOpIdx oi1 = make_memop_idx(MO_LEQ, mem_idx);
+
+    o0 = helper_le_ldq_mmu(env, addr + 0, oi0, ra);
+    o1 = helper_le_ldq_mmu(env, addr + 8, oi1, ra);
+    oldv = int128_make128(o0, o1);
+
+    success = int128_eq(oldv, cmpv);
+    if (success) {
+        helper_le_stq_mmu(env, addr + 0, int128_getlo(newv), oi1, ra);
+        helper_le_stq_mmu(env, addr + 8, int128_gethi(newv), oi1, ra);
+    }
+#endif
 
     return !success;
 }
 
-uint64_t HELPER(paired_cmpxchg64_le)(CPUARMState *env, uint64_t addr,
-                                              uint64_t new_lo, uint64_t new_hi)
-{
-    return do_paired_cmpxchg64_le(env, addr, new_lo, new_hi, false, GETPC());
-}
-
 uint64_t HELPER(paired_cmpxchg64_le_parallel)(CPUARMState *env, uint64_t addr,
                                               uint64_t new_lo, uint64_t new_hi)
-{
-    return do_paired_cmpxchg64_le(env, addr, new_lo, new_hi, true, GETPC());
-}
-
-static uint64_t do_paired_cmpxchg64_be(CPUARMState *env, uint64_t addr,
-                                       uint64_t new_lo, uint64_t new_hi,
-                                       bool parallel, uintptr_t ra)
 {
     Int128 oldv, cmpv, newv;
+    uintptr_t ra = GETPC();
     bool success;
+    int mem_idx;
+    TCGMemOpIdx oi;
 
-    /* high and low need to be switched here because this is not actually a
-     * 128bit store but two doublewords stored consecutively
-     */
-    cmpv = int128_make128(env->exclusive_high, env->exclusive_val);
-    newv = int128_make128(new_hi, new_lo);
-
-    if (parallel) {
-#ifndef CONFIG_ATOMIC128
+    if (!HAVE_CMPXCHG128) {
         cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
-#else
-        int mem_idx = cpu_mmu_index(env, false);
-        TCGMemOpIdx oi = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
-        oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
-        success = int128_eq(oldv, cmpv);
-#endif
-    } else {
-        uint64_t o0, o1;
-
-#ifdef CONFIG_USER_ONLY
-        /* ??? Enforce alignment.  */
-        uint64_t *haddr = g2h(addr);
-
-        helper_retaddr = ra;
-        o1 = ldq_be_p(haddr + 0);
-        o0 = ldq_be_p(haddr + 1);
-        oldv = int128_make128(o0, o1);
-
-        success = int128_eq(oldv, cmpv);
-        if (success) {
-            stq_be_p(haddr + 0, int128_gethi(newv));
-            stq_be_p(haddr + 1, int128_getlo(newv));
-        }
-        helper_retaddr = 0;
-#else
-        int mem_idx = cpu_mmu_index(env, false);
-        TCGMemOpIdx oi0 = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
-        TCGMemOpIdx oi1 = make_memop_idx(MO_BEQ, mem_idx);
-
-        o1 = helper_be_ldq_mmu(env, addr + 0, oi0, ra);
-        o0 = helper_be_ldq_mmu(env, addr + 8, oi1, ra);
-        oldv = int128_make128(o0, o1);
-
-        success = int128_eq(oldv, cmpv);
-        if (success) {
-            helper_be_stq_mmu(env, addr + 0, int128_gethi(newv), oi1, ra);
-            helper_be_stq_mmu(env, addr + 8, int128_getlo(newv), oi1, ra);
-        }
-#endif
     }
 
+    mem_idx = cpu_mmu_index(env, false);
+    oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
+
+    cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
+    newv = int128_make128(new_lo, new_hi);
+    oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra);
+
+    success = int128_eq(oldv, cmpv);
     return !success;
 }
 
 uint64_t HELPER(paired_cmpxchg64_be)(CPUARMState *env, uint64_t addr,
                                      uint64_t new_lo, uint64_t new_hi)
 {
-    return do_paired_cmpxchg64_be(env, addr, new_lo, new_hi, false, GETPC());
+    /*
+     * High and low need to be switched here because this is not actually a
+     * 128bit store but two doublewords stored consecutively
+     */
+    Int128 cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
+    Int128 newv = int128_make128(new_lo, new_hi);
+    Int128 oldv;
+    uintptr_t ra = GETPC();
+    uint64_t o0, o1;
+    bool success;
+
+#ifdef CONFIG_USER_ONLY
+    /* ??? Enforce alignment.  */
+    uint64_t *haddr = g2h(addr);
+
+    helper_retaddr = ra;
+    o1 = ldq_be_p(haddr + 0);
+    o0 = ldq_be_p(haddr + 1);
+    oldv = int128_make128(o0, o1);
+
+    success = int128_eq(oldv, cmpv);
+    if (success) {
+        stq_be_p(haddr + 0, int128_gethi(newv));
+        stq_be_p(haddr + 1, int128_getlo(newv));
+    }
+    helper_retaddr = 0;
+#else
+    int mem_idx = cpu_mmu_index(env, false);
+    TCGMemOpIdx oi0 = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
+    TCGMemOpIdx oi1 = make_memop_idx(MO_BEQ, mem_idx);
+
+    o1 = helper_be_ldq_mmu(env, addr + 0, oi0, ra);
+    o0 = helper_be_ldq_mmu(env, addr + 8, oi1, ra);
+    oldv = int128_make128(o0, o1);
+
+    success = int128_eq(oldv, cmpv);
+    if (success) {
+        helper_be_stq_mmu(env, addr + 0, int128_gethi(newv), oi1, ra);
+        helper_be_stq_mmu(env, addr + 8, int128_getlo(newv), oi1, ra);
+    }
+#endif
+
+    return !success;
 }
 
 uint64_t HELPER(paired_cmpxchg64_be_parallel)(CPUARMState *env, uint64_t addr,
-                                     uint64_t new_lo, uint64_t new_hi)
+                                              uint64_t new_lo, uint64_t new_hi)
 {
-    return do_paired_cmpxchg64_be(env, addr, new_lo, new_hi, true, GETPC());
+    Int128 oldv, cmpv, newv;
+    uintptr_t ra = GETPC();
+    bool success;
+    int mem_idx;
+    TCGMemOpIdx oi;
+
+    if (!HAVE_CMPXCHG128) {
+        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
+    }
+
+    mem_idx = cpu_mmu_index(env, false);
+    oi = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
+
+    /*
+     * High and low need to be switched here because this is not actually a
+     * 128bit store but two doublewords stored consecutively
+     */
+    cmpv = int128_make128(env->exclusive_high, env->exclusive_val);
+    newv = int128_make128(new_hi, new_lo);
+    oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
+
+    success = int128_eq(oldv, cmpv);
+    return !success;
 }
 
 /* Writes back the old data into Rs.  */
 void HELPER(casp_le_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
                               uint64_t new_lo, uint64_t new_hi)
 {
-    uintptr_t ra = GETPC();
-#ifndef CONFIG_ATOMIC128
-    cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
-#else
     Int128 oldv, cmpv, newv;
+    uintptr_t ra = GETPC();
+    int mem_idx;
+    TCGMemOpIdx oi;
+
+    if (!HAVE_CMPXCHG128) {
+        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
+    }
+
+    mem_idx = cpu_mmu_index(env, false);
+    oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
 
     cmpv = int128_make128(env->xregs[rs], env->xregs[rs + 1]);
     newv = int128_make128(new_lo, new_hi);
-
-    int mem_idx = cpu_mmu_index(env, false);
-    TCGMemOpIdx oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
     oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra);
 
     env->xregs[rs] = int128_getlo(oldv);
     env->xregs[rs + 1] = int128_gethi(oldv);
-#endif
 }
 
 void HELPER(casp_be_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
                               uint64_t new_hi, uint64_t new_lo)
 {
-    uintptr_t ra = GETPC();
-#ifndef CONFIG_ATOMIC128
-    cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
-#else
     Int128 oldv, cmpv, newv;
+    uintptr_t ra = GETPC();
+    int mem_idx;
+    TCGMemOpIdx oi;
+
+    if (!HAVE_CMPXCHG128) {
+        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
+    }
+
+    mem_idx = cpu_mmu_index(env, false);
+    oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
 
     cmpv = int128_make128(env->xregs[rs + 1], env->xregs[rs]);
     newv = int128_make128(new_lo, new_hi);
-
-    int mem_idx = cpu_mmu_index(env, false);
-    TCGMemOpIdx oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
     oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
 
     env->xregs[rs + 1] = int128_getlo(oldv);
     env->xregs[rs] = int128_gethi(oldv);
-#endif
 }
 
 /*
-- 
2.17.2

Reviewed-by: Emilio G. Cota <cota@braap.org>
Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/helper-a64.c    | 16 ++++------------
 target/arm/translate-a64.c | 38 ++++++++++++++++++++++----------------
 2 files changed, 26 insertions(+), 28 deletions(-)

diff --git a/target/arm/helper-a64.c b/target/arm/helper-a64.c
index XXXXXXX..XXXXXXX 100644
--- a/target/arm/helper-a64.c
+++ b/target/arm/helper-a64.c
@@ -XXX,XX +XXX,XX @@ uint64_t HELPER(paired_cmpxchg64_le_parallel)(CPUARMState *env, uint64_t addr,
     int mem_idx;
     TCGMemOpIdx oi;
 
-    if (!HAVE_CMPXCHG128) {
-        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
-    }
+    assert(HAVE_CMPXCHG128);
 
     mem_idx = cpu_mmu_index(env, false);
     oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
@@ -XXX,XX +XXX,XX @@ uint64_t HELPER(paired_cmpxchg64_be_parallel)(CPUARMState *env, uint64_t addr,
     int mem_idx;
     TCGMemOpIdx oi;
 
-    if (!HAVE_CMPXCHG128) {
-        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
-    }
+    assert(HAVE_CMPXCHG128);
 
     mem_idx = cpu_mmu_index(env, false);
     oi = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
@@ -XXX,XX +XXX,XX @@ void HELPER(casp_le_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
     int mem_idx;
     TCGMemOpIdx oi;
 
-    if (!HAVE_CMPXCHG128) {
-        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
-    }
+    assert(HAVE_CMPXCHG128);
 
     mem_idx = cpu_mmu_index(env, false);
     oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
@@ -XXX,XX +XXX,XX @@ void HELPER(casp_be_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
     int mem_idx;
     TCGMemOpIdx oi;
 
-    if (!HAVE_CMPXCHG128) {
-        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
-    }
+    assert(HAVE_CMPXCHG128);
 
     mem_idx = cpu_mmu_index(env, false);
     oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c
index XXXXXXX..XXXXXXX 100644
--- a/target/arm/translate-a64.c
+++ b/target/arm/translate-a64.c
@@ -XXX,XX +XXX,XX @@
 
 #include "trace-tcg.h"
 #include "translate-a64.h"
+#include "qemu/atomic128.h"
 
 static TCGv_i64 cpu_X[32];
 static TCGv_i64 cpu_pc;
@@ -XXX,XX +XXX,XX @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
                                        get_mem_index(s),
                                        MO_64 | MO_ALIGN | s->be_data);
             tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
-        } else if (s->be_data == MO_LE) {
-            if (tb_cflags(s->base.tb) & CF_PARALLEL) {
+        } else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
+            if (!HAVE_CMPXCHG128) {
+                gen_helper_exit_atomic(cpu_env);
+                s->base.is_jmp = DISAS_NORETURN;
+            } else if (s->be_data == MO_LE) {
                 gen_helper_paired_cmpxchg64_le_parallel(tmp, cpu_env,
                                                         cpu_exclusive_addr,
                                                         cpu_reg(s, rt),
                                                         cpu_reg(s, rt2));
             } else {
-                gen_helper_paired_cmpxchg64_le(tmp, cpu_env, cpu_exclusive_addr,
-                                               cpu_reg(s, rt), cpu_reg(s, rt2));
-            }
-        } else {
-            if (tb_cflags(s->base.tb) & CF_PARALLEL) {
                 gen_helper_paired_cmpxchg64_be_parallel(tmp, cpu_env,
                                                         cpu_exclusive_addr,
                                                         cpu_reg(s, rt),
                                                         cpu_reg(s, rt2));
-            } else {
-                gen_helper_paired_cmpxchg64_be(tmp, cpu_env, cpu_exclusive_addr,
-                                               cpu_reg(s, rt), cpu_reg(s, rt2));
             }
+        } else if (s->be_data == MO_LE) {
+            gen_helper_paired_cmpxchg64_le(tmp, cpu_env, cpu_exclusive_addr,
+                                           cpu_reg(s, rt), cpu_reg(s, rt2));
+        } else {
+            gen_helper_paired_cmpxchg64_be(tmp, cpu_env, cpu_exclusive_addr,
+                                           cpu_reg(s, rt), cpu_reg(s, rt2));
         }
     } else {
         tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr, cpu_exclusive_val,
@@ -XXX,XX +XXX,XX @@ static void gen_compare_and_swap_pair(DisasContext *s, int rs, int rt,
         }
         tcg_temp_free_i64(cmp);
     } else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
-        TCGv_i32 tcg_rs = tcg_const_i32(rs);
-
-        if (s->be_data == MO_LE) {
-            gen_helper_casp_le_parallel(cpu_env, tcg_rs, addr, t1, t2);
+        if (HAVE_CMPXCHG128) {
+            TCGv_i32 tcg_rs = tcg_const_i32(rs);
+            if (s->be_data == MO_LE) {
+                gen_helper_casp_le_parallel(cpu_env, tcg_rs, addr, t1, t2);
+            } else {
+                gen_helper_casp_be_parallel(cpu_env, tcg_rs, addr, t1, t2);
+            }
+            tcg_temp_free_i32(tcg_rs);
         } else {
-            gen_helper_casp_be_parallel(cpu_env, tcg_rs, addr, t1, t2);
+            gen_helper_exit_atomic(cpu_env);
+            s->base.is_jmp = DISAS_NORETURN;
         }
-        tcg_temp_free_i32(tcg_rs);
     } else {
         TCGv_i64 d1 = tcg_temp_new_i64();
         TCGv_i64 d2 = tcg_temp_new_i64();
-- 
2.17.2

Reviewed-by: Emilio G. Cota <cota@braap.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/ppc/helper.h     |   2 +-
 target/ppc/mem_helper.c |  33 ++++++++++--
 target/ppc/translate.c  | 115 +++++++++++++++++++++-------------------
 3 files changed, 88 insertions(+), 62 deletions(-)

diff --git a/target/ppc/helper.h b/target/ppc/helper.h
index XXXXXXX..XXXXXXX 100644
--- a/target/ppc/helper.h
+++ b/target/ppc/helper.h
@@ -XXX,XX +XXX,XX @@ DEF_HELPER_4(dscliq, void, env, fprp, fprp, i32)
 DEF_HELPER_1(tbegin, void, env)
 DEF_HELPER_FLAGS_1(fixup_thrm, TCG_CALL_NO_RWG, void, env)
 
-#if defined(TARGET_PPC64) && defined(CONFIG_ATOMIC128)
+#ifdef TARGET_PPC64
 DEF_HELPER_FLAGS_3(lq_le_parallel, TCG_CALL_NO_WG, i64, env, tl, i32)
 DEF_HELPER_FLAGS_3(lq_be_parallel, TCG_CALL_NO_WG, i64, env, tl, i32)
 DEF_HELPER_FLAGS_5(stq_le_parallel, TCG_CALL_NO_WG,
diff --git a/target/ppc/mem_helper.c b/target/ppc/mem_helper.c
index XXXXXXX..XXXXXXX 100644
--- a/target/ppc/mem_helper.c
+++ b/target/ppc/mem_helper.c
@@ -XXX,XX +XXX,XX @@
 #include "exec/cpu_ldst.h"
 #include "tcg.h"
 #include "internal.h"
+#include "qemu/atomic128.h"
 
 //#define DEBUG_OP
 
@@ -XXX,XX +XXX,XX @@ target_ulong helper_lscbx(CPUPPCState *env, target_ulong addr, uint32_t reg,
     return i;
 }
 
-#if defined(TARGET_PPC64) && defined(CONFIG_ATOMIC128)
+#ifdef TARGET_PPC64
 uint64_t helper_lq_le_parallel(CPUPPCState *env, target_ulong addr,
                                uint32_t opidx)
 {
-    Int128 ret = helper_atomic_ldo_le_mmu(env, addr, opidx, GETPC());
+    Int128 ret;
+
+    /* We will have raised EXCP_ATOMIC from the translator.  */
+    assert(HAVE_ATOMIC128);
+    ret = helper_atomic_ldo_le_mmu(env, addr, opidx, GETPC());
     env->retxh = int128_gethi(ret);
     return int128_getlo(ret);
 }
@@ -XXX,XX +XXX,XX @@ uint64_t helper_lq_le_parallel(CPUPPCState *env, target_ulong addr,
 uint64_t helper_lq_be_parallel(CPUPPCState *env, target_ulong addr,
                                uint32_t opidx)
 {
-    Int128 ret = helper_atomic_ldo_be_mmu(env, addr, opidx, GETPC());
+    Int128 ret;
+
+    /* We will have raised EXCP_ATOMIC from the translator.  */
+    assert(HAVE_ATOMIC128);
+    ret = helper_atomic_ldo_be_mmu(env, addr, opidx, GETPC());
     env->retxh = int128_gethi(ret);
     return int128_getlo(ret);
 }
@@ -XXX,XX +XXX,XX @@ uint64_t helper_lq_be_parallel(CPUPPCState *env, target_ulong addr,
 void helper_stq_le_parallel(CPUPPCState *env, target_ulong addr,
                             uint64_t lo, uint64_t hi, uint32_t opidx)
 {
-    Int128 val = int128_make128(lo, hi);
+    Int128 val;
+
+    /* We will have raised EXCP_ATOMIC from the translator.  */
+    assert(HAVE_ATOMIC128);
+    val = int128_make128(lo, hi);
     helper_atomic_sto_le_mmu(env, addr, val, opidx, GETPC());
 }
 
 void helper_stq_be_parallel(CPUPPCState *env, target_ulong addr,
                             uint64_t lo, uint64_t hi, uint32_t opidx)
 {
-    Int128 val = int128_make128(lo, hi);
+    Int128 val;
+
+    /* We will have raised EXCP_ATOMIC from the translator.  */
+    assert(HAVE_ATOMIC128);
+    val = int128_make128(lo, hi);
     helper_atomic_sto_be_mmu(env, addr, val, opidx, GETPC());
 }
 
@@ -XXX,XX +XXX,XX @@ uint32_t helper_stqcx_le_parallel(CPUPPCState *env, target_ulong addr,
 {
     bool success = false;
 
+    /* We will have raised EXCP_ATOMIC from the translator.  */
+    assert(HAVE_CMPXCHG128);
+
     if (likely(addr == env->reserve_addr)) {
         Int128 oldv, cmpv, newv;
 
@@ -XXX,XX +XXX,XX @@ uint32_t helper_stqcx_be_parallel(CPUPPCState *env, target_ulong addr,
 {
     bool success = false;
 
+    /* We will have raised EXCP_ATOMIC from the translator.  */
+    assert(HAVE_CMPXCHG128);
+
     if (likely(addr == env->reserve_addr)) {
         Int128 oldv, cmpv, newv;
 
diff --git a/target/ppc/translate.c b/target/ppc/translate.c
index XXXXXXX..XXXXXXX 100644
--- a/target/ppc/translate.c
+++ b/target/ppc/translate.c
@@ -XXX,XX +XXX,XX @@
 #include "trace-tcg.h"
 #include "exec/translator.h"
 #include "exec/log.h"
+#include "qemu/atomic128.h"
 
 
 #define CPU_SINGLE_STEP 0x1
@@ -XXX,XX +XXX,XX @@ static void gen_lq(DisasContext *ctx)
     hi = cpu_gpr[rd];
 
     if (tb_cflags(ctx->base.tb) & CF_PARALLEL) {
-#ifdef CONFIG_ATOMIC128
-        TCGv_i32 oi = tcg_temp_new_i32();
-        if (ctx->le_mode) {
-            tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ, ctx->mem_idx));
-            gen_helper_lq_le_parallel(lo, cpu_env, EA, oi);
+        if (HAVE_ATOMIC128) {
+            TCGv_i32 oi = tcg_temp_new_i32();
+            if (ctx->le_mode) {
+                tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ, ctx->mem_idx));
+                gen_helper_lq_le_parallel(lo, cpu_env, EA, oi);
+            } else {
+                tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ, ctx->mem_idx));
+                gen_helper_lq_be_parallel(lo, cpu_env, EA, oi);
+            }
+            tcg_temp_free_i32(oi);
+            tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh));
         } else {
-            tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ, ctx->mem_idx));
-            gen_helper_lq_be_parallel(lo, cpu_env, EA, oi);
+            /* Restart with exclusive lock.  */
+            gen_helper_exit_atomic(cpu_env);
+            ctx->base.is_jmp = DISAS_NORETURN;
         }
-        tcg_temp_free_i32(oi);
-        tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh));
-#else
-        /* Restart with exclusive lock.  */
-        gen_helper_exit_atomic(cpu_env);
-        ctx->base.is_jmp = DISAS_NORETURN;
-#endif
     } else if (ctx->le_mode) {
         tcg_gen_qemu_ld_i64(lo, EA, ctx->mem_idx, MO_LEQ);
         gen_addr_add(ctx, EA, EA, 8);
@@ -XXX,XX +XXX,XX @@ static void gen_std(DisasContext *ctx)
         hi = cpu_gpr[rs];
 
         if (tb_cflags(ctx->base.tb) & CF_PARALLEL) {
-#ifdef CONFIG_ATOMIC128
-            TCGv_i32 oi = tcg_temp_new_i32();
-            if (ctx->le_mode) {
-                tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ, ctx->mem_idx));
-                gen_helper_stq_le_parallel(cpu_env, EA, lo, hi, oi);
+            if (HAVE_ATOMIC128) {
+                TCGv_i32 oi = tcg_temp_new_i32();
+                if (ctx->le_mode) {
+                    tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ, ctx->mem_idx));
+                    gen_helper_stq_le_parallel(cpu_env, EA, lo, hi, oi);
+                } else {
+                    tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ, ctx->mem_idx));
+                    gen_helper_stq_be_parallel(cpu_env, EA, lo, hi, oi);
+                }
+                tcg_temp_free_i32(oi);
             } else {
-                tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ, ctx->mem_idx));
-                gen_helper_stq_be_parallel(cpu_env, EA, lo, hi, oi);
+                /* Restart with exclusive lock.  */
+                gen_helper_exit_atomic(cpu_env);
+                ctx->base.is_jmp = DISAS_NORETURN;
             }
-            tcg_temp_free_i32(oi);
-#else
-            /* Restart with exclusive lock.  */
-            gen_helper_exit_atomic(cpu_env);
-            ctx->base.is_jmp = DISAS_NORETURN;
-#endif
         } else if (ctx->le_mode) {
             tcg_gen_qemu_st_i64(lo, EA, ctx->mem_idx, MO_LEQ);
             gen_addr_add(ctx, EA, EA, 8);
@@ -XXX,XX +XXX,XX @@ static void gen_lqarx(DisasContext *ctx)
     hi = cpu_gpr[rd];
 
     if (tb_cflags(ctx->base.tb) & CF_PARALLEL) {
-#ifdef CONFIG_ATOMIC128
-        TCGv_i32 oi = tcg_temp_new_i32();
-        if (ctx->le_mode) {
-            tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ | MO_ALIGN_16,
-                                                ctx->mem_idx));
-            gen_helper_lq_le_parallel(lo, cpu_env, EA, oi);
+        if (HAVE_ATOMIC128) {
+            TCGv_i32 oi = tcg_temp_new_i32();
+            if (ctx->le_mode) {
+                tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ | MO_ALIGN_16,
+                                                    ctx->mem_idx));
+                gen_helper_lq_le_parallel(lo, cpu_env, EA, oi);
+            } else {
+                tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ | MO_ALIGN_16,
+                                                    ctx->mem_idx));
+                gen_helper_lq_be_parallel(lo, cpu_env, EA, oi);
+            }
+            tcg_temp_free_i32(oi);
+            tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh));
         } else {
-            tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ | MO_ALIGN_16,
-                                                ctx->mem_idx));
-            gen_helper_lq_be_parallel(lo, cpu_env, EA, oi);
+            /* Restart with exclusive lock.  */
+            gen_helper_exit_atomic(cpu_env);
+            ctx->base.is_jmp = DISAS_NORETURN;
+            tcg_temp_free(EA);
+            return;
         }
-        tcg_temp_free_i32(oi);
-        tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh));
-#else
-        /* Restart with exclusive lock.  */
-        gen_helper_exit_atomic(cpu_env);
-        ctx->base.is_jmp = DISAS_NORETURN;
-        tcg_temp_free(EA);
-        return;
-#endif
     } else if (ctx->le_mode) {
         tcg_gen_qemu_ld_i64(lo, EA, ctx->mem_idx, MO_LEQ | MO_ALIGN_16);
         tcg_gen_mov_tl(cpu_reserve, EA);
@@ -XXX,XX +XXX,XX @@ static void gen_stqcx_(DisasContext *ctx)
     hi = cpu_gpr[rs];
 
     if (tb_cflags(ctx->base.tb) & CF_PARALLEL) {
-        TCGv_i32 oi = tcg_const_i32(DEF_MEMOP(MO_Q) | MO_ALIGN_16);
-#ifdef CONFIG_ATOMIC128
-        if (ctx->le_mode) {
-            gen_helper_stqcx_le_parallel(cpu_crf[0], cpu_env, EA, lo, hi, oi);
+        if (HAVE_CMPXCHG128) {
+            TCGv_i32 oi = tcg_const_i32(DEF_MEMOP(MO_Q) | MO_ALIGN_16);
+            if (ctx->le_mode) {
+                gen_helper_stqcx_le_parallel(cpu_crf[0], cpu_env,
+                                             EA, lo, hi, oi);
+            } else {
+                gen_helper_stqcx_be_parallel(cpu_crf[0], cpu_env,
+                                             EA, lo, hi, oi);
+            }
+            tcg_temp_free_i32(oi);
         } else {
-            gen_helper_stqcx_le_parallel(cpu_crf[0], cpu_env, EA, lo, hi, oi);
+            /* Restart with exclusive lock.  */
+            gen_helper_exit_atomic(cpu_env);
+            ctx->base.is_jmp = DISAS_NORETURN;
         }
-#else
-        /* Restart with exclusive lock.  */
-        gen_helper_exit_atomic(cpu_env);
-        ctx->base.is_jmp = DISAS_NORETURN;
-#endif
         tcg_temp_free(EA);
-        tcg_temp_free_i32(oi);
     } else {
         TCGLabel *lab_fail = gen_new_label();
         TCGLabel *lab_over = gen_new_label();
-- 
2.17.2

Reviewed-by: David Hildenbrand <david@redhat.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/s390x/mem_helper.c | 92 +++++++++++++++++----------------------
 1 file changed, 41 insertions(+), 51 deletions(-)

diff --git a/target/s390x/mem_helper.c b/target/s390x/mem_helper.c
index XXXXXXX..XXXXXXX 100644
--- a/target/s390x/mem_helper.c
+++ b/target/s390x/mem_helper.c
@@ -XXX,XX +XXX,XX @@
 #include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 #include "qemu/int128.h"
+#include "qemu/atomic128.h"
 
 #if !defined(CONFIG_USER_ONLY)
 #include "hw/s390x/storage-keys.h"
@@ -XXX,XX +XXX,XX @@ static void do_cdsg(CPUS390XState *env, uint64_t addr,
     bool fail;
 
     if (parallel) {
-#ifndef CONFIG_ATOMIC128
+#if !HAVE_CMPXCHG128
         cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
 #else
         int mem_idx = cpu_mmu_index(env, false);
@@ -XXX,XX +XXX,XX @@ void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr,
 static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
                         uint64_t a2, bool parallel)
 {
-#if !defined(CONFIG_USER_ONLY) || defined(CONFIG_ATOMIC128)
     uint32_t mem_idx = cpu_mmu_index(env, false);
-#endif
     uintptr_t ra = GETPC();
     uint32_t fc = extract32(env->regs[0], 0, 8);
     uint32_t sc = extract32(env->regs[0], 8, 8);
@@ -XXX,XX +XXX,XX @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
     probe_write(env, a2, 0, mem_idx, ra);
 #endif
 
-    /* Note that the compare-and-swap is atomic, and the store is atomic, but
-       the complete operation is not.  Therefore we do not need to assert serial
-       context in order to implement this.  That said, restart early if we can't
-       support either operation that is supposed to be atomic.  */
+    /*
+     * Note that the compare-and-swap is atomic, and the store is atomic,
+     * but the complete operation is not.  Therefore we do not need to
+     * assert serial context in order to implement this.  That said,
+     * restart early if we can't support either operation that is supposed
+     * to be atomic.
+     */
     if (parallel) {
-        int mask = 0;
-#if !defined(CONFIG_ATOMIC64)
-        mask = -8;
-#elif !defined(CONFIG_ATOMIC128)
-        mask = -16;
+        uint32_t max = 2;
+#ifdef CONFIG_ATOMIC64
+        max = 3;
 #endif
-        if (((4 << fc) | (1 << sc)) & mask) {
+        if ((HAVE_CMPXCHG128 ? 0 : fc + 2 > max) ||
+            (HAVE_ATOMIC128  ? 0 : sc > max)) {
             cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
         }
     }
@@ -XXX,XX +XXX,XX @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
             Int128 cv = int128_make128(env->regs[r3 + 1], env->regs[r3]);
             Int128 ov;
 
-            if (parallel) {
-#ifdef CONFIG_ATOMIC128
-                TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
-                ov = helper_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi, ra);
-                cc = !int128_eq(ov, cv);
-#else
-                /* Note that we asserted !parallel above.  */
-                g_assert_not_reached();
-#endif
-            } else {
+            if (!parallel) {
                 uint64_t oh = cpu_ldq_data_ra(env, a1 + 0, ra);
                 uint64_t ol = cpu_ldq_data_ra(env, a1 + 8, ra);
 
@@ -XXX,XX +XXX,XX @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
 
                 cpu_stq_data_ra(env, a1 + 0, int128_gethi(nv), ra);
                 cpu_stq_data_ra(env, a1 + 8, int128_getlo(nv), ra);
+            } else if (HAVE_CMPXCHG128) {
+                TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
+                ov = helper_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi, ra);
+                cc = !int128_eq(ov, cv);
+            } else {
+                /* Note that we asserted !parallel above.  */
+                g_assert_not_reached();
             }
 
             env->regs[r3 + 0] = int128_gethi(ov);
@@ -XXX,XX +XXX,XX @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
             cpu_stq_data_ra(env, a2, svh, ra);
             break;
         case 4:
-            if (parallel) {
-#ifdef CONFIG_ATOMIC128
+            if (!parallel) {
+                cpu_stq_data_ra(env, a2 + 0, svh, ra);
+                cpu_stq_data_ra(env, a2 + 8, svl, ra);
+            } else if (HAVE_ATOMIC128) {
                 TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
                 Int128 sv = int128_make128(svl, svh);
                 helper_atomic_sto_be_mmu(env, a2, sv, oi, ra);
-#else
+            } else {
                 /* Note that we asserted !parallel above.  */
                 g_assert_not_reached();
-#endif
-            } else {
-                cpu_stq_data_ra(env, a2 + 0, svh, ra);
-                cpu_stq_data_ra(env, a2 + 8, svl, ra);
             }
             break;
         default:
@@ -XXX,XX +XXX,XX @@ static uint64_t do_lpq(CPUS390XState *env, uint64_t addr, bool parallel)
     uintptr_t ra = GETPC();
     uint64_t hi, lo;
 
-    if (parallel) {
-#ifndef CONFIG_ATOMIC128
-        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
-#else
+    if (!parallel) {
+        check_alignment(env, addr, 16, ra);
+        hi = cpu_ldq_data_ra(env, addr + 0, ra);
+        lo = cpu_ldq_data_ra(env, addr + 8, ra);
+    } else if (HAVE_ATOMIC128) {
         int mem_idx = cpu_mmu_index(env, false);
         TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
         Int128 v = helper_atomic_ldo_be_mmu(env, addr, oi, ra);
         hi = int128_gethi(v);
         lo = int128_getlo(v);
-#endif
     } else {
-        check_alignment(env, addr, 16, ra);
-
-        hi = cpu_ldq_data_ra(env, addr + 0, ra);
-        lo = cpu_ldq_data_ra(env, addr + 8, ra);
+        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
     }
 
     env->retxl = lo;
@@ -XXX,XX +XXX,XX @@ static void do_stpq(CPUS390XState *env, uint64_t addr,
 {
     uintptr_t ra = GETPC();
 
-    if (parallel) {
-#ifndef CONFIG_ATOMIC128
-        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
-#else
-        int mem_idx = cpu_mmu_index(env, false);
-        TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
-
-        Int128 v = int128_make128(low, high);
-        helper_atomic_sto_be_mmu(env, addr, v, oi, ra);
-#endif
-    } else {
+    if (!parallel) {
         check_alignment(env, addr, 16, ra);
-
         cpu_stq_data_ra(env, addr + 0, high, ra);
         cpu_stq_data_ra(env, addr + 8, low, ra);
+    } else if (HAVE_ATOMIC128) {
+        int mem_idx = cpu_mmu_index(env, false);
+        TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
+        Int128 v = int128_make128(low, high);
+        helper_atomic_sto_be_mmu(env, addr, v, oi, ra);
+    } else {
+        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
     }
 }
 
-- 
2.17.2

Reviewed-by: David Hildenbrand <david@redhat.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/s390x/mem_helper.c | 128 ++++++++++++++++++--------------------
 1 file changed, 61 insertions(+), 67 deletions(-)

diff --git a/target/s390x/mem_helper.c b/target/s390x/mem_helper.c
index XXXXXXX..XXXXXXX 100644
--- a/target/s390x/mem_helper.c
+++ b/target/s390x/mem_helper.c
@@ -XXX,XX +XXX,XX @@ uint32_t HELPER(trXX)(CPUS390XState *env, uint32_t r1, uint32_t r2,
     return cc;
 }
 
-static void do_cdsg(CPUS390XState *env, uint64_t addr,
-                    uint32_t r1, uint32_t r3, bool parallel)
+void HELPER(cdsg)(CPUS390XState *env, uint64_t addr,
+                  uint32_t r1, uint32_t r3)
 {
     uintptr_t ra = GETPC();
     Int128 cmpv = int128_make128(env->regs[r1 + 1], env->regs[r1]);
     Int128 newv = int128_make128(env->regs[r3 + 1], env->regs[r3]);
     Int128 oldv;
+    uint64_t oldh, oldl;
     bool fail;
 
-    if (parallel) {
-#if !HAVE_CMPXCHG128
-        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
-#else
-        int mem_idx = cpu_mmu_index(env, false);
-        TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
-        oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
-        fail = !int128_eq(oldv, cmpv);
-#endif
-    } else {
-        uint64_t oldh, oldl;
+    check_alignment(env, addr, 16, ra);
 
-        check_alignment(env, addr, 16, ra);
+    oldh = cpu_ldq_data_ra(env, addr + 0, ra);
+    oldl = cpu_ldq_data_ra(env, addr + 8, ra);
 
-        oldh = cpu_ldq_data_ra(env, addr + 0, ra);
-        oldl = cpu_ldq_data_ra(env, addr + 8, ra);
-
-        oldv = int128_make128(oldl, oldh);
-        fail = !int128_eq(oldv, cmpv);
-        if (fail) {
-            newv = oldv;
-        }
-
-        cpu_stq_data_ra(env, addr + 0, int128_gethi(newv), ra);
-        cpu_stq_data_ra(env, addr + 8, int128_getlo(newv), ra);
+    oldv = int128_make128(oldl, oldh);
+    fail = !int128_eq(oldv, cmpv);
+    if (fail) {
+        newv = oldv;
     }
 
+    cpu_stq_data_ra(env, addr + 0, int128_gethi(newv), ra);
+    cpu_stq_data_ra(env, addr + 8, int128_getlo(newv), ra);
+
     env->cc_op = fail;
     env->regs[r1] = int128_gethi(oldv);
     env->regs[r1 + 1] = int128_getlo(oldv);
 }
 
-void HELPER(cdsg)(CPUS390XState *env, uint64_t addr,
-                  uint32_t r1, uint32_t r3)
-{
-    do_cdsg(env, addr, r1, r3, false);
-}
-
 void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr,
                            uint32_t r1, uint32_t r3)
 {
-    do_cdsg(env, addr, r1, r3, true);
+    uintptr_t ra = GETPC();
+    Int128 cmpv = int128_make128(env->regs[r1 + 1], env->regs[r1]);
+    Int128 newv = int128_make128(env->regs[r3 + 1], env->regs[r3]);
+    int mem_idx;
+    TCGMemOpIdx oi;
+    Int128 oldv;
+    bool fail;
+
+    if (!HAVE_CMPXCHG128) {
+        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
+    }
+
+    mem_idx = cpu_mmu_index(env, false);
+    oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
+    oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
+    fail = !int128_eq(oldv, cmpv);
+
+    env->cc_op = fail;
+    env->regs[r1] = int128_gethi(oldv);
+    env->regs[r1 + 1] = int128_getlo(oldv);
 }
 
 static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
@@ -XXX,XX +XXX,XX @@ uint64_t HELPER(lra)(CPUS390XState *env, uint64_t addr)
 #endif
 
 /* load pair from quadword */
-static uint64_t do_lpq(CPUS390XState *env, uint64_t addr, bool parallel)
+uint64_t HELPER(lpq)(CPUS390XState *env, uint64_t addr)
 {
     uintptr_t ra = GETPC();
     uint64_t hi, lo;
 
-    if (!parallel) {
-        check_alignment(env, addr, 16, ra);
-        hi = cpu_ldq_data_ra(env, addr + 0, ra);
-        lo = cpu_ldq_data_ra(env, addr + 8, ra);
-    } else if (HAVE_ATOMIC128) {
+    check_alignment(env, addr, 16, ra);
+    hi = cpu_ldq_data_ra(env, addr + 0, ra);
+    lo = cpu_ldq_data_ra(env, addr + 8, ra);
+
+    env->retxl = lo;
+    return hi;
+}
+
+uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr)
+{
+    uintptr_t ra = GETPC();
+    uint64_t hi, lo;
+
+    if (HAVE_ATOMIC128) {
         int mem_idx = cpu_mmu_index(env, false);
         TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
         Int128 v = helper_atomic_ldo_be_mmu(env, addr, oi, ra);
@@ -XXX,XX +XXX,XX @@ static uint64_t do_lpq(CPUS390XState *env, uint64_t addr, bool parallel)
     return hi;
 }
 
-uint64_t HELPER(lpq)(CPUS390XState *env, uint64_t addr)
-{
-    return do_lpq(env, addr, false);
-}
-
-uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr)
-{
-    return do_lpq(env, addr, true);
-}
-
 /* store pair to quadword */
-static void do_stpq(CPUS390XState *env, uint64_t addr,
-                    uint64_t low, uint64_t high, bool parallel)
+void HELPER(stpq)(CPUS390XState *env, uint64_t addr,
+                  uint64_t low, uint64_t high)
 {
     uintptr_t ra = GETPC();
 
-    if (!parallel) {
-        check_alignment(env, addr, 16, ra);
-        cpu_stq_data_ra(env, addr + 0, high, ra);
-        cpu_stq_data_ra(env, addr + 8, low, ra);
-    } else if (HAVE_ATOMIC128) {
+    check_alignment(env, addr, 16, ra);
+    cpu_stq_data_ra(env, addr + 0, high, ra);
+    cpu_stq_data_ra(env, addr + 8, low, ra);
+}
+
+void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr,
+                           uint64_t low, uint64_t high)
+{
+    uintptr_t ra = GETPC();
+
+    if (HAVE_ATOMIC128) {
         int mem_idx = cpu_mmu_index(env, false);
         TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
         Int128 v = int128_make128(low, high);
@@ -XXX,XX +XXX,XX @@ static void do_stpq(CPUS390XState *env, uint64_t addr,
     }
 }
 
-void HELPER(stpq)(CPUS390XState *env, uint64_t addr,
-                  uint64_t low, uint64_t high)
-{
-    do_stpq(env, addr, low, high, false);
-}
-
-void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr,
-                           uint64_t low, uint64_t high)
-{
-    do_stpq(env, addr, low, high, true);
-}
-
 /* Execute instruction.  This instruction executes an insn modified with
    the contents of r1.  It does not change the executed instruction in memory;
    it does not change the program counter.
-- 
2.17.2

When op raises an exception, it may not have initialized the output
temps that would be written back by wout or cout.

Reviewed-by: David Hildenbrand <david@redhat.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/s390x/translate.c | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/target/s390x/translate.c b/target/s390x/translate.c
index XXXXXXX..XXXXXXX 100644
--- a/target/s390x/translate.c
+++ b/target/s390x/translate.c
@@ -XXX,XX +XXX,XX @@ struct DisasInsn {
 
     const char *name;
 
+    /* Pre-process arguments before HELP_OP.  */
     void (*help_in1)(DisasContext *, DisasFields *, DisasOps *);
     void (*help_in2)(DisasContext *, DisasFields *, DisasOps *);
     void (*help_prep)(DisasContext *, DisasFields *, DisasOps *);
+
+    /*
+     * Post-process output after HELP_OP.
+     * Note that these are not called if HELP_OP returns DISAS_NORETURN.
+     */
     void (*help_wout)(DisasContext *, DisasFields *, DisasOps *);
     void (*help_cout)(DisasContext *, DisasOps *);
+
+    /* Implement the operation itself.  */
     DisasJumpType (*help_op)(DisasContext *, DisasOps *);
 
     uint64_t data;
@@ -XXX,XX +XXX,XX @@ static DisasJumpType translate_one(CPUS390XState *env, DisasContext *s)
     if (insn->help_op) {
         ret = insn->help_op(s, &o);
     }
-    if (insn->help_wout) {
-        insn->help_wout(s, &f, &o);
-    }
-    if (insn->help_cout) {
-        insn->help_cout(s, &o);
+    if (ret != DISAS_NORETURN) {
+        if (insn->help_wout) {
+            insn->help_wout(s, &f, &o);
+        }
+        if (insn->help_cout) {
+            insn->help_cout(s, &o);
+        }
     }
 
     /* Free any temporaries created by the helpers.  */
-- 
2.17.2

Reviewed-by: David Hildenbrand <david@redhat.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/s390x/mem_helper.c | 40 +++++++++++++++++++--------------------
 target/s390x/translate.c  | 25 +++++++++++++++++-------
 2 files changed, 38 insertions(+), 27 deletions(-)

diff --git a/target/s390x/mem_helper.c b/target/s390x/mem_helper.c
index XXXXXXX..XXXXXXX 100644
--- a/target/s390x/mem_helper.c
+++ b/target/s390x/mem_helper.c
@@ -XXX,XX +XXX,XX @@ void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr,
     Int128 oldv;
     bool fail;
 
-    if (!HAVE_CMPXCHG128) {
-        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
-    }
+    assert(HAVE_CMPXCHG128);
 
     mem_idx = cpu_mmu_index(env, false);
     oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
@@ -XXX,XX +XXX,XX @@ uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr)
 {
     uintptr_t ra = GETPC();
     uint64_t hi, lo;
+    int mem_idx;
+    TCGMemOpIdx oi;
+    Int128 v;
 
-    if (HAVE_ATOMIC128) {
-        int mem_idx = cpu_mmu_index(env, false);
-        TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
-        Int128 v = helper_atomic_ldo_be_mmu(env, addr, oi, ra);
-        hi = int128_gethi(v);
-        lo = int128_getlo(v);
-    } else {
-        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
-    }
+    assert(HAVE_ATOMIC128);
+
+    mem_idx = cpu_mmu_index(env, false);
+    oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
+    v = helper_atomic_ldo_be_mmu(env, addr, oi, ra);
+    hi = int128_gethi(v);
+    lo = int128_getlo(v);
 
     env->retxl = lo;
     return hi;
@@ -XXX,XX +XXX,XX @@ void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr,
                            uint64_t low, uint64_t high)
 {
     uintptr_t ra = GETPC();
+    int mem_idx;
+    TCGMemOpIdx oi;
+    Int128 v;
 
-    if (HAVE_ATOMIC128) {
-        int mem_idx = cpu_mmu_index(env, false);
-        TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
-        Int128 v = int128_make128(low, high);
-        helper_atomic_sto_be_mmu(env, addr, v, oi, ra);
-    } else {
-        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
-    }
+    assert(HAVE_ATOMIC128);
+
+    mem_idx = cpu_mmu_index(env, false);
+    oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
+    v = int128_make128(low, high);
+    helper_atomic_sto_be_mmu(env, addr, v, oi, ra);
 }
 
 /* Execute instruction.  This instruction executes an insn modified with
diff --git a/target/s390x/translate.c b/target/s390x/translate.c
index XXXXXXX..XXXXXXX 100644
--- a/target/s390x/translate.c
+++ b/target/s390x/translate.c
@@ -XXX,XX +XXX,XX @@
 #include "trace-tcg.h"
 #include "exec/translator.h"
 #include "exec/log.h"
+#include "qemu/atomic128.h"
 
 
 /* Information that (most) every instruction needs to manipulate.  */
@@ -XXX,XX +XXX,XX @@ static DisasJumpType op_cdsg(DisasContext *s, DisasOps *o)
     int r3 = get_field(s->fields, r3);
     int d2 = get_field(s->fields, d2);
     int b2 = get_field(s->fields, b2);
+    DisasJumpType ret = DISAS_NEXT;
     TCGv_i64 addr;
     TCGv_i32 t_r1, t_r3;
 
@@ -XXX,XX +XXX,XX @@ static DisasJumpType op_cdsg(DisasContext *s, DisasOps *o)
     addr = get_address(s, 0, b2, d2);
     t_r1 = tcg_const_i32(r1);
     t_r3 = tcg_const_i32(r3);
-    if (tb_cflags(s->base.tb) & CF_PARALLEL) {
+    if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
+        gen_helper_cdsg(cpu_env, addr, t_r1, t_r3);
+    } else if (HAVE_CMPXCHG128) {
         gen_helper_cdsg_parallel(cpu_env, addr, t_r1, t_r3);
     } else {
-        gen_helper_cdsg(cpu_env, addr, t_r1, t_r3);
+        gen_helper_exit_atomic(cpu_env);
+        ret = DISAS_NORETURN;
     }
     tcg_temp_free_i64(addr);
     tcg_temp_free_i32(t_r1);
     tcg_temp_free_i32(t_r3);
 
     set_cc_static(s);
-    return DISAS_NEXT;
+    return ret;
 }
 
 static DisasJumpType op_csst(DisasContext *s, DisasOps *o)
@@ -XXX,XX +XXX,XX @@ static DisasJumpType op_lpd(DisasContext *s, DisasOps *o)
 
 static DisasJumpType op_lpq(DisasContext *s, DisasOps *o)
 {
-    if (tb_cflags(s->base.tb) & CF_PARALLEL) {
+    if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
+        gen_helper_lpq(o->out, cpu_env, o->in2);
+    } else if (HAVE_ATOMIC128) {
         gen_helper_lpq_parallel(o->out, cpu_env, o->in2);
     } else {
-        gen_helper_lpq(o->out, cpu_env, o->in2);
+        gen_helper_exit_atomic(cpu_env);
+        return DISAS_NORETURN;
     }
     return_low128(o->out2);
     return DISAS_NEXT;
@@ -XXX,XX +XXX,XX @@ static DisasJumpType op_stmh(DisasContext *s, DisasOps *o)
 
 static DisasJumpType op_stpq(DisasContext *s, DisasOps *o)
 {
-    if (tb_cflags(s->base.tb) & CF_PARALLEL) {
+    if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
+        gen_helper_stpq(cpu_env, o->in2, o->out2, o->out);
+    } else if (HAVE_ATOMIC128) {
         gen_helper_stpq_parallel(cpu_env, o->in2, o->out2, o->out);
     } else {
-        gen_helper_stpq(cpu_env, o->in2, o->out2, o->out);
+        gen_helper_exit_atomic(cpu_env);
+        return DISAS_NORETURN;
     }
     return DISAS_NEXT;
 }
-- 
2.17.2

From: "Emilio G. Cota" <cota@braap.org>

Updates can come from other threads, so readers that do not
take tlb_lock must use atomic_read to avoid undefined
behaviour (UB).

This completes the conversion to tlb_lock. This conversion results
on average in no performance loss, as the following experiments
(run on an Intel i7-6700K CPU @ 4.00GHz) show.

1. aarch64 bootup+shutdown test:

- Before:
 Performance counter stats for 'taskset -c 0 ../img/aarch64/die.sh' (10 runs):

7487.087786      task-clock (msec)         #    0.998 CPUs utilized            ( +-  0.12% )
    31,574,905,303      cycles                    #    4.217 GHz                      ( +-  0.12% )
    57,097,908,812      instructions              #    1.81  insns per cycle          ( +-  0.08% )
    10,255,415,367      branches                  # 1369.747 M/sec                    ( +-  0.08% )
       173,278,962      branch-misses             #    1.69% of all branches          ( +-  0.18% )

7.504481349 seconds time elapsed                                          ( +-  0.14% )

- After:
 Performance counter stats for 'taskset -c 0 ../img/aarch64/die.sh' (10 runs):

7462.441328      task-clock (msec)         #    0.998 CPUs utilized            ( +-  0.07% )
    31,478,476,520      cycles                    #    4.218 GHz                      ( +-  0.07% )
    57,017,330,084      instructions              #    1.81  insns per cycle          ( +-  0.05% )
    10,251,929,667      branches                  # 1373.804 M/sec                    ( +-  0.05% )
       173,023,787      branch-misses             #    1.69% of all branches          ( +-  0.11% )

7.474970463 seconds time elapsed                                          ( +-  0.07% )

2. SPEC06int:
                                              SPEC06int (test set)
                                           [Y axis: Speedup over master]
  1.15 +-+----+------+------+------+------+------+-------+------+------+------+------+------+------+----+-+
       |                                                                                                  |
   1.1 +-+.................................+++.............................+  tlb-lock-v2 (m+++x)       +-+
       |                                +++ |                   +++        tlb-lock-v3 (spinl|ck)         |
       |                    +++          |  |     +++    +++     |                           |            |
  1.05 +-+....+++...........####.........|####.+++.|......|.....###....+++...........+++....###.........+-+
       |      ###         ++#| #         |# |# ***### +++### +++#+#     |     +++     |     #|#    ###    |
     1 +-+++***+#++++####+++#++#++++++++++#++#+*+*++#++++#+#+****+#++++###++++###++++###++++#+#++++#+#+++-+
       |    *+* #    #++# ***  #   #### ***  # * *++# ****+# *| * # ****|#   |# #    #|#    #+#    # #    |
  0.95 +-+..*.*.#....#..#.*|*..#...#..#.*|*..#.*.*..#.*|.*.#.*++*.#.*++*+#.****.#....#+#....#.#..++#.#..+-+
       |    * * #    #  # *|*  #   #  # *|*  # * *  # *++* # *  * # *  * # * |* #  ++# #    # #  *** #    |
       |    * * #  ++#  # *+*  #   #  # *|*  # * *  # *  * # *  * # *  * # *++* # **** #  ++# #  * * #    |
   0.9 +-+..*.*.#...|#..#.*.*..#.++#..#.*|*..#.*.*..#.*..*.#.*..*.#.*..*.#.*..*.#.*.|*.#...|#.#..*.*.#..+-+
       |    * * #  ***  # * *  #  |#  # *+*  # * *  # *  * # *  * # *  * # *  * # *++* #   |# #  * * #    |
  0.85 +-+..*.*.#..*|*..#.*.*..#.***..#.*.*..#.*.*..#.*..*.#.*..*.#.*..*.#.*..*.#.*..*.#.****.#..*.*.#..+-+
       |    * * #  *+*  # * *  # *|*  # * *  # * *  # *  * # *  * # *  * # *  * # *  * # * |* #  * * #    |
       |    * * #  * *  # * *  # *+*  # * *  # * *  # *  * # *  * # *  * # *  * # *  * # * |* #  * * #    |
   0.8 +-+..*.*.#..*.*..#.*.*..#.*.*..#.*.*..#.*.*..#.*..*.#.*..*.#.*..*.#.*..*.#.*..*.#.*++*.#..*.*.#..+-+
       |    * * #  * *  # * *  # * *  # * *  # * *  # *  * # *  * # *  * # *  * # *  * # *  * #  * * #    |
  0.75 +-+--***##--***###-***###-***###-***###-***###-****##-****##-****##-****##-****##-****##--***##--+-+
 400.perlben401.bzip2403.gcc429.m445.gob456.hmme45462.libqua464.h26471.omnet473483.xalancbmkgeomean

png: https://imgur.com/a/BHzpPTW

Notes:
- tlb-lock-v2 corresponds to an implementation with a mutex.
- tlb-lock-v3 corresponds to the current implementation, i.e.
  a spinlock and a single lock acquisition in tlb_set_page_with_attrs.

Signed-off-by: Emilio G. Cota <cota@braap.org>
Message-Id: <20181016153840.25877-1-cota@braap.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 accel/tcg/softmmu_template.h     | 12 ++++++------
 include/exec/cpu_ldst.h          | 11 ++++++++++-
 include/exec/cpu_ldst_template.h |  2 +-
 accel/tcg/cputlb.c               | 19 +++++++++++++------
 4 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/accel/tcg/softmmu_template.h b/accel/tcg/softmmu_template.h
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/softmmu_template.h
+++ b/accel/tcg/softmmu_template.h
@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
     uintptr_t mmu_idx = get_mmuidx(oi);
     uintptr_t index = tlb_index(env, mmu_idx, addr);
     CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
-    target_ulong tlb_addr = entry->addr_write;
+    target_ulong tlb_addr = tlb_addr_write(entry);
     unsigned a_bits = get_alignment_bits(get_memop(oi));
     uintptr_t haddr;
 
@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE,
                      mmu_idx, retaddr);
         }
-        tlb_addr = entry->addr_write & ~TLB_INVALID_MASK;
+        tlb_addr = tlb_addr_write(entry) & ~TLB_INVALID_MASK;
     }
 
     /* Handle an IO access.  */
@@ -XXX,XX +XXX,XX @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
            cannot evict the first.  */
         page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
         entry2 = tlb_entry(env, mmu_idx, page2);
-        if (!tlb_hit_page(entry2->addr_write, page2)
+        if (!tlb_hit_page(tlb_addr_write(entry2), page2)
             && !VICTIM_TLB_HIT(addr_write, page2)) {
             tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE,
                      mmu_idx, retaddr);
@@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
     uintptr_t mmu_idx = get_mmuidx(oi);
     uintptr_t index = tlb_index(env, mmu_idx, addr);
     CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
-    target_ulong tlb_addr = entry->addr_write;
+    target_ulong tlb_addr = tlb_addr_write(entry);
     unsigned a_bits = get_alignment_bits(get_memop(oi));
     uintptr_t haddr;
 
@@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
             tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE,
                      mmu_idx, retaddr);
         }
-        tlb_addr = entry->addr_write & ~TLB_INVALID_MASK;
+        tlb_addr = tlb_addr_write(entry) & ~TLB_INVALID_MASK;
     }
 
     /* Handle an IO access.  */
@@ -XXX,XX +XXX,XX @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
            cannot evict the first.  */
         page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
         entry2 = tlb_entry(env, mmu_idx, page2);
-        if (!tlb_hit_page(entry2->addr_write, page2)
+        if (!tlb_hit_page(tlb_addr_write(entry2), page2)
             && !VICTIM_TLB_HIT(addr_write, page2)) {
             tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE,
                      mmu_idx, retaddr);
diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h
index XXXXXXX..XXXXXXX 100644
--- a/include/exec/cpu_ldst.h
+++ b/include/exec/cpu_ldst.h
@@ -XXX,XX +XXX,XX @@ extern __thread uintptr_t helper_retaddr;
 /* The memory helpers for tcg-generated code need tcg_target_long etc.  */
 #include "tcg.h"
 
+static inline target_ulong tlb_addr_write(const CPUTLBEntry *entry)
+{
+#if TCG_OVERSIZED_GUEST
+    return entry->addr_write;
+#else
+    return atomic_read(&entry->addr_write);
+#endif
+}
+
 /* Find the TLB index corresponding to the mmu_idx + address pair.  */
 static inline uintptr_t tlb_index(CPUArchState *env, uintptr_t mmu_idx,
                                   target_ulong addr)
@@ -XXX,XX +XXX,XX @@ static inline void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr,
         tlb_addr = tlbentry->addr_read;
         break;
     case 1:
-        tlb_addr = tlbentry->addr_write;
+        tlb_addr = tlb_addr_write(tlbentry);
         break;
     case 2:
         tlb_addr = tlbentry->addr_code;
diff --git a/include/exec/cpu_ldst_template.h b/include/exec/cpu_ldst_template.h
index XXXXXXX..XXXXXXX 100644
--- a/include/exec/cpu_ldst_template.h
+++ b/include/exec/cpu_ldst_template.h
@@ -XXX,XX +XXX,XX @@ glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
     addr = ptr;
     mmu_idx = CPU_MMU_INDEX;
     entry = tlb_entry(env, mmu_idx, addr);
-    if (unlikely(entry->addr_write !=
+    if (unlikely(tlb_addr_write(entry) !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
         oi = make_memop_idx(SHIFT, mmu_idx);
         glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(env, addr, v, oi,
diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
index XXXXXXX..XXXXXXX 100644
--- a/accel/tcg/cputlb.c
+++ b/accel/tcg/cputlb.c
@@ -XXX,XX +XXX,XX @@ static inline bool tlb_hit_page_anyprot(CPUTLBEntry *tlb_entry,
                                         target_ulong page)
 {
     return tlb_hit_page(tlb_entry->addr_read, page) ||
-           tlb_hit_page(tlb_entry->addr_write, page) ||
+           tlb_hit_page(tlb_addr_write(tlb_entry), page) ||
            tlb_hit_page(tlb_entry->addr_code, page);
 }
 
@@ -XXX,XX +XXX,XX @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry,
         tlb_fill(cpu, addr, size, MMU_DATA_STORE, mmu_idx, retaddr);
 
         entry = tlb_entry(env, mmu_idx, addr);
-        tlb_addr = entry->addr_write;
+        tlb_addr = tlb_addr_write(entry);
         if (!(tlb_addr & ~(TARGET_PAGE_MASK | TLB_RECHECK))) {
             /* RAM access */
             uintptr_t haddr = addr + entry->addend;
@@ -XXX,XX +XXX,XX @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
     assert_cpu_is_self(ENV_GET_CPU(env));
     for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) {
         CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx];
-        target_ulong cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs);
+        target_ulong cmp;
+
+        /* elt_ofs might correspond to .addr_write, so use atomic_read */
+#if TCG_OVERSIZED_GUEST
+        cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs);
+#else
+        cmp = atomic_read((target_ulong *)((uintptr_t)vtlb + elt_ofs));
+#endif
 
         if (cmp == page) {
             /* Found entry in victim tlb, swap tlb and iotlb.  */
@@ -XXX,XX +XXX,XX @@ void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx,
     uintptr_t index = tlb_index(env, mmu_idx, addr);
     CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
 
-    if (!tlb_hit(entry->addr_write, addr)) {
+    if (!tlb_hit(tlb_addr_write(entry), addr)) {
         /* TLB entry is for a different page */
         if (!VICTIM_TLB_HIT(addr_write, addr)) {
             tlb_fill(ENV_GET_CPU(env), addr, size, MMU_DATA_STORE,
@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
     size_t mmu_idx = get_mmuidx(oi);
     uintptr_t index = tlb_index(env, mmu_idx, addr);
     CPUTLBEntry *tlbe = tlb_entry(env, mmu_idx, addr);
-    target_ulong tlb_addr = tlbe->addr_write;
+    target_ulong tlb_addr = tlb_addr_write(tlbe);
     TCGMemOp mop = get_memop(oi);
     int a_bits = get_alignment_bits(mop);
     int s_bits = mop & MO_SIZE;
@@ -XXX,XX +XXX,XX @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
             tlb_fill(ENV_GET_CPU(env), addr, 1 << s_bits, MMU_DATA_STORE,
                      mmu_idx, retaddr);
         }
-        tlb_addr = tlbe->addr_write & ~TLB_INVALID_MASK;
+        tlb_addr = tlb_addr_write(tlbe) & ~TLB_INVALID_MASK;
     }
 
     /* Notice an IO access or a needs-MMU-lookup access */
-- 
2.17.2

Merge the first set of reviewed patches from my queue.

The following changes since commit 6dd06214892d71cbbdd25daed7693e58afcb1093:

Merge tag 'pull-hex-20230421' of https://github.com/quic/qemu into staging (2023-04-22 08:31:38 +0100)

are available in the Git repository at:

https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20230423

for you to fetch changes up to 3ea9be33400f14305565a9a094cb6031c07183d5:

tcg/riscv: Conditionalize tcg_out_exts_i32_i64 (2023-04-23 08:46:45 +0100)

----------------------------------------------------------------
tcg cleanups:
  - Remove tcg_abort()
  - Split out extensions as known backend interfaces
  - Put the separate extensions together as tcg_out_movext
  - Introduce tcg_out_xchg as a backend interface
  - Clear TCGLabelQemuLdst on allocation
  - Avoid redundant extensions for riscv

----------------------------------------------------------------
Richard Henderson (15):
      tcg: Replace if + tcg_abort with tcg_debug_assert
      tcg: Replace tcg_abort with g_assert_not_reached
      tcg: Split out tcg_out_ext8s
      tcg: Split out tcg_out_ext8u
      tcg: Split out tcg_out_ext16s
      tcg: Split out tcg_out_ext16u
      tcg: Split out tcg_out_ext32s
      tcg: Split out tcg_out_ext32u
      tcg: Split out tcg_out_exts_i32_i64
      tcg: Split out tcg_out_extu_i32_i64
      tcg: Split out tcg_out_extrl_i64_i32
      tcg: Introduce tcg_out_movext
      tcg: Introduce tcg_out_xchg
      tcg: Clear TCGLabelQemuLdst on allocation
      tcg/riscv: Conditionalize tcg_out_exts_i32_i64

include/tcg/tcg.h                |   6 --
 target/i386/tcg/translate.c      |  20 +++---
 target/s390x/tcg/translate.c     |   4 +-
 tcg/optimize.c                   |  10 ++-
 tcg/tcg.c                        | 135 +++++++++++++++++++++++++++++++++++----
 tcg/aarch64/tcg-target.c.inc     | 106 +++++++++++++++++++-----------
 tcg/arm/tcg-target.c.inc         |  93 +++++++++++++++++----------
 tcg/i386/tcg-target.c.inc        | 129 ++++++++++++++++++-------------------
 tcg/loongarch64/tcg-target.c.inc | 123 +++++++++++++----------------------
 tcg/mips/tcg-target.c.inc        |  94 +++++++++++++++++++--------
 tcg/ppc/tcg-target.c.inc         | 119 ++++++++++++++++++----------------
 tcg/riscv/tcg-target.c.inc       |  83 +++++++++++-------------
 tcg/s390x/tcg-target.c.inc       | 128 +++++++++++++++++--------------------
 tcg/sparc64/tcg-target.c.inc     | 117 +++++++++++++++++++++------------
 tcg/tcg-ldst.c.inc               |   1 +
 tcg/tci/tcg-target.c.inc         | 116 ++++++++++++++++++++++++++++++---
 16 files changed, 786 insertions(+), 498 deletions(-)

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/tcg.c                 | 4 +---
 tcg/i386/tcg-target.c.inc | 8 +++-----
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
 {
     TCGTemp *ts;
 
-    if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
-        tcg_abort();
-    }
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
 
     ts = tcg_global_alloc(s);
     ts->base_type = type;
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
     }
 }
 
-/* Use SMALL != 0 to force a short forward branch.  */
-static void tcg_out_jxx(TCGContext *s, int opc, TCGLabel *l, int small)
+/* Set SMALL to force a short forward branch.  */
+static void tcg_out_jxx(TCGContext *s, int opc, TCGLabel *l, bool small)
 {
     int32_t val, val1;
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_jxx(TCGContext *s, int opc, TCGLabel *l, int small)
             }
             tcg_out8(s, val1);
         } else {
-            if (small) {
-                tcg_abort();
-            }
+            tcg_debug_assert(!small);
             if (opc == -1) {
                 tcg_out8(s, OPC_JMP_long);
                 tcg_out32(s, val - 5);
-- 
2.34.1

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 include/tcg/tcg.h            |  6 ------
 target/i386/tcg/translate.c  | 20 ++++++++++----------
 target/s390x/tcg/translate.c |  4 ++--
 tcg/optimize.c               | 10 ++++------
 tcg/tcg.c                    |  8 ++++----
 tcg/aarch64/tcg-target.c.inc |  4 ++--
 tcg/arm/tcg-target.c.inc     |  2 +-
 tcg/i386/tcg-target.c.inc    | 14 +++++++-------
 tcg/mips/tcg-target.c.inc    | 14 +++++++-------
 tcg/ppc/tcg-target.c.inc     |  8 ++++----
 tcg/s390x/tcg-target.c.inc   |  8 ++++----
 tcg/sparc64/tcg-target.c.inc |  2 +-
 tcg/tci/tcg-target.c.inc     |  2 +-
 13 files changed, 47 insertions(+), 55 deletions(-)

diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h
index XXXXXXX..XXXXXXX 100644
--- a/include/tcg/tcg.h
+++ b/include/tcg/tcg.h
@@ -XXX,XX +XXX,XX @@ typedef struct TCGTargetOpDef {
     const char *args_ct_str[TCG_MAX_OP_ARGS];
 } TCGTargetOpDef;
 
-#define tcg_abort() \
-do {\
-    fprintf(stderr, "%s:%d: tcg fatal error\n", __FILE__, __LINE__);\
-    abort();\
-} while (0)
-
 bool tcg_op_supported(TCGOpcode op);
 
 void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args);
diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c
index XXXXXXX..XXXXXXX 100644
--- a/target/i386/tcg/translate.c
+++ b/target/i386/tcg/translate.c
@@ -XXX,XX +XXX,XX @@ static TCGv gen_op_deposit_reg_v(DisasContext *s, MemOp ot, int reg, TCGv dest,
         break;
 #endif
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
     return cpu_regs[reg];
 }
@@ -XXX,XX +XXX,XX @@ static void gen_lea_v_seg(DisasContext *s, MemOp aflag, TCGv a0,
         }
         break;
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 
     if (ovr_seg >= 0) {
@@ -XXX,XX +XXX,XX @@ static void gen_helper_in_func(MemOp ot, TCGv v, TCGv_i32 n)
         gen_helper_inl(v, cpu_env, n);
         break;
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static void gen_helper_out_func(MemOp ot, TCGv_i32 v, TCGv_i32 n)
         gen_helper_outl(cpu_env, v, n);
         break;
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static void gen_rotc_rm_T1(DisasContext *s, MemOp ot, int op1,
             break;
 #endif
         default:
-            tcg_abort();
+            g_assert_not_reached();
         }
     } else {
         switch (ot) {
@@ -XXX,XX +XXX,XX @@ static void gen_rotc_rm_T1(DisasContext *s, MemOp ot, int op1,
             break;
 #endif
         default:
-            tcg_abort();
+            g_assert_not_reached();
         }
     }
     /* store */
@@ -XXX,XX +XXX,XX @@ static AddressParts gen_lea_modrm_0(CPUX86State *env, DisasContext *s,
         break;
 
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 
  done:
@@ -XXX,XX +XXX,XX @@ static inline uint32_t insn_get(CPUX86State *env, DisasContext *s, MemOp ot)
         ret = x86_ldl_code(env, s);
         break;
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
     return ret;
 }
@@ -XXX,XX +XXX,XX @@ static bool disas_insn(DisasContext *s, CPUState *cpu)
             gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0);
             break;
         default:
-            tcg_abort();
+            g_assert_not_reached();
         }
         break;
     case 0x99: /* CDQ/CWD */
@@ -XXX,XX +XXX,XX @@ static bool disas_insn(DisasContext *s, CPUState *cpu)
             gen_op_mov_reg_v(s, MO_16, R_EDX, s->T0);
             break;
         default:
-            tcg_abort();
+            g_assert_not_reached();
         }
         break;
     case 0x1af: /* imul Gv, Ev */
diff --git a/target/s390x/tcg/translate.c b/target/s390x/tcg/translate.c
index XXXXXXX..XXXXXXX 100644
--- a/target/s390x/tcg/translate.c
+++ b/target/s390x/tcg/translate.c
@@ -XXX,XX +XXX,XX @@ static int get_mem_index(DisasContext *s)
     case PSW_ASC_HOME >> FLAG_MASK_PSW_SHIFT:
         return MMU_HOME_IDX;
     default:
-        tcg_abort();
+        g_assert_not_reached();
         break;
     }
 #endif
@@ -XXX,XX +XXX,XX @@ static void gen_op_calc_cc(DisasContext *s)
         gen_helper_calc_cc(cc_op, cpu_env, cc_op, cc_src, cc_dst, cc_vr);
         break;
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 
     /* We now have cc in cc_op as constant */
diff --git a/tcg/optimize.c b/tcg/optimize.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/optimize.c
+++ b/tcg/optimize.c
@@ -XXX,XX +XXX,XX @@ static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y)
         return (uint64_t)x % ((uint64_t)y ? : 1);
 
     default:
-        fprintf(stderr,
-                "Unrecognized operation %d in do_constant_folding.\n", op);
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static bool do_constant_folding_cond_32(uint32_t x, uint32_t y, TCGCond c)
     case TCG_COND_GTU:
         return x > y;
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static bool do_constant_folding_cond_64(uint64_t x, uint64_t y, TCGCond c)
     case TCG_COND_GTU:
         return x > y;
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static bool do_constant_folding_cond_eq(TCGCond c)
     case TCG_COND_EQ:
         return 1;
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
 
         case TEMP_VAL_DEAD:
         default:
-            tcg_abort();
+            g_assert_not_reached();
         }
         ts->mem_coherent = 1;
     }
@@ -XXX,XX +XXX,XX @@ static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
         }
     }
 
-    tcg_abort();
+    g_assert_not_reached();
 }
 
 static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs,
@@ -XXX,XX +XXX,XX @@ static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs,
             }
         }
     }
-    tcg_abort();
+    g_assert_not_reached();
 }
 
 /* Make sure the temporary is in a register.  If needed, allocate the register
@@ -XXX,XX +XXX,XX @@ static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
         break;
     case TEMP_VAL_DEAD:
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
     set_temp_val_reg(s, ts, reg);
 }
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp memop, TCGType ext,
         tcg_out_ldst_r(s, I3312_LDRX, data_r, addr_r, otype, off_r);
         break;
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp memop,
         tcg_out_ldst_r(s, I3312_STRX, data_r, addr_r, otype, off_r);
         break;
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
         tcg_patch8(code_ptr, value);
         break;
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
     return true;
 }
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_pushi(TCGContext *s, tcg_target_long val)
         tcg_out_opc(s, OPC_PUSH_Iv, 0, 0, 0);
         tcg_out32(s, val);
     } else {
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static void tgen_arithi(TCGContext *s, int c, int r0,
         return;
     }
 
-    tcg_abort();
+    g_assert_not_reached();
 }
 
 static void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
@@ -XXX,XX +XXX,XX @@ static void tcg_out_brcond2(TCGContext *s, const TCGArg *args,
                          label_this, small);
         break;
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
     tcg_out_label(s, label_next);
 }
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
         }
         break;
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 
     /* Jump to the code corresponding to next IR of qemu_st */
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
             /* load bits 0..15 */
             tcg_out_modrm(s, OPC_MOVL_EvGv | P_DATA16, a2, a0);
         } else {
-            tcg_abort();
+            g_assert_not_reached();
         }
         break;
 
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 
 #undef OP_32_64
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret,
         break;
 
      default:
-         tcg_abort();
+         g_assert_not_reached();
          break;
      }
 }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1,
         break;
 
     default:
-        tcg_abort();
+        g_assert_not_reached();
         break;
     }
 
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
         }
         break;
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
     i = tcg_out_call_iarg_imm(s, i, oi);
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi,
         }
         break;
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi,
         break;
 
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_unalign(TCGContext *s, TCGReg lo, TCGReg hi,
         break;
 
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2,
         break;
 
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
     op |= BF(cr) | ((type == TCG_TYPE_I64) << 21);
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond,
         break;
 
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_cmp2(TCGContext *s, const TCGArg *args,
         break;
 
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_exit_tb:   /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:   /* Always emitted via tcg_out_goto_tb.  */
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data,
         break;
 
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg data,
         break;
 
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
         tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
         break;
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
     tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R5, oi);
     tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R6, (uintptr_t)lb->raddr);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
 }
 
-- 
2.34.1

We will need a backend interface for performing 8-bit sign-extend.
Use it in tcg_reg_alloc_op in the meantime.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/tcg.c                        | 21 ++++++++++++++++-----
 tcg/aarch64/tcg-target.c.inc     | 11 +++++++----
 tcg/arm/tcg-target.c.inc         | 10 ++++------
 tcg/i386/tcg-target.c.inc        | 10 +++++-----
 tcg/loongarch64/tcg-target.c.inc | 11 ++++-------
 tcg/mips/tcg-target.c.inc        | 12 ++++++++----
 tcg/ppc/tcg-target.c.inc         | 10 ++++------
 tcg/riscv/tcg-target.c.inc       |  9 +++------
 tcg/s390x/tcg-target.c.inc       | 10 +++-------
 tcg/sparc64/tcg-target.c.inc     |  7 +++++++
 tcg/tci/tcg-target.c.inc         | 21 ++++++++++++++++++++-
 11 files changed, 81 insertions(+), 51 deletions(-)

diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
 static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
 static void tcg_out_movi(TCGContext *s, TCGType type,
                          TCGReg ret, tcg_target_long arg);
+static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
 static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
 static void tcg_out_goto_tb(TCGContext *s, int which);
@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
     }
 
     /* emit instruction */
-    if (def->flags & TCG_OPF_VECTOR) {
-        tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
-                       new_args, const_args);
-    } else {
-        tcg_out_op(s, op->opc, new_args, const_args);
+    switch (op->opc) {
+    case INDEX_op_ext8s_i32:
+        tcg_out_ext8s(s, TCG_TYPE_I32, new_args[0], new_args[1]);
+        break;
+    case INDEX_op_ext8s_i64:
+        tcg_out_ext8s(s, TCG_TYPE_I64, new_args[0], new_args[1]);
+        break;
+    default:
+        if (def->flags & TCG_OPF_VECTOR) {
+            tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
+                           new_args, const_args);
+        } else {
+            tcg_out_op(s, op->opc, new_args, const_args);
+        }
+        break;
     }
 
     /* move the outputs in the correct register if needed */
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_sxt(TCGContext *s, TCGType ext, MemOp s_bits,
     tcg_out_sbfm(s, ext, rd, rn, 0, bits);
 }
 
+static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rn)
+{
+    tcg_out_sxt(s, type, MO_8, rd, rn);
+}
+
 static inline void tcg_out_uxt(TCGContext *s, MemOp s_bits,
                                TCGReg rd, TCGReg rn)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         }
         break;
 
-    case INDEX_op_ext8s_i64:
-    case INDEX_op_ext8s_i32:
-        tcg_out_sxt(s, ext, MO_8, a0, a1);
-        break;
     case INDEX_op_ext16s_i64:
     case INDEX_op_ext16s_i32:
         tcg_out_sxt(s, ext, MO_16, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
+    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
+    case INDEX_op_ext8s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_udiv(TCGContext *s, ARMCond cond,
     tcg_out32(s, 0x0730f010 | (cond << 28) | (rd << 16) | rn | (rm << 8));
 }
 
-static void tcg_out_ext8s(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn)
+static void tcg_out_ext8s(TCGContext *s, TCGType t, TCGReg rd, TCGReg rn)
 {
     /* sxtb */
-    tcg_out32(s, 0x06af0070 | (cond << 28) | (rd << 12) | rn);
+    tcg_out32(s, 0x06af0070 | (COND_AL << 28) | (rd << 12) | rn);
 }
 
 static void __attribute__((unused))
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
     datahi = lb->datahi_reg;
     switch (opc & MO_SSIZE) {
     case MO_SB:
-        tcg_out_ext8s(s, COND_AL, datalo, TCG_REG_R0);
+        tcg_out_ext8s(s, TCG_TYPE_I32, datalo, TCG_REG_R0);
         break;
     case MO_SW:
         tcg_out_ext16s(s, COND_AL, datalo, TCG_REG_R0);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_bswap32(s, COND_AL, args[0], args[1]);
         break;
 
-    case INDEX_op_ext8s_i32:
-        tcg_out_ext8s(s, COND_AL, args[0], args[1]);
-        break;
     case INDEX_op_ext16s_i32:
         tcg_out_ext16s(s, COND_AL, args[0], args[1]);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
+    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_ext8u(TCGContext *s, int dest, int src)
     tcg_out_modrm(s, OPC_MOVZBL + P_REXB_RM, dest, src);
 }
 
-static void tcg_out_ext8s(TCGContext *s, int dest, int src, int rexw)
+static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
 {
+    int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
     /* movsbl */
     tcg_debug_assert(src < 4 || TCG_TARGET_REG_BITS == 64);
     tcg_out_modrm(s, OPC_MOVSBL + P_REXB_RM + rexw, dest, src);
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
     data_reg = l->datalo_reg;
     switch (opc & MO_SSIZE) {
     case MO_SB:
-        tcg_out_ext8s(s, data_reg, TCG_REG_EAX, rexw);
+        tcg_out_ext8s(s, l->type, data_reg, TCG_REG_EAX);
         break;
     case MO_SW:
         tcg_out_ext16s(s, data_reg, TCG_REG_EAX, rexw);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, a0);
         break;
 
-    OP_32_64(ext8s):
-        tcg_out_ext8s(s, a0, a1, rexw);
-        break;
     OP_32_64(ext16s):
         tcg_out_ext16s(s, a0, a1, rexw);
         break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
+    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
+    case INDEX_op_ext8s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.c.inc
+++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg)
     tcg_out_opc_bstrpick_d(s, ret, arg, 0, 31);
 }
 
-static void tcg_out_ext8s(TCGContext *s, TCGReg ret, TCGReg arg)
+static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
 {
     tcg_out_opc_sext_b(s, ret, arg);
 }
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
 
     switch (opc & MO_SSIZE) {
     case MO_SB:
-        tcg_out_ext8s(s, l->datalo_reg, TCG_REG_A0);
+        tcg_out_ext8s(s, type, l->datalo_reg, TCG_REG_A0);
         break;
     case MO_SW:
         tcg_out_ext16s(s, l->datalo_reg, TCG_REG_A0);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_brcond(s, a2, a0, a1, arg_label(args[3]));
         break;
 
-    case INDEX_op_ext8s_i32:
-    case INDEX_op_ext8s_i64:
-        tcg_out_ext8s(s, a0, a1);
-        break;
-
     case INDEX_op_ext8u_i32:
     case INDEX_op_ext8u_i64:
         tcg_out_ext8u(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
+    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
+    case INDEX_op_ext8s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type,
     }
 }
 
+static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
+{
+    tcg_debug_assert(TCG_TARGET_HAS_ext8s_i32);
+    tcg_out_opc_reg(s, OPC_SEB, rd, TCG_REG_ZERO, rs);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_not_i64:
         i1 = OPC_NOR;
         goto do_unary;
-    case INDEX_op_ext8s_i32:
-    case INDEX_op_ext8s_i64:
-        i1 = OPC_SEB;
-        goto do_unary;
     case INDEX_op_ext16s_i32:
     case INDEX_op_ext16s_i64:
         i1 = OPC_SEH;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
+    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
+    case INDEX_op_ext8s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_rlw(TCGContext *s, int op, TCGReg ra, TCGReg rs,
     tcg_out32(s, op | RA(ra) | RS(rs) | SH(sh) | MB(mb) | ME(me));
 }
 
-static inline void tcg_out_ext8s(TCGContext *s, TCGReg dst, TCGReg src)
+static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
 {
     tcg_out32(s, EXTSB | RA(dst) | RS(src));
 }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ld8s_i32:
     case INDEX_op_ld8s_i64:
         tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]);
-        tcg_out_ext8s(s, args[0], args[0]);
+        tcg_out_ext8s(s, TCG_TYPE_REG, args[0], args[0]);
         break;
     case INDEX_op_ld16u_i32:
     case INDEX_op_ld16u_i64:
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_qemu_st(s, args, true);
         break;
 
-    case INDEX_op_ext8s_i32:
-    case INDEX_op_ext8s_i64:
-        tcg_out_ext8s(s, args[0], args[1]);
-        break;
     case INDEX_op_ext16s_i32:
     case INDEX_op_ext16s_i64:
         tcg_out_ext16s(s, args[0], args[1]);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_call:      /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:   /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:   /* Always emitted via tcg_out_goto_tb.  */
+    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
+    case INDEX_op_ext8s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg)
     tcg_out_opc_imm(s, OPC_SRLI, ret, ret, 32);
 }
 
-static void tcg_out_ext8s(TCGContext *s, TCGReg ret, TCGReg arg)
+static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
 {
     tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 24);
     tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 24);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_ext32u(s, a0, a1);
         break;
 
-    case INDEX_op_ext8s_i32:
-    case INDEX_op_ext8s_i64:
-        tcg_out_ext8s(s, a0, a1);
-        break;
-
     case INDEX_op_ext16s_i32:
     case INDEX_op_ext16s_i64:
         tcg_out_ext16s(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
+    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
+    case INDEX_op_ext8s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_risbg(TCGContext *s, TCGReg dest, TCGReg src,
     tcg_out16(s, (ofs << 8) | (RIEf_RISBG & 0xff));
 }
 
-static void tgen_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
+static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
 {
     tcg_out_insn(s, RRE, LGBR, dest, src);
 }
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         }
         break;
 
-    case INDEX_op_ext8s_i32:
-        tgen_ext8s(s, TCG_TYPE_I32, args[0], args[1]);
-        break;
     case INDEX_op_ext16s_i32:
         tgen_ext16s(s, TCG_TYPE_I32, args[0], args[1]);
         break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         }
         break;
 
-    case INDEX_op_ext8s_i64:
-        tgen_ext8s(s, TCG_TYPE_I64, args[0], args[1]);
-        break;
     case INDEX_op_ext16s_i64:
         tgen_ext16s(s, TCG_TYPE_I64, args[0], args[1]);
         break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
+    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
+    case INDEX_op_ext8s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type,
     tcg_out_movi_int(s, type, ret, arg, false, TCG_REG_T2);
 }
 
+static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
+{
+    g_assert_not_reached();
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
+    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
+    case INDEX_op_ext8s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type,
     }
 }
 
+static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
+{
+    switch (type) {
+    case TCG_TYPE_I32:
+        tcg_debug_assert(TCG_TARGET_HAS_ext8s_i32);
+        tcg_out_op_rr(s, INDEX_op_ext8s_i32, rd, rs);
+        break;
+#if TCG_TARGET_REG_BITS == 64
+    case TCG_TYPE_I64:
+        tcg_debug_assert(TCG_TARGET_HAS_ext8s_i64);
+        tcg_out_op_rr(s, INDEX_op_ext8s_i64, rd, rs);
+        break;
+#endif
+    default:
+        g_assert_not_reached();
+    }
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     CASE_32_64(neg)      /* Optional (TCG_TARGET_HAS_neg_*). */
     CASE_32_64(not)      /* Optional (TCG_TARGET_HAS_not_*). */
-    CASE_32_64(ext8s)    /* Optional (TCG_TARGET_HAS_ext8s_*). */
     CASE_32_64(ext8u)    /* Optional (TCG_TARGET_HAS_ext8u_*). */
     CASE_32_64(ext16s)   /* Optional (TCG_TARGET_HAS_ext16s_*). */
     CASE_32_64(ext16u)   /* Optional (TCG_TARGET_HAS_ext16u_*). */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
+    case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
+    case INDEX_op_ext8s_i64:
     default:
         g_assert_not_reached();
     }
-- 
2.34.1

We will need a backend interface for performing 8-bit zero-extend.
Use it in tcg_reg_alloc_op in the meantime.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/tcg.c                        |  5 +++++
 tcg/aarch64/tcg-target.c.inc     | 11 +++++++----
 tcg/arm/tcg-target.c.inc         | 12 +++++++++---
 tcg/i386/tcg-target.c.inc        |  7 +++----
 tcg/loongarch64/tcg-target.c.inc |  7 ++-----
 tcg/mips/tcg-target.c.inc        |  9 ++++++++-
 tcg/ppc/tcg-target.c.inc         |  7 +++++++
 tcg/riscv/tcg-target.c.inc       |  7 ++-----
 tcg/s390x/tcg-target.c.inc       | 14 +++++---------
 tcg/sparc64/tcg-target.c.inc     |  9 ++++++++-
 tcg/tci/tcg-target.c.inc         | 14 +++++++++++++-
 11 files changed, 69 insertions(+), 33 deletions(-)

diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
 static void tcg_out_movi(TCGContext *s, TCGType type,
                          TCGReg ret, tcg_target_long arg);
 static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
+static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
 static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
 static void tcg_out_goto_tb(TCGContext *s, int which);
@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
     case INDEX_op_ext8s_i64:
         tcg_out_ext8s(s, TCG_TYPE_I64, new_args[0], new_args[1]);
         break;
+    case INDEX_op_ext8u_i32:
+    case INDEX_op_ext8u_i64:
+        tcg_out_ext8u(s, new_args[0], new_args[1]);
+        break;
     default:
         if (def->flags & TCG_OPF_VECTOR) {
             tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_uxt(TCGContext *s, MemOp s_bits,
     tcg_out_ubfm(s, 0, rd, rn, 0, bits);
 }
 
+static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    tcg_out_uxt(s, MO_8, rd, rn);
+}
+
 static void tcg_out_addsubi(TCGContext *s, int ext, TCGReg rd,
                             TCGReg rn, int64_t aimm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32s_i64:
         tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
         break;
-    case INDEX_op_ext8u_i64:
-    case INDEX_op_ext8u_i32:
-        tcg_out_uxt(s, MO_8, a0, a1);
-        break;
     case INDEX_op_ext16u_i64:
     case INDEX_op_ext16u_i32:
         tcg_out_uxt(s, MO_16, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
     case INDEX_op_ext8s_i64:
+    case INDEX_op_ext8u_i32:
+    case INDEX_op_ext8u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType t, TCGReg rd, TCGReg rn)
     tcg_out32(s, 0x06af0070 | (COND_AL << 28) | (rd << 12) | rn);
 }
 
+static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    tcg_out_dat_imm(s, COND_AL, ARITH_AND, rd, rn, 0xff);
+}
+
 static void __attribute__((unused))
-tcg_out_ext8u(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn)
+tcg_out_ext8u_cond(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn)
 {
     tcg_out_dat_imm(s, cond, ARITH_AND, rd, rn, 0xff);
 }
@@ -XXX,XX +XXX,XX @@ static TCGReg NAME(TCGContext *s, TCGReg argreg, ARGTYPE arg)              \
 
 DEFINE_TCG_OUT_ARG(tcg_out_arg_imm32, uint32_t, tcg_out_movi32,
     (tcg_out_movi32(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
-DEFINE_TCG_OUT_ARG(tcg_out_arg_reg8, TCGReg, tcg_out_ext8u,
-    (tcg_out_ext8u(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
+DEFINE_TCG_OUT_ARG(tcg_out_arg_reg8, TCGReg, tcg_out_ext8u_cond,
+    (tcg_out_ext8u_cond(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
 DEFINE_TCG_OUT_ARG(tcg_out_arg_reg16, TCGReg, tcg_out_ext16u,
     (tcg_out_ext16u(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
 DEFINE_TCG_OUT_ARG(tcg_out_arg_reg32, TCGReg, tcg_out_mov_reg, )
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_exit_tb:  /* Always emitted via tcg_out_exit_tb.  */
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
+    case INDEX_op_ext8u_i32:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_rolw_8(TCGContext *s, int reg)
     tcg_out_shifti(s, SHIFT_ROL + P_DATA16, reg, 8);
 }
 
-static inline void tcg_out_ext8u(TCGContext *s, int dest, int src)
+static void tcg_out_ext8u(TCGContext *s, TCGReg dest, TCGReg src)
 {
     /* movzbl */
     tcg_debug_assert(src < 4 || TCG_TARGET_REG_BITS == 64);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     OP_32_64(ext16s):
         tcg_out_ext16s(s, a0, a1, rexw);
         break;
-    OP_32_64(ext8u):
-        tcg_out_ext8u(s, a0, a1);
-        break;
     OP_32_64(ext16u):
         tcg_out_ext16u(s, a0, a1);
         break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
     case INDEX_op_ext8s_i64:
+    case INDEX_op_ext8u_i32:
+    case INDEX_op_ext8u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.c.inc
+++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_brcond(s, a2, a0, a1, arg_label(args[3]));
         break;
 
-    case INDEX_op_ext8u_i32:
-    case INDEX_op_ext8u_i64:
-        tcg_out_ext8u(s, a0, a1);
-        break;
-
     case INDEX_op_ext16s_i32:
     case INDEX_op_ext16s_i64:
         tcg_out_ext16s(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
     case INDEX_op_ext8s_i64:
+    case INDEX_op_ext8u_i32:
+    case INDEX_op_ext8u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
     tcg_out_opc_reg(s, OPC_SEB, rd, TCG_REG_ZERO, rs);
 }
 
+static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_out_opc_imm(s, OPC_ANDI, rd, rs, 0xff);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static int tcg_out_call_iarg_reg8(TCGContext *s, int i, TCGReg arg)
     if (i < ARRAY_SIZE(tcg_target_call_iarg_regs)) {
         tmp = tcg_target_call_iarg_regs[i];
     }
-    tcg_out_opc_imm(s, OPC_ANDI, tmp, arg, 0xff);
+    tcg_out_ext8u(s, tmp, arg);
     return tcg_out_call_iarg_reg(s, i, tmp);
 }
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
     case INDEX_op_ext8s_i64:
+    case INDEX_op_ext8u_i32:
+    case INDEX_op_ext8u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
     tcg_out32(s, EXTSB | RA(dst) | RS(src));
 }
 
+static void tcg_out_ext8u(TCGContext *s, TCGReg dst, TCGReg src)
+{
+    tcg_out32(s, ANDI | SAI(src, dst, 0xff));
+}
+
 static inline void tcg_out_ext16s(TCGContext *s, TCGReg dst, TCGReg src)
 {
     tcg_out32(s, EXTSH | RA(dst) | RS(src));
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_goto_tb:   /* Always emitted via tcg_out_goto_tb.  */
     case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
     case INDEX_op_ext8s_i64:
+    case INDEX_op_ext8u_i32:
+    case INDEX_op_ext8u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_qemu_st(s, args, true);
         break;
 
-    case INDEX_op_ext8u_i32:
-    case INDEX_op_ext8u_i64:
-        tcg_out_ext8u(s, a0, a1);
-        break;
-
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext16u_i64:
         tcg_out_ext16u(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
     case INDEX_op_ext8s_i64:
+    case INDEX_op_ext8u_i32:
+    case INDEX_op_ext8u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
     tcg_out_insn(s, RRE, LGBR, dest, src);
 }
 
-static void tgen_ext8u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
+static void tcg_out_ext8u(TCGContext *s, TCGReg dest, TCGReg src)
 {
     tcg_out_insn(s, RRE, LLGCR, dest, src);
 }
@@ -XXX,XX +XXX,XX @@ static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
         return;
     }
     if ((val & valid) == 0xff) {
-        tgen_ext8u(s, TCG_TYPE_I64, dest, dest);
+        tcg_out_ext8u(s, dest, dest);
         return;
     }
     if ((val & valid) == 0xffff) {
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
     }
     switch (opc & MO_SIZE) {
     case MO_UB:
-        tgen_ext8u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
+        tcg_out_ext8u(s, TCG_REG_R4, data_reg);
         break;
     case MO_UW:
         tgen_ext16u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16s_i32:
         tgen_ext16s(s, TCG_TYPE_I32, args[0], args[1]);
         break;
-    case INDEX_op_ext8u_i32:
-        tgen_ext8u(s, TCG_TYPE_I32, args[0], args[1]);
-        break;
     case INDEX_op_ext16u_i32:
         tgen_ext16u(s, TCG_TYPE_I32, args[0], args[1]);
         break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32s_i64:
         tgen_ext32s(s, args[0], args[1]);
         break;
-    case INDEX_op_ext8u_i64:
-        tgen_ext8u(s, TCG_TYPE_I64, args[0], args[1]);
-        break;
     case INDEX_op_ext16u_i64:
         tgen_ext16u(s, TCG_TYPE_I64, args[0], args[1]);
         break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
     case INDEX_op_ext8s_i64:
+    case INDEX_op_ext8u_i32:
+    case INDEX_op_ext8u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
     g_assert_not_reached();
 }
 
+static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_out_arithi(s, rd, rs, 0xff, ARITH_AND);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void emit_extend(TCGContext *s, TCGReg r, int op)
      */
     switch (op & MO_SIZE) {
     case MO_8:
-        tcg_out_arithi(s, r, r, 0xff, ARITH_AND);
+        tcg_out_ext8u(s, r, r);
         break;
     case MO_16:
         tcg_out_arithi(s, r, r, 16, SHIFT_SLL);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
     case INDEX_op_ext8s_i64:
+    case INDEX_op_ext8u_i32:
+    case INDEX_op_ext8u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
     }
 }
 
+static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    if (TCG_TARGET_REG_BITS == 64) {
+        tcg_debug_assert(TCG_TARGET_HAS_ext8u_i64);
+        tcg_out_op_rr(s, INDEX_op_ext8u_i64, rd, rs);
+    } else {
+        tcg_debug_assert(TCG_TARGET_HAS_ext8u_i32);
+        tcg_out_op_rr(s, INDEX_op_ext8u_i32, rd, rs);
+    }
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     CASE_32_64(neg)      /* Optional (TCG_TARGET_HAS_neg_*). */
     CASE_32_64(not)      /* Optional (TCG_TARGET_HAS_not_*). */
-    CASE_32_64(ext8u)    /* Optional (TCG_TARGET_HAS_ext8u_*). */
     CASE_32_64(ext16s)   /* Optional (TCG_TARGET_HAS_ext16s_*). */
     CASE_32_64(ext16u)   /* Optional (TCG_TARGET_HAS_ext16u_*). */
     CASE_64(ext32s)      /* Optional (TCG_TARGET_HAS_ext32s_i64). */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
     case INDEX_op_ext8s_i64:
+    case INDEX_op_ext8u_i32:
+    case INDEX_op_ext8u_i64:
     default:
         g_assert_not_reached();
     }
-- 
2.34.1

We will need a backend interface for performing 16-bit sign-extend.
Use it in tcg_reg_alloc_op in the meantime.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/tcg.c                        |  7 +++++++
 tcg/aarch64/tcg-target.c.inc     | 13 ++++++++-----
 tcg/arm/tcg-target.c.inc         | 10 ++++------
 tcg/i386/tcg-target.c.inc        | 16 ++++++++--------
 tcg/loongarch64/tcg-target.c.inc | 13 +++++--------
 tcg/mips/tcg-target.c.inc        | 11 ++++++++---
 tcg/ppc/tcg-target.c.inc         | 12 +++++-------
 tcg/riscv/tcg-target.c.inc       |  9 +++------
 tcg/s390x/tcg-target.c.inc       | 12 ++++--------
 tcg/sparc64/tcg-target.c.inc     |  7 +++++++
 tcg/tci/tcg-target.c.inc         | 21 ++++++++++++++++++++-
 11 files changed, 79 insertions(+), 52 deletions(-)

diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
 static void tcg_out_movi(TCGContext *s, TCGType type,
                          TCGReg ret, tcg_target_long arg);
 static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
+static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
 static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
 static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
     case INDEX_op_ext8u_i64:
         tcg_out_ext8u(s, new_args[0], new_args[1]);
         break;
+    case INDEX_op_ext16s_i32:
+        tcg_out_ext16s(s, TCG_TYPE_I32, new_args[0], new_args[1]);
+        break;
+    case INDEX_op_ext16s_i64:
+        tcg_out_ext16s(s, TCG_TYPE_I64, new_args[0], new_args[1]);
+        break;
     default:
         if (def->flags & TCG_OPF_VECTOR) {
             tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rn)
     tcg_out_sxt(s, type, MO_8, rd, rn);
 }
 
+static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rn)
+{
+    tcg_out_sxt(s, type, MO_16, rd, rn);
+}
+
 static inline void tcg_out_uxt(TCGContext *s, MemOp s_bits,
                                TCGReg rd, TCGReg rn)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_rev(s, TCG_TYPE_I32, MO_16, a0, a1);
         if (a2 & TCG_BSWAP_OS) {
             /* Output must be sign-extended. */
-            tcg_out_sxt(s, ext, MO_16, a0, a0);
+            tcg_out_ext16s(s, ext, a0, a0);
         } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
             /* Output must be zero-extended, but input isn't. */
             tcg_out_uxt(s, MO_16, a0, a0);
         }
         break;
 
-    case INDEX_op_ext16s_i64:
-    case INDEX_op_ext16s_i32:
-        tcg_out_sxt(s, ext, MO_16, a0, a1);
-        break;
     case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8s_i64:
     case INDEX_op_ext8u_i32:
     case INDEX_op_ext8u_i64:
+    case INDEX_op_ext16s_i64:
+    case INDEX_op_ext16s_i32:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ tcg_out_ext8u_cond(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn)
     tcg_out_dat_imm(s, cond, ARITH_AND, rd, rn, 0xff);
 }
 
-static void tcg_out_ext16s(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn)
+static void tcg_out_ext16s(TCGContext *s, TCGType t, TCGReg rd, TCGReg rn)
 {
     /* sxth */
-    tcg_out32(s, 0x06bf0070 | (cond << 28) | (rd << 12) | rn);
+    tcg_out32(s, 0x06bf0070 | (COND_AL << 28) | (rd << 12) | rn);
 }
 
 static void tcg_out_ext16u(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn)
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
         tcg_out_ext8s(s, TCG_TYPE_I32, datalo, TCG_REG_R0);
         break;
     case MO_SW:
-        tcg_out_ext16s(s, COND_AL, datalo, TCG_REG_R0);
+        tcg_out_ext16s(s, TCG_TYPE_I32, datalo, TCG_REG_R0);
         break;
     default:
         tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_R0);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_bswap32(s, COND_AL, args[0], args[1]);
         break;
 
-    case INDEX_op_ext16s_i32:
-        tcg_out_ext16s(s, COND_AL, args[0], args[1]);
-        break;
     case INDEX_op_ext16u_i32:
         tcg_out_ext16u(s, COND_AL, args[0], args[1]);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_goto_tb:  /* Always emitted via tcg_out_goto_tb.  */
     case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
     case INDEX_op_ext8u_i32:
+    case INDEX_op_ext16s_i32:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_ext16u(TCGContext *s, int dest, int src)
     tcg_out_modrm(s, OPC_MOVZWL, dest, src);
 }
 
-static inline void tcg_out_ext16s(TCGContext *s, int dest, int src, int rexw)
+static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
 {
+    int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
     /* movsw[lq] */
     tcg_out_modrm(s, OPC_MOVSWL + rexw, dest, src);
 }
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
     MemOp opc = get_memop(oi);
     TCGReg data_reg;
     tcg_insn_unit **label_ptr = &l->label_ptr[0];
-    int rexw = (l->type == TCG_TYPE_I64 ? P_REXW : 0);
 
     /* resolve label address */
     tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4);
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
         tcg_out_ext8s(s, l->type, data_reg, TCG_REG_EAX);
         break;
     case MO_SW:
-        tcg_out_ext16s(s, data_reg, TCG_REG_EAX, rexw);
+        tcg_out_ext16s(s, l->type, data_reg, TCG_REG_EAX);
         break;
 #if TCG_TARGET_REG_BITS == 64
     case MO_SL:
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
                                    TCGReg base, int index, intptr_t ofs,
                                    int seg, bool is64, MemOp memop)
 {
+    TCGType type = is64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
     bool use_movbe = false;
     int rexw = is64 * P_REXW;
     int movop = OPC_MOVL_GvEv;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
         if (use_movbe) {
             tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg,
                                      datalo, base, index, 0, ofs);
-            tcg_out_ext16s(s, datalo, datalo, rexw);
+            tcg_out_ext16s(s, type, datalo, datalo);
         } else {
             tcg_out_modrm_sib_offset(s, OPC_MOVSWL + rexw + seg,
                                      datalo, base, index, 0, ofs);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, a0);
         break;
 
-    OP_32_64(ext16s):
-        tcg_out_ext16s(s, a0, a1, rexw);
-        break;
     OP_32_64(ext16u):
         tcg_out_ext16u(s, a0, a1);
         break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         if (a1 < 4 && a0 < 8) {
             tcg_out_modrm(s, OPC_MOVSBL, a0, a1 + 4);
         } else {
-            tcg_out_ext16s(s, a0, a1, 0);
+            tcg_out_ext16s(s, TCG_TYPE_I32, a0, a1);
             tcg_out_shifti(s, SHIFT_SAR, a0, 8);
         }
         break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8s_i64:
     case INDEX_op_ext8u_i32:
     case INDEX_op_ext8u_i64:
+    case INDEX_op_ext16s_i32:
+    case INDEX_op_ext16s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.c.inc
+++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
     tcg_out_opc_sext_b(s, ret, arg);
 }
 
-static void tcg_out_ext16s(TCGContext *s, TCGReg ret, TCGReg arg)
+static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
 {
     tcg_out_opc_sext_h(s, ret, arg);
 }
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
         tcg_out_ext8s(s, type, l->datalo_reg, TCG_REG_A0);
         break;
     case MO_SW:
-        tcg_out_ext16s(s, l->datalo_reg, TCG_REG_A0);
+        tcg_out_ext16s(s, type, l->datalo_reg, TCG_REG_A0);
         break;
     case MO_SL:
         tcg_out_ext32s(s, l->datalo_reg, TCG_REG_A0);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_brcond(s, a2, a0, a1, arg_label(args[3]));
         break;
 
-    case INDEX_op_ext16s_i32:
-    case INDEX_op_ext16s_i64:
-        tcg_out_ext16s(s, a0, a1);
-        break;
-
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext16u_i64:
         tcg_out_ext16u(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_bswap16_i64:
         tcg_out_opc_revb_2h(s, a0, a1);
         if (a2 & TCG_BSWAP_OS) {
-            tcg_out_ext16s(s, a0, a0);
+            tcg_out_ext16s(s, TCG_TYPE_REG, a0, a0);
         } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
             tcg_out_ext16u(s, a0, a0);
         }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8s_i64:
     case INDEX_op_ext8u_i32:
     case INDEX_op_ext8u_i64:
+    case INDEX_op_ext16s_i32:
+    case INDEX_op_ext16s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_opc_imm(s, OPC_ANDI, rd, rs, 0xff);
 }
 
+static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
+{
+    tcg_debug_assert(TCG_TARGET_HAS_ext16s_i32);
+    tcg_out_opc_reg(s, OPC_SEH, rd, TCG_REG_ZERO, rs);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_not_i64:
         i1 = OPC_NOR;
         goto do_unary;
-    case INDEX_op_ext16s_i32:
-    case INDEX_op_ext16s_i64:
-        i1 = OPC_SEH;
     do_unary:
         tcg_out_opc_reg(s, i1, a0, TCG_REG_ZERO, a1);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8s_i64:
     case INDEX_op_ext8u_i32:
     case INDEX_op_ext8u_i64:
+    case INDEX_op_ext16s_i32:
+    case INDEX_op_ext16s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8u(TCGContext *s, TCGReg dst, TCGReg src)
     tcg_out32(s, ANDI | SAI(src, dst, 0xff));
 }
 
-static inline void tcg_out_ext16s(TCGContext *s, TCGReg dst, TCGReg src)
+static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
 {
     tcg_out32(s, EXTSH | RA(dst) | RS(src));
 }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_bswap16(TCGContext *s, TCGReg dst, TCGReg src, int flags)
     if (have_isa_3_10) {
         tcg_out32(s, BRH | RA(dst) | RS(src));
         if (flags & TCG_BSWAP_OS) {
-            tcg_out_ext16s(s, dst, dst);
+            tcg_out_ext16s(s, TCG_TYPE_REG, dst, dst);
         } else if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
             tcg_out_ext16u(s, dst, dst);
         }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_bswap16(TCGContext *s, TCGReg dst, TCGReg src, int flags)
     tcg_out_rlw(s, RLWIMI, tmp, src, 8, 16, 23);
 
     if (flags & TCG_BSWAP_OS) {
-        tcg_out_ext16s(s, dst, tmp);
+        tcg_out_ext16s(s, TCG_TYPE_REG, dst, tmp);
     } else {
         tcg_out_mov(s, TCG_TYPE_REG, dst, tmp);
     }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_qemu_st(s, args, true);
         break;
 
-    case INDEX_op_ext16s_i32:
-    case INDEX_op_ext16s_i64:
-        tcg_out_ext16s(s, args[0], args[1]);
-        break;
     case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         tcg_out_ext32s(s, args[0], args[1]);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8s_i64:
     case INDEX_op_ext8u_i32:
     case INDEX_op_ext8u_i64:
+    case INDEX_op_ext16s_i32:
+    case INDEX_op_ext16s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
     tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 24);
 }
 
-static void tcg_out_ext16s(TCGContext *s, TCGReg ret, TCGReg arg)
+static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
 {
     tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16);
     tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 16);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_ext32u(s, a0, a1);
         break;
 
-    case INDEX_op_ext16s_i32:
-    case INDEX_op_ext16s_i64:
-        tcg_out_ext16s(s, a0, a1);
-        break;
-
     case INDEX_op_ext32s_i64:
     case INDEX_op_extrl_i64_i32:
     case INDEX_op_ext_i32_i64:
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8s_i64:
     case INDEX_op_ext8u_i32:
     case INDEX_op_ext8u_i64:
+    case INDEX_op_ext16s_i32:
+    case INDEX_op_ext16s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8u(TCGContext *s, TCGReg dest, TCGReg src)
     tcg_out_insn(s, RRE, LLGCR, dest, src);
 }
 
-static void tgen_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
+static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
 {
     tcg_out_insn(s, RRE, LGHR, dest, src);
 }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data,
     case MO_SW | MO_BSWAP:
         /* swapped sign-extended halfword load */
         tcg_out_insn(s, RXY, LRVH, data, base, index, disp);
-        tgen_ext16s(s, TCG_TYPE_I64, data, data);
+        tcg_out_ext16s(s, TCG_TYPE_REG, data, data);
         break;
     case MO_SW:
         tcg_out_insn(s, RXY, LGH, data, base, index, disp);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         }
         break;
 
-    case INDEX_op_ext16s_i32:
-        tgen_ext16s(s, TCG_TYPE_I32, args[0], args[1]);
-        break;
     case INDEX_op_ext16u_i32:
         tgen_ext16u(s, TCG_TYPE_I32, args[0], args[1]);
         break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         }
         break;
 
-    case INDEX_op_ext16s_i64:
-        tgen_ext16s(s, TCG_TYPE_I64, args[0], args[1]);
-        break;
     case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         tgen_ext32s(s, args[0], args[1]);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8s_i64:
     case INDEX_op_ext8u_i32:
     case INDEX_op_ext8u_i64:
+    case INDEX_op_ext16s_i32:
+    case INDEX_op_ext16s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
     g_assert_not_reached();
 }
 
+static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
+{
+    g_assert_not_reached();
+}
+
 static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs)
 {
     tcg_out_arithi(s, rd, rs, 0xff, ARITH_AND);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8s_i64:
     case INDEX_op_ext8u_i32:
     case INDEX_op_ext8u_i64:
+    case INDEX_op_ext16s_i32:
+    case INDEX_op_ext16s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs)
     }
 }
 
+static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
+{
+    switch (type) {
+    case TCG_TYPE_I32:
+        tcg_debug_assert(TCG_TARGET_HAS_ext16s_i32);
+        tcg_out_op_rr(s, INDEX_op_ext16s_i32, rd, rs);
+        break;
+#if TCG_TARGET_REG_BITS == 64
+    case TCG_TYPE_I64:
+        tcg_debug_assert(TCG_TARGET_HAS_ext16s_i64);
+        tcg_out_op_rr(s, INDEX_op_ext16s_i64, rd, rs);
+        break;
+#endif
+    default:
+        g_assert_not_reached();
+    }
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     CASE_32_64(neg)      /* Optional (TCG_TARGET_HAS_neg_*). */
     CASE_32_64(not)      /* Optional (TCG_TARGET_HAS_not_*). */
-    CASE_32_64(ext16s)   /* Optional (TCG_TARGET_HAS_ext16s_*). */
     CASE_32_64(ext16u)   /* Optional (TCG_TARGET_HAS_ext16u_*). */
     CASE_64(ext32s)      /* Optional (TCG_TARGET_HAS_ext32s_i64). */
     CASE_64(ext32u)      /* Optional (TCG_TARGET_HAS_ext32u_i64). */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8s_i64:
     case INDEX_op_ext8u_i32:
     case INDEX_op_ext8u_i64:
+    case INDEX_op_ext16s_i32:
+    case INDEX_op_ext16s_i64:
     default:
         g_assert_not_reached();
     }
-- 
2.34.1

We will need a backend interface for performing 16-bit zero-extend.
Use it in tcg_reg_alloc_op in the meantime.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/tcg.c                        |  5 +++++
 tcg/aarch64/tcg-target.c.inc     | 13 ++++++++-----
 tcg/arm/tcg-target.c.inc         | 17 ++++++++++-------
 tcg/i386/tcg-target.c.inc        |  8 +++-----
 tcg/loongarch64/tcg-target.c.inc |  7 ++-----
 tcg/mips/tcg-target.c.inc        |  5 +++++
 tcg/ppc/tcg-target.c.inc         |  4 +++-
 tcg/riscv/tcg-target.c.inc       |  7 ++-----
 tcg/s390x/tcg-target.c.inc       | 17 ++++++-----------
 tcg/sparc64/tcg-target.c.inc     | 11 +++++++++--
 tcg/tci/tcg-target.c.inc         | 14 +++++++++++++-
 11 files changed, 66 insertions(+), 42 deletions(-)

diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type,
 static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
 static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
 static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
+static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
 static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
 static void tcg_out_goto_tb(TCGContext *s, int which);
@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
     case INDEX_op_ext16s_i64:
         tcg_out_ext16s(s, TCG_TYPE_I64, new_args[0], new_args[1]);
         break;
+    case INDEX_op_ext16u_i32:
+    case INDEX_op_ext16u_i64:
+        tcg_out_ext16u(s, new_args[0], new_args[1]);
+        break;
     default:
         if (def->flags & TCG_OPF_VECTOR) {
             tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rn)
     tcg_out_uxt(s, MO_8, rd, rn);
 }
 
+static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    tcg_out_uxt(s, MO_16, rd, rn);
+}
+
 static void tcg_out_addsubi(TCGContext *s, int ext, TCGReg rd,
                             TCGReg rn, int64_t aimm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
             tcg_out_ext16s(s, ext, a0, a0);
         } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
             /* Output must be zero-extended, but input isn't. */
-            tcg_out_uxt(s, MO_16, a0, a0);
+            tcg_out_ext16u(s, a0, a0);
         }
         break;
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32s_i64:
         tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
         break;
-    case INDEX_op_ext16u_i64:
-    case INDEX_op_ext16u_i32:
-        tcg_out_uxt(s, MO_16, a0, a1);
-        break;
     case INDEX_op_extu_i32_i64:
     case INDEX_op_ext32u_i64:
         tcg_out_movr(s, TCG_TYPE_I32, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8u_i64:
     case INDEX_op_ext16s_i64:
     case INDEX_op_ext16s_i32:
+    case INDEX_op_ext16u_i64:
+    case INDEX_op_ext16u_i32:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16s(TCGContext *s, TCGType t, TCGReg rd, TCGReg rn)
     tcg_out32(s, 0x06bf0070 | (COND_AL << 28) | (rd << 12) | rn);
 }
 
-static void tcg_out_ext16u(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn)
+static void tcg_out_ext16u_cond(TCGContext *s, ARMCond cond,
+                                TCGReg rd, TCGReg rn)
 {
     /* uxth */
     tcg_out32(s, 0x06ff0070 | (cond << 28) | (rd << 12) | rn);
 }
 
+static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    tcg_out_ext16u_cond(s, COND_AL, rd, rn);
+}
+
 static void tcg_out_bswap16(TCGContext *s, ARMCond cond,
                             TCGReg rd, TCGReg rn, int flags)
 {
@@ -XXX,XX +XXX,XX @@ DEFINE_TCG_OUT_ARG(tcg_out_arg_imm32, uint32_t, tcg_out_movi32,
     (tcg_out_movi32(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
 DEFINE_TCG_OUT_ARG(tcg_out_arg_reg8, TCGReg, tcg_out_ext8u_cond,
     (tcg_out_ext8u_cond(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
-DEFINE_TCG_OUT_ARG(tcg_out_arg_reg16, TCGReg, tcg_out_ext16u,
-    (tcg_out_ext16u(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
+DEFINE_TCG_OUT_ARG(tcg_out_arg_reg16, TCGReg, tcg_out_ext16u_cond,
+    (tcg_out_ext16u_cond(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
 DEFINE_TCG_OUT_ARG(tcg_out_arg_reg32, TCGReg, tcg_out_mov_reg, )
 
 static TCGReg tcg_out_arg_reg64(TCGContext *s, TCGReg argreg,
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_bswap32(s, COND_AL, args[0], args[1]);
         break;
 
-    case INDEX_op_ext16u_i32:
-        tcg_out_ext16u(s, COND_AL, args[0], args[1]);
-        break;
-
     case INDEX_op_deposit_i32:
         tcg_out_deposit(s, COND_AL, args[0], args[2],
                         args[3], args[4], const_args[2]);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8s_i32:  /* Always emitted via tcg_reg_alloc_op.  */
     case INDEX_op_ext8u_i32:
     case INDEX_op_ext16s_i32:
+    case INDEX_op_ext16u_i32:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
     tcg_out_modrm(s, OPC_MOVSBL + P_REXB_RM + rexw, dest, src);
 }
 
-static inline void tcg_out_ext16u(TCGContext *s, int dest, int src)
+static void tcg_out_ext16u(TCGContext *s, TCGReg dest, TCGReg src)
 {
     /* movzwl */
     tcg_out_modrm(s, OPC_MOVZWL, dest, src);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, a0);
         break;
 
-    OP_32_64(ext16u):
-        tcg_out_ext16u(s, a0, a1);
-        break;
-
     case INDEX_op_qemu_ld_i32:
         tcg_out_qemu_ld(s, args, 0);
         break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8u_i64:
     case INDEX_op_ext16s_i32:
     case INDEX_op_ext16s_i64:
+    case INDEX_op_ext16u_i32:
+    case INDEX_op_ext16u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.c.inc
+++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_brcond(s, a2, a0, a1, arg_label(args[3]));
         break;
 
-    case INDEX_op_ext16u_i32:
-    case INDEX_op_ext16u_i64:
-        tcg_out_ext16u(s, a0, a1);
-        break;
-
     case INDEX_op_ext32u_i64:
     case INDEX_op_extu_i32_i64:
         tcg_out_ext32u(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8u_i64:
     case INDEX_op_ext16s_i32:
     case INDEX_op_ext16s_i64:
+    case INDEX_op_ext16u_i32:
+    case INDEX_op_ext16u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
     tcg_out_opc_reg(s, OPC_SEH, rd, TCG_REG_ZERO, rs);
 }
 
+static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_out_opc_imm(s, OPC_ANDI, rd, rs, 0xffff);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
     tcg_out32(s, EXTSH | RA(dst) | RS(src));
 }
 
-static inline void tcg_out_ext16u(TCGContext *s, TCGReg dst, TCGReg src)
+static void tcg_out_ext16u(TCGContext *s, TCGReg dst, TCGReg src)
 {
     tcg_out32(s, ANDI | SAI(src, dst, 0xffff));
 }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8u_i64:
     case INDEX_op_ext16s_i32:
     case INDEX_op_ext16s_i64:
+    case INDEX_op_ext16u_i32:
+    case INDEX_op_ext16u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_qemu_st(s, args, true);
         break;
 
-    case INDEX_op_ext16u_i32:
-    case INDEX_op_ext16u_i64:
-        tcg_out_ext16u(s, a0, a1);
-        break;
-
     case INDEX_op_ext32u_i64:
     case INDEX_op_extu_i32_i64:
         tcg_out_ext32u(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8u_i64:
     case INDEX_op_ext16s_i32:
     case INDEX_op_ext16s_i64:
+    case INDEX_op_ext16u_i32:
+    case INDEX_op_ext16u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
     tcg_out_insn(s, RRE, LGHR, dest, src);
 }
 
-static void tgen_ext16u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
+static void tcg_out_ext16u(TCGContext *s, TCGReg dest, TCGReg src)
 {
     tcg_out_insn(s, RRE, LLGHR, dest, src);
 }
@@ -XXX,XX +XXX,XX @@ static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
         return;
     }
     if ((val & valid) == 0xffff) {
-        tgen_ext16u(s, TCG_TYPE_I64, dest, dest);
+        tcg_out_ext16u(s, dest, dest);
         return;
     }
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data,
     case MO_UW | MO_BSWAP:
         /* swapped unsigned halfword load with upper bits zeroed */
         tcg_out_insn(s, RXY, LRVH, data, base, index, disp);
-        tgen_ext16u(s, TCG_TYPE_I64, data, data);
+        tcg_out_ext16u(s, data, data);
         break;
     case MO_UW:
         tcg_out_insn(s, RXY, LLGH, data, base, index, disp);
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
         tcg_out_ext8u(s, TCG_REG_R4, data_reg);
         break;
     case MO_UW:
-        tgen_ext16u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
+        tcg_out_ext16u(s, TCG_REG_R4, data_reg);
         break;
     case MO_UL:
         tgen_ext32u(s, TCG_REG_R4, data_reg);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         }
         break;
 
-    case INDEX_op_ext16u_i32:
-        tgen_ext16u(s, TCG_TYPE_I32, args[0], args[1]);
-        break;
-
     case INDEX_op_bswap16_i32:
         a0 = args[0], a1 = args[1], a2 = args[2];
         tcg_out_insn(s, RRE, LRVR, a0, a1);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32s_i64:
         tgen_ext32s(s, args[0], args[1]);
         break;
-    case INDEX_op_ext16u_i64:
-        tgen_ext16u(s, TCG_TYPE_I64, args[0], args[1]);
-        break;
     case INDEX_op_extu_i32_i64:
     case INDEX_op_ext32u_i64:
         tgen_ext32u(s, args[0], args[1]);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8u_i64:
     case INDEX_op_ext16s_i32:
     case INDEX_op_ext16s_i64:
+    case INDEX_op_ext16u_i32:
+    case INDEX_op_ext16u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_arithi(s, rd, rs, 0xff, ARITH_AND);
 }
 
+static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_out_arithi(s, rd, rs, 16, SHIFT_SLL);
+    tcg_out_arithi(s, rd, rd, 16, SHIFT_SRL);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void emit_extend(TCGContext *s, TCGReg r, int op)
         tcg_out_ext8u(s, r, r);
         break;
     case MO_16:
-        tcg_out_arithi(s, r, r, 16, SHIFT_SLL);
-        tcg_out_arithi(s, r, r, 16, SHIFT_SRL);
+        tcg_out_ext16u(s, r, r);
         break;
     case MO_32:
         tcg_out_arith(s, r, r, 0, SHIFT_SRL);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8u_i64:
     case INDEX_op_ext16s_i32:
     case INDEX_op_ext16s_i64:
+    case INDEX_op_ext16u_i32:
+    case INDEX_op_ext16u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
     }
 }
 
+static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    if (TCG_TARGET_REG_BITS == 64) {
+        tcg_debug_assert(TCG_TARGET_HAS_ext16u_i64);
+        tcg_out_op_rr(s, INDEX_op_ext16u_i64, rd, rs);
+    } else {
+        tcg_debug_assert(TCG_TARGET_HAS_ext16u_i32);
+        tcg_out_op_rr(s, INDEX_op_ext16u_i32, rd, rs);
+    }
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     CASE_32_64(neg)      /* Optional (TCG_TARGET_HAS_neg_*). */
     CASE_32_64(not)      /* Optional (TCG_TARGET_HAS_not_*). */
-    CASE_32_64(ext16u)   /* Optional (TCG_TARGET_HAS_ext16u_*). */
     CASE_64(ext32s)      /* Optional (TCG_TARGET_HAS_ext32s_i64). */
     CASE_64(ext32u)      /* Optional (TCG_TARGET_HAS_ext32u_i64). */
     CASE_64(ext_i32)
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8u_i64:
     case INDEX_op_ext16s_i32:
     case INDEX_op_ext16s_i64:
+    case INDEX_op_ext16u_i32:
+    case INDEX_op_ext16u_i64:
     default:
         g_assert_not_reached();
     }
-- 
2.34.1

We will need a backend interface for performing 32-bit sign-extend.
Use it in tcg_reg_alloc_op in the meantime.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/tcg.c                        |  4 ++++
 tcg/aarch64/tcg-target.c.inc     |  9 +++++++--
 tcg/arm/tcg-target.c.inc         |  5 +++++
 tcg/i386/tcg-target.c.inc        |  5 +++--
 tcg/loongarch64/tcg-target.c.inc |  2 +-
 tcg/mips/tcg-target.c.inc        | 12 +++++++++---
 tcg/ppc/tcg-target.c.inc         |  5 +++--
 tcg/riscv/tcg-target.c.inc       |  2 +-
 tcg/s390x/tcg-target.c.inc       | 10 +++++-----
 tcg/sparc64/tcg-target.c.inc     | 11 ++++++++---
 tcg/tci/tcg-target.c.inc         |  9 ++++++++-
 11 files changed, 54 insertions(+), 20 deletions(-)

diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
 static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
 static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
+static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
 static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
 static void tcg_out_goto_tb(TCGContext *s, int which);
@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
     case INDEX_op_ext16u_i64:
         tcg_out_ext16u(s, new_args[0], new_args[1]);
         break;
+    case INDEX_op_ext32s_i64:
+        tcg_out_ext32s(s, new_args[0], new_args[1]);
+        break;
     default:
         if (def->flags & TCG_OPF_VECTOR) {
             tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rn)
     tcg_out_sxt(s, type, MO_16, rd, rn);
 }
 
+static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    tcg_out_sxt(s, TCG_TYPE_I64, MO_32, rd, rn);
+}
+
 static inline void tcg_out_uxt(TCGContext *s, MemOp s_bits,
                                TCGReg rd, TCGReg rn)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_bswap32_i64:
         tcg_out_rev(s, TCG_TYPE_I32, MO_32, a0, a1);
         if (a2 & TCG_BSWAP_OS) {
-            tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a0);
+            tcg_out_ext32s(s, a0, a0);
         }
         break;
     case INDEX_op_bswap32_i32:
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 
     case INDEX_op_ext_i32_i64:
-    case INDEX_op_ext32s_i64:
         tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
         break;
     case INDEX_op_extu_i32_i64:
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16s_i32:
     case INDEX_op_ext16u_i64:
     case INDEX_op_ext16u_i32:
+    case INDEX_op_ext32s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rn)
     tcg_out_ext16u_cond(s, COND_AL, rd, rn);
 }
 
+static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    g_assert_not_reached();
+}
+
 static void tcg_out_bswap16(TCGContext *s, ARMCond cond,
                             TCGReg rd, TCGReg rn, int flags)
 {
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_ext32u(TCGContext *s, int dest, int src)
     tcg_out_modrm(s, OPC_MOVL_GvEv, dest, src);
 }
 
-static inline void tcg_out_ext32s(TCGContext *s, int dest, int src)
+static void tcg_out_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
 {
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
     tcg_out_modrm(s, OPC_MOVSLQ, dest, src);
 }
 
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_ext32u(s, a0, a1);
         break;
     case INDEX_op_ext_i32_i64:
-    case INDEX_op_ext32s_i64:
         tcg_out_ext32s(s, a0, a1);
         break;
     case INDEX_op_extrh_i64_i32:
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16s_i64:
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext16u_i64:
+    case INDEX_op_ext32s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.c.inc
+++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_ext32u(s, a0, a1);
         break;
 
-    case INDEX_op_ext32s_i64:
     case INDEX_op_extrl_i64_i32:
     case INDEX_op_ext_i32_i64:
         tcg_out_ext32s(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16s_i64:
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext16u_i64:
+    case INDEX_op_ext32s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_opc_imm(s, OPC_ANDI, rd, rs, 0xffff);
 }
 
+static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
+    tcg_out_opc_sa(s, OPC_SLL, rd, rs, 0);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
     /* delay slot */
     if (TCG_TARGET_REG_BITS == 64 && l->type == TCG_TYPE_I32) {
         /* we always sign-extend 32-bit loads */
-        tcg_out_opc_sa(s, OPC_SLL, v0, TCG_REG_V0, 0);
+        tcg_out_ext32s(s, v0, TCG_REG_V0);
     } else {
         tcg_out_opc_reg(s, OPC_OR, v0, TCG_REG_V0, TCG_REG_ZERO);
     }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_extrh_i64_i32:
         tcg_out_dsra(s, a0, a1, 32);
         break;
-    case INDEX_op_ext32s_i64:
     case INDEX_op_ext_i32_i64:
     case INDEX_op_extrl_i64_i32:
-        tcg_out_opc_sa(s, OPC_SLL, a0, a1, 0);
+        tcg_out_ext32s(s, a0, a1);
         break;
     case INDEX_op_ext32u_i64:
     case INDEX_op_extu_i32_i64:
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext8u_i64:
     case INDEX_op_ext16s_i32:
     case INDEX_op_ext16s_i64:
+    case INDEX_op_ext32s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16u(TCGContext *s, TCGReg dst, TCGReg src)
     tcg_out32(s, ANDI | SAI(src, dst, 0xffff));
 }
 
-static inline void tcg_out_ext32s(TCGContext *s, TCGReg dst, TCGReg src)
+static void tcg_out_ext32s(TCGContext *s, TCGReg dst, TCGReg src)
 {
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
     tcg_out32(s, EXTSW | RA(dst) | RS(src));
 }
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 
     case INDEX_op_ext_i32_i64:
-    case INDEX_op_ext32s_i64:
         tcg_out_ext32s(s, args[0], args[1]);
         break;
     case INDEX_op_extu_i32_i64:
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16s_i64:
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext16u_i64:
+    case INDEX_op_ext32s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_ext32u(s, a0, a1);
         break;
 
-    case INDEX_op_ext32s_i64:
     case INDEX_op_extrl_i64_i32:
     case INDEX_op_ext_i32_i64:
         tcg_out_ext32s(s, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16s_i64:
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext16u_i64:
+    case INDEX_op_ext32s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16u(TCGContext *s, TCGReg dest, TCGReg src)
     tcg_out_insn(s, RRE, LLGHR, dest, src);
 }
 
-static inline void tgen_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
+static void tcg_out_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
 {
     tcg_out_insn(s, RRE, LGFR, dest, src);
 }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data,
     case MO_SL | MO_BSWAP:
         /* swapped sign-extended int load */
         tcg_out_insn(s, RXY, LRV, data, base, index, disp);
-        tgen_ext32s(s, data, data);
+        tcg_out_ext32s(s, data, data);
         break;
     case MO_SL:
         tcg_out_insn(s, RXY, LGF, data, base, index, disp);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         a0 = args[0], a1 = args[1], a2 = args[2];
         tcg_out_insn(s, RRE, LRVR, a0, a1);
         if (a2 & TCG_BSWAP_OS) {
-            tgen_ext32s(s, a0, a0);
+            tcg_out_ext32s(s, a0, a0);
         } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
             tgen_ext32u(s, a0, a0);
         }
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 
     case INDEX_op_ext_i32_i64:
-    case INDEX_op_ext32s_i64:
-        tgen_ext32s(s, args[0], args[1]);
+        tcg_out_ext32s(s, args[0], args[1]);
         break;
     case INDEX_op_extu_i32_i64:
     case INDEX_op_ext32u_i64:
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16s_i64:
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext16u_i64:
+    case INDEX_op_ext32s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_arithi(s, rd, rd, 16, SHIFT_SRL);
 }
 
+static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_out_arithi(s, rd, rs, 0, SHIFT_SRA);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr,
 
     /* We let the helper sign-extend SB and SW, but leave SL for here.  */
     if (is_64 && (memop & MO_SSIZE) == MO_SL) {
-        tcg_out_arithi(s, data, TCG_REG_O0, 0, SHIFT_SRA);
+        tcg_out_ext32s(s, data, TCG_REG_O0);
     } else {
         tcg_out_mov(s, TCG_TYPE_REG, data, TCG_REG_O0);
     }
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         c = ARITH_UDIVX;
         goto gen_arith;
     case INDEX_op_ext_i32_i64:
-    case INDEX_op_ext32s_i64:
-        tcg_out_arithi(s, a0, a1, 0, SHIFT_SRA);
+        tcg_out_ext32s(s, a0, a1);
         break;
     case INDEX_op_extu_i32_i64:
     case INDEX_op_ext32u_i64:
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16s_i64:
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext16u_i64:
+    case INDEX_op_ext32s_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rs)
     }
 }
 
+static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
+    tcg_debug_assert(TCG_TARGET_HAS_ext32s_i64);
+    tcg_out_op_rr(s, INDEX_op_ext32s_i64, rd, rs);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     CASE_32_64(neg)      /* Optional (TCG_TARGET_HAS_neg_*). */
     CASE_32_64(not)      /* Optional (TCG_TARGET_HAS_not_*). */
-    CASE_64(ext32s)      /* Optional (TCG_TARGET_HAS_ext32s_i64). */
     CASE_64(ext32u)      /* Optional (TCG_TARGET_HAS_ext32u_i64). */
     CASE_64(ext_i32)
     CASE_64(extu_i32)
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16s_i64:
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext16u_i64:
+    case INDEX_op_ext32s_i64:
     default:
         g_assert_not_reached();
     }
-- 
2.34.1

We will need a backend interface for performing 32-bit zero-extend.
Use it in tcg_reg_alloc_op in the meantime.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/tcg.c                        |  4 ++++
 tcg/aarch64/tcg-target.c.inc     |  9 +++++++--
 tcg/arm/tcg-target.c.inc         |  5 +++++
 tcg/i386/tcg-target.c.inc        |  4 ++--
 tcg/loongarch64/tcg-target.c.inc |  2 +-
 tcg/mips/tcg-target.c.inc        |  3 ++-
 tcg/ppc/tcg-target.c.inc         |  4 +++-
 tcg/riscv/tcg-target.c.inc       |  2 +-
 tcg/s390x/tcg-target.c.inc       | 20 ++++++++++----------
 tcg/sparc64/tcg-target.c.inc     | 17 +++++++++++------
 tcg/tci/tcg-target.c.inc         |  9 ++++++++-
 11 files changed, 54 insertions(+), 25 deletions(-)

diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
 static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
+static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
 static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
 static void tcg_out_goto_tb(TCGContext *s, int which);
@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
     case INDEX_op_ext32s_i64:
         tcg_out_ext32s(s, new_args[0], new_args[1]);
         break;
+    case INDEX_op_ext32u_i64:
+        tcg_out_ext32u(s, new_args[0], new_args[1]);
+        break;
     default:
         if (def->flags & TCG_OPF_VECTOR) {
             tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rn)
     tcg_out_uxt(s, MO_16, rd, rn);
 }
 
+static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    tcg_out_movr(s, TCG_TYPE_I32, rd, rn);
+}
+
 static void tcg_out_addsubi(TCGContext *s, int ext, TCGReg rd,
                             TCGReg rn, int64_t aimm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
         break;
     case INDEX_op_extu_i32_i64:
-    case INDEX_op_ext32u_i64:
-        tcg_out_movr(s, TCG_TYPE_I32, a0, a1);
+        tcg_out_ext32u(s, a0, a1);
         break;
 
     case INDEX_op_deposit_i64:
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i64:
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext32s_i64:
+    case INDEX_op_ext32u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rn)
     g_assert_not_reached();
 }
 
+static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    g_assert_not_reached();
+}
+
 static void tcg_out_bswap16(TCGContext *s, ARMCond cond,
                             TCGReg rd, TCGReg rn, int flags)
 {
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
     tcg_out_modrm(s, OPC_MOVSWL + rexw, dest, src);
 }
 
-static inline void tcg_out_ext32u(TCGContext *s, int dest, int src)
+static void tcg_out_ext32u(TCGContext *s, TCGReg dest, TCGReg src)
 {
     /* 32-bit mov zero extends.  */
     tcg_out_modrm(s, OPC_MOVL_GvEv, dest, src);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_bswap64(s, a0);
         break;
     case INDEX_op_extu_i32_i64:
-    case INDEX_op_ext32u_i64:
     case INDEX_op_extrl_i64_i32:
         tcg_out_ext32u(s, a0, a1);
         break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext16u_i64:
     case INDEX_op_ext32s_i64:
+    case INDEX_op_ext32u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.c.inc
+++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_brcond(s, a2, a0, a1, arg_label(args[3]));
         break;
 
-    case INDEX_op_ext32u_i64:
     case INDEX_op_extu_i32_i64:
         tcg_out_ext32u(s, a0, a1);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext16u_i64:
     case INDEX_op_ext32s_i64:
+    case INDEX_op_ext32u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_bswap64(TCGContext *s, TCGReg ret, TCGReg arg)
 
 static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg)
 {
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
     if (use_mips32r2_instructions) {
         tcg_out_opc_bf(s, OPC_DEXT, ret, arg, 31, 0);
     } else {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_extrl_i64_i32:
         tcg_out_ext32s(s, a0, a1);
         break;
-    case INDEX_op_ext32u_i64:
     case INDEX_op_extu_i32_i64:
         tcg_out_ext32u(s, a0, a1);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16s_i32:
     case INDEX_op_ext16s_i64:
     case INDEX_op_ext32s_i64:
+    case INDEX_op_ext32u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg dst, TCGReg src)
     tcg_out32(s, EXTSW | RA(dst) | RS(src));
 }
 
-static inline void tcg_out_ext32u(TCGContext *s, TCGReg dst, TCGReg src)
+static void tcg_out_ext32u(TCGContext *s, TCGReg dst, TCGReg src)
 {
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
     tcg_out_rld(s, RLDICL, dst, src, 0, 32);
 }
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext16u_i64:
     case INDEX_op_ext32s_i64:
+    case INDEX_op_ext32u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_qemu_st(s, args, true);
         break;
 
-    case INDEX_op_ext32u_i64:
     case INDEX_op_extu_i32_i64:
         tcg_out_ext32u(s, a0, a1);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext16u_i64:
     case INDEX_op_ext32s_i64:
+    case INDEX_op_ext32u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
     tcg_out_insn(s, RRE, LGFR, dest, src);
 }
 
-static inline void tgen_ext32u(TCGContext *s, TCGReg dest, TCGReg src)
+static void tcg_out_ext32u(TCGContext *s, TCGReg dest, TCGReg src)
 {
     tcg_out_insn(s, RRE, LLGFR, dest, src);
 }
@@ -XXX,XX +XXX,XX @@ static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
 
     /* Look for the zero-extensions.  */
     if ((val & valid) == 0xffffffff) {
-        tgen_ext32u(s, dest, dest);
+        tcg_out_ext32u(s, dest, dest);
         return;
     }
     if ((val & valid) == 0xff) {
@@ -XXX,XX +XXX,XX @@ static void tgen_ctpop(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
     /* With MIE3, and bit 0 of m4 set, we get the complete result. */
     if (HAVE_FACILITY(MISC_INSN_EXT3)) {
         if (type == TCG_TYPE_I32) {
-            tgen_ext32u(s, dest, src);
+            tcg_out_ext32u(s, dest, src);
             src = dest;
         }
         tcg_out_insn(s, RRFc, POPCNT, dest, src, 8);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data,
     case MO_UL | MO_BSWAP:
         /* swapped unsigned int load with upper bits zeroed */
         tcg_out_insn(s, RXY, LRV, data, base, index, disp);
-        tgen_ext32u(s, data, data);
+        tcg_out_ext32u(s, data, data);
         break;
     case MO_UL:
         tcg_out_insn(s, RXY, LLGF, data, base, index, disp);
@@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, MemOp opc,
                  offsetof(CPUTLBEntry, addend));
 
     if (TARGET_LONG_BITS == 32) {
-        tgen_ext32u(s, TCG_REG_R3, addr_reg);
+        tcg_out_ext32u(s, TCG_REG_R3, addr_reg);
         return TCG_REG_R3;
     }
     return addr_reg;
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
         tcg_out_ext16u(s, TCG_REG_R4, data_reg);
         break;
     case MO_UL:
-        tgen_ext32u(s, TCG_REG_R4, data_reg);
+        tcg_out_ext32u(s, TCG_REG_R4, data_reg);
         break;
     case MO_UQ:
         tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
@@ -XXX,XX +XXX,XX @@ static void tcg_prepare_user_ldst(TCGContext *s, TCGReg *addr_reg,
                                   TCGReg *index_reg, tcg_target_long *disp)
 {
     if (TARGET_LONG_BITS == 32) {
-        tgen_ext32u(s, TCG_TMP0, *addr_reg);
+        tcg_out_ext32u(s, TCG_TMP0, *addr_reg);
         *addr_reg = TCG_TMP0;
     }
     if (guest_base < 0x80000) {
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         if (a2 & TCG_BSWAP_OS) {
             tcg_out_ext32s(s, a0, a0);
         } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
-            tgen_ext32u(s, a0, a0);
+            tcg_out_ext32u(s, a0, a0);
         }
         break;
 
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_ext32s(s, args[0], args[1]);
         break;
     case INDEX_op_extu_i32_i64:
-    case INDEX_op_ext32u_i64:
-        tgen_ext32u(s, args[0], args[1]);
+        tcg_out_ext32u(s, args[0], args[1]);
         break;
 
     case INDEX_op_add2_i64:
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext16u_i64:
     case INDEX_op_ext32s_i64:
+    case INDEX_op_ext32u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_arithi(s, rd, rs, 0, SHIFT_SRA);
 }
 
+static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_out_arithi(s, rd, rs, 0, SHIFT_SRL);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void emit_extend(TCGContext *s, TCGReg r, int op)
         tcg_out_ext16u(s, r, r);
         break;
     case MO_32:
-        tcg_out_arith(s, r, r, 0, SHIFT_SRL);
+        tcg_out_ext32u(s, r, r);
         break;
     case MO_64:
         break;
@@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addr, int mem_index,
 
     /* If the guest address must be zero-extended, do so now.  */
     if (TARGET_LONG_BITS == 32) {
-        tcg_out_arithi(s, r0, addr, 0, SHIFT_SRL);
+        tcg_out_ext32u(s, r0, addr);
         return r0;
     }
     return addr;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr,
     unsigned t_bits;
 
     if (TARGET_LONG_BITS == 32) {
-        tcg_out_arithi(s, TCG_REG_T1, addr, 0, SHIFT_SRL);
+        tcg_out_ext32u(s, TCG_REG_T1, addr);
         addr = TCG_REG_T1;
     }
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr,
     unsigned t_bits;
 
     if (TARGET_LONG_BITS == 32) {
-        tcg_out_arithi(s, TCG_REG_T1, addr, 0, SHIFT_SRL);
+        tcg_out_ext32u(s, TCG_REG_T1, addr);
         addr = TCG_REG_T1;
     }
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_ext32s(s, a0, a1);
         break;
     case INDEX_op_extu_i32_i64:
-    case INDEX_op_ext32u_i64:
-        tcg_out_arithi(s, a0, a1, 0, SHIFT_SRL);
+        tcg_out_ext32u(s, a0, a1);
         break;
     case INDEX_op_extrl_i64_i32:
         tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext16u_i64:
     case INDEX_op_ext32s_i64:
+    case INDEX_op_ext32u_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_op_rr(s, INDEX_op_ext32s_i64, rd, rs);
 }
 
+static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
+    tcg_debug_assert(TCG_TARGET_HAS_ext32u_i64);
+    tcg_out_op_rr(s, INDEX_op_ext32u_i64, rd, rs);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     CASE_32_64(neg)      /* Optional (TCG_TARGET_HAS_neg_*). */
     CASE_32_64(not)      /* Optional (TCG_TARGET_HAS_not_*). */
-    CASE_64(ext32u)      /* Optional (TCG_TARGET_HAS_ext32u_i64). */
     CASE_64(ext_i32)
     CASE_64(extu_i32)
     CASE_32_64(ctpop)    /* Optional (TCG_TARGET_HAS_ctpop_*). */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext16u_i64:
     case INDEX_op_ext32s_i64:
+    case INDEX_op_ext32u_i64:
     default:
         g_assert_not_reached();
     }
-- 
2.34.1

We will need a backend interface for type extension with sign.
Use it in tcg_reg_alloc_op in the meantime.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/tcg.c                        | 4 ++++
 tcg/aarch64/tcg-target.c.inc     | 9 ++++++---
 tcg/arm/tcg-target.c.inc         | 5 +++++
 tcg/i386/tcg-target.c.inc        | 9 ++++++---
 tcg/loongarch64/tcg-target.c.inc | 7 ++++++-
 tcg/mips/tcg-target.c.inc        | 7 ++++++-
 tcg/ppc/tcg-target.c.inc         | 9 ++++++---
 tcg/riscv/tcg-target.c.inc       | 7 ++++++-
 tcg/s390x/tcg-target.c.inc       | 9 ++++++---
 tcg/sparc64/tcg-target.c.inc     | 9 ++++++---
 tcg/tci/tcg-target.c.inc         | 7 ++++++-
 11 files changed, 63 insertions(+), 19 deletions(-)

diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg);
+static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
 static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
 static void tcg_out_goto_tb(TCGContext *s, int which);
@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
     case INDEX_op_ext32u_i64:
         tcg_out_ext32u(s, new_args[0], new_args[1]);
         break;
+    case INDEX_op_ext_i32_i64:
+        tcg_out_exts_i32_i64(s, new_args[0], new_args[1]);
+        break;
     default:
         if (def->flags & TCG_OPF_VECTOR) {
             tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rn)
     tcg_out_sxt(s, TCG_TYPE_I64, MO_32, rd, rn);
 }
 
+static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    tcg_out_ext32s(s, rd, rn);
+}
+
 static inline void tcg_out_uxt(TCGContext *s, MemOp s_bits,
                                TCGReg rd, TCGReg rn)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         }
         break;
 
-    case INDEX_op_ext_i32_i64:
-        tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
-        break;
     case INDEX_op_extu_i32_i64:
         tcg_out_ext32u(s, a0, a1);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i32:
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
+    case INDEX_op_ext_i32_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rn)
     g_assert_not_reached();
 }
 
+static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    g_assert_not_reached();
+}
+
 static void tcg_out_bswap16(TCGContext *s, ARMCond cond,
                             TCGReg rd, TCGReg rn, int flags)
 {
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
     tcg_out_modrm(s, OPC_MOVSLQ, dest, src);
 }
 
+static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
+{
+    tcg_out_ext32s(s, dest, src);
+}
+
 static inline void tcg_out_bswap64(TCGContext *s, int reg)
 {
     tcg_out_opc(s, OPC_BSWAP + P_REXW + LOWREGMASK(reg), 0, reg, 0);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_extrl_i64_i32:
         tcg_out_ext32u(s, a0, a1);
         break;
-    case INDEX_op_ext_i32_i64:
-        tcg_out_ext32s(s, a0, a1);
-        break;
     case INDEX_op_extrh_i64_i32:
         tcg_out_shifti(s, SHIFT_SHR + P_REXW, a0, 32);
         break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i64:
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
+    case INDEX_op_ext_i32_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.c.inc
+++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg)
     tcg_out_opc_addi_w(s, ret, arg, 0);
 }
 
+static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
+{
+    tcg_out_ext32s(s, ret, arg);
+}
+
 static void tcg_out_clzctz(TCGContext *s, LoongArchInsn opc,
                            TCGReg a0, TCGReg a1, TCGReg a2,
                            bool c2, bool is_32bit)
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 
     case INDEX_op_extrl_i64_i32:
-    case INDEX_op_ext_i32_i64:
         tcg_out_ext32s(s, a0, a1);
         break;
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i64:
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
+    case INDEX_op_ext_i32_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_opc_sa(s, OPC_SLL, rd, rs, 0);
 }
 
+static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_out_ext32s(s, rd, rs);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_extrh_i64_i32:
         tcg_out_dsra(s, a0, a1, 32);
         break;
-    case INDEX_op_ext_i32_i64:
     case INDEX_op_extrl_i64_i32:
         tcg_out_ext32s(s, a0, a1);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16s_i64:
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
+    case INDEX_op_ext_i32_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32u(TCGContext *s, TCGReg dst, TCGReg src)
     tcg_out_rld(s, RLDICL, dst, src, 0, 32);
 }
 
+static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg dst, TCGReg src)
+{
+    tcg_out_ext32s(s, dst, src);
+}
+
 static inline void tcg_out_shli32(TCGContext *s, TCGReg dst, TCGReg src, int c)
 {
     tcg_out_rlw(s, RLWINM, dst, src, c, 0, 31 - c);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_qemu_st(s, args, true);
         break;
 
-    case INDEX_op_ext_i32_i64:
-        tcg_out_ext32s(s, args[0], args[1]);
-        break;
     case INDEX_op_extu_i32_i64:
         tcg_out_ext32u(s, args[0], args[1]);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i64:
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
+    case INDEX_op_ext_i32_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg)
     tcg_out_opc_imm(s, OPC_ADDIW, ret, arg, 0);
 }
 
+static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
+{
+    tcg_out_ext32s(s, ret, arg);
+}
+
 static void tcg_out_ldst(TCGContext *s, RISCVInsn opc, TCGReg data,
                          TCGReg addr, intptr_t offset)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 
     case INDEX_op_extrl_i64_i32:
-    case INDEX_op_ext_i32_i64:
         tcg_out_ext32s(s, a0, a1);
         break;
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i64:
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
+    case INDEX_op_ext_i32_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32u(TCGContext *s, TCGReg dest, TCGReg src)
     tcg_out_insn(s, RRE, LLGFR, dest, src);
 }
 
+static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
+{
+    tcg_out_ext32s(s, dest, src);
+}
+
 static void tgen_andi_risbg(TCGContext *s, TCGReg out, TCGReg in, uint64_t val)
 {
     int msb, lsb;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         }
         break;
 
-    case INDEX_op_ext_i32_i64:
-        tcg_out_ext32s(s, args[0], args[1]);
-        break;
     case INDEX_op_extu_i32_i64:
         tcg_out_ext32u(s, args[0], args[1]);
         break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i64:
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
+    case INDEX_op_ext_i32_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_arithi(s, rd, rs, 0, SHIFT_SRL);
 }
 
+static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_out_ext32s(s, rd, rs);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_divu_i64:
         c = ARITH_UDIVX;
         goto gen_arith;
-    case INDEX_op_ext_i32_i64:
-        tcg_out_ext32s(s, a0, a1);
-        break;
     case INDEX_op_extu_i32_i64:
         tcg_out_ext32u(s, a0, a1);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i64:
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
+    case INDEX_op_ext_i32_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_op_rr(s, INDEX_op_ext32u_i64, rd, rs);
 }
 
+static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_out_ext32s(s, rd, rs);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     CASE_32_64(neg)      /* Optional (TCG_TARGET_HAS_neg_*). */
     CASE_32_64(not)      /* Optional (TCG_TARGET_HAS_not_*). */
-    CASE_64(ext_i32)
     CASE_64(extu_i32)
     CASE_32_64(ctpop)    /* Optional (TCG_TARGET_HAS_ctpop_*). */
     case INDEX_op_bswap32_i32: /* Optional (TCG_TARGET_HAS_bswap32_i32). */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i64:
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
+    case INDEX_op_ext_i32_i64:
     default:
         g_assert_not_reached();
     }
-- 
2.34.1

We will need a backend interface for type extension with zero.
Use it in tcg_reg_alloc_op in the meantime.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/tcg.c                        |  4 ++++
 tcg/aarch64/tcg-target.c.inc     | 10 ++++++----
 tcg/arm/tcg-target.c.inc         |  5 +++++
 tcg/i386/tcg-target.c.inc        |  7 ++++++-
 tcg/loongarch64/tcg-target.c.inc | 10 ++++++----
 tcg/mips/tcg-target.c.inc        |  9 ++++++---
 tcg/ppc/tcg-target.c.inc         | 10 ++++++----
 tcg/riscv/tcg-target.c.inc       | 10 ++++++----
 tcg/s390x/tcg-target.c.inc       | 10 ++++++----
 tcg/sparc64/tcg-target.c.inc     |  9 ++++++---
 tcg/tci/tcg-target.c.inc         |  7 ++++++-
 11 files changed, 63 insertions(+), 28 deletions(-)

diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
+static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
 static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
 static void tcg_out_goto_tb(TCGContext *s, int which);
@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
     case INDEX_op_ext_i32_i64:
         tcg_out_exts_i32_i64(s, new_args[0], new_args[1]);
         break;
+    case INDEX_op_extu_i32_i64:
+        tcg_out_extu_i32_i64(s, new_args[0], new_args[1]);
+        break;
     default:
         if (def->flags & TCG_OPF_VECTOR) {
             tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rn)
     tcg_out_movr(s, TCG_TYPE_I32, rd, rn);
 }
 
+static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    tcg_out_ext32u(s, rd, rn);
+}
+
 static void tcg_out_addsubi(TCGContext *s, int ext, TCGReg rd,
                             TCGReg rn, int64_t aimm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         }
         break;
 
-    case INDEX_op_extu_i32_i64:
-        tcg_out_ext32u(s, a0, a1);
-        break;
-
     case INDEX_op_deposit_i64:
     case INDEX_op_deposit_i32:
         tcg_out_dep(s, ext, a0, REG0(2), args[3], args[4]);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
+    case INDEX_op_extu_i32_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn)
     g_assert_not_reached();
 }
 
+static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    g_assert_not_reached();
+}
+
 static void tcg_out_bswap16(TCGContext *s, ARMCond cond,
                             TCGReg rd, TCGReg rn, int flags)
 {
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
     tcg_out_ext32s(s, dest, src);
 }
 
+static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
+{
+    tcg_out_ext32u(s, dest, src);
+}
+
 static inline void tcg_out_bswap64(TCGContext *s, int reg)
 {
     tcg_out_opc(s, OPC_BSWAP + P_REXW + LOWREGMASK(reg), 0, reg, 0);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_bswap64_i64:
         tcg_out_bswap64(s, a0);
         break;
-    case INDEX_op_extu_i32_i64:
     case INDEX_op_extrl_i64_i32:
         tcg_out_ext32u(s, a0, a1);
         break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
+    case INDEX_op_extu_i32_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.c.inc
+++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
     tcg_out_ext32s(s, ret, arg);
 }
 
+static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
+{
+    tcg_out_ext32u(s, ret, arg);
+}
+
 static void tcg_out_clzctz(TCGContext *s, LoongArchInsn opc,
                            TCGReg a0, TCGReg a1, TCGReg a2,
                            bool c2, bool is_32bit)
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_brcond(s, a2, a0, a1, arg_label(args[3]));
         break;
 
-    case INDEX_op_extu_i32_i64:
-        tcg_out_ext32u(s, a0, a1);
-        break;
-
     case INDEX_op_extrl_i64_i32:
         tcg_out_ext32s(s, a0, a1);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
+    case INDEX_op_extu_i32_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_ext32s(s, rd, rs);
 }
 
+static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_out_ext32u(s, rd, rs);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_extrl_i64_i32:
         tcg_out_ext32s(s, a0, a1);
         break;
-    case INDEX_op_extu_i32_i64:
-        tcg_out_ext32u(s, a0, a1);
-        break;
 
     case INDEX_op_sar_i32:
         i1 = OPC_SRAV, i2 = OPC_SRA;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
+    case INDEX_op_extu_i32_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg dst, TCGReg src)
     tcg_out_ext32s(s, dst, src);
 }
 
+static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg dst, TCGReg src)
+{
+    tcg_out_ext32u(s, dst, src);
+}
+
 static inline void tcg_out_shli32(TCGContext *s, TCGReg dst, TCGReg src, int c)
 {
     tcg_out_rlw(s, RLWINM, dst, src, c, 0, 31 - c);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_qemu_st(s, args, true);
         break;
 
-    case INDEX_op_extu_i32_i64:
-        tcg_out_ext32u(s, args[0], args[1]);
-        break;
-
     case INDEX_op_setcond_i32:
         tcg_out_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1], args[2],
                         const_args[2]);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
+    case INDEX_op_extu_i32_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
     tcg_out_ext32s(s, ret, arg);
 }
 
+static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
+{
+    tcg_out_ext32u(s, ret, arg);
+}
+
 static void tcg_out_ldst(TCGContext *s, RISCVInsn opc, TCGReg data,
                          TCGReg addr, intptr_t offset)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_qemu_st(s, args, true);
         break;
 
-    case INDEX_op_extu_i32_i64:
-        tcg_out_ext32u(s, a0, a1);
-        break;
-
     case INDEX_op_extrl_i64_i32:
         tcg_out_ext32s(s, a0, a1);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
+    case INDEX_op_extu_i32_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
     tcg_out_ext32s(s, dest, src);
 }
 
+static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
+{
+    tcg_out_ext32u(s, dest, src);
+}
+
 static void tgen_andi_risbg(TCGContext *s, TCGReg out, TCGReg in, uint64_t val)
 {
     int msb, lsb;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         }
         break;
 
-    case INDEX_op_extu_i32_i64:
-        tcg_out_ext32u(s, args[0], args[1]);
-        break;
-
     case INDEX_op_add2_i64:
         if (const_args[4]) {
             if ((int64_t)args[4] >= 0) {
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
+    case INDEX_op_extu_i32_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_ext32s(s, rd, rs);
 }
 
+static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_out_ext32u(s, rd, rs);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_divu_i64:
         c = ARITH_UDIVX;
         goto gen_arith;
-    case INDEX_op_extu_i32_i64:
-        tcg_out_ext32u(s, a0, a1);
-        break;
     case INDEX_op_extrl_i64_i32:
         tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
+    case INDEX_op_extu_i32_i64:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_ext32s(s, rd, rs);
 }
 
+static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_out_ext32u(s, rd, rs);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 
     CASE_32_64(neg)      /* Optional (TCG_TARGET_HAS_neg_*). */
     CASE_32_64(not)      /* Optional (TCG_TARGET_HAS_not_*). */
-    CASE_64(extu_i32)
     CASE_32_64(ctpop)    /* Optional (TCG_TARGET_HAS_ctpop_*). */
     case INDEX_op_bswap32_i32: /* Optional (TCG_TARGET_HAS_bswap32_i32). */
     case INDEX_op_bswap64_i64: /* Optional (TCG_TARGET_HAS_bswap64_i64). */
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32s_i64:
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
+    case INDEX_op_extu_i32_i64:
     default:
         g_assert_not_reached();
     }
-- 
2.34.1

We will need a backend interface for type truncation.  For those backends
that did not enable TCG_TARGET_HAS_extrl_i64_i32, use tcg_out_mov.
Use it in tcg_reg_alloc_op in the meantime.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/tcg.c                        |  4 ++++
 tcg/aarch64/tcg-target.c.inc     |  6 ++++++
 tcg/arm/tcg-target.c.inc         |  5 +++++
 tcg/i386/tcg-target.c.inc        |  9 ++++++---
 tcg/loongarch64/tcg-target.c.inc | 10 ++++++----
 tcg/mips/tcg-target.c.inc        |  9 ++++++---
 tcg/ppc/tcg-target.c.inc         |  7 +++++++
 tcg/riscv/tcg-target.c.inc       | 10 ++++++----
 tcg/s390x/tcg-target.c.inc       |  6 ++++++
 tcg/sparc64/tcg-target.c.inc     |  9 ++++++---
 tcg/tci/tcg-target.c.inc         |  7 +++++++
 11 files changed, 65 insertions(+), 17 deletions(-)

diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
+static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
 static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
 static void tcg_out_goto_tb(TCGContext *s, int which);
@@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
     case INDEX_op_extu_i32_i64:
         tcg_out_extu_i32_i64(s, new_args[0], new_args[1]);
         break;
+    case INDEX_op_extrl_i64_i32:
+        tcg_out_extrl_i64_i32(s, new_args[0], new_args[1]);
+        break;
     default:
         if (def->flags & TCG_OPF_VECTOR) {
             tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn)
     tcg_out_ext32u(s, rd, rn);
 }
 
+static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    tcg_out_mov(s, TCG_TYPE_I32, rd, rn);
+}
+
 static void tcg_out_addsubi(TCGContext *s, int ext, TCGReg rd,
                             TCGReg rn, int64_t aimm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
     case INDEX_op_extu_i32_i64:
+    case INDEX_op_extrl_i64_i32:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn)
     g_assert_not_reached();
 }
 
+static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    g_assert_not_reached();
+}
+
 static void tcg_out_bswap16(TCGContext *s, ARMCond cond,
                             TCGReg rd, TCGReg rn, int flags)
 {
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
     tcg_out_ext32u(s, dest, src);
 }
 
+static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg dest, TCGReg src)
+{
+    tcg_out_ext32u(s, dest, src);
+}
+
 static inline void tcg_out_bswap64(TCGContext *s, int reg)
 {
     tcg_out_opc(s, OPC_BSWAP + P_REXW + LOWREGMASK(reg), 0, reg, 0);
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_bswap64_i64:
         tcg_out_bswap64(s, a0);
         break;
-    case INDEX_op_extrl_i64_i32:
-        tcg_out_ext32u(s, a0, a1);
-        break;
     case INDEX_op_extrh_i64_i32:
         tcg_out_shifti(s, SHIFT_SHR + P_REXW, a0, 32);
         break;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
     case INDEX_op_extu_i32_i64:
+    case INDEX_op_extrl_i64_i32:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.c.inc
+++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
     tcg_out_ext32u(s, ret, arg);
 }
 
+static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg)
+{
+    tcg_out_ext32s(s, ret, arg);
+}
+
 static void tcg_out_clzctz(TCGContext *s, LoongArchInsn opc,
                            TCGReg a0, TCGReg a1, TCGReg a2,
                            bool c2, bool is_32bit)
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_brcond(s, a2, a0, a1, arg_label(args[3]));
         break;
 
-    case INDEX_op_extrl_i64_i32:
-        tcg_out_ext32s(s, a0, a1);
-        break;
-
     case INDEX_op_extrh_i64_i32:
         tcg_out_opc_srai_d(s, a0, a1, 32);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
     case INDEX_op_extu_i32_i64:
+    case INDEX_op_extrl_i64_i32:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_ext32u(s, rd, rs);
 }
 
+static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_out_ext32s(s, rd, rs);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_extrh_i64_i32:
         tcg_out_dsra(s, a0, a1, 32);
         break;
-    case INDEX_op_extrl_i64_i32:
-        tcg_out_ext32s(s, a0, a1);
-        break;
 
     case INDEX_op_sar_i32:
         i1 = OPC_SRAV, i2 = OPC_SRA;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
     case INDEX_op_extu_i32_i64:
+    case INDEX_op_extrl_i64_i32:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg dst, TCGReg src)
     tcg_out_ext32u(s, dst, src);
 }
 
+static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
+    tcg_out_mov(s, TCG_TYPE_I32, rd, rn);
+}
+
 static inline void tcg_out_shli32(TCGContext *s, TCGReg dst, TCGReg src, int c)
 {
     tcg_out_rlw(s, RLWINM, dst, src, c, 0, 31 - c);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
     case INDEX_op_extu_i32_i64:
+    case INDEX_op_extrl_i64_i32:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
     tcg_out_ext32u(s, ret, arg);
 }
 
+static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg)
+{
+    tcg_out_ext32s(s, ret, arg);
+}
+
 static void tcg_out_ldst(TCGContext *s, RISCVInsn opc, TCGReg data,
                          TCGReg addr, intptr_t offset)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_qemu_st(s, args, true);
         break;
 
-    case INDEX_op_extrl_i64_i32:
-        tcg_out_ext32s(s, a0, a1);
-        break;
-
     case INDEX_op_extrh_i64_i32:
         tcg_out_opc_imm(s, OPC_SRAI, a0, a1, 32);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
     case INDEX_op_extu_i32_i64:
+    case INDEX_op_extrl_i64_i32:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
     tcg_out_ext32u(s, dest, src);
 }
 
+static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg dest, TCGReg src)
+{
+    tcg_out_mov(s, TCG_TYPE_I32, dest, src);
+}
+
 static void tgen_andi_risbg(TCGContext *s, TCGReg out, TCGReg in, uint64_t val)
 {
     int msb, lsb;
@@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
     case INDEX_op_extu_i32_i64:
+    case INDEX_op_extrl_i64_i32:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_ext32u(s, rd, rs);
 }
 
+static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_out_mov(s, TCG_TYPE_I32, rd, rs);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_divu_i64:
         c = ARITH_UDIVX;
         goto gen_arith;
-    case INDEX_op_extrl_i64_i32:
-        tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
-        break;
     case INDEX_op_extrh_i64_i32:
         tcg_out_arithi(s, a0, a1, 32, SHIFT_SRLX);
         break;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
     case INDEX_op_extu_i32_i64:
+    case INDEX_op_extrl_i64_i32:
     default:
         g_assert_not_reached();
     }
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_ext32u(s, rd, rs);
 }
 
+static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs)
+{
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
+    tcg_out_mov(s, TCG_TYPE_I32, rd, rs);
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext32u_i64:
     case INDEX_op_ext_i32_i64:
     case INDEX_op_extu_i32_i64:
+    case INDEX_op_extrl_i64_i32:
     default:
         g_assert_not_reached();
     }
-- 
2.34.1

This is common code in most qemu_{ld,st} slow paths, extending the
input value for the store helper data argument or extending the
return value from the load helper.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/tcg.c                        | 63 ++++++++++++++++++++++++++++++++
 tcg/aarch64/tcg-target.c.inc     |  8 +---
 tcg/arm/tcg-target.c.inc         | 16 ++------
 tcg/i386/tcg-target.c.inc        | 30 +++------------
 tcg/loongarch64/tcg-target.c.inc | 53 ++++-----------------------
 tcg/ppc/tcg-target.c.inc         | 38 +++++--------------
 tcg/riscv/tcg-target.c.inc       | 13 +------
 tcg/s390x/tcg-target.c.inc       | 19 ++--------
 tcg/sparc64/tcg-target.c.inc     | 31 +++-------------
 9 files changed, 103 insertions(+), 168 deletions(-)

diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ void tcg_raise_tb_overflow(TCGContext *s)
     siglongjmp(s->jmp_trans, -2);
 }
 
+/**
+ * tcg_out_movext -- move and extend
+ * @s: tcg context
+ * @dst_type: integral type for destination
+ * @dst: destination register
+ * @src_type: integral type for source
+ * @src_ext: extension to apply to source
+ * @src: source register
+ *
+ * Move or extend @src into @dst, depending on @src_ext and the types.
+ */
+static void __attribute__((unused))
+tcg_out_movext(TCGContext *s, TCGType dst_type, TCGReg dst,
+               TCGType src_type, MemOp src_ext, TCGReg src)
+{
+    switch (src_ext) {
+    case MO_UB:
+        tcg_out_ext8u(s, dst, src);
+        break;
+    case MO_SB:
+        tcg_out_ext8s(s, dst_type, dst, src);
+        break;
+    case MO_UW:
+        tcg_out_ext16u(s, dst, src);
+        break;
+    case MO_SW:
+        tcg_out_ext16s(s, dst_type, dst, src);
+        break;
+    case MO_UL:
+    case MO_SL:
+        if (dst_type == TCG_TYPE_I32) {
+            if (src_type == TCG_TYPE_I32) {
+                tcg_out_mov(s, TCG_TYPE_I32, dst, src);
+            } else {
+                tcg_out_extrl_i64_i32(s, dst, src);
+            }
+        } else if (src_type == TCG_TYPE_I32) {
+            if (src_ext & MO_SIGN) {
+                tcg_out_exts_i32_i64(s, dst, src);
+            } else {
+                tcg_out_extu_i32_i64(s, dst, src);
+            }
+        } else {
+            if (src_ext & MO_SIGN) {
+                tcg_out_ext32s(s, dst, src);
+            } else {
+                tcg_out_ext32u(s, dst, src);
+            }
+        }
+        break;
+    case MO_UQ:
+        tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
+        if (dst_type == TCG_TYPE_I32) {
+            tcg_out_extrl_i64_i32(s, dst, src);
+        } else {
+            tcg_out_mov(s, TCG_TYPE_I64, dst, src);
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
 #define C_PFX1(P, A)                    P##A
 #define C_PFX2(P, A, B)                 P##A##_##B
 #define C_PFX3(P, A, B, C)              P##A##_##B##_##C
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
 {
     MemOpIdx oi = lb->oi;
     MemOp opc = get_memop(oi);
-    MemOp size = opc & MO_SIZE;
 
     if (!reloc_pc19(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
         return false;
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
     tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X2, oi);
     tcg_out_adr(s, TCG_REG_X3, lb->raddr);
     tcg_out_call_int(s, qemu_ld_helpers[opc & MO_SIZE]);
-    if (opc & MO_SIGN) {
-        tcg_out_sxt(s, lb->type, size, lb->datalo_reg, TCG_REG_X0);
-    } else {
-        tcg_out_mov(s, size == MO_64, lb->datalo_reg, TCG_REG_X0);
-    }
 
+    tcg_out_movext(s, lb->type, lb->datalo_reg,
+                   TCG_TYPE_REG, opc & MO_SSIZE, TCG_REG_X0);
     tcg_out_goto(s, lb->raddr);
     return true;
 }
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
 
     datalo = lb->datalo_reg;
     datahi = lb->datahi_reg;
-    switch (opc & MO_SSIZE) {
-    case MO_SB:
-        tcg_out_ext8s(s, TCG_TYPE_I32, datalo, TCG_REG_R0);
-        break;
-    case MO_SW:
-        tcg_out_ext16s(s, TCG_TYPE_I32, datalo, TCG_REG_R0);
-        break;
-    default:
-        tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_R0);
-        break;
-    case MO_UQ:
+    if ((opc & MO_SIZE) == MO_64) {
         if (datalo != TCG_REG_R1) {
             tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_R0);
             tcg_out_mov_reg(s, COND_AL, datahi, TCG_REG_R1);
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
             tcg_out_mov_reg(s, COND_AL, datahi, TCG_REG_R1);
             tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_TMP);
         }
-        break;
+    } else {
+        tcg_out_movext(s, TCG_TYPE_I32, datalo,
+                       TCG_TYPE_I32, opc & MO_SSIZE, TCG_REG_R0);
     }
 
     tcg_out_goto(s, COND_AL, lb->raddr);
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
     tcg_out_branch(s, 1, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]);
 
     data_reg = l->datalo_reg;
-    switch (opc & MO_SSIZE) {
-    case MO_SB:
-        tcg_out_ext8s(s, l->type, data_reg, TCG_REG_EAX);
-        break;
-    case MO_SW:
-        tcg_out_ext16s(s, l->type, data_reg, TCG_REG_EAX);
-        break;
-#if TCG_TARGET_REG_BITS == 64
-    case MO_SL:
-        tcg_out_ext32s(s, data_reg, TCG_REG_EAX);
-        break;
-#endif
-    case MO_UB:
-    case MO_UW:
-        /* Note that the helpers have zero-extended to tcg_target_long.  */
-    case MO_UL:
-        tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
-        break;
-    case MO_UQ:
-        if (TCG_TARGET_REG_BITS == 64) {
-            tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_RAX);
-        } else if (data_reg == TCG_REG_EDX) {
+    if (TCG_TARGET_REG_BITS == 32 && (opc & MO_SIZE) == MO_64) {
+        if (data_reg == TCG_REG_EDX) {
             /* xchg %edx, %eax */
             tcg_out_opc(s, OPC_XCHG_ax_r32 + TCG_REG_EDX, 0, 0, 0);
             tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EAX);
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
             tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
             tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EDX);
         }
-        break;
-    default:
-        g_assert_not_reached();
+    } else {
+        tcg_out_movext(s, l->type, data_reg,
+                       TCG_TYPE_REG, opc & MO_SSIZE, TCG_REG_EAX);
     }
 
     /* Jump to the code corresponding to next IR of qemu_st */
diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.c.inc
+++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
     MemOpIdx oi = l->oi;
     MemOp opc = get_memop(oi);
     MemOp size = opc & MO_SIZE;
-    TCGType type = l->type;
 
     /* resolve label address */
     if (!reloc_br_sk16(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
 
     tcg_out_call_int(s, qemu_ld_helpers[size], false);
 
-    switch (opc & MO_SSIZE) {
-    case MO_SB:
-        tcg_out_ext8s(s, type, l->datalo_reg, TCG_REG_A0);
-        break;
-    case MO_SW:
-        tcg_out_ext16s(s, type, l->datalo_reg, TCG_REG_A0);
-        break;
-    case MO_SL:
-        tcg_out_ext32s(s, l->datalo_reg, TCG_REG_A0);
-        break;
-    case MO_UL:
-        if (type == TCG_TYPE_I32) {
-            /* MO_UL loads of i32 should be sign-extended too */
-            tcg_out_ext32s(s, l->datalo_reg, TCG_REG_A0);
-            break;
-        }
-        /* fallthrough */
-    default:
-        tcg_out_mov(s, type, l->datalo_reg, TCG_REG_A0);
-        break;
-    }
-
+    tcg_out_movext(s, l->type, l->datalo_reg,
+                   TCG_TYPE_REG, opc & MO_SSIZE, TCG_REG_A0);
     return tcg_out_goto(s, l->raddr);
 }
 
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
     /* call store helper */
     tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_A0, TCG_AREG0);
     tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_A1, l->addrlo_reg);
-    switch (size) {
-    case MO_8:
-        tcg_out_ext8u(s, TCG_REG_A2, l->datalo_reg);
-        break;
-    case MO_16:
-        tcg_out_ext16u(s, TCG_REG_A2, l->datalo_reg);
-        break;
-    case MO_32:
-        tcg_out_ext32u(s, TCG_REG_A2, l->datalo_reg);
-        break;
-    case MO_64:
-        tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_A2, l->datalo_reg);
-        break;
-    default:
-        g_assert_not_reached();
-        break;
-    }
+    tcg_out_movext(s, size == MO_64 ? TCG_TYPE_I32 : TCG_TYPE_I32, TCG_REG_A2,
+                   l->type, size, l->datalo_reg);
     tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A3, oi);
     tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A4, (tcg_target_long)l->raddr);
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_indexed(TCGContext *s, TCGReg data,
     }
 }
 
-static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args)
+static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, TCGType type)
 {
     TCGReg addr_regl;
     TCGReg data_regl;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args)
     tcg_out_tlb_load(s, addr_regl, oi, label_ptr, 0);
     base = tcg_out_zext_addr_if_32_bit(s, addr_regl, TCG_REG_TMP0);
     tcg_out_qemu_st_indexed(s, data_regl, base, TCG_REG_TMP2, opc);
-    add_qemu_ldst_label(s, 0, oi,
-                        0, /* type param is unused for stores */
+    add_qemu_ldst_label(s, 0, oi, type,
                         data_regl, addr_regl,
                         s->code_ptr, label_ptr);
 #else
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_qemu_ld(s, args, TCG_TYPE_I64);
         break;
     case INDEX_op_qemu_st_i32:
-        tcg_out_qemu_st(s, args);
+        tcg_out_qemu_st(s, args, TCG_TYPE_I32);
         break;
     case INDEX_op_qemu_st_i64:
-        tcg_out_qemu_st(s, args);
+        tcg_out_qemu_st(s, args, TCG_TYPE_I64);
         break;
 
     case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static const uint32_t qemu_stx_opc[(MO_SIZE + MO_BSWAP) + 1] = {
     [MO_BSWAP | MO_UQ] = STDBRX,
 };
 
-static const uint32_t qemu_exts_opc[4] = {
-    EXTSB, EXTSH, EXTSW, 0
-};
-
 #if defined (CONFIG_SOFTMMU)
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
  *                                 int mmu_idx, uintptr_t ra)
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
     if (TCG_TARGET_REG_BITS == 32 && (opc & MO_SIZE) == MO_64) {
         tcg_out_mov(s, TCG_TYPE_I32, lo, TCG_REG_R4);
         tcg_out_mov(s, TCG_TYPE_I32, hi, TCG_REG_R3);
-    } else if (opc & MO_SIGN) {
-        uint32_t insn = qemu_exts_opc[opc & MO_SIZE];
-        tcg_out32(s, insn | RA(lo) | RS(TCG_REG_R3));
     } else {
-        tcg_out_mov(s, TCG_TYPE_REG, lo, TCG_REG_R3);
+        tcg_out_movext(s, lb->type, lo,
+                       TCG_TYPE_REG, opc & MO_SSIZE, TCG_REG_R3);
     }
 
     tcg_out_b(s, 0, lb->raddr);
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
 
     lo = lb->datalo_reg;
     hi = lb->datahi_reg;
-    if (TCG_TARGET_REG_BITS == 32) {
-        switch (s_bits) {
-        case MO_64:
-            arg |= (TCG_TARGET_CALL_ARG_I64 == TCG_CALL_ARG_EVEN);
-            tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
-            /* FALLTHRU */
-        case MO_32:
-            tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
-            break;
-        default:
-            tcg_out_rlw(s, RLWINM, arg++, lo, 0, 32 - (8 << s_bits), 31);
-            break;
-        }
+    if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) {
+        arg |= (TCG_TARGET_CALL_ARG_I64 == TCG_CALL_ARG_EVEN);
+        tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
+        tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
     } else {
-        if (s_bits == MO_64) {
-            tcg_out_mov(s, TCG_TYPE_I64, arg++, lo);
-        } else {
-            tcg_out_rld(s, RLDICL, arg++, lo, 0, 64 - (8 << s_bits));
-        }
+        tcg_out_movext(s, s_bits == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32,
+                       arg++, lb->type, s_bits, lo);
     }
 
     tcg_out_movi(s, TCG_TYPE_I32, arg++, oi);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
         } else {
             insn = qemu_ldx_opc[opc & (MO_SIZE | MO_BSWAP)];
             tcg_out32(s, insn | TAB(datalo, rbase, addrlo));
-            insn = qemu_exts_opc[s_bits];
-            tcg_out32(s, insn | RA(datalo) | RS(datalo));
+            tcg_out_movext(s, TCG_TYPE_REG, datalo,
+                           TCG_TYPE_REG, opc & MO_SSIZE, datalo);
         }
     }
 
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
     /* call store helper */
     tcg_out_mov(s, TCG_TYPE_PTR, a0, TCG_AREG0);
     tcg_out_mov(s, TCG_TYPE_PTR, a1, l->addrlo_reg);
-    tcg_out_mov(s, TCG_TYPE_PTR, a2, l->datalo_reg);
-    switch (s_bits) {
-    case MO_8:
-        tcg_out_ext8u(s, a2, a2);
-        break;
-    case MO_16:
-        tcg_out_ext16u(s, a2, a2);
-        break;
-    default:
-        break;
-    }
+    tcg_out_movext(s, s_bits == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32, a2,
+                   l->type, s_bits, l->datalo_reg);
     tcg_out_movi(s, TCG_TYPE_PTR, a3, oi);
     tcg_out_movi(s, TCG_TYPE_PTR, a4, (tcg_target_long)l->raddr);
 
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
     TCGReg data_reg = lb->datalo_reg;
     MemOpIdx oi = lb->oi;
     MemOp opc = get_memop(oi);
+    MemOp size = opc & MO_SIZE;
 
     if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL,
                      (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) {
@@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
     if (TARGET_LONG_BITS == 64) {
         tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R3, addr_reg);
     }
-    switch (opc & MO_SIZE) {
-    case MO_UB:
-        tcg_out_ext8u(s, TCG_REG_R4, data_reg);
-        break;
-    case MO_UW:
-        tcg_out_ext16u(s, TCG_REG_R4, data_reg);
-        break;
-    case MO_UL:
-        tcg_out_ext32u(s, TCG_REG_R4, data_reg);
-        break;
-    case MO_UQ:
-        tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
-        break;
-    default:
-        g_assert_not_reached();
-    }
+    tcg_out_movext(s, size == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32,
+                   TCG_REG_R4, lb->type, size, data_reg);
     tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R5, oi);
     tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R6, (uintptr_t)lb->raddr);
     tcg_out_call_int(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_mb(TCGContext *s, TCGArg a0)
 static const tcg_insn_unit *qemu_ld_trampoline[(MO_SSIZE | MO_BSWAP) + 1];
 static const tcg_insn_unit *qemu_st_trampoline[(MO_SIZE | MO_BSWAP) + 1];
 
-static void emit_extend(TCGContext *s, TCGReg r, int op)
-{
-    /* Emit zero extend of 8, 16 or 32 bit data as
-     * required by the MO_* value op; do nothing for 64 bit.
-     */
-    switch (op & MO_SIZE) {
-    case MO_8:
-        tcg_out_ext8u(s, r, r);
-        break;
-    case MO_16:
-        tcg_out_ext16u(s, r, r);
-        break;
-    case MO_32:
-        tcg_out_ext32u(s, r, r);
-        break;
-    case MO_64:
-        break;
-    }
-}
-
 static void build_trampolines(TCGContext *s)
 {
     static void * const qemu_ld_helpers[] = {
@@ -XXX,XX +XXX,XX @@ static void build_trampolines(TCGContext *s)
         }
         qemu_st_trampoline[i] = tcg_splitwx_to_rx(s->code_ptr);
 
-        emit_extend(s, TCG_REG_O2, i);
-
         /* Set the retaddr operand.  */
         tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O4, TCG_REG_O7);
 
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr,
 }
 
 static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr,
-                            MemOpIdx oi)
+                            MemOpIdx oi, TCGType data_type)
 {
     MemOp memop = get_memop(oi);
     tcg_insn_unit *label_ptr;
@@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr,
     /* TLB Miss.  */
 
     tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_O1, addrz);
-    tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_O2, data);
+    tcg_out_movext(s, (memop & MO_SIZE) == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32,
+                   TCG_REG_O2, data_type, memop & MO_SIZE, data);
 
     func = qemu_st_trampoline[memop & (MO_BSWAP | MO_SIZE)];
     tcg_debug_assert(func != NULL);
@@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_qemu_ld(s, a0, a1, a2, true);
         break;
     case INDEX_op_qemu_st_i32:
+        tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32);
+        break;
     case INDEX_op_qemu_st_i64:
-        tcg_out_qemu_st(s, a0, a1, a2);
+        tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I64);
         break;
 
     case INDEX_op_ld32s_i64:
-- 
2.34.1

We will want a backend interface for register swapping.
This is only properly defined for x86; all others get a
stub version that always indicates failure.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/tcg.c                        | 2 ++
 tcg/aarch64/tcg-target.c.inc     | 5 +++++
 tcg/arm/tcg-target.c.inc         | 5 +++++
 tcg/i386/tcg-target.c.inc        | 8 ++++++++
 tcg/loongarch64/tcg-target.c.inc | 5 +++++
 tcg/mips/tcg-target.c.inc        | 5 +++++
 tcg/ppc/tcg-target.c.inc         | 5 +++++
 tcg/riscv/tcg-target.c.inc       | 5 +++++
 tcg/s390x/tcg-target.c.inc       | 5 +++++
 tcg/sparc64/tcg-target.c.inc     | 5 +++++
 tcg/tci/tcg-target.c.inc         | 5 +++++
 11 files changed, 55 insertions(+)

diff --git a/tcg/tcg.c b/tcg/tcg.c
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -XXX,XX +XXX,XX @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg);
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
+static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
+    __attribute__((unused));
 static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
 static void tcg_out_goto_tb(TCGContext *s, int which);
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
     tcg_out_insn(s, 3305, LDR, 0, rd);
 }
 
+static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
+{
+    return false;
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type,
     tcg_out_movi32(s, COND_AL, ret, arg);
 }
 
+static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
+{
+    return false;
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct)
 #define OPC_VPTERNLOGQ  (0x25 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX)
 #define OPC_VZEROUPPER  (0x77 | P_EXT)
 #define OPC_XCHG_ax_r32	(0x90)
+#define OPC_XCHG_EvGv   (0x87)
 
 #define OPC_GRP3_Eb     (0xf6)
 #define OPC_GRP3_Ev     (0xf7)
@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type,
     }
 }
 
+static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
+{
+    int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
+    tcg_out_modrm(s, OPC_XCHG_EvGv + rexw, r1, r2);
+    return true;
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/loongarch64/tcg-target.c.inc
+++ b/tcg/loongarch64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_addi(TCGContext *s, TCGType type, TCGReg rd,
     }
 }
 
+static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
+{
+    return false;
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_ext32s(s, rd, rs);
 }
 
+static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
+{
+    return false;
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg ret,
     }
 }
 
+static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
+{
+    return false;
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
     tcg_out_opc_imm(s, OPC_LD, rd, rd, 0);
 }
 
+static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
+{
+    return false;
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
     return false;
 }
 
+static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
+{
+    return false;
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_mov(s, TCG_TYPE_I32, rd, rs);
 }
 
+static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
+{
+    return false;
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index XXXXXXX..XXXXXXX 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -XXX,XX +XXX,XX @@ static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs)
     tcg_out_mov(s, TCG_TYPE_I32, rd, rs);
 }
 
+static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
+{
+    return false;
+}
+
 static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
                              tcg_target_long imm)
 {
-- 
2.34.1