The following changes since commit c52d69e7dbaaed0ffdef8125e79218672c30161d: Merge remote-tracking branch 'remotes/cschoenebeck/tags/pull-9p-20211027' into staging (2021-10-27 11:45:18 -0700) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20211027 for you to fetch changes up to 820c025f0dcacf2f3c12735b1f162893fbfa7bc6: tcg/optimize: Propagate sign info for shifting (2021-10-27 17:11:23 -0700) ---------------------------------------------------------------- Improvements to qemu/int128 Fixes for 128/64 division. Cleanup tcg/optimize.c Optimize redundant sign extensions ---------------------------------------------------------------- Frédéric Pétrot (1): qemu/int128: Add int128_{not,xor} Luis Pires (4): host-utils: move checks out of divu128/divs128 host-utils: move udiv_qrnnd() to host-utils host-utils: add 128-bit quotient support to divu128/divs128 host-utils: add unit tests for divu128/divs128 Richard Henderson (51): tcg/optimize: Rename "mask" to "z_mask" tcg/optimize: Split out OptContext tcg/optimize: Remove do_default label tcg/optimize: Change tcg_opt_gen_{mov,movi} interface tcg/optimize: Move prev_mb into OptContext tcg/optimize: Split out init_arguments tcg/optimize: Split out copy_propagate tcg/optimize: Split out fold_call tcg/optimize: Drop nb_oargs, nb_iargs locals tcg/optimize: Change fail return for do_constant_folding_cond* tcg/optimize: Return true from tcg_opt_gen_{mov,movi} tcg/optimize: Split out finish_folding tcg/optimize: Use a boolean to avoid a mass of continues tcg/optimize: Split out fold_mb, fold_qemu_{ld,st} tcg/optimize: Split out fold_const{1,2} tcg/optimize: Split out fold_setcond2 tcg/optimize: Split out fold_brcond2 tcg/optimize: Split out fold_brcond tcg/optimize: Split out fold_setcond tcg/optimize: Split out fold_mulu2_i32 tcg/optimize: Split out fold_addsub2_i32 tcg/optimize: Split out fold_movcond tcg/optimize: Split out fold_extract2 tcg/optimize: Split out fold_extract, fold_sextract tcg/optimize: Split out fold_deposit tcg/optimize: Split out fold_count_zeros tcg/optimize: Split out fold_bswap tcg/optimize: Split out fold_dup, fold_dup2 tcg/optimize: Split out fold_mov tcg/optimize: Split out fold_xx_to_i tcg/optimize: Split out fold_xx_to_x tcg/optimize: Split out fold_xi_to_i tcg/optimize: Add type to OptContext tcg/optimize: Split out fold_to_not tcg/optimize: Split out fold_sub_to_neg tcg/optimize: Split out fold_xi_to_x tcg/optimize: Split out fold_ix_to_i tcg/optimize: Split out fold_masks tcg/optimize: Expand fold_mulu2_i32 to all 4-arg multiplies tcg/optimize: Expand fold_addsub2_i32 to 64-bit ops tcg/optimize: Sink commutative operand swapping into fold functions tcg/optimize: Stop forcing z_mask to "garbage" for 32-bit values tcg/optimize: Use fold_xx_to_i for orc tcg/optimize: Use fold_xi_to_x for mul tcg/optimize: Use fold_xi_to_x for div tcg/optimize: Use fold_xx_to_i for rem tcg/optimize: Optimize sign extensions tcg/optimize: Propagate sign info for logical operations tcg/optimize: Propagate sign info for setcond tcg/optimize: Propagate sign info for bit counting tcg/optimize: Propagate sign info for shifting include/fpu/softfloat-macros.h | 82 -- include/hw/clock.h | 5 +- include/qemu/host-utils.h | 121 +- include/qemu/int128.h | 20 + target/ppc/int_helper.c | 23 +- tcg/optimize.c | 2644 ++++++++++++++++++++++++---------------- tests/unit/test-div128.c | 197 +++ util/host-utils.c | 147 ++- tests/unit/meson.build | 1 + 9 files changed, 2053 insertions(+), 1187 deletions(-) create mode 100644 tests/unit/test-div128.c From: Frédéric Pétrot <frederic.petrot@univ-grenoble-alpes.fr> Addition of not and xor on 128-bit integers. Signed-off-by: Frédéric Pétrot <frederic.petrot@univ-grenoble-alpes.fr> Co-authored-by: Fabien Portas <fabien.portas@grenoble-inp.org> Message-Id: <20211025122818.168890-3-frederic.petrot@univ-grenoble-alpes.fr> [rth: Split out logical operations.] Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/qemu/int128.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/include/qemu/int128.h b/include/qemu/int128.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/int128.h +++ b/include/qemu/int128.h @@ -XXX,XX +XXX,XX @@ static inline Int128 int128_exts64(int64_t a) return a; } +static inline Int128 int128_not(Int128 a) +{ + return ~a; +} + static inline Int128 int128_and(Int128 a, Int128 b) { return a & b; @@ -XXX,XX +XXX,XX @@ static inline Int128 int128_or(Int128 a, Int128 b) return a | b; } +static inline Int128 int128_xor(Int128 a, Int128 b) +{ + return a ^ b; +} + static inline Int128 int128_rshift(Int128 a, int n) { return a >> n; @@ -XXX,XX +XXX,XX @@ static inline Int128 int128_exts64(int64_t a) return int128_make128(a, (a < 0) ? -1 : 0); } +static inline Int128 int128_not(Int128 a) +{ + return int128_make128(~a.lo, ~a.hi); +} + static inline Int128 int128_and(Int128 a, Int128 b) { return int128_make128(a.lo & b.lo, a.hi & b.hi); @@ -XXX,XX +XXX,XX @@ static inline Int128 int128_or(Int128 a, Int128 b) return int128_make128(a.lo | b.lo, a.hi | b.hi); } +static inline Int128 int128_xor(Int128 a, Int128 b) +{ + return int128_make128(a.lo ^ b.lo, a.hi ^ b.hi); +} + static inline Int128 int128_rshift(Int128 a, int n) { int64_t h; -- 2.25.1 From: Luis Pires <luis.pires@eldorado.org.br> In preparation for changing the divu128/divs128 implementations to allow for quotients larger than 64 bits, move the div-by-zero and overflow checks to the callers. Signed-off-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20211025191154.350831-2-luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/hw/clock.h | 5 +++-- include/qemu/host-utils.h | 34 ++++++++++++--------------------- target/ppc/int_helper.c | 14 +++++++++----- util/host-utils.c | 40 ++++++++++++++++++--------------------- 4 files changed, 42 insertions(+), 51 deletions(-) diff --git a/include/hw/clock.h b/include/hw/clock.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/clock.h +++ b/include/hw/clock.h @@ -XXX,XX +XXX,XX @@ static inline uint64_t clock_ns_to_ticks(const Clock *clk, uint64_t ns) return 0; } /* - * Ignore divu128() return value as we've caught div-by-zero and don't - * need different behaviour for overflow. + * BUG: when CONFIG_INT128 is not defined, the current implementation of + * divu128 does not return a valid truncated quotient, so the result will + * be wrong. */ divu128(&lo, &hi, clk->period); return lo; diff --git a/include/qemu/host-utils.h b/include/qemu/host-utils.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/host-utils.h +++ b/include/qemu/host-utils.h @@ -XXX,XX +XXX,XX @@ static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c) return (__int128_t)a * b / c; } -static inline int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) +static inline void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) { - if (divisor == 0) { - return 1; - } else { - __uint128_t dividend = ((__uint128_t)*phigh << 64) | *plow; - __uint128_t result = dividend / divisor; - *plow = result; - *phigh = dividend % divisor; - return result > UINT64_MAX; - } + __uint128_t dividend = ((__uint128_t)*phigh << 64) | *plow; + __uint128_t result = dividend / divisor; + *plow = result; + *phigh = dividend % divisor; } -static inline int divs128(int64_t *plow, int64_t *phigh, int64_t divisor) +static inline void divs128(int64_t *plow, int64_t *phigh, int64_t divisor) { - if (divisor == 0) { - return 1; - } else { - __int128_t dividend = ((__int128_t)*phigh << 64) | (uint64_t)*plow; - __int128_t result = dividend / divisor; - *plow = result; - *phigh = dividend % divisor; - return result != *plow; - } + __int128_t dividend = ((__int128_t)*phigh << 64) | (uint64_t)*plow; + __int128_t result = dividend / divisor; + *plow = result; + *phigh = dividend % divisor; } #else void muls64(uint64_t *plow, uint64_t *phigh, int64_t a, int64_t b); void mulu64(uint64_t *plow, uint64_t *phigh, uint64_t a, uint64_t b); -int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor); -int divs128(int64_t *plow, int64_t *phigh, int64_t divisor); +void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor); +void divs128(int64_t *plow, int64_t *phigh, int64_t divisor); static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c) { diff --git a/target/ppc/int_helper.c b/target/ppc/int_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/int_helper.c +++ b/target/ppc/int_helper.c @@ -XXX,XX +XXX,XX @@ uint64_t helper_divdeu(CPUPPCState *env, uint64_t ra, uint64_t rb, uint32_t oe) uint64_t rt = 0; int overflow = 0; - overflow = divu128(&rt, &ra, rb); - - if (unlikely(overflow)) { + if (unlikely(rb == 0 || ra >= rb)) { + overflow = 1; rt = 0; /* Undefined */ + } else { + divu128(&rt, &ra, rb); } if (oe) { @@ -XXX,XX +XXX,XX @@ uint64_t helper_divde(CPUPPCState *env, uint64_t rau, uint64_t rbu, uint32_t oe) int64_t rt = 0; int64_t ra = (int64_t)rau; int64_t rb = (int64_t)rbu; - int overflow = divs128(&rt, &ra, rb); + int overflow = 0; - if (unlikely(overflow)) { + if (unlikely(rb == 0 || uabs64(ra) >= uabs64(rb))) { + overflow = 1; rt = 0; /* Undefined */ + } else { + divs128(&rt, &ra, rb); } if (oe) { diff --git a/util/host-utils.c b/util/host-utils.c index XXXXXXX..XXXXXXX 100644 --- a/util/host-utils.c +++ b/util/host-utils.c @@ -XXX,XX +XXX,XX @@ void muls64 (uint64_t *plow, uint64_t *phigh, int64_t a, int64_t b) *phigh = rh; } -/* Unsigned 128x64 division. Returns 1 if overflow (divide by zero or */ -/* quotient exceeds 64 bits). Otherwise returns quotient via plow and */ -/* remainder via phigh. */ -int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) +/* + * Unsigned 128-by-64 division. Returns quotient via plow and + * remainder via phigh. + * The result must fit in 64 bits (plow) - otherwise, the result + * is undefined. + * This function will cause a division by zero if passed a zero divisor. + */ +void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) { uint64_t dhi = *phigh; uint64_t dlo = *plow; unsigned i; uint64_t carry = 0; - if (divisor == 0) { - return 1; - } else if (dhi == 0) { + if (divisor == 0 || dhi == 0) { *plow = dlo / divisor; *phigh = dlo % divisor; - return 0; - } else if (dhi >= divisor) { - return 1; } else { for (i = 0; i < 64; i++) { @@ -XXX,XX +XXX,XX @@ int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) *plow = dlo; *phigh = dhi; - return 0; } } -int divs128(int64_t *plow, int64_t *phigh, int64_t divisor) +/* + * Signed 128-by-64 division. Returns quotient via plow and + * remainder via phigh. + * The result must fit in 64 bits (plow) - otherwise, the result + * is undefined. + * This function will cause a division by zero if passed a zero divisor. + */ +void divs128(int64_t *plow, int64_t *phigh, int64_t divisor) { int sgn_dvdnd = *phigh < 0; int sgn_divsr = divisor < 0; - int overflow = 0; if (sgn_dvdnd) { *plow = ~(*plow); @@ -XXX,XX +XXX,XX @@ int divs128(int64_t *plow, int64_t *phigh, int64_t divisor) divisor = 0 - divisor; } - overflow = divu128((uint64_t *)plow, (uint64_t *)phigh, (uint64_t)divisor); + divu128((uint64_t *)plow, (uint64_t *)phigh, (uint64_t)divisor); if (sgn_dvdnd ^ sgn_divsr) { *plow = 0 - *plow; } - - if (!overflow) { - if ((*plow < 0) ^ (sgn_dvdnd ^ sgn_divsr)) { - overflow = 1; - } - } - - return overflow; } #endif -- 2.25.1 From: Luis Pires <luis.pires@eldorado.org.br> Move udiv_qrnnd() from include/fpu/softfloat-macros.h to host-utils, so it can be reused by divu128(). Signed-off-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20211025191154.350831-3-luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-macros.h | 82 ---------------------------------- include/qemu/host-utils.h | 81 +++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 82 deletions(-) diff --git a/include/fpu/softfloat-macros.h b/include/fpu/softfloat-macros.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-macros.h +++ b/include/fpu/softfloat-macros.h @@ -XXX,XX +XXX,XX @@ * so some portions are provided under: * the SoftFloat-2a license * the BSD license - * GPL-v2-or-later * * Any future contributions to this file after December 1st 2014 will be * taken to be licensed under the Softfloat-2a license unless specifically @@ -XXX,XX +XXX,XX @@ this code that are retained. * THE POSSIBILITY OF SUCH DAMAGE. */ -/* Portions of this work are licensed under the terms of the GNU GPL, - * version 2 or later. See the COPYING file in the top-level directory. - */ - #ifndef FPU_SOFTFLOAT_MACROS_H #define FPU_SOFTFLOAT_MACROS_H @@ -XXX,XX +XXX,XX @@ static inline uint64_t estimateDiv128To64(uint64_t a0, uint64_t a1, uint64_t b) } -/* From the GNU Multi Precision Library - longlong.h __udiv_qrnnd - * (https://gmplib.org/repo/gmp/file/tip/longlong.h) - * - * Licensed under the GPLv2/LGPLv3 - */ -static inline uint64_t udiv_qrnnd(uint64_t *r, uint64_t n1, - uint64_t n0, uint64_t d) -{ -#if defined(__x86_64__) - uint64_t q; - asm("divq %4" : "=a"(q), "=d"(*r) : "0"(n0), "1"(n1), "rm"(d)); - return q; -#elif defined(__s390x__) && !defined(__clang__) - /* Need to use a TImode type to get an even register pair for DLGR. */ - unsigned __int128 n = (unsigned __int128)n1 << 64 | n0; - asm("dlgr %0, %1" : "+r"(n) : "r"(d)); - *r = n >> 64; - return n; -#elif defined(_ARCH_PPC64) && defined(_ARCH_PWR7) - /* From Power ISA 2.06, programming note for divdeu. */ - uint64_t q1, q2, Q, r1, r2, R; - asm("divdeu %0,%2,%4; divdu %1,%3,%4" - : "=&r"(q1), "=r"(q2) - : "r"(n1), "r"(n0), "r"(d)); - r1 = -(q1 * d); /* low part of (n1<<64) - (q1 * d) */ - r2 = n0 - (q2 * d); - Q = q1 + q2; - R = r1 + r2; - if (R >= d || R < r2) { /* overflow implies R > d */ - Q += 1; - R -= d; - } - *r = R; - return Q; -#else - uint64_t d0, d1, q0, q1, r1, r0, m; - - d0 = (uint32_t)d; - d1 = d >> 32; - - r1 = n1 % d1; - q1 = n1 / d1; - m = q1 * d0; - r1 = (r1 << 32) | (n0 >> 32); - if (r1 < m) { - q1 -= 1; - r1 += d; - if (r1 >= d) { - if (r1 < m) { - q1 -= 1; - r1 += d; - } - } - } - r1 -= m; - - r0 = r1 % d1; - q0 = r1 / d1; - m = q0 * d0; - r0 = (r0 << 32) | (uint32_t)n0; - if (r0 < m) { - q0 -= 1; - r0 += d; - if (r0 >= d) { - if (r0 < m) { - q0 -= 1; - r0 += d; - } - } - } - r0 -= m; - - *r = r0; - return (q1 << 32) | q0; -#endif -} - /*---------------------------------------------------------------------------- | Returns an approximation to the square root of the 32-bit significand given | by `a'. Considered as an integer, `a' must be at least 2^31. If bit 0 of diff --git a/include/qemu/host-utils.h b/include/qemu/host-utils.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/host-utils.h +++ b/include/qemu/host-utils.h @@ -XXX,XX +XXX,XX @@ * THE SOFTWARE. */ +/* Portions of this work are licensed under the terms of the GNU GPL, + * version 2 or later. See the COPYING file in the top-level directory. + */ + #ifndef HOST_UTILS_H #define HOST_UTILS_H @@ -XXX,XX +XXX,XX @@ void urshift(uint64_t *plow, uint64_t *phigh, int32_t shift); */ void ulshift(uint64_t *plow, uint64_t *phigh, int32_t shift, bool *overflow); +/* From the GNU Multi Precision Library - longlong.h __udiv_qrnnd + * (https://gmplib.org/repo/gmp/file/tip/longlong.h) + * + * Licensed under the GPLv2/LGPLv3 + */ +static inline uint64_t udiv_qrnnd(uint64_t *r, uint64_t n1, + uint64_t n0, uint64_t d) +{ +#if defined(__x86_64__) + uint64_t q; + asm("divq %4" : "=a"(q), "=d"(*r) : "0"(n0), "1"(n1), "rm"(d)); + return q; +#elif defined(__s390x__) && !defined(__clang__) + /* Need to use a TImode type to get an even register pair for DLGR. */ + unsigned __int128 n = (unsigned __int128)n1 << 64 | n0; + asm("dlgr %0, %1" : "+r"(n) : "r"(d)); + *r = n >> 64; + return n; +#elif defined(_ARCH_PPC64) && defined(_ARCH_PWR7) + /* From Power ISA 2.06, programming note for divdeu. */ + uint64_t q1, q2, Q, r1, r2, R; + asm("divdeu %0,%2,%4; divdu %1,%3,%4" + : "=&r"(q1), "=r"(q2) + : "r"(n1), "r"(n0), "r"(d)); + r1 = -(q1 * d); /* low part of (n1<<64) - (q1 * d) */ + r2 = n0 - (q2 * d); + Q = q1 + q2; + R = r1 + r2; + if (R >= d || R < r2) { /* overflow implies R > d */ + Q += 1; + R -= d; + } + *r = R; + return Q; +#else + uint64_t d0, d1, q0, q1, r1, r0, m; + + d0 = (uint32_t)d; + d1 = d >> 32; + + r1 = n1 % d1; + q1 = n1 / d1; + m = q1 * d0; + r1 = (r1 << 32) | (n0 >> 32); + if (r1 < m) { + q1 -= 1; + r1 += d; + if (r1 >= d) { + if (r1 < m) { + q1 -= 1; + r1 += d; + } + } + } + r1 -= m; + + r0 = r1 % d1; + q0 = r1 / d1; + m = q0 * d0; + r0 = (r0 << 32) | (uint32_t)n0; + if (r0 < m) { + q0 -= 1; + r0 += d; + if (r0 >= d) { + if (r0 < m) { + q0 -= 1; + r0 += d; + } + } + } + r0 -= m; + + *r = r0; + return (q1 << 32) | q0; +#endif +} + #endif -- 2.25.1 From: Luis Pires <luis.pires@eldorado.org.br> These will be used to implement new decimal floating point instructions from Power ISA 3.1. The remainder is now returned directly by divu128/divs128, freeing up phigh to receive the high 64 bits of the quotient. Signed-off-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20211025191154.350831-4-luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/hw/clock.h | 6 +- include/qemu/host-utils.h | 20 ++++-- target/ppc/int_helper.c | 9 +-- util/host-utils.c | 133 +++++++++++++++++++++++++------------- 4 files changed, 108 insertions(+), 60 deletions(-) diff --git a/include/hw/clock.h b/include/hw/clock.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/clock.h +++ b/include/hw/clock.h @@ -XXX,XX +XXX,XX @@ static inline uint64_t clock_ns_to_ticks(const Clock *clk, uint64_t ns) if (clk->period == 0) { return 0; } - /* - * BUG: when CONFIG_INT128 is not defined, the current implementation of - * divu128 does not return a valid truncated quotient, so the result will - * be wrong. - */ + divu128(&lo, &hi, clk->period); return lo; } diff --git a/include/qemu/host-utils.h b/include/qemu/host-utils.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/host-utils.h +++ b/include/qemu/host-utils.h @@ -XXX,XX +XXX,XX @@ static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c) return (__int128_t)a * b / c; } -static inline void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) +static inline uint64_t divu128(uint64_t *plow, uint64_t *phigh, + uint64_t divisor) { __uint128_t dividend = ((__uint128_t)*phigh << 64) | *plow; __uint128_t result = dividend / divisor; + *plow = result; - *phigh = dividend % divisor; + *phigh = result >> 64; + return dividend % divisor; } -static inline void divs128(int64_t *plow, int64_t *phigh, int64_t divisor) +static inline int64_t divs128(uint64_t *plow, int64_t *phigh, + int64_t divisor) { - __int128_t dividend = ((__int128_t)*phigh << 64) | (uint64_t)*plow; + __int128_t dividend = ((__int128_t)*phigh << 64) | *plow; __int128_t result = dividend / divisor; + *plow = result; - *phigh = dividend % divisor; + *phigh = result >> 64; + return dividend % divisor; } #else void muls64(uint64_t *plow, uint64_t *phigh, int64_t a, int64_t b); void mulu64(uint64_t *plow, uint64_t *phigh, uint64_t a, uint64_t b); -void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor); -void divs128(int64_t *plow, int64_t *phigh, int64_t divisor); +uint64_t divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor); +int64_t divs128(uint64_t *plow, int64_t *phigh, int64_t divisor); static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c) { diff --git a/target/ppc/int_helper.c b/target/ppc/int_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/int_helper.c +++ b/target/ppc/int_helper.c @@ -XXX,XX +XXX,XX @@ uint64_t helper_divdeu(CPUPPCState *env, uint64_t ra, uint64_t rb, uint32_t oe) uint64_t helper_divde(CPUPPCState *env, uint64_t rau, uint64_t rbu, uint32_t oe) { - int64_t rt = 0; + uint64_t rt = 0; int64_t ra = (int64_t)rau; int64_t rb = (int64_t)rbu; int overflow = 0; @@ -XXX,XX +XXX,XX @@ uint32_t helper_bcdcfsq(ppc_avr_t *r, ppc_avr_t *b, uint32_t ps) int cr; uint64_t lo_value; uint64_t hi_value; + uint64_t rem; ppc_avr_t ret = { .u64 = { 0, 0 } }; if (b->VsrSD(0) < 0) { @@ -XXX,XX +XXX,XX @@ uint32_t helper_bcdcfsq(ppc_avr_t *r, ppc_avr_t *b, uint32_t ps) * In that case, we leave r unchanged. */ } else { - divu128(&lo_value, &hi_value, 1000000000000000ULL); + rem = divu128(&lo_value, &hi_value, 1000000000000000ULL); - for (i = 1; i < 16; hi_value /= 10, i++) { - bcd_put_digit(&ret, hi_value % 10, i); + for (i = 1; i < 16; rem /= 10, i++) { + bcd_put_digit(&ret, rem % 10, i); } for (; i < 32; lo_value /= 10, i++) { diff --git a/util/host-utils.c b/util/host-utils.c index XXXXXXX..XXXXXXX 100644 --- a/util/host-utils.c +++ b/util/host-utils.c @@ -XXX,XX +XXX,XX @@ void muls64 (uint64_t *plow, uint64_t *phigh, int64_t a, int64_t b) } /* - * Unsigned 128-by-64 division. Returns quotient via plow and - * remainder via phigh. - * The result must fit in 64 bits (plow) - otherwise, the result - * is undefined. - * This function will cause a division by zero if passed a zero divisor. + * Unsigned 128-by-64 division. + * Returns the remainder. + * Returns quotient via plow and phigh. + * Also returns the remainder via the function return value. */ -void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) +uint64_t divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) { uint64_t dhi = *phigh; uint64_t dlo = *plow; - unsigned i; - uint64_t carry = 0; + uint64_t rem, dhighest; + int sh; if (divisor == 0 || dhi == 0) { *plow = dlo / divisor; - *phigh = dlo % divisor; + *phigh = 0; + return dlo % divisor; } else { + sh = clz64(divisor); - for (i = 0; i < 64; i++) { - carry = dhi >> 63; - dhi = (dhi << 1) | (dlo >> 63); - if (carry || (dhi >= divisor)) { - dhi -= divisor; - carry = 1; - } else { - carry = 0; + if (dhi < divisor) { + if (sh != 0) { + /* normalize the divisor, shifting the dividend accordingly */ + divisor <<= sh; + dhi = (dhi << sh) | (dlo >> (64 - sh)); + dlo <<= sh; } - dlo = (dlo << 1) | carry; + + *phigh = 0; + *plow = udiv_qrnnd(&rem, dhi, dlo, divisor); + } else { + if (sh != 0) { + /* normalize the divisor, shifting the dividend accordingly */ + divisor <<= sh; + dhighest = dhi >> (64 - sh); + dhi = (dhi << sh) | (dlo >> (64 - sh)); + dlo <<= sh; + + *phigh = udiv_qrnnd(&dhi, dhighest, dhi, divisor); + } else { + /** + * dhi >= divisor + * Since the MSB of divisor is set (sh == 0), + * (dhi - divisor) < divisor + * + * Thus, the high part of the quotient is 1, and we can + * calculate the low part with a single call to udiv_qrnnd + * after subtracting divisor from dhi + */ + dhi -= divisor; + *phigh = 1; + } + + *plow = udiv_qrnnd(&rem, dhi, dlo, divisor); } - *plow = dlo; - *phigh = dhi; + /* + * since the dividend/divisor might have been normalized, + * the remainder might also have to be shifted back + */ + return rem >> sh; } } /* - * Signed 128-by-64 division. Returns quotient via plow and - * remainder via phigh. - * The result must fit in 64 bits (plow) - otherwise, the result - * is undefined. - * This function will cause a division by zero if passed a zero divisor. + * Signed 128-by-64 division. + * Returns quotient via plow and phigh. + * Also returns the remainder via the function return value. */ -void divs128(int64_t *plow, int64_t *phigh, int64_t divisor) +int64_t divs128(uint64_t *plow, int64_t *phigh, int64_t divisor) { - int sgn_dvdnd = *phigh < 0; - int sgn_divsr = divisor < 0; + bool neg_quotient = false, neg_remainder = false; + uint64_t unsig_hi = *phigh, unsig_lo = *plow; + uint64_t rem; - if (sgn_dvdnd) { - *plow = ~(*plow); - *phigh = ~(*phigh); - if (*plow == (int64_t)-1) { + if (*phigh < 0) { + neg_quotient = !neg_quotient; + neg_remainder = !neg_remainder; + + if (unsig_lo == 0) { + unsig_hi = -unsig_hi; + } else { + unsig_hi = ~unsig_hi; + unsig_lo = -unsig_lo; + } + } + + if (divisor < 0) { + neg_quotient = !neg_quotient; + + divisor = -divisor; + } + + rem = divu128(&unsig_lo, &unsig_hi, (uint64_t)divisor); + + if (neg_quotient) { + if (unsig_lo == 0) { + *phigh = -unsig_hi; *plow = 0; - (*phigh)++; - } else { - (*plow)++; - } + } else { + *phigh = ~unsig_hi; + *plow = -unsig_lo; + } + } else { + *phigh = unsig_hi; + *plow = unsig_lo; } - if (sgn_divsr) { - divisor = 0 - divisor; - } - - divu128((uint64_t *)plow, (uint64_t *)phigh, (uint64_t)divisor); - - if (sgn_dvdnd ^ sgn_divsr) { - *plow = 0 - *plow; + if (neg_remainder) { + return -rem; + } else { + return rem; } } #endif -- 2.25.1 From: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20211025191154.350831-5-luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tests/unit/test-div128.c | 197 +++++++++++++++++++++++++++++++++++++++ tests/unit/meson.build | 1 + 2 files changed, 198 insertions(+) create mode 100644 tests/unit/test-div128.c diff --git a/tests/unit/test-div128.c b/tests/unit/test-div128.c new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/unit/test-div128.c @@ -XXX,XX +XXX,XX @@ +/* + * Test 128-bit division functions + * + * Copyright (c) 2021 Instituto de Pesquisas Eldorado (eldorado.org.br) + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ + +#include "qemu/osdep.h" +#include "qemu/host-utils.h" + +typedef struct { + uint64_t high; + uint64_t low; + uint64_t rhigh; + uint64_t rlow; + uint64_t divisor; + uint64_t remainder; +} test_data_unsigned; + +typedef struct { + int64_t high; + uint64_t low; + int64_t rhigh; + uint64_t rlow; + int64_t divisor; + int64_t remainder; +} test_data_signed; + +static const test_data_unsigned test_table_unsigned[] = { + /* Dividend fits in 64 bits */ + { 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL}, + { 0x0000000000000000ULL, 0x0000000000000001ULL, + 0x0000000000000000ULL, 0x0000000000000001ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL}, + { 0x0000000000000000ULL, 0x0000000000000003ULL, + 0x0000000000000000ULL, 0x0000000000000001ULL, + 0x0000000000000002ULL, 0x0000000000000001ULL}, + { 0x0000000000000000ULL, 0x8000000000000000ULL, + 0x0000000000000000ULL, 0x8000000000000000ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL}, + { 0x0000000000000000ULL, 0xa000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000002ULL, + 0x4000000000000000ULL, 0x2000000000000000ULL}, + { 0x0000000000000000ULL, 0x8000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000001ULL, + 0x8000000000000000ULL, 0x0000000000000000ULL}, + + /* Dividend > 64 bits, with MSB 0 */ + { 0x123456789abcdefeULL, 0xefedcba987654321ULL, + 0x123456789abcdefeULL, 0xefedcba987654321ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL}, + { 0x123456789abcdefeULL, 0xefedcba987654321ULL, + 0x0000000000000001ULL, 0x000000000000000dULL, + 0x123456789abcdefeULL, 0x03456789abcdf03bULL}, + { 0x123456789abcdefeULL, 0xefedcba987654321ULL, + 0x0123456789abcdefULL, 0xeefedcba98765432ULL, + 0x0000000000000010ULL, 0x0000000000000001ULL}, + + /* Dividend > 64 bits, with MSB 1 */ + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x0000000000000001ULL, 0x0000000000000000ULL}, + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x0000000000000001ULL, 0x0000000000000000ULL, + 0xfeeddccbbaa99887ULL, 0x766554433221100fULL}, + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x0feeddccbbaa9988ULL, 0x7766554433221100ULL, + 0x0000000000000010ULL, 0x000000000000000fULL}, + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x000000000000000eULL, 0x00f0f0f0f0f0f35aULL, + 0x123456789abcdefeULL, 0x0f8922bc55ef90c3ULL}, + + /** + * Divisor == 64 bits, with MSB 1 + * and high 64 bits of dividend >= divisor + * (for testing normalization) + */ + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x0000000000000001ULL, 0x0000000000000000ULL, + 0xfeeddccbbaa99887ULL, 0x766554433221100fULL}, + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x0000000000000001ULL, 0xfddbb9977553310aULL, + 0x8000000000000001ULL, 0x78899aabbccddf05ULL}, + + /* Dividend > 64 bits, divisor almost as big */ + { 0x0000000000000001ULL, 0x23456789abcdef01ULL, + 0x0000000000000000ULL, 0x000000000000000fULL, + 0x123456789abcdefeULL, 0x123456789abcde1fULL}, +}; + +static const test_data_signed test_table_signed[] = { + /* Positive dividend, positive/negative divisors */ + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0x0000000000000000LL, 0x0000000000bc614eULL, + 0x0000000000000001LL, 0x0000000000000000LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0xffffffffffffffffLL, 0x0000000000000000LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0x0000000000000000LL, 0x00000000005e30a7ULL, + 0x0000000000000002LL, 0x0000000000000000LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0xffffffffffffffffLL, 0xffffffffffa1cf59ULL, + 0xfffffffffffffffeLL, 0x0000000000000000LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0x0000000000000000LL, 0x0000000000178c29ULL, + 0x0000000000000008LL, 0x0000000000000006LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0xffffffffffffffffLL, 0xffffffffffe873d7ULL, + 0xfffffffffffffff8LL, 0x0000000000000006LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0x0000000000000000LL, 0x000000000000550dULL, + 0x0000000000000237LL, 0x0000000000000183LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0xffffffffffffffffLL, 0xffffffffffffaaf3ULL, + 0xfffffffffffffdc9LL, 0x0000000000000183LL}, + + /* Negative dividend, positive/negative divisors */ + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0x0000000000000001LL, 0x0000000000000000LL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0x0000000000000000LL, 0x0000000000bc614eULL, + 0xffffffffffffffffLL, 0x0000000000000000LL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0xffffffffffffffffLL, 0xffffffffffa1cf59ULL, + 0x0000000000000002LL, 0x0000000000000000LL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0x0000000000000000LL, 0x00000000005e30a7ULL, + 0xfffffffffffffffeLL, 0x0000000000000000LL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0xffffffffffffffffLL, 0xffffffffffe873d7ULL, + 0x0000000000000008LL, 0xfffffffffffffffaLL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0x0000000000000000LL, 0x0000000000178c29ULL, + 0xfffffffffffffff8LL, 0xfffffffffffffffaLL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0xffffffffffffffffLL, 0xffffffffffffaaf3ULL, + 0x0000000000000237LL, 0xfffffffffffffe7dLL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0x0000000000000000LL, 0x000000000000550dULL, + 0xfffffffffffffdc9LL, 0xfffffffffffffe7dLL}, +}; + +static void test_divu128(void) +{ + int i; + uint64_t rem; + test_data_unsigned tmp; + + for (i = 0; i < ARRAY_SIZE(test_table_unsigned); ++i) { + tmp = test_table_unsigned[i]; + + rem = divu128(&tmp.low, &tmp.high, tmp.divisor); + g_assert_cmpuint(tmp.low, ==, tmp.rlow); + g_assert_cmpuint(tmp.high, ==, tmp.rhigh); + g_assert_cmpuint(rem, ==, tmp.remainder); + } +} + +static void test_divs128(void) +{ + int i; + int64_t rem; + test_data_signed tmp; + + for (i = 0; i < ARRAY_SIZE(test_table_signed); ++i) { + tmp = test_table_signed[i]; + + rem = divs128(&tmp.low, &tmp.high, tmp.divisor); + g_assert_cmpuint(tmp.low, ==, tmp.rlow); + g_assert_cmpuint(tmp.high, ==, tmp.rhigh); + g_assert_cmpuint(rem, ==, tmp.remainder); + } +} + +int main(int argc, char **argv) +{ + g_test_init(&argc, &argv, NULL); + g_test_add_func("/host-utils/test_divu128", test_divu128); + g_test_add_func("/host-utils/test_divs128", test_divs128); + return g_test_run(); +} diff --git a/tests/unit/meson.build b/tests/unit/meson.build index XXXXXXX..XXXXXXX 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -XXX,XX +XXX,XX @@ tests = { # all code tested by test-x86-cpuid is inside topology.h 'test-x86-cpuid': [], 'test-cutils': [], + 'test-div128': [], 'test-shift128': [], 'test-mul64': [], # all code tested by test-int128 is inside int128.h -- 2.25.1 Prepare for tracking different masks by renaming this one. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 142 +++++++++++++++++++++++++------------------------ 1 file changed, 72 insertions(+), 70 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { TCGTemp *prev_copy; TCGTemp *next_copy; uint64_t val; - uint64_t mask; + uint64_t z_mask; /* mask bit is 0 if and only if value bit is 0 */ } TempOptInfo; static inline TempOptInfo *ts_info(TCGTemp *ts) @@ -XXX,XX +XXX,XX @@ static void reset_ts(TCGTemp *ts) ti->next_copy = ts; ti->prev_copy = ts; ti->is_const = false; - ti->mask = -1; + ti->z_mask = -1; } static void reset_temp(TCGArg arg) @@ -XXX,XX +XXX,XX @@ static void init_ts_info(TCGTempSet *temps_used, TCGTemp *ts) if (ts->kind == TEMP_CONST) { ti->is_const = true; ti->val = ts->val; - ti->mask = ts->val; + ti->z_mask = ts->val; if (TCG_TARGET_REG_BITS > 32 && ts->type == TCG_TYPE_I32) { /* High bits of a 32-bit quantity are garbage. */ - ti->mask |= ~0xffffffffull; + ti->z_mask |= ~0xffffffffull; } } else { ti->is_const = false; - ti->mask = -1; + ti->z_mask = -1; } } @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) const TCGOpDef *def; TempOptInfo *di; TempOptInfo *si; - uint64_t mask; + uint64_t z_mask; TCGOpcode new_op; if (ts_are_copies(dst_ts, src_ts)) { @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) op->args[0] = dst; op->args[1] = src; - mask = si->mask; + z_mask = si->z_mask; if (TCG_TARGET_REG_BITS > 32 && new_op == INDEX_op_mov_i32) { /* High bits of the destination are now garbage. */ - mask |= ~0xffffffffull; + z_mask |= ~0xffffffffull; } - di->mask = mask; + di->z_mask = z_mask; if (src_ts->type == dst_ts->type) { TempOptInfo *ni = ts_info(si->next_copy); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { - uint64_t mask, partmask, affected, tmp; + uint64_t z_mask, partmask, affected, tmp; int nb_oargs, nb_iargs; TCGOpcode opc = op->opc; const TCGOpDef *def = &tcg_op_defs[opc]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* Simplify using known-zero bits. Currently only ops with a single output argument is supported. */ - mask = -1; + z_mask = -1; affected = -1; switch (opc) { CASE_OP_32_64(ext8s): - if ((arg_info(op->args[1])->mask & 0x80) != 0) { + if ((arg_info(op->args[1])->z_mask & 0x80) != 0) { break; } QEMU_FALLTHROUGH; CASE_OP_32_64(ext8u): - mask = 0xff; + z_mask = 0xff; goto and_const; CASE_OP_32_64(ext16s): - if ((arg_info(op->args[1])->mask & 0x8000) != 0) { + if ((arg_info(op->args[1])->z_mask & 0x8000) != 0) { break; } QEMU_FALLTHROUGH; CASE_OP_32_64(ext16u): - mask = 0xffff; + z_mask = 0xffff; goto and_const; case INDEX_op_ext32s_i64: - if ((arg_info(op->args[1])->mask & 0x80000000) != 0) { + if ((arg_info(op->args[1])->z_mask & 0x80000000) != 0) { break; } QEMU_FALLTHROUGH; case INDEX_op_ext32u_i64: - mask = 0xffffffffU; + z_mask = 0xffffffffU; goto and_const; CASE_OP_32_64(and): - mask = arg_info(op->args[2])->mask; + z_mask = arg_info(op->args[2])->z_mask; if (arg_is_const(op->args[2])) { and_const: - affected = arg_info(op->args[1])->mask & ~mask; + affected = arg_info(op->args[1])->z_mask & ~z_mask; } - mask = arg_info(op->args[1])->mask & mask; + z_mask = arg_info(op->args[1])->z_mask & z_mask; break; case INDEX_op_ext_i32_i64: - if ((arg_info(op->args[1])->mask & 0x80000000) != 0) { + if ((arg_info(op->args[1])->z_mask & 0x80000000) != 0) { break; } QEMU_FALLTHROUGH; case INDEX_op_extu_i32_i64: /* We do not compute affected as it is a size changing op. */ - mask = (uint32_t)arg_info(op->args[1])->mask; + z_mask = (uint32_t)arg_info(op->args[1])->z_mask; break; CASE_OP_32_64(andc): /* Known-zeros does not imply known-ones. Therefore unless op->args[2] is constant, we can't infer anything from it. */ if (arg_is_const(op->args[2])) { - mask = ~arg_info(op->args[2])->mask; + z_mask = ~arg_info(op->args[2])->z_mask; goto and_const; } /* But we certainly know nothing outside args[1] may be set. */ - mask = arg_info(op->args[1])->mask; + z_mask = arg_info(op->args[1])->z_mask; break; case INDEX_op_sar_i32: if (arg_is_const(op->args[2])) { tmp = arg_info(op->args[2])->val & 31; - mask = (int32_t)arg_info(op->args[1])->mask >> tmp; + z_mask = (int32_t)arg_info(op->args[1])->z_mask >> tmp; } break; case INDEX_op_sar_i64: if (arg_is_const(op->args[2])) { tmp = arg_info(op->args[2])->val & 63; - mask = (int64_t)arg_info(op->args[1])->mask >> tmp; + z_mask = (int64_t)arg_info(op->args[1])->z_mask >> tmp; } break; case INDEX_op_shr_i32: if (arg_is_const(op->args[2])) { tmp = arg_info(op->args[2])->val & 31; - mask = (uint32_t)arg_info(op->args[1])->mask >> tmp; + z_mask = (uint32_t)arg_info(op->args[1])->z_mask >> tmp; } break; case INDEX_op_shr_i64: if (arg_is_const(op->args[2])) { tmp = arg_info(op->args[2])->val & 63; - mask = (uint64_t)arg_info(op->args[1])->mask >> tmp; + z_mask = (uint64_t)arg_info(op->args[1])->z_mask >> tmp; } break; case INDEX_op_extrl_i64_i32: - mask = (uint32_t)arg_info(op->args[1])->mask; + z_mask = (uint32_t)arg_info(op->args[1])->z_mask; break; case INDEX_op_extrh_i64_i32: - mask = (uint64_t)arg_info(op->args[1])->mask >> 32; + z_mask = (uint64_t)arg_info(op->args[1])->z_mask >> 32; break; CASE_OP_32_64(shl): if (arg_is_const(op->args[2])) { tmp = arg_info(op->args[2])->val & (TCG_TARGET_REG_BITS - 1); - mask = arg_info(op->args[1])->mask << tmp; + z_mask = arg_info(op->args[1])->z_mask << tmp; } break; CASE_OP_32_64(neg): /* Set to 1 all bits to the left of the rightmost. */ - mask = -(arg_info(op->args[1])->mask - & -arg_info(op->args[1])->mask); + z_mask = -(arg_info(op->args[1])->z_mask + & -arg_info(op->args[1])->z_mask); break; CASE_OP_32_64(deposit): - mask = deposit64(arg_info(op->args[1])->mask, - op->args[3], op->args[4], - arg_info(op->args[2])->mask); + z_mask = deposit64(arg_info(op->args[1])->z_mask, + op->args[3], op->args[4], + arg_info(op->args[2])->z_mask); break; CASE_OP_32_64(extract): - mask = extract64(arg_info(op->args[1])->mask, - op->args[2], op->args[3]); + z_mask = extract64(arg_info(op->args[1])->z_mask, + op->args[2], op->args[3]); if (op->args[2] == 0) { - affected = arg_info(op->args[1])->mask & ~mask; + affected = arg_info(op->args[1])->z_mask & ~z_mask; } break; CASE_OP_32_64(sextract): - mask = sextract64(arg_info(op->args[1])->mask, - op->args[2], op->args[3]); - if (op->args[2] == 0 && (tcg_target_long)mask >= 0) { - affected = arg_info(op->args[1])->mask & ~mask; + z_mask = sextract64(arg_info(op->args[1])->z_mask, + op->args[2], op->args[3]); + if (op->args[2] == 0 && (tcg_target_long)z_mask >= 0) { + affected = arg_info(op->args[1])->z_mask & ~z_mask; } break; CASE_OP_32_64(or): CASE_OP_32_64(xor): - mask = arg_info(op->args[1])->mask | arg_info(op->args[2])->mask; + z_mask = arg_info(op->args[1])->z_mask + | arg_info(op->args[2])->z_mask; break; case INDEX_op_clz_i32: case INDEX_op_ctz_i32: - mask = arg_info(op->args[2])->mask | 31; + z_mask = arg_info(op->args[2])->z_mask | 31; break; case INDEX_op_clz_i64: case INDEX_op_ctz_i64: - mask = arg_info(op->args[2])->mask | 63; + z_mask = arg_info(op->args[2])->z_mask | 63; break; case INDEX_op_ctpop_i32: - mask = 32 | 31; + z_mask = 32 | 31; break; case INDEX_op_ctpop_i64: - mask = 64 | 63; + z_mask = 64 | 63; break; CASE_OP_32_64(setcond): case INDEX_op_setcond2_i32: - mask = 1; + z_mask = 1; break; CASE_OP_32_64(movcond): - mask = arg_info(op->args[3])->mask | arg_info(op->args[4])->mask; + z_mask = arg_info(op->args[3])->z_mask + | arg_info(op->args[4])->z_mask; break; CASE_OP_32_64(ld8u): - mask = 0xff; + z_mask = 0xff; break; CASE_OP_32_64(ld16u): - mask = 0xffff; + z_mask = 0xffff; break; case INDEX_op_ld32u_i64: - mask = 0xffffffffu; + z_mask = 0xffffffffu; break; CASE_OP_32_64(qemu_ld): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) MemOpIdx oi = op->args[nb_oargs + nb_iargs]; MemOp mop = get_memop(oi); if (!(mop & MO_SIGN)) { - mask = (2ULL << ((8 << (mop & MO_SIZE)) - 1)) - 1; + z_mask = (2ULL << ((8 << (mop & MO_SIZE)) - 1)) - 1; } } break; CASE_OP_32_64(bswap16): - mask = arg_info(op->args[1])->mask; - if (mask <= 0xffff) { + z_mask = arg_info(op->args[1])->z_mask; + if (z_mask <= 0xffff) { op->args[2] |= TCG_BSWAP_IZ; } - mask = bswap16(mask); + z_mask = bswap16(z_mask); switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { case TCG_BSWAP_OZ: break; case TCG_BSWAP_OS: - mask = (int16_t)mask; + z_mask = (int16_t)z_mask; break; default: /* undefined high bits */ - mask |= MAKE_64BIT_MASK(16, 48); + z_mask |= MAKE_64BIT_MASK(16, 48); break; } break; case INDEX_op_bswap32_i64: - mask = arg_info(op->args[1])->mask; - if (mask <= 0xffffffffu) { + z_mask = arg_info(op->args[1])->z_mask; + if (z_mask <= 0xffffffffu) { op->args[2] |= TCG_BSWAP_IZ; } - mask = bswap32(mask); + z_mask = bswap32(z_mask); switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { case TCG_BSWAP_OZ: break; case TCG_BSWAP_OS: - mask = (int32_t)mask; + z_mask = (int32_t)z_mask; break; default: /* undefined high bits */ - mask |= MAKE_64BIT_MASK(32, 32); + z_mask |= MAKE_64BIT_MASK(32, 32); break; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* 32-bit ops generate 32-bit results. For the result is zero test below, we can ignore high bits, but for further optimizations we need to record that the high bits contain garbage. */ - partmask = mask; + partmask = z_mask; if (!(def->flags & TCG_OPF_64BIT)) { - mask |= ~(tcg_target_ulong)0xffffffffu; + z_mask |= ~(tcg_target_ulong)0xffffffffu; partmask &= 0xffffffffu; affected &= 0xffffffffu; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) vs the high word of the input. */ do_setcond_high: reset_temp(op->args[0]); - arg_info(op->args[0])->mask = 1; + arg_info(op->args[0])->z_mask = 1; op->opc = INDEX_op_setcond_i32; op->args[1] = op->args[2]; op->args[2] = op->args[4]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } do_setcond_low: reset_temp(op->args[0]); - arg_info(op->args[0])->mask = 1; + arg_info(op->args[0])->z_mask = 1; op->opc = INDEX_op_setcond_i32; op->args[2] = op->args[3]; op->args[3] = op->args[5]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* Default case: we know nothing about operation (or were unable to compute the operation result) so no propagation is done. We trash everything if the operation is the end of a basic - block, otherwise we only trash the output args. "mask" is + block, otherwise we only trash the output args. "z_mask" is the non-zero bits mask for the first output arg. */ if (def->flags & TCG_OPF_BB_END) { memset(&temps_used, 0, sizeof(temps_used)); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* Save the corresponding known-zero bits mask for the first output argument (only one supported so far). */ if (i == 0) { - arg_info(op->args[i])->mask = mask; + arg_info(op->args[i])->z_mask = z_mask; } } } -- 2.25.1 Provide what will become a larger context for splitting the very large tcg_optimize function. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 77 ++++++++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { uint64_t z_mask; /* mask bit is 0 if and only if value bit is 0 */ } TempOptInfo; +typedef struct OptContext { + TCGTempSet temps_used; +} OptContext; + static inline TempOptInfo *ts_info(TCGTemp *ts) { return ts->state_ptr; @@ -XXX,XX +XXX,XX @@ static void reset_temp(TCGArg arg) } /* Initialize and activate a temporary. */ -static void init_ts_info(TCGTempSet *temps_used, TCGTemp *ts) +static void init_ts_info(OptContext *ctx, TCGTemp *ts) { size_t idx = temp_idx(ts); TempOptInfo *ti; - if (test_bit(idx, temps_used->l)) { + if (test_bit(idx, ctx->temps_used.l)) { return; } - set_bit(idx, temps_used->l); + set_bit(idx, ctx->temps_used.l); ti = ts->state_ptr; if (ti == NULL) { @@ -XXX,XX +XXX,XX @@ static void init_ts_info(TCGTempSet *temps_used, TCGTemp *ts) } } -static void init_arg_info(TCGTempSet *temps_used, TCGArg arg) +static void init_arg_info(OptContext *ctx, TCGArg arg) { - init_ts_info(temps_used, arg_temp(arg)); + init_ts_info(ctx, arg_temp(arg)); } static TCGTemp *find_better_copy(TCGContext *s, TCGTemp *ts) @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) } } -static void tcg_opt_gen_movi(TCGContext *s, TCGTempSet *temps_used, +static void tcg_opt_gen_movi(TCGContext *s, OptContext *ctx, TCGOp *op, TCGArg dst, uint64_t val) { const TCGOpDef *def = &tcg_op_defs[op->opc]; @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_movi(TCGContext *s, TCGTempSet *temps_used, /* Convert movi to mov with constant temp. */ tv = tcg_constant_internal(type, val); - init_ts_info(temps_used, tv); + init_ts_info(ctx, tv); tcg_opt_gen_mov(s, op, dst, temp_arg(tv)); } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) { int nb_temps, nb_globals, i; TCGOp *op, *op_next, *prev_mb = NULL; - TCGTempSet temps_used; + OptContext ctx = {}; /* Array VALS has an element for each temp. If this temp holds a constant then its value is kept in VALS' element. @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) nb_temps = s->nb_temps; nb_globals = s->nb_globals; - memset(&temps_used, 0, sizeof(temps_used)); for (i = 0; i < nb_temps; ++i) { s->temps[i].state_ptr = NULL; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) for (i = 0; i < nb_oargs + nb_iargs; i++) { TCGTemp *ts = arg_temp(op->args[i]); if (ts) { - init_ts_info(&temps_used, ts); + init_ts_info(&ctx, ts); } } } else { nb_oargs = def->nb_oargs; nb_iargs = def->nb_iargs; for (i = 0; i < nb_oargs + nb_iargs; i++) { - init_arg_info(&temps_used, op->args[i]); + init_arg_info(&ctx, op->args[i]); } } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(rotr): if (arg_is_const(op->args[1]) && arg_info(op->args[1])->val == 0) { - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], 0); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (partmask == 0) { tcg_debug_assert(nb_oargs == 1); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], 0); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); continue; } if (affected == 0) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(mulsh): if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == 0) { - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], 0); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(sub): CASE_OP_32_64_VEC(xor): if (args_are_copies(op->args[1], op->args[2])) { - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], 0); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = arg_info(op->args[1])->val; tmp = dup_const(TCGOP_VECE(op), tmp); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_dup2_vec: assert(TCG_TARGET_REG_BITS == 32); if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], + tcg_opt_gen_movi(s, &ctx, op, op->args[0], deposit64(arg_info(op->args[1])->val, 32, 32, arg_info(op->args[2])->val)); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_extrh_i64_i32: if (arg_is_const(op->args[1])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, 0); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, op->args[2]); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, arg_info(op->args[2])->val); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) TCGArg v = arg_info(op->args[1])->val; if (v != 0) { tmp = do_constant_folding(opc, v, 0); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); } else { tcg_opt_gen_mov(s, op, op->args[0], op->args[2]); } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = deposit64(arg_info(op->args[1])->val, op->args[3], op->args[4], arg_info(op->args[2])->val); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = extract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = sextract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = (int32_t)(((uint32_t)v1 >> shr) | ((uint32_t)v2 << (32 - shr))); } - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[3]); if (tmp != 2) { - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[1], op->args[2]); if (tmp != 2) { if (tmp) { - memset(&temps_used, 0, sizeof(temps_used)); + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); op->opc = INDEX_op_br; op->args[0] = op->args[3]; } else { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(s, &temps_used, op, rl, (int32_t)a); - tcg_opt_gen_movi(s, &temps_used, op2, rh, (int32_t)(a >> 32)); + tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)a); + tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(a >> 32)); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(s, &temps_used, op, rl, (int32_t)r); - tcg_opt_gen_movi(s, &temps_used, op2, rh, (int32_t)(r >> 32)); + tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)r); + tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(r >> 32)); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (tmp != 2) { if (tmp) { do_brcond_true: - memset(&temps_used, 0, sizeof(temps_used)); + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); op->opc = INDEX_op_br; op->args[0] = op->args[5]; } else { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* Simplify LT/GE comparisons vs zero to a single compare vs the high word of the input. */ do_brcond_high: - memset(&temps_used, 0, sizeof(temps_used)); + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); op->opc = INDEX_op_brcond_i32; op->args[0] = op->args[1]; op->args[1] = op->args[3]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) goto do_default; } do_brcond_low: - memset(&temps_used, 0, sizeof(temps_used)); + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); op->opc = INDEX_op_brcond_i32; op->args[1] = op->args[2]; op->args[2] = op->args[4]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[5]); if (tmp != 2) { do_setcond_const: - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); } else if ((op->args[5] == TCG_COND_LT || op->args[5] == TCG_COND_GE) && arg_is_const(op->args[3]) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (!(tcg_call_flags(op) & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) { for (i = 0; i < nb_globals; i++) { - if (test_bit(i, temps_used.l)) { + if (test_bit(i, ctx.temps_used.l)) { reset_ts(&s->temps[i]); } } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) block, otherwise we only trash the output args. "z_mask" is the non-zero bits mask for the first output arg. */ if (def->flags & TCG_OPF_BB_END) { - memset(&temps_used, 0, sizeof(temps_used)); + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); } else { do_reset_output: for (i = 0; i < nb_oargs; i++) { -- 2.25.1 Break the final cleanup clause out of the main switch statement. When fully folding an opcode to mov/movi, use "continue" to process the next opcode, else break to fall into the final cleanup. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 190 ++++++++++++++++++++++++------------------------- 1 file changed, 94 insertions(+), 96 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) switch (opc) { CASE_OP_32_64_VEC(mov): tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); - break; + continue; case INDEX_op_dup_vec: if (arg_is_const(op->args[1])) { tmp = arg_info(op->args[1])->val; tmp = dup_const(TCGOP_VECE(op), tmp); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; case INDEX_op_dup2_vec: assert(TCG_TARGET_REG_BITS == 32); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tcg_opt_gen_movi(s, &ctx, op, op->args[0], deposit64(arg_info(op->args[1])->val, 32, 32, arg_info(op->args[2])->val)); - break; + continue; } else if (args_are_copies(op->args[1], op->args[2])) { op->opc = INDEX_op_dup_vec; TCGOP_VECE(op) = MO_32; nb_iargs = 1; } - goto do_default; + break; CASE_OP_32_64(not): CASE_OP_32_64(neg): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, 0); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(bswap16): CASE_OP_32_64(bswap32): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = do_constant_folding(opc, arg_info(op->args[1])->val, op->args[2]); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(add): CASE_OP_32_64(sub): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = do_constant_folding(opc, arg_info(op->args[1])->val, arg_info(op->args[2])->val); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(clz): CASE_OP_32_64(ctz): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } else { tcg_opt_gen_mov(s, op, op->args[0], op->args[2]); } - break; + continue; } - goto do_default; + break; CASE_OP_32_64(deposit): if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[3], op->args[4], arg_info(op->args[2])->val); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(extract): if (arg_is_const(op->args[1])) { tmp = extract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(sextract): if (arg_is_const(op->args[1])) { tmp = sextract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(extract2): if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) ((uint32_t)v2 << (32 - shr))); } tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(setcond): tmp = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[3]); if (tmp != 2) { tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(brcond): tmp = do_constant_folding_cond(opc, op->args[0], op->args[1], op->args[2]); - if (tmp != 2) { - if (tmp) { - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - op->opc = INDEX_op_br; - op->args[0] = op->args[3]; - } else { - tcg_op_remove(s, op); - } + switch (tmp) { + case 0: + tcg_op_remove(s, op); + continue; + case 1: + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); + op->opc = opc = INDEX_op_br; + op->args[0] = op->args[3]; break; } - goto do_default; + break; CASE_OP_32_64(movcond): tmp = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[5]); if (tmp != 2) { tcg_opt_gen_mov(s, op, op->args[0], op->args[4-tmp]); - break; + continue; } if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { uint64_t tv = arg_info(op->args[3])->val; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (fv == 1 && tv == 0) { cond = tcg_invert_cond(cond); } else if (!(tv == 1 && fv == 0)) { - goto do_default; + break; } op->args[3] = cond; op->opc = opc = (opc == INDEX_op_movcond_i32 @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) : INDEX_op_setcond_i64); nb_iargs = 2; } - goto do_default; + break; case INDEX_op_add2_i32: case INDEX_op_sub2_i32: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rh = op->args[1]; tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)a); tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(a >> 32)); - break; + continue; } - goto do_default; + break; case INDEX_op_mulu2_i32: if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rh = op->args[1]; tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)r); tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(r >> 32)); - break; + continue; } - goto do_default; + break; case INDEX_op_brcond2_i32: tmp = do_constant_folding_cond2(&op->args[0], &op->args[2], op->args[4]); - if (tmp != 2) { - if (tmp) { - do_brcond_true: - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - op->opc = INDEX_op_br; - op->args[0] = op->args[5]; - } else { + if (tmp == 0) { do_brcond_false: - tcg_op_remove(s, op); - } - } else if ((op->args[4] == TCG_COND_LT - || op->args[4] == TCG_COND_GE) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == 0 - && arg_is_const(op->args[3]) - && arg_info(op->args[3])->val == 0) { + tcg_op_remove(s, op); + continue; + } + if (tmp == 1) { + do_brcond_true: + op->opc = opc = INDEX_op_br; + op->args[0] = op->args[5]; + break; + } + if ((op->args[4] == TCG_COND_LT || op->args[4] == TCG_COND_GE) + && arg_is_const(op->args[2]) + && arg_info(op->args[2])->val == 0 + && arg_is_const(op->args[3]) + && arg_info(op->args[3])->val == 0) { /* Simplify LT/GE comparisons vs zero to a single compare vs the high word of the input. */ do_brcond_high: - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - op->opc = INDEX_op_brcond_i32; + op->opc = opc = INDEX_op_brcond_i32; op->args[0] = op->args[1]; op->args[1] = op->args[3]; op->args[2] = op->args[4]; op->args[3] = op->args[5]; - } else if (op->args[4] == TCG_COND_EQ) { + break; + } + if (op->args[4] == TCG_COND_EQ) { /* Simplify EQ comparisons where one of the pairs can be simplified. */ tmp = do_constant_folding_cond(INDEX_op_brcond_i32, @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (tmp == 0) { goto do_brcond_false; } else if (tmp != 1) { - goto do_default; + break; } do_brcond_low: memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[1] = op->args[2]; op->args[2] = op->args[4]; op->args[3] = op->args[5]; - } else if (op->args[4] == TCG_COND_NE) { + break; + } + if (op->args[4] == TCG_COND_NE) { /* Simplify NE comparisons where one of the pairs can be simplified. */ tmp = do_constant_folding_cond(INDEX_op_brcond_i32, @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } else if (tmp == 1) { goto do_brcond_true; } - goto do_default; - } else { - goto do_default; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (tmp != 2) { do_setcond_const: tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - } else if ((op->args[5] == TCG_COND_LT - || op->args[5] == TCG_COND_GE) - && arg_is_const(op->args[3]) - && arg_info(op->args[3])->val == 0 - && arg_is_const(op->args[4]) - && arg_info(op->args[4])->val == 0) { + continue; + } + if ((op->args[5] == TCG_COND_LT || op->args[5] == TCG_COND_GE) + && arg_is_const(op->args[3]) + && arg_info(op->args[3])->val == 0 + && arg_is_const(op->args[4]) + && arg_info(op->args[4])->val == 0) { /* Simplify LT/GE comparisons vs zero to a single compare vs the high word of the input. */ do_setcond_high: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[1] = op->args[2]; op->args[2] = op->args[4]; op->args[3] = op->args[5]; - } else if (op->args[5] == TCG_COND_EQ) { + break; + } + if (op->args[5] == TCG_COND_EQ) { /* Simplify EQ comparisons where one of the pairs can be simplified. */ tmp = do_constant_folding_cond(INDEX_op_setcond_i32, @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (tmp == 0) { goto do_setcond_high; } else if (tmp != 1) { - goto do_default; + break; } do_setcond_low: reset_temp(op->args[0]); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->opc = INDEX_op_setcond_i32; op->args[2] = op->args[3]; op->args[3] = op->args[5]; - } else if (op->args[5] == TCG_COND_NE) { + break; + } + if (op->args[5] == TCG_COND_NE) { /* Simplify NE comparisons where one of the pairs can be simplified. */ tmp = do_constant_folding_cond(INDEX_op_setcond_i32, @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } else if (tmp == 1) { goto do_setcond_const; } - goto do_default; - } else { - goto do_default; } break; - case INDEX_op_call: - if (!(tcg_call_flags(op) + default: + break; + } + + /* Some of the folding above can change opc. */ + opc = op->opc; + def = &tcg_op_defs[opc]; + if (def->flags & TCG_OPF_BB_END) { + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); + } else { + if (opc == INDEX_op_call && + !(tcg_call_flags(op) & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) { for (i = 0; i < nb_globals; i++) { if (test_bit(i, ctx.temps_used.l)) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } } } - goto do_reset_output; - default: - do_default: - /* Default case: we know nothing about operation (or were unable - to compute the operation result) so no propagation is done. - We trash everything if the operation is the end of a basic - block, otherwise we only trash the output args. "z_mask" is - the non-zero bits mask for the first output arg. */ - if (def->flags & TCG_OPF_BB_END) { - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - } else { - do_reset_output: - for (i = 0; i < nb_oargs; i++) { - reset_temp(op->args[i]); - /* Save the corresponding known-zero bits mask for the - first output argument (only one supported so far). */ - if (i == 0) { - arg_info(op->args[i])->z_mask = z_mask; - } + for (i = 0; i < nb_oargs; i++) { + reset_temp(op->args[i]); + /* Save the corresponding known-zero bits mask for the + first output argument (only one supported so far). */ + if (i == 0) { + arg_info(op->args[i])->z_mask = z_mask; } } - break; } /* Eliminate duplicate and redundant fence instructions. */ -- 2.25.1 Adjust the interface to take the OptContext parameter instead of TCGContext or both. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 67 +++++++++++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { } TempOptInfo; typedef struct OptContext { + TCGContext *tcg; TCGTempSet temps_used; } OptContext; @@ -XXX,XX +XXX,XX @@ static bool args_are_copies(TCGArg arg1, TCGArg arg2) return ts_are_copies(arg_temp(arg1), arg_temp(arg2)); } -static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) +static void tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) { TCGTemp *dst_ts = arg_temp(dst); TCGTemp *src_ts = arg_temp(src); @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) TCGOpcode new_op; if (ts_are_copies(dst_ts, src_ts)) { - tcg_op_remove(s, op); + tcg_op_remove(ctx->tcg, op); return; } @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) } } -static void tcg_opt_gen_movi(TCGContext *s, OptContext *ctx, - TCGOp *op, TCGArg dst, uint64_t val) +static void tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, + TCGArg dst, uint64_t val) { const TCGOpDef *def = &tcg_op_defs[op->opc]; TCGType type; @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_movi(TCGContext *s, OptContext *ctx, /* Convert movi to mov with constant temp. */ tv = tcg_constant_internal(type, val); init_ts_info(ctx, tv); - tcg_opt_gen_mov(s, op, dst, temp_arg(tv)); + tcg_opt_gen_mov(ctx, op, dst, temp_arg(tv)); } static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) { int nb_temps, nb_globals, i; TCGOp *op, *op_next, *prev_mb = NULL; - OptContext ctx = {}; + OptContext ctx = { .tcg = s }; /* Array VALS has an element for each temp. If this temp holds a constant then its value is kept in VALS' element. @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(rotr): if (arg_is_const(op->args[1]) && arg_info(op->args[1])->val == 0) { - tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); + tcg_opt_gen_movi(&ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (!arg_is_const(op->args[1]) && arg_is_const(op->args[2]) && arg_info(op->args[2])->val == 0) { - tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (!arg_is_const(op->args[1]) && arg_is_const(op->args[2]) && arg_info(op->args[2])->val == -1) { - tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (partmask == 0) { tcg_debug_assert(nb_oargs == 1); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); + tcg_opt_gen_movi(&ctx, op, op->args[0], 0); continue; } if (affected == 0) { tcg_debug_assert(nb_oargs == 1); - tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(mulsh): if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == 0) { - tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); + tcg_opt_gen_movi(&ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(or): CASE_OP_32_64_VEC(and): if (args_are_copies(op->args[1], op->args[2])) { - tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(sub): CASE_OP_32_64_VEC(xor): if (args_are_copies(op->args[1], op->args[2])) { - tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); + tcg_opt_gen_movi(&ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) allocator where needed and possible. Also detect copies. */ switch (opc) { CASE_OP_32_64_VEC(mov): - tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; case INDEX_op_dup_vec: if (arg_is_const(op->args[1])) { tmp = arg_info(op->args[1])->val; tmp = dup_const(TCGOP_VECE(op), tmp); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_dup2_vec: assert(TCG_TARGET_REG_BITS == 32); if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - tcg_opt_gen_movi(s, &ctx, op, op->args[0], + tcg_opt_gen_movi(&ctx, op, op->args[0], deposit64(arg_info(op->args[1])->val, 32, 32, arg_info(op->args[2])->val)); continue; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_extrh_i64_i32: if (arg_is_const(op->args[1])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, 0); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, op->args[2]); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, arg_info(op->args[2])->val); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) TCGArg v = arg_info(op->args[1])->val; if (v != 0) { tmp = do_constant_folding(opc, v, 0); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); } else { - tcg_opt_gen_mov(s, op, op->args[0], op->args[2]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[2]); } continue; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = deposit64(arg_info(op->args[1])->val, op->args[3], op->args[4], arg_info(op->args[2])->val); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = extract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = sextract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = (int32_t)(((uint32_t)v1 >> shr) | ((uint32_t)v2 << (32 - shr))); } - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[3]); if (tmp != 2) { - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[5]); if (tmp != 2) { - tcg_opt_gen_mov(s, op, op->args[0], op->args[4-tmp]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[4-tmp]); continue; } if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)a); - tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(a >> 32)); + tcg_opt_gen_movi(&ctx, op, rl, (int32_t)a); + tcg_opt_gen_movi(&ctx, op2, rh, (int32_t)(a >> 32)); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)r); - tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(r >> 32)); + tcg_opt_gen_movi(&ctx, op, rl, (int32_t)r); + tcg_opt_gen_movi(&ctx, op2, rh, (int32_t)(r >> 32)); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[5]); if (tmp != 2) { do_setcond_const: - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } if ((op->args[5] == TCG_COND_LT || op->args[5] == TCG_COND_GE) -- 2.25.1 This will expose the variable to subroutines that will be broken out of tcg_optimize. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { typedef struct OptContext { TCGContext *tcg; + TCGOp *prev_mb; TCGTempSet temps_used; } OptContext; @@ -XXX,XX +XXX,XX @@ static bool swap_commutative2(TCGArg *p1, TCGArg *p2) void tcg_optimize(TCGContext *s) { int nb_temps, nb_globals, i; - TCGOp *op, *op_next, *prev_mb = NULL; + TCGOp *op, *op_next; OptContext ctx = { .tcg = s }; /* Array VALS has an element for each temp. @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } /* Eliminate duplicate and redundant fence instructions. */ - if (prev_mb) { + if (ctx.prev_mb) { switch (opc) { case INDEX_op_mb: /* Merge two barriers of the same type into one, @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) * barrier. This is stricter than specified but for * the purposes of TCG is better than not optimizing. */ - prev_mb->args[0] |= op->args[0]; + ctx.prev_mb->args[0] |= op->args[0]; tcg_op_remove(s, op); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_qemu_st_i64: case INDEX_op_call: /* Opcodes that touch guest memory stop the optimization. */ - prev_mb = NULL; + ctx.prev_mb = NULL; break; } } else if (opc == INDEX_op_mb) { - prev_mb = op; + ctx.prev_mb = op; } } } -- 2.25.1 There was no real reason for calls to have separate code here. Unify init for calls vs non-calls using the call path, which handles TCG_CALL_DUMMY_ARG. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void init_ts_info(OptContext *ctx, TCGTemp *ts) } } -static void init_arg_info(OptContext *ctx, TCGArg arg) -{ - init_ts_info(ctx, arg_temp(arg)); -} - static TCGTemp *find_better_copy(TCGContext *s, TCGTemp *ts) { TCGTemp *i, *g, *l; @@ -XXX,XX +XXX,XX @@ static bool swap_commutative2(TCGArg *p1, TCGArg *p2) return false; } +static void init_arguments(OptContext *ctx, TCGOp *op, int nb_args) +{ + for (int i = 0; i < nb_args; i++) { + TCGTemp *ts = arg_temp(op->args[i]); + if (ts) { + init_ts_info(ctx, ts); + } + } +} + /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (opc == INDEX_op_call) { nb_oargs = TCGOP_CALLO(op); nb_iargs = TCGOP_CALLI(op); - for (i = 0; i < nb_oargs + nb_iargs; i++) { - TCGTemp *ts = arg_temp(op->args[i]); - if (ts) { - init_ts_info(&ctx, ts); - } - } } else { nb_oargs = def->nb_oargs; nb_iargs = def->nb_iargs; - for (i = 0; i < nb_oargs + nb_iargs; i++) { - init_arg_info(&ctx, op->args[i]); - } } + init_arguments(&ctx, op, nb_oargs + nb_iargs); /* Do copy propagation */ for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { -- 2.25.1 Continue splitting tcg_optimize. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void init_arguments(OptContext *ctx, TCGOp *op, int nb_args) } } +static void copy_propagate(OptContext *ctx, TCGOp *op, + int nb_oargs, int nb_iargs) +{ + TCGContext *s = ctx->tcg; + + for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) { + TCGTemp *ts = arg_temp(op->args[i]); + if (ts && ts_is_copy(ts)) { + op->args[i] = temp_arg(find_better_copy(s, ts)); + } + } +} + /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) nb_iargs = def->nb_iargs; } init_arguments(&ctx, op, nb_oargs + nb_iargs); - - /* Do copy propagation */ - for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { - TCGTemp *ts = arg_temp(op->args[i]); - if (ts && ts_is_copy(ts)) { - op->args[i] = temp_arg(find_better_copy(s, ts)); - } - } + copy_propagate(&ctx, op, nb_oargs, nb_iargs); /* For commutative operations make constant second argument */ switch (opc) { -- 2.25.1 Calls are special in that they have a variable number of arguments, and need to be able to clobber globals. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 63 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void copy_propagate(OptContext *ctx, TCGOp *op, } } +static bool fold_call(OptContext *ctx, TCGOp *op) +{ + TCGContext *s = ctx->tcg; + int nb_oargs = TCGOP_CALLO(op); + int nb_iargs = TCGOP_CALLI(op); + int flags, i; + + init_arguments(ctx, op, nb_oargs + nb_iargs); + copy_propagate(ctx, op, nb_oargs, nb_iargs); + + /* If the function reads or writes globals, reset temp data. */ + flags = tcg_call_flags(op); + if (!(flags & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) { + int nb_globals = s->nb_globals; + + for (i = 0; i < nb_globals; i++) { + if (test_bit(i, ctx->temps_used.l)) { + reset_ts(&ctx->tcg->temps[i]); + } + } + } + + /* Reset temp data for outputs. */ + for (i = 0; i < nb_oargs; i++) { + reset_temp(op->args[i]); + } + + /* Stop optimizing MB across calls. */ + ctx->prev_mb = NULL; + return true; +} + /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { - int nb_temps, nb_globals, i; + int nb_temps, i; TCGOp *op, *op_next; OptContext ctx = { .tcg = s }; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) available through the doubly linked circular list. */ nb_temps = s->nb_temps; - nb_globals = s->nb_globals; - for (i = 0; i < nb_temps; ++i) { s->temps[i].state_ptr = NULL; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) uint64_t z_mask, partmask, affected, tmp; int nb_oargs, nb_iargs; TCGOpcode opc = op->opc; - const TCGOpDef *def = &tcg_op_defs[opc]; + const TCGOpDef *def; - /* Count the arguments, and initialize the temps that are - going to be used */ + /* Calls are special. */ if (opc == INDEX_op_call) { - nb_oargs = TCGOP_CALLO(op); - nb_iargs = TCGOP_CALLI(op); - } else { - nb_oargs = def->nb_oargs; - nb_iargs = def->nb_iargs; + fold_call(&ctx, op); + continue; } + + def = &tcg_op_defs[opc]; + nb_oargs = def->nb_oargs; + nb_iargs = def->nb_iargs; init_arguments(&ctx, op, nb_oargs + nb_iargs); copy_propagate(&ctx, op, nb_oargs, nb_iargs); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (def->flags & TCG_OPF_BB_END) { memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); } else { - if (opc == INDEX_op_call && - !(tcg_call_flags(op) - & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) { - for (i = 0; i < nb_globals; i++) { - if (test_bit(i, ctx.temps_used.l)) { - reset_ts(&s->temps[i]); - } - } - } - for (i = 0; i < nb_oargs; i++) { reset_temp(op->args[i]); /* Save the corresponding known-zero bits mask for the @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_qemu_st_i32: case INDEX_op_qemu_st8_i32: case INDEX_op_qemu_st_i64: - case INDEX_op_call: /* Opcodes that touch guest memory stop the optimization. */ ctx.prev_mb = NULL; break; -- 2.25.1 Rather than try to keep these up-to-date across folding, re-read nb_oargs at the end, after re-reading the opcode. A couple of asserts need dropping, but that will take care of itself as we split the function further. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { uint64_t z_mask, partmask, affected, tmp; - int nb_oargs, nb_iargs; TCGOpcode opc = op->opc; const TCGOpDef *def; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } def = &tcg_op_defs[opc]; - nb_oargs = def->nb_oargs; - nb_iargs = def->nb_iargs; - init_arguments(&ctx, op, nb_oargs + nb_iargs); - copy_propagate(&ctx, op, nb_oargs, nb_iargs); + init_arguments(&ctx, op, def->nb_oargs + def->nb_iargs); + copy_propagate(&ctx, op, def->nb_oargs, def->nb_iargs); /* For commutative operations make constant second argument */ switch (opc) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(qemu_ld): { - MemOpIdx oi = op->args[nb_oargs + nb_iargs]; + MemOpIdx oi = op->args[def->nb_oargs + def->nb_iargs]; MemOp mop = get_memop(oi); if (!(mop & MO_SIGN)) { z_mask = (2ULL << ((8 << (mop & MO_SIZE)) - 1)) - 1; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } if (partmask == 0) { - tcg_debug_assert(nb_oargs == 1); tcg_opt_gen_movi(&ctx, op, op->args[0], 0); continue; } if (affected == 0) { - tcg_debug_assert(nb_oargs == 1); tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } else if (args_are_copies(op->args[1], op->args[2])) { op->opc = INDEX_op_dup_vec; TCGOP_VECE(op) = MO_32; - nb_iargs = 1; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->opc = opc = (opc == INDEX_op_movcond_i32 ? INDEX_op_setcond_i32 : INDEX_op_setcond_i64); - nb_iargs = 2; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (def->flags & TCG_OPF_BB_END) { memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); } else { + int nb_oargs = def->nb_oargs; for (i = 0; i < nb_oargs; i++) { reset_temp(op->args[i]); /* Save the corresponding known-zero bits mask for the -- 2.25.1 Return -1 instead of 2 for failure, so that we can use comparisons against 0 for all cases. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 145 +++++++++++++++++++++++++------------------------ 1 file changed, 74 insertions(+), 71 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool do_constant_folding_cond_eq(TCGCond c) } } -/* Return 2 if the condition can't be simplified, and the result - of the condition (0 or 1) if it can */ -static TCGArg do_constant_folding_cond(TCGOpcode op, TCGArg x, - TCGArg y, TCGCond c) +/* + * Return -1 if the condition can't be simplified, + * and the result of the condition (0 or 1) if it can. + */ +static int do_constant_folding_cond(TCGOpcode op, TCGArg x, + TCGArg y, TCGCond c) { uint64_t xv = arg_info(x)->val; uint64_t yv = arg_info(y)->val; @@ -XXX,XX +XXX,XX @@ static TCGArg do_constant_folding_cond(TCGOpcode op, TCGArg x, case TCG_COND_GEU: return 1; default: - return 2; + return -1; } } - return 2; + return -1; } -/* Return 2 if the condition can't be simplified, and the result - of the condition (0 or 1) if it can */ -static TCGArg do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c) +/* + * Return -1 if the condition can't be simplified, + * and the result of the condition (0 or 1) if it can. + */ +static int do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c) { TCGArg al = p1[0], ah = p1[1]; TCGArg bl = p2[0], bh = p2[1]; @@ -XXX,XX +XXX,XX @@ static TCGArg do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c) if (args_are_copies(al, bl) && args_are_copies(ah, bh)) { return do_constant_folding_cond_eq(c); } - return 2; + return -1; } static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; CASE_OP_32_64(setcond): - tmp = do_constant_folding_cond(opc, op->args[1], - op->args[2], op->args[3]); - if (tmp != 2) { - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); + i = do_constant_folding_cond(opc, op->args[1], + op->args[2], op->args[3]); + if (i >= 0) { + tcg_opt_gen_movi(&ctx, op, op->args[0], i); continue; } break; CASE_OP_32_64(brcond): - tmp = do_constant_folding_cond(opc, op->args[0], - op->args[1], op->args[2]); - switch (tmp) { - case 0: + i = do_constant_folding_cond(opc, op->args[0], + op->args[1], op->args[2]); + if (i == 0) { tcg_op_remove(s, op); continue; - case 1: + } else if (i > 0) { memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); op->opc = opc = INDEX_op_br; op->args[0] = op->args[3]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; CASE_OP_32_64(movcond): - tmp = do_constant_folding_cond(opc, op->args[1], - op->args[2], op->args[5]); - if (tmp != 2) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[4-tmp]); + i = do_constant_folding_cond(opc, op->args[1], + op->args[2], op->args[5]); + if (i >= 0) { + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[4 - i]); continue; } if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; case INDEX_op_brcond2_i32: - tmp = do_constant_folding_cond2(&op->args[0], &op->args[2], - op->args[4]); - if (tmp == 0) { + i = do_constant_folding_cond2(&op->args[0], &op->args[2], + op->args[4]); + if (i == 0) { do_brcond_false: tcg_op_remove(s, op); continue; } - if (tmp == 1) { + if (i > 0) { do_brcond_true: op->opc = opc = INDEX_op_br; op->args[0] = op->args[5]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (op->args[4] == TCG_COND_EQ) { /* Simplify EQ comparisons where one of the pairs can be simplified. */ - tmp = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[0], op->args[2], - TCG_COND_EQ); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_brcond_i32, + op->args[0], op->args[2], + TCG_COND_EQ); + if (i == 0) { goto do_brcond_false; - } else if (tmp == 1) { + } else if (i > 0) { goto do_brcond_high; } - tmp = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[1], op->args[3], - TCG_COND_EQ); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_brcond_i32, + op->args[1], op->args[3], + TCG_COND_EQ); + if (i == 0) { goto do_brcond_false; - } else if (tmp != 1) { + } else if (i < 0) { break; } do_brcond_low: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (op->args[4] == TCG_COND_NE) { /* Simplify NE comparisons where one of the pairs can be simplified. */ - tmp = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[0], op->args[2], - TCG_COND_NE); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_brcond_i32, + op->args[0], op->args[2], + TCG_COND_NE); + if (i == 0) { goto do_brcond_high; - } else if (tmp == 1) { + } else if (i > 0) { goto do_brcond_true; } - tmp = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[1], op->args[3], - TCG_COND_NE); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_brcond_i32, + op->args[1], op->args[3], + TCG_COND_NE); + if (i == 0) { goto do_brcond_low; - } else if (tmp == 1) { + } else if (i > 0) { goto do_brcond_true; } } break; case INDEX_op_setcond2_i32: - tmp = do_constant_folding_cond2(&op->args[1], &op->args[3], - op->args[5]); - if (tmp != 2) { + i = do_constant_folding_cond2(&op->args[1], &op->args[3], + op->args[5]); + if (i >= 0) { do_setcond_const: - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], i); continue; } if ((op->args[5] == TCG_COND_LT || op->args[5] == TCG_COND_GE) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (op->args[5] == TCG_COND_EQ) { /* Simplify EQ comparisons where one of the pairs can be simplified. */ - tmp = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[1], op->args[3], - TCG_COND_EQ); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_setcond_i32, + op->args[1], op->args[3], + TCG_COND_EQ); + if (i == 0) { goto do_setcond_const; - } else if (tmp == 1) { + } else if (i > 0) { goto do_setcond_high; } - tmp = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[2], op->args[4], - TCG_COND_EQ); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_setcond_i32, + op->args[2], op->args[4], + TCG_COND_EQ); + if (i == 0) { goto do_setcond_high; - } else if (tmp != 1) { + } else if (i < 0) { break; } do_setcond_low: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (op->args[5] == TCG_COND_NE) { /* Simplify NE comparisons where one of the pairs can be simplified. */ - tmp = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[1], op->args[3], - TCG_COND_NE); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_setcond_i32, + op->args[1], op->args[3], + TCG_COND_NE); + if (i == 0) { goto do_setcond_high; - } else if (tmp == 1) { + } else if (i > 0) { goto do_setcond_const; } - tmp = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[2], op->args[4], - TCG_COND_NE); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_setcond_i32, + op->args[2], op->args[4], + TCG_COND_NE); + if (i == 0) { goto do_setcond_low; - } else if (tmp == 1) { + } else if (i > 0) { goto do_setcond_const; } } -- 2.25.1 This will allow callers to tail call to these functions and return true indicating processing complete. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool args_are_copies(TCGArg arg1, TCGArg arg2) return ts_are_copies(arg_temp(arg1), arg_temp(arg2)); } -static void tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) +static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) { TCGTemp *dst_ts = arg_temp(dst); TCGTemp *src_ts = arg_temp(src); @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) if (ts_are_copies(dst_ts, src_ts)) { tcg_op_remove(ctx->tcg, op); - return; + return true; } reset_ts(dst_ts); @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) di->is_const = si->is_const; di->val = si->val; } + return true; } -static void tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, +static bool tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, TCGArg dst, uint64_t val) { const TCGOpDef *def = &tcg_op_defs[op->opc]; @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, /* Convert movi to mov with constant temp. */ tv = tcg_constant_internal(type, val); init_ts_info(ctx, tv); - tcg_opt_gen_mov(ctx, op, dst, temp_arg(tv)); + return tcg_opt_gen_mov(ctx, op, dst, temp_arg(tv)); } static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y) -- 2.25.1 Copy z_mask into OptContext, for writeback to the first output within the new function. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 49 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { TCGContext *tcg; TCGOp *prev_mb; TCGTempSet temps_used; + + /* In flight values from optimization. */ + uint64_t z_mask; } OptContext; static inline TempOptInfo *ts_info(TCGTemp *ts) @@ -XXX,XX +XXX,XX @@ static void copy_propagate(OptContext *ctx, TCGOp *op, } } +static void finish_folding(OptContext *ctx, TCGOp *op) +{ + const TCGOpDef *def = &tcg_op_defs[op->opc]; + int i, nb_oargs; + + /* + * For an opcode that ends a BB, reset all temp data. + * We do no cross-BB optimization. + */ + if (def->flags & TCG_OPF_BB_END) { + memset(&ctx->temps_used, 0, sizeof(ctx->temps_used)); + ctx->prev_mb = NULL; + return; + } + + nb_oargs = def->nb_oargs; + for (i = 0; i < nb_oargs; i++) { + reset_temp(op->args[i]); + /* + * Save the corresponding known-zero bits mask for the + * first output argument (only one supported so far). + */ + if (i == 0) { + arg_info(op->args[i])->z_mask = ctx->z_mask; + } + } +} + static bool fold_call(OptContext *ctx, TCGOp *op) { TCGContext *s = ctx->tcg; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) partmask &= 0xffffffffu; affected &= 0xffffffffu; } + ctx.z_mask = z_mask; if (partmask == 0) { tcg_opt_gen_movi(&ctx, op, op->args[0], 0); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Some of the folding above can change opc. */ - opc = op->opc; - def = &tcg_op_defs[opc]; - if (def->flags & TCG_OPF_BB_END) { - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - } else { - int nb_oargs = def->nb_oargs; - for (i = 0; i < nb_oargs; i++) { - reset_temp(op->args[i]); - /* Save the corresponding known-zero bits mask for the - first output argument (only one supported so far). */ - if (i == 0) { - arg_info(op->args[i])->z_mask = z_mask; - } - } - } + finish_folding(&ctx, op); /* Eliminate duplicate and redundant fence instructions. */ if (ctx.prev_mb) { -- 2.25.1 Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) uint64_t z_mask, partmask, affected, tmp; TCGOpcode opc = op->opc; const TCGOpDef *def; + bool done = false; /* Calls are special. */ if (opc == INDEX_op_call) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) allocator where needed and possible. Also detect copies. */ switch (opc) { CASE_OP_32_64_VEC(mov): - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - continue; + done = tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); + break; case INDEX_op_dup_vec: if (arg_is_const(op->args[1])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - finish_folding(&ctx, op); + if (!done) { + finish_folding(&ctx, op); + } /* Eliminate duplicate and redundant fence instructions. */ if (ctx.prev_mb) { -- 2.25.1 This puts the separate mb optimization into the same framework as the others. While fold_qemu_{ld,st} are currently identical, that won't last as more code gets moved. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 89 +++++++++++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_call(OptContext *ctx, TCGOp *op) return true; } +static bool fold_mb(OptContext *ctx, TCGOp *op) +{ + /* Eliminate duplicate and redundant fence instructions. */ + if (ctx->prev_mb) { + /* + * Merge two barriers of the same type into one, + * or a weaker barrier into a stronger one, + * or two weaker barriers into a stronger one. + * mb X; mb Y => mb X|Y + * mb; strl => mb; st + * ldaq; mb => ld; mb + * ldaq; strl => ld; mb; st + * Other combinations are also merged into a strong + * barrier. This is stricter than specified but for + * the purposes of TCG is better than not optimizing. + */ + ctx->prev_mb->args[0] |= op->args[0]; + tcg_op_remove(ctx->tcg, op); + } else { + ctx->prev_mb = op; + } + return true; +} + +static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) +{ + /* Opcodes that touch guest memory stop the mb optimization. */ + ctx->prev_mb = NULL; + return false; +} + +static bool fold_qemu_st(OptContext *ctx, TCGOp *op) +{ + /* Opcodes that touch guest memory stop the mb optimization. */ + ctx->prev_mb = NULL; + return false; +} + /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; + case INDEX_op_mb: + done = fold_mb(&ctx, op); + break; + case INDEX_op_qemu_ld_i32: + case INDEX_op_qemu_ld_i64: + done = fold_qemu_ld(&ctx, op); + break; + case INDEX_op_qemu_st_i32: + case INDEX_op_qemu_st8_i32: + case INDEX_op_qemu_st_i64: + done = fold_qemu_st(&ctx, op); + break; + default: break; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (!done) { finish_folding(&ctx, op); } - - /* Eliminate duplicate and redundant fence instructions. */ - if (ctx.prev_mb) { - switch (opc) { - case INDEX_op_mb: - /* Merge two barriers of the same type into one, - * or a weaker barrier into a stronger one, - * or two weaker barriers into a stronger one. - * mb X; mb Y => mb X|Y - * mb; strl => mb; st - * ldaq; mb => ld; mb - * ldaq; strl => ld; mb; st - * Other combinations are also merged into a strong - * barrier. This is stricter than specified but for - * the purposes of TCG is better than not optimizing. - */ - ctx.prev_mb->args[0] |= op->args[0]; - tcg_op_remove(s, op); - break; - - default: - /* Opcodes that end the block stop the optimization. */ - if ((def->flags & TCG_OPF_BB_END) == 0) { - break; - } - /* fallthru */ - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_ld_i64: - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st8_i32: - case INDEX_op_qemu_st_i64: - /* Opcodes that touch guest memory stop the optimization. */ - ctx.prev_mb = NULL; - break; - } - } else if (opc == INDEX_op_mb) { - ctx.prev_mb = op; - } } } -- 2.25.1 Split out a whole bunch of placeholder functions, which are currently identical. That won't last as more code gets moved. Use CASE_32_64_VEC for some logical operators that previously missed the addition of vectors. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 271 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 219 insertions(+), 52 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void finish_folding(OptContext *ctx, TCGOp *op) } } +/* + * The fold_* functions return true when processing is complete, + * usually by folding the operation to a constant or to a copy, + * and calling tcg_opt_gen_{mov,movi}. They may do other things, + * like collect information about the value produced, for use in + * optimizing a subsequent operation. + * + * These first fold_* functions are all helpers, used by other + * folders for more specific operations. + */ + +static bool fold_const1(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t; + + t = arg_info(op->args[1])->val; + t = do_constant_folding(op->opc, t, 0); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return false; +} + +static bool fold_const2(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { + uint64_t t1 = arg_info(op->args[1])->val; + uint64_t t2 = arg_info(op->args[2])->val; + + t1 = do_constant_folding(op->opc, t1, t2); + return tcg_opt_gen_movi(ctx, op, op->args[0], t1); + } + return false; +} + +/* + * These outermost fold_<op> functions are sorted alphabetically. + */ + +static bool fold_add(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_and(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_andc(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + static bool fold_call(OptContext *ctx, TCGOp *op) { TCGContext *s = ctx->tcg; @@ -XXX,XX +XXX,XX @@ static bool fold_call(OptContext *ctx, TCGOp *op) return true; } +static bool fold_ctpop(OptContext *ctx, TCGOp *op) +{ + return fold_const1(ctx, op); +} + +static bool fold_divide(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_eqv(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_exts(OptContext *ctx, TCGOp *op) +{ + return fold_const1(ctx, op); +} + +static bool fold_extu(OptContext *ctx, TCGOp *op) +{ + return fold_const1(ctx, op); +} + static bool fold_mb(OptContext *ctx, TCGOp *op) { /* Eliminate duplicate and redundant fence instructions. */ @@ -XXX,XX +XXX,XX @@ static bool fold_mb(OptContext *ctx, TCGOp *op) return true; } +static bool fold_mul(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_nand(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_neg(OptContext *ctx, TCGOp *op) +{ + return fold_const1(ctx, op); +} + +static bool fold_nor(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_not(OptContext *ctx, TCGOp *op) +{ + return fold_const1(ctx, op); +} + +static bool fold_or(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_orc(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) { /* Opcodes that touch guest memory stop the mb optimization. */ @@ -XXX,XX +XXX,XX @@ static bool fold_qemu_st(OptContext *ctx, TCGOp *op) return false; } +static bool fold_remainder(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_shift(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_sub(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_xor(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(not): - CASE_OP_32_64(neg): - CASE_OP_32_64(ext8s): - CASE_OP_32_64(ext8u): - CASE_OP_32_64(ext16s): - CASE_OP_32_64(ext16u): - CASE_OP_32_64(ctpop): - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_extrl_i64_i32: - case INDEX_op_extrh_i64_i32: - if (arg_is_const(op->args[1])) { - tmp = do_constant_folding(opc, arg_info(op->args[1])->val, 0); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - CASE_OP_32_64(bswap16): CASE_OP_32_64(bswap32): case INDEX_op_bswap64_i64: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(add): - CASE_OP_32_64(sub): - CASE_OP_32_64(mul): - CASE_OP_32_64(or): - CASE_OP_32_64(and): - CASE_OP_32_64(xor): - CASE_OP_32_64(shl): - CASE_OP_32_64(shr): - CASE_OP_32_64(sar): - CASE_OP_32_64(rotl): - CASE_OP_32_64(rotr): - CASE_OP_32_64(andc): - CASE_OP_32_64(orc): - CASE_OP_32_64(eqv): - CASE_OP_32_64(nand): - CASE_OP_32_64(nor): - CASE_OP_32_64(muluh): - CASE_OP_32_64(mulsh): - CASE_OP_32_64(div): - CASE_OP_32_64(divu): - CASE_OP_32_64(rem): - CASE_OP_32_64(remu): - if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - tmp = do_constant_folding(opc, arg_info(op->args[1])->val, - arg_info(op->args[2])->val); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - CASE_OP_32_64(clz): CASE_OP_32_64(ctz): if (arg_is_const(op->args[1])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; + default: + break; + + /* ---------------------------------------------------------- */ + /* Sorted alphabetically by opcode as much as possible. */ + + CASE_OP_32_64_VEC(add): + done = fold_add(&ctx, op); + break; + CASE_OP_32_64_VEC(and): + done = fold_and(&ctx, op); + break; + CASE_OP_32_64_VEC(andc): + done = fold_andc(&ctx, op); + break; + CASE_OP_32_64(ctpop): + done = fold_ctpop(&ctx, op); + break; + CASE_OP_32_64(div): + CASE_OP_32_64(divu): + done = fold_divide(&ctx, op); + break; + CASE_OP_32_64(eqv): + done = fold_eqv(&ctx, op); + break; + CASE_OP_32_64(ext8s): + CASE_OP_32_64(ext16s): + case INDEX_op_ext32s_i64: + case INDEX_op_ext_i32_i64: + done = fold_exts(&ctx, op); + break; + CASE_OP_32_64(ext8u): + CASE_OP_32_64(ext16u): + case INDEX_op_ext32u_i64: + case INDEX_op_extu_i32_i64: + case INDEX_op_extrl_i64_i32: + case INDEX_op_extrh_i64_i32: + done = fold_extu(&ctx, op); + break; case INDEX_op_mb: done = fold_mb(&ctx, op); break; + CASE_OP_32_64(mul): + done = fold_mul(&ctx, op); + break; + CASE_OP_32_64(mulsh): + CASE_OP_32_64(muluh): + done = fold_mul_highpart(&ctx, op); + break; + CASE_OP_32_64(nand): + done = fold_nand(&ctx, op); + break; + CASE_OP_32_64(neg): + done = fold_neg(&ctx, op); + break; + CASE_OP_32_64(nor): + done = fold_nor(&ctx, op); + break; + CASE_OP_32_64_VEC(not): + done = fold_not(&ctx, op); + break; + CASE_OP_32_64_VEC(or): + done = fold_or(&ctx, op); + break; + CASE_OP_32_64_VEC(orc): + done = fold_orc(&ctx, op); + break; case INDEX_op_qemu_ld_i32: case INDEX_op_qemu_ld_i64: done = fold_qemu_ld(&ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_qemu_st_i64: done = fold_qemu_st(&ctx, op); break; - - default: + CASE_OP_32_64(rem): + CASE_OP_32_64(remu): + done = fold_remainder(&ctx, op); + break; + CASE_OP_32_64(rotl): + CASE_OP_32_64(rotr): + CASE_OP_32_64(sar): + CASE_OP_32_64(shl): + CASE_OP_32_64(shr): + done = fold_shift(&ctx, op); + break; + CASE_OP_32_64_VEC(sub): + done = fold_sub(&ctx, op); + break; + CASE_OP_32_64_VEC(xor): + done = fold_xor(&ctx, op); break; } -- 2.25.1 Reduce some code duplication by folding the NE and EQ cases. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 145 ++++++++++++++++++++++++------------------------- 1 file changed, 72 insertions(+), 73 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_remainder(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_setcond2(OptContext *ctx, TCGOp *op) +{ + TCGCond cond = op->args[5]; + int i = do_constant_folding_cond2(&op->args[1], &op->args[3], cond); + int inv = 0; + + if (i >= 0) { + goto do_setcond_const; + } + + switch (cond) { + case TCG_COND_LT: + case TCG_COND_GE: + /* + * Simplify LT/GE comparisons vs zero to a single compare + * vs the high word of the input. + */ + if (arg_is_const(op->args[3]) && arg_info(op->args[3])->val == 0 && + arg_is_const(op->args[4]) && arg_info(op->args[4])->val == 0) { + goto do_setcond_high; + } + break; + + case TCG_COND_NE: + inv = 1; + QEMU_FALLTHROUGH; + case TCG_COND_EQ: + /* + * Simplify EQ/NE comparisons where one of the pairs + * can be simplified. + */ + i = do_constant_folding_cond(INDEX_op_setcond_i32, op->args[1], + op->args[3], cond); + switch (i ^ inv) { + case 0: + goto do_setcond_const; + case 1: + goto do_setcond_high; + } + + i = do_constant_folding_cond(INDEX_op_setcond_i32, op->args[2], + op->args[4], cond); + switch (i ^ inv) { + case 0: + goto do_setcond_const; + case 1: + op->args[2] = op->args[3]; + op->args[3] = cond; + op->opc = INDEX_op_setcond_i32; + break; + } + break; + + default: + break; + + do_setcond_high: + op->args[1] = op->args[2]; + op->args[2] = op->args[4]; + op->args[3] = cond; + op->opc = INDEX_op_setcond_i32; + break; + } + return false; + + do_setcond_const: + return tcg_opt_gen_movi(ctx, op, op->args[0], i); +} + static bool fold_shift(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - case INDEX_op_setcond2_i32: - i = do_constant_folding_cond2(&op->args[1], &op->args[3], - op->args[5]); - if (i >= 0) { - do_setcond_const: - tcg_opt_gen_movi(&ctx, op, op->args[0], i); - continue; - } - if ((op->args[5] == TCG_COND_LT || op->args[5] == TCG_COND_GE) - && arg_is_const(op->args[3]) - && arg_info(op->args[3])->val == 0 - && arg_is_const(op->args[4]) - && arg_info(op->args[4])->val == 0) { - /* Simplify LT/GE comparisons vs zero to a single compare - vs the high word of the input. */ - do_setcond_high: - reset_temp(op->args[0]); - arg_info(op->args[0])->z_mask = 1; - op->opc = INDEX_op_setcond_i32; - op->args[1] = op->args[2]; - op->args[2] = op->args[4]; - op->args[3] = op->args[5]; - break; - } - if (op->args[5] == TCG_COND_EQ) { - /* Simplify EQ comparisons where one of the pairs - can be simplified. */ - i = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[1], op->args[3], - TCG_COND_EQ); - if (i == 0) { - goto do_setcond_const; - } else if (i > 0) { - goto do_setcond_high; - } - i = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[2], op->args[4], - TCG_COND_EQ); - if (i == 0) { - goto do_setcond_high; - } else if (i < 0) { - break; - } - do_setcond_low: - reset_temp(op->args[0]); - arg_info(op->args[0])->z_mask = 1; - op->opc = INDEX_op_setcond_i32; - op->args[2] = op->args[3]; - op->args[3] = op->args[5]; - break; - } - if (op->args[5] == TCG_COND_NE) { - /* Simplify NE comparisons where one of the pairs - can be simplified. */ - i = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[1], op->args[3], - TCG_COND_NE); - if (i == 0) { - goto do_setcond_high; - } else if (i > 0) { - goto do_setcond_const; - } - i = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[2], op->args[4], - TCG_COND_NE); - if (i == 0) { - goto do_setcond_low; - } else if (i > 0) { - goto do_setcond_const; - } - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(shr): done = fold_shift(&ctx, op); break; + case INDEX_op_setcond2_i32: + done = fold_setcond2(&ctx, op); + break; CASE_OP_32_64_VEC(sub): done = fold_sub(&ctx, op); break; -- 2.25.1 Reduce some code duplication by folding the NE and EQ cases. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 159 +++++++++++++++++++++++++------------------------ 1 file changed, 81 insertions(+), 78 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_brcond2(OptContext *ctx, TCGOp *op) +{ + TCGCond cond = op->args[4]; + int i = do_constant_folding_cond2(&op->args[0], &op->args[2], cond); + TCGArg label = op->args[5]; + int inv = 0; + + if (i >= 0) { + goto do_brcond_const; + } + + switch (cond) { + case TCG_COND_LT: + case TCG_COND_GE: + /* + * Simplify LT/GE comparisons vs zero to a single compare + * vs the high word of the input. + */ + if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == 0 && + arg_is_const(op->args[3]) && arg_info(op->args[3])->val == 0) { + goto do_brcond_high; + } + break; + + case TCG_COND_NE: + inv = 1; + QEMU_FALLTHROUGH; + case TCG_COND_EQ: + /* + * Simplify EQ/NE comparisons where one of the pairs + * can be simplified. + */ + i = do_constant_folding_cond(INDEX_op_brcond_i32, op->args[0], + op->args[2], cond); + switch (i ^ inv) { + case 0: + goto do_brcond_const; + case 1: + goto do_brcond_high; + } + + i = do_constant_folding_cond(INDEX_op_brcond_i32, op->args[1], + op->args[3], cond); + switch (i ^ inv) { + case 0: + goto do_brcond_const; + case 1: + op->opc = INDEX_op_brcond_i32; + op->args[1] = op->args[2]; + op->args[2] = cond; + op->args[3] = label; + break; + } + break; + + default: + break; + + do_brcond_high: + op->opc = INDEX_op_brcond_i32; + op->args[0] = op->args[1]; + op->args[1] = op->args[3]; + op->args[2] = cond; + op->args[3] = label; + break; + + do_brcond_const: + if (i == 0) { + tcg_op_remove(ctx->tcg, op); + return true; + } + op->opc = INDEX_op_br; + op->args[0] = label; + break; + } + return false; +} + static bool fold_call(OptContext *ctx, TCGOp *op) { TCGContext *s = ctx->tcg; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - case INDEX_op_brcond2_i32: - i = do_constant_folding_cond2(&op->args[0], &op->args[2], - op->args[4]); - if (i == 0) { - do_brcond_false: - tcg_op_remove(s, op); - continue; - } - if (i > 0) { - do_brcond_true: - op->opc = opc = INDEX_op_br; - op->args[0] = op->args[5]; - break; - } - if ((op->args[4] == TCG_COND_LT || op->args[4] == TCG_COND_GE) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == 0 - && arg_is_const(op->args[3]) - && arg_info(op->args[3])->val == 0) { - /* Simplify LT/GE comparisons vs zero to a single compare - vs the high word of the input. */ - do_brcond_high: - op->opc = opc = INDEX_op_brcond_i32; - op->args[0] = op->args[1]; - op->args[1] = op->args[3]; - op->args[2] = op->args[4]; - op->args[3] = op->args[5]; - break; - } - if (op->args[4] == TCG_COND_EQ) { - /* Simplify EQ comparisons where one of the pairs - can be simplified. */ - i = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[0], op->args[2], - TCG_COND_EQ); - if (i == 0) { - goto do_brcond_false; - } else if (i > 0) { - goto do_brcond_high; - } - i = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[1], op->args[3], - TCG_COND_EQ); - if (i == 0) { - goto do_brcond_false; - } else if (i < 0) { - break; - } - do_brcond_low: - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - op->opc = INDEX_op_brcond_i32; - op->args[1] = op->args[2]; - op->args[2] = op->args[4]; - op->args[3] = op->args[5]; - break; - } - if (op->args[4] == TCG_COND_NE) { - /* Simplify NE comparisons where one of the pairs - can be simplified. */ - i = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[0], op->args[2], - TCG_COND_NE); - if (i == 0) { - goto do_brcond_high; - } else if (i > 0) { - goto do_brcond_true; - } - i = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[1], op->args[3], - TCG_COND_NE); - if (i == 0) { - goto do_brcond_low; - } else if (i > 0) { - goto do_brcond_true; - } - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(andc): done = fold_andc(&ctx, op); break; + case INDEX_op_brcond2_i32: + done = fold_brcond2(&ctx, op); + break; CASE_OP_32_64(ctpop): done = fold_ctpop(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_brcond(OptContext *ctx, TCGOp *op) +{ + TCGCond cond = op->args[2]; + int i = do_constant_folding_cond(op->opc, op->args[0], op->args[1], cond); + + if (i == 0) { + tcg_op_remove(ctx->tcg, op); + return true; + } + if (i > 0) { + op->opc = INDEX_op_br; + op->args[0] = op->args[3]; + } + return false; +} + static bool fold_brcond2(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[4]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(brcond): - i = do_constant_folding_cond(opc, op->args[0], - op->args[1], op->args[2]); - if (i == 0) { - tcg_op_remove(s, op); - continue; - } else if (i > 0) { - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - op->opc = opc = INDEX_op_br; - op->args[0] = op->args[3]; - break; - } - break; - CASE_OP_32_64(movcond): i = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[5]); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(andc): done = fold_andc(&ctx, op); break; + CASE_OP_32_64(brcond): + done = fold_brcond(&ctx, op); + break; case INDEX_op_brcond2_i32: done = fold_brcond2(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_remainder(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_setcond(OptContext *ctx, TCGOp *op) +{ + TCGCond cond = op->args[3]; + int i = do_constant_folding_cond(op->opc, op->args[1], op->args[2], cond); + + if (i >= 0) { + return tcg_opt_gen_movi(ctx, op, op->args[0], i); + } + return false; +} + static bool fold_setcond2(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[5]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(setcond): - i = do_constant_folding_cond(opc, op->args[1], - op->args[2], op->args[3]); - if (i >= 0) { - tcg_opt_gen_movi(&ctx, op, op->args[0], i); - continue; - } - break; - CASE_OP_32_64(movcond): i = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[5]); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(shr): done = fold_shift(&ctx, op); break; + CASE_OP_32_64(setcond): + done = fold_setcond(&ctx, op); + break; case INDEX_op_setcond2_i32: done = fold_setcond2(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_mulu2_i32(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { + uint32_t a = arg_info(op->args[2])->val; + uint32_t b = arg_info(op->args[3])->val; + uint64_t r = (uint64_t)a * b; + TCGArg rl, rh; + TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_mov_i32); + + rl = op->args[0]; + rh = op->args[1]; + tcg_opt_gen_movi(ctx, op, rl, (int32_t)r); + tcg_opt_gen_movi(ctx, op2, rh, (int32_t)(r >> 32)); + return true; + } + return false; +} + static bool fold_nand(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - case INDEX_op_mulu2_i32: - if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { - uint32_t a = arg_info(op->args[2])->val; - uint32_t b = arg_info(op->args[3])->val; - uint64_t r = (uint64_t)a * b; - TCGArg rl, rh; - TCGOp *op2 = tcg_op_insert_before(s, op, INDEX_op_mov_i32); - - rl = op->args[0]; - rh = op->args[1]; - tcg_opt_gen_movi(&ctx, op, rl, (int32_t)r); - tcg_opt_gen_movi(&ctx, op2, rh, (int32_t)(r >> 32)); - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(muluh): done = fold_mul_highpart(&ctx, op); break; + case INDEX_op_mulu2_i32: + done = fold_mulu2_i32(&ctx, op); + break; CASE_OP_32_64(nand): done = fold_nand(&ctx, op); break; -- 2.25.1 Add two additional helpers, fold_add2_i32 and fold_sub2_i32 which will not be simple wrappers forever. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 70 +++++++++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_add(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_addsub2_i32(OptContext *ctx, TCGOp *op, bool add) +{ + if (arg_is_const(op->args[2]) && arg_is_const(op->args[3]) && + arg_is_const(op->args[4]) && arg_is_const(op->args[5])) { + uint32_t al = arg_info(op->args[2])->val; + uint32_t ah = arg_info(op->args[3])->val; + uint32_t bl = arg_info(op->args[4])->val; + uint32_t bh = arg_info(op->args[5])->val; + uint64_t a = ((uint64_t)ah << 32) | al; + uint64_t b = ((uint64_t)bh << 32) | bl; + TCGArg rl, rh; + TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_mov_i32); + + if (add) { + a += b; + } else { + a -= b; + } + + rl = op->args[0]; + rh = op->args[1]; + tcg_opt_gen_movi(ctx, op, rl, (int32_t)a); + tcg_opt_gen_movi(ctx, op2, rh, (int32_t)(a >> 32)); + return true; + } + return false; +} + +static bool fold_add2_i32(OptContext *ctx, TCGOp *op) +{ + return fold_addsub2_i32(ctx, op, true); +} + static bool fold_and(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ static bool fold_sub(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) +{ + return fold_addsub2_i32(ctx, op, false); +} + static bool fold_xor(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - case INDEX_op_add2_i32: - case INDEX_op_sub2_i32: - if (arg_is_const(op->args[2]) && arg_is_const(op->args[3]) - && arg_is_const(op->args[4]) && arg_is_const(op->args[5])) { - uint32_t al = arg_info(op->args[2])->val; - uint32_t ah = arg_info(op->args[3])->val; - uint32_t bl = arg_info(op->args[4])->val; - uint32_t bh = arg_info(op->args[5])->val; - uint64_t a = ((uint64_t)ah << 32) | al; - uint64_t b = ((uint64_t)bh << 32) | bl; - TCGArg rl, rh; - TCGOp *op2 = tcg_op_insert_before(s, op, INDEX_op_mov_i32); - - if (opc == INDEX_op_add2_i32) { - a += b; - } else { - a -= b; - } - - rl = op->args[0]; - rh = op->args[1]; - tcg_opt_gen_movi(&ctx, op, rl, (int32_t)a); - tcg_opt_gen_movi(&ctx, op2, rh, (int32_t)(a >> 32)); - continue; - } - break; default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(add): done = fold_add(&ctx, op); break; + case INDEX_op_add2_i32: + done = fold_add2_i32(&ctx, op); + break; CASE_OP_32_64_VEC(and): done = fold_and(&ctx, op); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(sub): done = fold_sub(&ctx, op); break; + case INDEX_op_sub2_i32: + done = fold_sub2_i32(&ctx, op); + break; CASE_OP_32_64_VEC(xor): done = fold_xor(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 56 ++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_mb(OptContext *ctx, TCGOp *op) return true; } +static bool fold_movcond(OptContext *ctx, TCGOp *op) +{ + TCGOpcode opc = op->opc; + TCGCond cond = op->args[5]; + int i = do_constant_folding_cond(opc, op->args[1], op->args[2], cond); + + if (i >= 0) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]); + } + + if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { + uint64_t tv = arg_info(op->args[3])->val; + uint64_t fv = arg_info(op->args[4])->val; + + opc = (opc == INDEX_op_movcond_i32 + ? INDEX_op_setcond_i32 : INDEX_op_setcond_i64); + + if (tv == 1 && fv == 0) { + op->opc = opc; + op->args[3] = cond; + } else if (fv == 1 && tv == 0) { + op->opc = opc; + op->args[3] = tcg_invert_cond(cond); + } + } + return false; +} + static bool fold_mul(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(movcond): - i = do_constant_folding_cond(opc, op->args[1], - op->args[2], op->args[5]); - if (i >= 0) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[4 - i]); - continue; - } - if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { - uint64_t tv = arg_info(op->args[3])->val; - uint64_t fv = arg_info(op->args[4])->val; - TCGCond cond = op->args[5]; - - if (fv == 1 && tv == 0) { - cond = tcg_invert_cond(cond); - } else if (!(tv == 1 && fv == 0)) { - break; - } - op->args[3] = cond; - op->opc = opc = (opc == INDEX_op_movcond_i32 - ? INDEX_op_setcond_i32 - : INDEX_op_setcond_i64); - } - break; - - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_mb: done = fold_mb(&ctx, op); break; + CASE_OP_32_64(movcond): + done = fold_movcond(&ctx, op); + break; CASE_OP_32_64(mul): done = fold_mul(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_extract2(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { + uint64_t v1 = arg_info(op->args[1])->val; + uint64_t v2 = arg_info(op->args[2])->val; + int shr = op->args[3]; + + if (op->opc == INDEX_op_extract2_i64) { + v1 >>= shr; + v2 <<= 64 - shr; + } else { + v1 = (uint32_t)v1 >> shr; + v2 = (int32_t)v2 << (32 - shr); + } + return tcg_opt_gen_movi(ctx, op, op->args[0], v1 | v2); + } + return false; +} + static bool fold_exts(OptContext *ctx, TCGOp *op) { return fold_const1(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(extract2): - if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - uint64_t v1 = arg_info(op->args[1])->val; - uint64_t v2 = arg_info(op->args[2])->val; - int shr = op->args[3]; - - if (opc == INDEX_op_extract2_i64) { - tmp = (v1 >> shr) | (v2 << (64 - shr)); - } else { - tmp = (int32_t)(((uint32_t)v1 >> shr) | - ((uint32_t)v2 << (32 - shr))); - } - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(eqv): done = fold_eqv(&ctx, op); break; + CASE_OP_32_64(extract2): + done = fold_extract2(&ctx, op); + break; CASE_OP_32_64(ext8s): CASE_OP_32_64(ext16s): case INDEX_op_ext32s_i64: -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_extract(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t; + + t = arg_info(op->args[1])->val; + t = extract64(t, op->args[2], op->args[3]); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return false; +} + static bool fold_extract2(OptContext *ctx, TCGOp *op) { if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) return tcg_opt_gen_movi(ctx, op, op->args[0], i); } +static bool fold_sextract(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t; + + t = arg_info(op->args[1])->val; + t = sextract64(t, op->args[2], op->args[3]); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return false; +} + static bool fold_shift(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(extract): - if (arg_is_const(op->args[1])) { - tmp = extract64(arg_info(op->args[1])->val, - op->args[2], op->args[3]); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - - CASE_OP_32_64(sextract): - if (arg_is_const(op->args[1])) { - tmp = sextract64(arg_info(op->args[1])->val, - op->args[2], op->args[3]); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(eqv): done = fold_eqv(&ctx, op); break; + CASE_OP_32_64(extract): + done = fold_extract(&ctx, op); + break; CASE_OP_32_64(extract2): done = fold_extract2(&ctx, op); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_setcond2_i32: done = fold_setcond2(&ctx, op); break; + CASE_OP_32_64(sextract): + done = fold_sextract(&ctx, op); + break; CASE_OP_32_64_VEC(sub): done = fold_sub(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_ctpop(OptContext *ctx, TCGOp *op) return fold_const1(ctx, op); } +static bool fold_deposit(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { + uint64_t t1 = arg_info(op->args[1])->val; + uint64_t t2 = arg_info(op->args[2])->val; + + t1 = deposit64(t1, op->args[3], op->args[4], t2); + return tcg_opt_gen_movi(ctx, op, op->args[0], t1); + } + return false; +} + static bool fold_divide(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(deposit): - if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - tmp = deposit64(arg_info(op->args[1])->val, - op->args[3], op->args[4], - arg_info(op->args[2])->val); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(ctpop): done = fold_ctpop(&ctx, op); break; + CASE_OP_32_64(deposit): + done = fold_deposit(&ctx, op); + break; CASE_OP_32_64(div): CASE_OP_32_64(divu): done = fold_divide(&ctx, op); -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_call(OptContext *ctx, TCGOp *op) return true; } +static bool fold_count_zeros(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t = arg_info(op->args[1])->val; + + if (t != 0) { + t = do_constant_folding(op->opc, t, 0); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]); + } + return false; +} + static bool fold_ctpop(OptContext *ctx, TCGOp *op) { return fold_const1(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(clz): - CASE_OP_32_64(ctz): - if (arg_is_const(op->args[1])) { - TCGArg v = arg_info(op->args[1])->val; - if (v != 0) { - tmp = do_constant_folding(opc, v, 0); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - } else { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[2]); - } - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_brcond2_i32: done = fold_brcond2(&ctx, op); break; + CASE_OP_32_64(clz): + CASE_OP_32_64(ctz): + done = fold_count_zeros(&ctx, op); + break; CASE_OP_32_64(ctpop): done = fold_ctpop(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) return false; } +static bool fold_bswap(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t = arg_info(op->args[1])->val; + + t = do_constant_folding(op->opc, t, op->args[2]); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return false; +} + static bool fold_call(OptContext *ctx, TCGOp *op) { TCGContext *s = ctx->tcg; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(bswap16): - CASE_OP_32_64(bswap32): - case INDEX_op_bswap64_i64: - if (arg_is_const(op->args[1])) { - tmp = do_constant_folding(opc, arg_info(op->args[1])->val, - op->args[2]); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_brcond2_i32: done = fold_brcond2(&ctx, op); break; + CASE_OP_32_64(bswap16): + CASE_OP_32_64(bswap32): + case INDEX_op_bswap64_i64: + done = fold_bswap(&ctx, op); + break; CASE_OP_32_64(clz): CASE_OP_32_64(ctz): done = fold_count_zeros(&ctx, op); -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 53 +++++++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_divide(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_dup(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t = arg_info(op->args[1])->val; + t = dup_const(TCGOP_VECE(op), t); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return false; +} + +static bool fold_dup2(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { + uint64_t t = deposit64(arg_info(op->args[1])->val, 32, 32, + arg_info(op->args[2])->val); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + + if (args_are_copies(op->args[1], op->args[2])) { + op->opc = INDEX_op_dup_vec; + TCGOP_VECE(op) = MO_32; + } + return false; +} + static bool fold_eqv(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) done = tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); break; - case INDEX_op_dup_vec: - if (arg_is_const(op->args[1])) { - tmp = arg_info(op->args[1])->val; - tmp = dup_const(TCGOP_VECE(op), tmp); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - - case INDEX_op_dup2_vec: - assert(TCG_TARGET_REG_BITS == 32); - if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - tcg_opt_gen_movi(&ctx, op, op->args[0], - deposit64(arg_info(op->args[1])->val, 32, 32, - arg_info(op->args[2])->val)); - continue; - } else if (args_are_copies(op->args[1], op->args[2])) { - op->opc = INDEX_op_dup_vec; - TCGOP_VECE(op) = MO_32; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(divu): done = fold_divide(&ctx, op); break; + case INDEX_op_dup_vec: + done = fold_dup(&ctx, op); + break; + case INDEX_op_dup2_vec: + done = fold_dup2(&ctx, op); + break; CASE_OP_32_64(eqv): done = fold_eqv(&ctx, op); break; -- 2.25.1 This is the final entry in the main switch that was in a different form. After this, we have the option to convert the switch into a function dispatch table. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_mb(OptContext *ctx, TCGOp *op) return true; } +static bool fold_mov(OptContext *ctx, TCGOp *op) +{ + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); +} + static bool fold_movcond(OptContext *ctx, TCGOp *op) { TCGOpcode opc = op->opc; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Propagate constants through copy operations and do constant - folding. Constants will be substituted to arguments by register - allocator where needed and possible. Also detect copies. */ + /* + * Process each opcode. + * Sorted alphabetically by opcode as much as possible. + */ switch (opc) { - CASE_OP_32_64_VEC(mov): - done = tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - break; - - default: - break; - - /* ---------------------------------------------------------- */ - /* Sorted alphabetically by opcode as much as possible. */ - CASE_OP_32_64_VEC(add): done = fold_add(&ctx, op); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_mb: done = fold_mb(&ctx, op); break; + CASE_OP_32_64_VEC(mov): + done = fold_mov(&ctx, op); + break; CASE_OP_32_64(movcond): done = fold_movcond(&ctx, op); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(xor): done = fold_xor(&ctx, op); break; + default: + break; } if (!done) { -- 2.25.1 Pull the "op r, a, a => movi r, 0" optimization into a function, and use it in the outer opcode fold functions. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) return false; } +/* If the binary operation has both arguments equal, fold to @i. */ +static bool fold_xx_to_i(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (args_are_copies(op->args[1], op->args[2])) { + return tcg_opt_gen_movi(ctx, op, op->args[0], i); + } + return false; +} + /* * These outermost fold_<op> functions are sorted alphabetically. */ @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) static bool fold_andc(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_i(ctx, op, 0)) { + return true; + } + return false; } static bool fold_brcond(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_shift(OptContext *ctx, TCGOp *op) static bool fold_sub(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_i(ctx, op, 0)) { + return true; + } + return false; } static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) static bool fold_xor(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_i(ctx, op, 0)) { + return true; + } + return false; } /* Propagate constants and copies, fold constant expressions. */ @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Simplify expression for "op r, a, a => movi r, 0" cases */ - switch (opc) { - CASE_OP_32_64_VEC(andc): - CASE_OP_32_64_VEC(sub): - CASE_OP_32_64_VEC(xor): - if (args_are_copies(op->args[1], op->args[2])) { - tcg_opt_gen_movi(&ctx, op, op->args[0], 0); - continue; - } - break; - default: - break; - } - /* * Process each opcode. * Sorted alphabetically by opcode as much as possible. -- 2.25.1 Pull the "op r, a, a => mov r, a" optimization into a function, and use it in the outer opcode fold functions. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_xx_to_i(OptContext *ctx, TCGOp *op, uint64_t i) return false; } +/* If the binary operation has both arguments equal, fold to identity. */ +static bool fold_xx_to_x(OptContext *ctx, TCGOp *op) +{ + if (args_are_copies(op->args[1], op->args[2])) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); + } + return false; +} + /* * These outermost fold_<op> functions are sorted alphabetically. + * + * The ordering of the transformations should be: + * 1) those that produce a constant + * 2) those that produce a copy + * 3) those that produce information about the result value. */ static bool fold_add(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_add2_i32(OptContext *ctx, TCGOp *op) static bool fold_and(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_x(ctx, op)) { + return true; + } + return false; } static bool fold_andc(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_not(OptContext *ctx, TCGOp *op) static bool fold_or(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_x(ctx, op)) { + return true; + } + return false; } static bool fold_orc(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Simplify expression for "op r, a, a => mov r, a" cases */ - switch (opc) { - CASE_OP_32_64_VEC(or): - CASE_OP_32_64_VEC(and): - if (args_are_copies(op->args[1], op->args[2])) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - continue; - } - break; - default: - break; - } - /* * Process each opcode. * Sorted alphabetically by opcode as much as possible. -- 2.25.1 Pull the "op r, a, 0 => movi r, 0" optimization into a function, and use it in the outer opcode fold functions. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) return false; } +/* If the binary operation has second argument @i, fold to @i. */ +static bool fold_xi_to_i(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == i) { + return tcg_opt_gen_movi(ctx, op, op->args[0], i); + } + return false; +} + /* If the binary operation has both arguments equal, fold to @i. */ static bool fold_xx_to_i(OptContext *ctx, TCGOp *op, uint64_t i) { @@ -XXX,XX +XXX,XX @@ static bool fold_add2_i32(OptContext *ctx, TCGOp *op) static bool fold_and(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_xi_to_i(ctx, op, 0) || fold_xx_to_x(ctx, op)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) static bool fold_mul(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_i(ctx, op, 0)) { + return true; + } + return false; } static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_i(ctx, op, 0)) { + return true; + } + return false; } static bool fold_mulu2_i32(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) continue; } - /* Simplify expression for "op r, a, 0 => movi r, 0" cases */ - switch (opc) { - CASE_OP_32_64_VEC(and): - CASE_OP_32_64_VEC(mul): - CASE_OP_32_64(muluh): - CASE_OP_32_64(mulsh): - if (arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == 0) { - tcg_opt_gen_movi(&ctx, op, op->args[0], 0); - continue; - } - break; - default: - break; - } - /* * Process each opcode. * Sorted alphabetically by opcode as much as possible. -- 2.25.1 Compute the type of the operation early. There are at least 4 places that used a def->flags ladder to determine the type of the operation being optimized. There were two places that assumed !TCG_OPF_64BIT means TCG_TYPE_I32, and so could potentially compute incorrect results for vector operations. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 149 +++++++++++++++++++++++++++++-------------------- 1 file changed, 89 insertions(+), 60 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { /* In flight values from optimization. */ uint64_t z_mask; + TCGType type; } OptContext; static inline TempOptInfo *ts_info(TCGTemp *ts) @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) { TCGTemp *dst_ts = arg_temp(dst); TCGTemp *src_ts = arg_temp(src); - const TCGOpDef *def; TempOptInfo *di; TempOptInfo *si; uint64_t z_mask; @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) reset_ts(dst_ts); di = ts_info(dst_ts); si = ts_info(src_ts); - def = &tcg_op_defs[op->opc]; - if (def->flags & TCG_OPF_VECTOR) { - new_op = INDEX_op_mov_vec; - } else if (def->flags & TCG_OPF_64BIT) { - new_op = INDEX_op_mov_i64; - } else { + + switch (ctx->type) { + case TCG_TYPE_I32: new_op = INDEX_op_mov_i32; + break; + case TCG_TYPE_I64: + new_op = INDEX_op_mov_i64; + break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + /* TCGOP_VECL and TCGOP_VECE remain unchanged. */ + new_op = INDEX_op_mov_vec; + break; + default: + g_assert_not_reached(); } op->opc = new_op; - /* TCGOP_VECL and TCGOP_VECE remain unchanged. */ op->args[0] = dst; op->args[1] = src; @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) static bool tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, TCGArg dst, uint64_t val) { - const TCGOpDef *def = &tcg_op_defs[op->opc]; - TCGType type; - TCGTemp *tv; - - if (def->flags & TCG_OPF_VECTOR) { - type = TCGOP_VECL(op) + TCG_TYPE_V64; - } else if (def->flags & TCG_OPF_64BIT) { - type = TCG_TYPE_I64; - } else { - type = TCG_TYPE_I32; - } - /* Convert movi to mov with constant temp. */ - tv = tcg_constant_internal(type, val); + TCGTemp *tv = tcg_constant_internal(ctx->type, val); + init_ts_info(ctx, tv); return tcg_opt_gen_mov(ctx, op, dst, temp_arg(tv)); } @@ -XXX,XX +XXX,XX @@ static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y) } } -static uint64_t do_constant_folding(TCGOpcode op, uint64_t x, uint64_t y) +static uint64_t do_constant_folding(TCGOpcode op, TCGType type, + uint64_t x, uint64_t y) { - const TCGOpDef *def = &tcg_op_defs[op]; uint64_t res = do_constant_folding_2(op, x, y); - if (!(def->flags & TCG_OPF_64BIT)) { + if (type == TCG_TYPE_I32) { res = (int32_t)res; } return res; @@ -XXX,XX +XXX,XX @@ static bool do_constant_folding_cond_eq(TCGCond c) * Return -1 if the condition can't be simplified, * and the result of the condition (0 or 1) if it can. */ -static int do_constant_folding_cond(TCGOpcode op, TCGArg x, +static int do_constant_folding_cond(TCGType type, TCGArg x, TCGArg y, TCGCond c) { uint64_t xv = arg_info(x)->val; uint64_t yv = arg_info(y)->val; if (arg_is_const(x) && arg_is_const(y)) { - const TCGOpDef *def = &tcg_op_defs[op]; - tcg_debug_assert(!(def->flags & TCG_OPF_VECTOR)); - if (def->flags & TCG_OPF_64BIT) { - return do_constant_folding_cond_64(xv, yv, c); - } else { + switch (type) { + case TCG_TYPE_I32: return do_constant_folding_cond_32(xv, yv, c); + case TCG_TYPE_I64: + return do_constant_folding_cond_64(xv, yv, c); + default: + /* Only scalar comparisons are optimizable */ + return -1; } } else if (args_are_copies(x, y)) { return do_constant_folding_cond_eq(c); @@ -XXX,XX +XXX,XX @@ static bool fold_const1(OptContext *ctx, TCGOp *op) uint64_t t; t = arg_info(op->args[1])->val; - t = do_constant_folding(op->opc, t, 0); + t = do_constant_folding(op->opc, ctx->type, t, 0); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } return false; @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) uint64_t t1 = arg_info(op->args[1])->val; uint64_t t2 = arg_info(op->args[2])->val; - t1 = do_constant_folding(op->opc, t1, t2); + t1 = do_constant_folding(op->opc, ctx->type, t1, t2); return tcg_opt_gen_movi(ctx, op, op->args[0], t1); } return false; @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) static bool fold_brcond(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[2]; - int i = do_constant_folding_cond(op->opc, op->args[0], op->args[1], cond); + int i = do_constant_folding_cond(ctx->type, op->args[0], op->args[1], cond); if (i == 0) { tcg_op_remove(ctx->tcg, op); @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) * Simplify EQ/NE comparisons where one of the pairs * can be simplified. */ - i = do_constant_folding_cond(INDEX_op_brcond_i32, op->args[0], + i = do_constant_folding_cond(TCG_TYPE_I32, op->args[0], op->args[2], cond); switch (i ^ inv) { case 0: @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) goto do_brcond_high; } - i = do_constant_folding_cond(INDEX_op_brcond_i32, op->args[1], + i = do_constant_folding_cond(TCG_TYPE_I32, op->args[1], op->args[3], cond); switch (i ^ inv) { case 0: @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) if (arg_is_const(op->args[1])) { uint64_t t = arg_info(op->args[1])->val; - t = do_constant_folding(op->opc, t, op->args[2]); + t = do_constant_folding(op->opc, ctx->type, t, op->args[2]); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } return false; @@ -XXX,XX +XXX,XX @@ static bool fold_count_zeros(OptContext *ctx, TCGOp *op) uint64_t t = arg_info(op->args[1])->val; if (t != 0) { - t = do_constant_folding(op->opc, t, 0); + t = do_constant_folding(op->opc, ctx->type, t, 0); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]); @@ -XXX,XX +XXX,XX @@ static bool fold_mov(OptContext *ctx, TCGOp *op) static bool fold_movcond(OptContext *ctx, TCGOp *op) { - TCGOpcode opc = op->opc; TCGCond cond = op->args[5]; - int i = do_constant_folding_cond(opc, op->args[1], op->args[2], cond); + int i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); if (i >= 0) { return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]); @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { uint64_t tv = arg_info(op->args[3])->val; uint64_t fv = arg_info(op->args[4])->val; + TCGOpcode opc; - opc = (opc == INDEX_op_movcond_i32 - ? INDEX_op_setcond_i32 : INDEX_op_setcond_i64); + switch (ctx->type) { + case TCG_TYPE_I32: + opc = INDEX_op_setcond_i32; + break; + case TCG_TYPE_I64: + opc = INDEX_op_setcond_i64; + break; + default: + g_assert_not_reached(); + } if (tv == 1 && fv == 0) { op->opc = opc; @@ -XXX,XX +XXX,XX @@ static bool fold_remainder(OptContext *ctx, TCGOp *op) static bool fold_setcond(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[3]; - int i = do_constant_folding_cond(op->opc, op->args[1], op->args[2], cond); + int i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); if (i >= 0) { return tcg_opt_gen_movi(ctx, op, op->args[0], i); @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) * Simplify EQ/NE comparisons where one of the pairs * can be simplified. */ - i = do_constant_folding_cond(INDEX_op_setcond_i32, op->args[1], + i = do_constant_folding_cond(TCG_TYPE_I32, op->args[1], op->args[3], cond); switch (i ^ inv) { case 0: @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) goto do_setcond_high; } - i = do_constant_folding_cond(INDEX_op_setcond_i32, op->args[2], + i = do_constant_folding_cond(TCG_TYPE_I32, op->args[2], op->args[4], cond); switch (i ^ inv) { case 0: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) init_arguments(&ctx, op, def->nb_oargs + def->nb_iargs); copy_propagate(&ctx, op, def->nb_oargs, def->nb_iargs); + /* Pre-compute the type of the operation. */ + if (def->flags & TCG_OPF_VECTOR) { + ctx.type = TCG_TYPE_V64 + TCGOP_VECL(op); + } else if (def->flags & TCG_OPF_64BIT) { + ctx.type = TCG_TYPE_I64; + } else { + ctx.type = TCG_TYPE_I32; + } + /* For commutative operations make constant second argument */ switch (opc) { CASE_OP_32_64_VEC(add): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* Proceed with possible constant folding. */ break; } - if (opc == INDEX_op_sub_i32) { + switch (ctx.type) { + case TCG_TYPE_I32: neg_op = INDEX_op_neg_i32; have_neg = TCG_TARGET_HAS_neg_i32; - } else if (opc == INDEX_op_sub_i64) { + break; + case TCG_TYPE_I64: neg_op = INDEX_op_neg_i64; have_neg = TCG_TARGET_HAS_neg_i64; - } else if (TCG_TARGET_HAS_neg_vec) { - TCGType type = TCGOP_VECL(op) + TCG_TYPE_V64; - unsigned vece = TCGOP_VECE(op); - neg_op = INDEX_op_neg_vec; - have_neg = tcg_can_emit_vec_op(neg_op, type, vece) > 0; - } else { break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + neg_op = INDEX_op_neg_vec; + have_neg = tcg_can_emit_vec_op(neg_op, ctx.type, + TCGOP_VECE(op)) > 0; + break; + default: + g_assert_not_reached(); } if (!have_neg) { break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) TCGOpcode not_op; bool have_not; - if (def->flags & TCG_OPF_VECTOR) { - not_op = INDEX_op_not_vec; - have_not = TCG_TARGET_HAS_not_vec; - } else if (def->flags & TCG_OPF_64BIT) { - not_op = INDEX_op_not_i64; - have_not = TCG_TARGET_HAS_not_i64; - } else { + switch (ctx.type) { + case TCG_TYPE_I32: not_op = INDEX_op_not_i32; have_not = TCG_TARGET_HAS_not_i32; + break; + case TCG_TYPE_I64: + not_op = INDEX_op_not_i64; + have_not = TCG_TARGET_HAS_not_i64; + break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + not_op = INDEX_op_not_vec; + have_not = TCG_TARGET_HAS_not_vec; + break; + default: + g_assert_not_reached(); } if (!have_not) { break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) below, we can ignore high bits, but for further optimizations we need to record that the high bits contain garbage. */ partmask = z_mask; - if (!(def->flags & TCG_OPF_64BIT)) { + if (ctx.type == TCG_TYPE_I32) { z_mask |= ~(tcg_target_ulong)0xffffffffu; partmask &= 0xffffffffu; affected &= 0xffffffffu; -- 2.25.1 Split out the conditional conversion from a more complex logical operation to a simple NOT. Create a couple more helpers to make this easy for the outer-most logical operations. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 158 +++++++++++++++++++++++++++---------------------- 1 file changed, 86 insertions(+), 72 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) return false; } +/* + * Convert @op to NOT, if NOT is supported by the host. + * Return true f the conversion is successful, which will still + * indicate that the processing is complete. + */ +static bool fold_not(OptContext *ctx, TCGOp *op); +static bool fold_to_not(OptContext *ctx, TCGOp *op, int idx) +{ + TCGOpcode not_op; + bool have_not; + + switch (ctx->type) { + case TCG_TYPE_I32: + not_op = INDEX_op_not_i32; + have_not = TCG_TARGET_HAS_not_i32; + break; + case TCG_TYPE_I64: + not_op = INDEX_op_not_i64; + have_not = TCG_TARGET_HAS_not_i64; + break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + not_op = INDEX_op_not_vec; + have_not = TCG_TARGET_HAS_not_vec; + break; + default: + g_assert_not_reached(); + } + if (have_not) { + op->opc = not_op; + op->args[1] = op->args[idx]; + return fold_not(ctx, op); + } + return false; +} + +/* If the binary operation has first argument @i, fold to NOT. */ +static bool fold_ix_to_not(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (arg_is_const(op->args[1]) && arg_info(op->args[1])->val == i) { + return fold_to_not(ctx, op, 2); + } + return false; +} + /* If the binary operation has second argument @i, fold to @i. */ static bool fold_xi_to_i(OptContext *ctx, TCGOp *op, uint64_t i) { @@ -XXX,XX +XXX,XX @@ static bool fold_xi_to_i(OptContext *ctx, TCGOp *op, uint64_t i) return false; } +/* If the binary operation has second argument @i, fold to NOT. */ +static bool fold_xi_to_not(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == i) { + return fold_to_not(ctx, op, 1); + } + return false; +} + /* If the binary operation has both arguments equal, fold to @i. */ static bool fold_xx_to_i(OptContext *ctx, TCGOp *op, uint64_t i) { @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) static bool fold_andc(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || - fold_xx_to_i(ctx, op, 0)) { + fold_xx_to_i(ctx, op, 0) || + fold_ix_to_not(ctx, op, -1)) { return true; } return false; @@ -XXX,XX +XXX,XX @@ static bool fold_dup2(OptContext *ctx, TCGOp *op) static bool fold_eqv(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_not(ctx, op, 0)) { + return true; + } + return false; } static bool fold_extract(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_mulu2_i32(OptContext *ctx, TCGOp *op) static bool fold_nand(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_not(ctx, op, -1)) { + return true; + } + return false; } static bool fold_neg(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_neg(OptContext *ctx, TCGOp *op) static bool fold_nor(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_not(ctx, op, 0)) { + return true; + } + return false; } static bool fold_not(OptContext *ctx, TCGOp *op) { - return fold_const1(ctx, op); + if (fold_const1(ctx, op)) { + return true; + } + + /* Because of fold_to_not, we want to always return true, via finish. */ + finish_folding(ctx, op); + return true; } static bool fold_or(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_or(OptContext *ctx, TCGOp *op) static bool fold_orc(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_ix_to_not(ctx, op, 0)) { + return true; + } + return false; } static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) static bool fold_xor(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || - fold_xx_to_i(ctx, op, 0)) { + fold_xx_to_i(ctx, op, 0) || + fold_xi_to_not(ctx, op, -1)) { return true; } return false; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } } break; - CASE_OP_32_64_VEC(xor): - CASE_OP_32_64(nand): - if (!arg_is_const(op->args[1]) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == -1) { - i = 1; - goto try_not; - } - break; - CASE_OP_32_64(nor): - if (!arg_is_const(op->args[1]) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == 0) { - i = 1; - goto try_not; - } - break; - CASE_OP_32_64_VEC(andc): - if (!arg_is_const(op->args[2]) - && arg_is_const(op->args[1]) - && arg_info(op->args[1])->val == -1) { - i = 2; - goto try_not; - } - break; - CASE_OP_32_64_VEC(orc): - CASE_OP_32_64(eqv): - if (!arg_is_const(op->args[2]) - && arg_is_const(op->args[1]) - && arg_info(op->args[1])->val == 0) { - i = 2; - goto try_not; - } - break; - try_not: - { - TCGOpcode not_op; - bool have_not; - - switch (ctx.type) { - case TCG_TYPE_I32: - not_op = INDEX_op_not_i32; - have_not = TCG_TARGET_HAS_not_i32; - break; - case TCG_TYPE_I64: - not_op = INDEX_op_not_i64; - have_not = TCG_TARGET_HAS_not_i64; - break; - case TCG_TYPE_V64: - case TCG_TYPE_V128: - case TCG_TYPE_V256: - not_op = INDEX_op_not_vec; - have_not = TCG_TARGET_HAS_not_vec; - break; - default: - g_assert_not_reached(); - } - if (!have_not) { - break; - } - op->opc = not_op; - reset_temp(op->args[0]); - op->args[1] = op->args[i]; - continue; - } default: break; } -- 2.25.1 Even though there is only one user, place this more complex conversion into its own helper. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 89 ++++++++++++++++++++++++++------------------------ 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_nand(OptContext *ctx, TCGOp *op) static bool fold_neg(OptContext *ctx, TCGOp *op) { - return fold_const1(ctx, op); + if (fold_const1(ctx, op)) { + return true; + } + /* + * Because of fold_sub_to_neg, we want to always return true, + * via finish_folding. + */ + finish_folding(ctx, op); + return true; } static bool fold_nor(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_shift(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_sub_to_neg(OptContext *ctx, TCGOp *op) +{ + TCGOpcode neg_op; + bool have_neg; + + if (!arg_is_const(op->args[1]) || arg_info(op->args[1])->val != 0) { + return false; + } + + switch (ctx->type) { + case TCG_TYPE_I32: + neg_op = INDEX_op_neg_i32; + have_neg = TCG_TARGET_HAS_neg_i32; + break; + case TCG_TYPE_I64: + neg_op = INDEX_op_neg_i64; + have_neg = TCG_TARGET_HAS_neg_i64; + break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + neg_op = INDEX_op_neg_vec; + have_neg = (TCG_TARGET_HAS_neg_vec && + tcg_can_emit_vec_op(neg_op, ctx->type, TCGOP_VECE(op)) > 0); + break; + default: + g_assert_not_reached(); + } + if (have_neg) { + op->opc = neg_op; + op->args[1] = op->args[2]; + return fold_neg(ctx, op); + } + return false; +} + static bool fold_sub(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || - fold_xx_to_i(ctx, op, 0)) { + fold_xx_to_i(ctx, op, 0) || + fold_sub_to_neg(ctx, op)) { return true; } return false; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) continue; } break; - CASE_OP_32_64_VEC(sub): - { - TCGOpcode neg_op; - bool have_neg; - - if (arg_is_const(op->args[2])) { - /* Proceed with possible constant folding. */ - break; - } - switch (ctx.type) { - case TCG_TYPE_I32: - neg_op = INDEX_op_neg_i32; - have_neg = TCG_TARGET_HAS_neg_i32; - break; - case TCG_TYPE_I64: - neg_op = INDEX_op_neg_i64; - have_neg = TCG_TARGET_HAS_neg_i64; - break; - case TCG_TYPE_V64: - case TCG_TYPE_V128: - case TCG_TYPE_V256: - neg_op = INDEX_op_neg_vec; - have_neg = tcg_can_emit_vec_op(neg_op, ctx.type, - TCGOP_VECE(op)) > 0; - break; - default: - g_assert_not_reached(); - } - if (!have_neg) { - break; - } - if (arg_is_const(op->args[1]) - && arg_info(op->args[1])->val == 0) { - op->opc = neg_op; - reset_temp(op->args[0]); - op->args[1] = op->args[2]; - continue; - } - } - break; default: break; } -- 2.25.1 Pull the "op r, a, i => mov r, a" optimization into a function, and use them in the outer-most logical operations. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 61 +++++++++++++++++++++----------------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_xi_to_i(OptContext *ctx, TCGOp *op, uint64_t i) return false; } +/* If the binary operation has second argument @i, fold to identity. */ +static bool fold_xi_to_x(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == i) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); + } + return false; +} + /* If the binary operation has second argument @i, fold to NOT. */ static bool fold_xi_to_not(OptContext *ctx, TCGOp *op, uint64_t i) { @@ -XXX,XX +XXX,XX @@ static bool fold_xx_to_x(OptContext *ctx, TCGOp *op) static bool fold_add(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, 0)) { + return true; + } + return false; } static bool fold_addsub2_i32(OptContext *ctx, TCGOp *op, bool add) @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || fold_xi_to_i(ctx, op, 0) || + fold_xi_to_x(ctx, op, -1) || fold_xx_to_x(ctx, op)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || fold_xx_to_i(ctx, op, 0) || + fold_xi_to_x(ctx, op, 0) || fold_ix_to_not(ctx, op, -1)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_dup2(OptContext *ctx, TCGOp *op) static bool fold_eqv(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, -1) || fold_xi_to_not(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_not(OptContext *ctx, TCGOp *op) static bool fold_or(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, 0) || fold_xx_to_x(ctx, op)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_or(OptContext *ctx, TCGOp *op) static bool fold_orc(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, -1) || fold_ix_to_not(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) static bool fold_shift(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, 0)) { + return true; + } + return false; } static bool fold_sub_to_neg(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_sub(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || fold_xx_to_i(ctx, op, 0) || + fold_xi_to_x(ctx, op, 0) || fold_sub_to_neg(ctx, op)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_xor(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || fold_xx_to_i(ctx, op, 0) || + fold_xi_to_x(ctx, op, 0) || fold_xi_to_not(ctx, op, -1)) { return true; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Simplify expression for "op r, a, const => mov r, a" cases */ - switch (opc) { - CASE_OP_32_64_VEC(add): - CASE_OP_32_64_VEC(sub): - CASE_OP_32_64_VEC(or): - CASE_OP_32_64_VEC(xor): - CASE_OP_32_64_VEC(andc): - CASE_OP_32_64(shl): - CASE_OP_32_64(shr): - CASE_OP_32_64(sar): - CASE_OP_32_64(rotl): - CASE_OP_32_64(rotr): - if (!arg_is_const(op->args[1]) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == 0) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - continue; - } - break; - CASE_OP_32_64_VEC(and): - CASE_OP_32_64_VEC(orc): - CASE_OP_32_64(eqv): - if (!arg_is_const(op->args[1]) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == -1) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - continue; - } - break; - default: - break; - } - /* Simplify using known-zero bits. Currently only ops with a single output argument is supported. */ z_mask = -1; -- 2.25.1 Pull the "op r, 0, b => movi r, 0" optimization into a function, and use it in fold_shift. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_to_not(OptContext *ctx, TCGOp *op, int idx) return false; } +/* If the binary operation has first argument @i, fold to @i. */ +static bool fold_ix_to_i(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (arg_is_const(op->args[1]) && arg_info(op->args[1])->val == i) { + return tcg_opt_gen_movi(ctx, op, op->args[0], i); + } + return false; +} + /* If the binary operation has first argument @i, fold to NOT. */ static bool fold_ix_to_not(OptContext *ctx, TCGOp *op, uint64_t i) { @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) static bool fold_shift(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_ix_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Simplify expressions for "shift/rot r, 0, a => movi r, 0", - and "sub r, 0, a => neg r, a" case. */ - switch (opc) { - CASE_OP_32_64(shl): - CASE_OP_32_64(shr): - CASE_OP_32_64(sar): - CASE_OP_32_64(rotl): - CASE_OP_32_64(rotr): - if (arg_is_const(op->args[1]) - && arg_info(op->args[1])->val == 0) { - tcg_opt_gen_movi(&ctx, op, op->args[0], 0); - continue; - } - break; - default: - break; - } - /* Simplify using known-zero bits. Currently only ops with a single output argument is supported. */ z_mask = -1; -- 2.25.1 Move all of the known-zero optimizations into the per-opcode functions. Use fold_masks when there is a possibility of the result being determined, and simply set ctx->z_mask otherwise. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 545 ++++++++++++++++++++++++++----------------------- 1 file changed, 294 insertions(+), 251 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { TCGTempSet temps_used; /* In flight values from optimization. */ - uint64_t z_mask; + uint64_t a_mask; /* mask bit is 0 iff value identical to first input */ + uint64_t z_mask; /* mask bit is 0 iff value bit is 0 */ TCGType type; } OptContext; @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) return false; } +static bool fold_masks(OptContext *ctx, TCGOp *op) +{ + uint64_t a_mask = ctx->a_mask; + uint64_t z_mask = ctx->z_mask; + + /* + * 32-bit ops generate 32-bit results. For the result is zero test + * below, we can ignore high bits, but for further optimizations we + * need to record that the high bits contain garbage. + */ + if (ctx->type == TCG_TYPE_I32) { + ctx->z_mask |= MAKE_64BIT_MASK(32, 32); + a_mask &= MAKE_64BIT_MASK(0, 32); + z_mask &= MAKE_64BIT_MASK(0, 32); + } + + if (z_mask == 0) { + return tcg_opt_gen_movi(ctx, op, op->args[0], 0); + } + if (a_mask == 0) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); + } + return false; +} + /* * Convert @op to NOT, if NOT is supported by the host. * Return true f the conversion is successful, which will still @@ -XXX,XX +XXX,XX @@ static bool fold_add2_i32(OptContext *ctx, TCGOp *op) static bool fold_and(OptContext *ctx, TCGOp *op) { + uint64_t z1, z2; + if (fold_const2(ctx, op) || fold_xi_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, -1) || fold_xx_to_x(ctx, op)) { return true; } - return false; + + z1 = arg_info(op->args[1])->z_mask; + z2 = arg_info(op->args[2])->z_mask; + ctx->z_mask = z1 & z2; + + /* + * Known-zeros does not imply known-ones. Therefore unless + * arg2 is constant, we can't infer affected bits from it. + */ + if (arg_is_const(op->args[2])) { + ctx->a_mask = z1 & ~z2; + } + + return fold_masks(ctx, op); } static bool fold_andc(OptContext *ctx, TCGOp *op) { + uint64_t z1; + if (fold_const2(ctx, op) || fold_xx_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, 0) || fold_ix_to_not(ctx, op, -1)) { return true; } - return false; + + z1 = arg_info(op->args[1])->z_mask; + + /* + * Known-zeros does not imply known-ones. Therefore unless + * arg2 is constant, we can't infer anything from it. + */ + if (arg_is_const(op->args[2])) { + uint64_t z2 = ~arg_info(op->args[2])->z_mask; + ctx->a_mask = z1 & ~z2; + z1 &= z2; + } + ctx->z_mask = z1; + + return fold_masks(ctx, op); } static bool fold_brcond(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) static bool fold_bswap(OptContext *ctx, TCGOp *op) { + uint64_t z_mask, sign; + if (arg_is_const(op->args[1])) { uint64_t t = arg_info(op->args[1])->val; t = do_constant_folding(op->opc, ctx->type, t, op->args[2]); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } - return false; + + z_mask = arg_info(op->args[1])->z_mask; + switch (op->opc) { + case INDEX_op_bswap16_i32: + case INDEX_op_bswap16_i64: + z_mask = bswap16(z_mask); + sign = INT16_MIN; + break; + case INDEX_op_bswap32_i32: + case INDEX_op_bswap32_i64: + z_mask = bswap32(z_mask); + sign = INT32_MIN; + break; + case INDEX_op_bswap64_i64: + z_mask = bswap64(z_mask); + sign = INT64_MIN; + break; + default: + g_assert_not_reached(); + } + + switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { + case TCG_BSWAP_OZ: + break; + case TCG_BSWAP_OS: + /* If the sign bit may be 1, force all the bits above to 1. */ + if (z_mask & sign) { + z_mask |= sign; + } + break; + default: + /* The high bits are undefined: force all bits above the sign to 1. */ + z_mask |= sign << 1; + break; + } + ctx->z_mask = z_mask; + + return fold_masks(ctx, op); } static bool fold_call(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_call(OptContext *ctx, TCGOp *op) static bool fold_count_zeros(OptContext *ctx, TCGOp *op) { + uint64_t z_mask; + if (arg_is_const(op->args[1])) { uint64_t t = arg_info(op->args[1])->val; @@ -XXX,XX +XXX,XX @@ static bool fold_count_zeros(OptContext *ctx, TCGOp *op) } return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]); } + + switch (ctx->type) { + case TCG_TYPE_I32: + z_mask = 31; + break; + case TCG_TYPE_I64: + z_mask = 63; + break; + default: + g_assert_not_reached(); + } + ctx->z_mask = arg_info(op->args[2])->z_mask | z_mask; + return false; } static bool fold_ctpop(OptContext *ctx, TCGOp *op) { - return fold_const1(ctx, op); + if (fold_const1(ctx, op)) { + return true; + } + + switch (ctx->type) { + case TCG_TYPE_I32: + ctx->z_mask = 32 | 31; + break; + case TCG_TYPE_I64: + ctx->z_mask = 64 | 63; + break; + default: + g_assert_not_reached(); + } + return false; } static bool fold_deposit(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) t1 = deposit64(t1, op->args[3], op->args[4], t2); return tcg_opt_gen_movi(ctx, op, op->args[0], t1); } + + ctx->z_mask = deposit64(arg_info(op->args[1])->z_mask, + op->args[3], op->args[4], + arg_info(op->args[2])->z_mask); return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) static bool fold_extract(OptContext *ctx, TCGOp *op) { + uint64_t z_mask_old, z_mask; + if (arg_is_const(op->args[1])) { uint64_t t; @@ -XXX,XX +XXX,XX @@ static bool fold_extract(OptContext *ctx, TCGOp *op) t = extract64(t, op->args[2], op->args[3]); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } - return false; + + z_mask_old = arg_info(op->args[1])->z_mask; + z_mask = extract64(z_mask_old, op->args[2], op->args[3]); + if (op->args[2] == 0) { + ctx->a_mask = z_mask_old ^ z_mask; + } + ctx->z_mask = z_mask; + + return fold_masks(ctx, op); } static bool fold_extract2(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_extract2(OptContext *ctx, TCGOp *op) static bool fold_exts(OptContext *ctx, TCGOp *op) { - return fold_const1(ctx, op); + uint64_t z_mask_old, z_mask, sign; + bool type_change = false; + + if (fold_const1(ctx, op)) { + return true; + } + + z_mask_old = z_mask = arg_info(op->args[1])->z_mask; + + switch (op->opc) { + CASE_OP_32_64(ext8s): + sign = INT8_MIN; + z_mask = (uint8_t)z_mask; + break; + CASE_OP_32_64(ext16s): + sign = INT16_MIN; + z_mask = (uint16_t)z_mask; + break; + case INDEX_op_ext_i32_i64: + type_change = true; + QEMU_FALLTHROUGH; + case INDEX_op_ext32s_i64: + sign = INT32_MIN; + z_mask = (uint32_t)z_mask; + break; + default: + g_assert_not_reached(); + } + + if (z_mask & sign) { + z_mask |= sign; + } else if (!type_change) { + ctx->a_mask = z_mask_old ^ z_mask; + } + ctx->z_mask = z_mask; + + return fold_masks(ctx, op); } static bool fold_extu(OptContext *ctx, TCGOp *op) { - return fold_const1(ctx, op); + uint64_t z_mask_old, z_mask; + bool type_change = false; + + if (fold_const1(ctx, op)) { + return true; + } + + z_mask_old = z_mask = arg_info(op->args[1])->z_mask; + + switch (op->opc) { + CASE_OP_32_64(ext8u): + z_mask = (uint8_t)z_mask; + break; + CASE_OP_32_64(ext16u): + z_mask = (uint16_t)z_mask; + break; + case INDEX_op_extrl_i64_i32: + case INDEX_op_extu_i32_i64: + type_change = true; + QEMU_FALLTHROUGH; + case INDEX_op_ext32u_i64: + z_mask = (uint32_t)z_mask; + break; + case INDEX_op_extrh_i64_i32: + type_change = true; + z_mask >>= 32; + break; + default: + g_assert_not_reached(); + } + + ctx->z_mask = z_mask; + if (!type_change) { + ctx->a_mask = z_mask_old ^ z_mask; + } + return fold_masks(ctx, op); } static bool fold_mb(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]); } + ctx->z_mask = arg_info(op->args[3])->z_mask + | arg_info(op->args[4])->z_mask; + if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { uint64_t tv = arg_info(op->args[3])->val; uint64_t fv = arg_info(op->args[4])->val; @@ -XXX,XX +XXX,XX @@ static bool fold_nand(OptContext *ctx, TCGOp *op) static bool fold_neg(OptContext *ctx, TCGOp *op) { + uint64_t z_mask; + if (fold_const1(ctx, op)) { return true; } + + /* Set to 1 all bits to the left of the rightmost. */ + z_mask = arg_info(op->args[1])->z_mask; + ctx->z_mask = -(z_mask & -z_mask); + /* * Because of fold_sub_to_neg, we want to always return true, * via finish_folding. @@ -XXX,XX +XXX,XX @@ static bool fold_or(OptContext *ctx, TCGOp *op) fold_xx_to_x(ctx, op)) { return true; } - return false; + + ctx->z_mask = arg_info(op->args[1])->z_mask + | arg_info(op->args[2])->z_mask; + return fold_masks(ctx, op); } static bool fold_orc(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_orc(OptContext *ctx, TCGOp *op) static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) { + const TCGOpDef *def = &tcg_op_defs[op->opc]; + MemOpIdx oi = op->args[def->nb_oargs + def->nb_iargs]; + MemOp mop = get_memop(oi); + int width = 8 * memop_size(mop); + + if (!(mop & MO_SIGN) && width < 64) { + ctx->z_mask = MAKE_64BIT_MASK(0, width); + } + /* Opcodes that touch guest memory stop the mb optimization. */ ctx->prev_mb = NULL; return false; @@ -XXX,XX +XXX,XX @@ static bool fold_setcond(OptContext *ctx, TCGOp *op) if (i >= 0) { return tcg_opt_gen_movi(ctx, op, op->args[0], i); } + + ctx->z_mask = 1; return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) op->opc = INDEX_op_setcond_i32; break; } + + ctx->z_mask = 1; return false; do_setcond_const: @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) static bool fold_sextract(OptContext *ctx, TCGOp *op) { + int64_t z_mask_old, z_mask; + if (arg_is_const(op->args[1])) { uint64_t t; @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) t = sextract64(t, op->args[2], op->args[3]); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } - return false; + + z_mask_old = arg_info(op->args[1])->z_mask; + z_mask = sextract64(z_mask_old, op->args[2], op->args[3]); + if (op->args[2] == 0 && z_mask >= 0) { + ctx->a_mask = z_mask_old ^ z_mask; + } + ctx->z_mask = z_mask; + + return fold_masks(ctx, op); } static bool fold_shift(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_shift(OptContext *ctx, TCGOp *op) fold_xi_to_x(ctx, op, 0)) { return true; } + + if (arg_is_const(op->args[2])) { + ctx->z_mask = do_constant_folding(op->opc, ctx->type, + arg_info(op->args[1])->z_mask, + arg_info(op->args[2])->val); + return fold_masks(ctx, op); + } return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) return fold_addsub2_i32(ctx, op, false); } +static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) +{ + /* We can't do any folding with a load, but we can record bits. */ + switch (op->opc) { + CASE_OP_32_64(ld8u): + ctx->z_mask = MAKE_64BIT_MASK(0, 8); + break; + CASE_OP_32_64(ld16u): + ctx->z_mask = MAKE_64BIT_MASK(0, 16); + break; + case INDEX_op_ld32u_i64: + ctx->z_mask = MAKE_64BIT_MASK(0, 32); + break; + default: + g_assert_not_reached(); + } + return false; +} + static bool fold_xor(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || @@ -XXX,XX +XXX,XX @@ static bool fold_xor(OptContext *ctx, TCGOp *op) fold_xi_to_not(ctx, op, -1)) { return true; } - return false; + + ctx->z_mask = arg_info(op->args[1])->z_mask + | arg_info(op->args[2])->z_mask; + return fold_masks(ctx, op); } /* Propagate constants and copies, fold constant expressions. */ @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { - uint64_t z_mask, partmask, affected, tmp; TCGOpcode opc = op->opc; const TCGOpDef *def; bool done = false; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Simplify using known-zero bits. Currently only ops with a single - output argument is supported. */ - z_mask = -1; - affected = -1; - switch (opc) { - CASE_OP_32_64(ext8s): - if ((arg_info(op->args[1])->z_mask & 0x80) != 0) { - break; - } - QEMU_FALLTHROUGH; - CASE_OP_32_64(ext8u): - z_mask = 0xff; - goto and_const; - CASE_OP_32_64(ext16s): - if ((arg_info(op->args[1])->z_mask & 0x8000) != 0) { - break; - } - QEMU_FALLTHROUGH; - CASE_OP_32_64(ext16u): - z_mask = 0xffff; - goto and_const; - case INDEX_op_ext32s_i64: - if ((arg_info(op->args[1])->z_mask & 0x80000000) != 0) { - break; - } - QEMU_FALLTHROUGH; - case INDEX_op_ext32u_i64: - z_mask = 0xffffffffU; - goto and_const; - - CASE_OP_32_64(and): - z_mask = arg_info(op->args[2])->z_mask; - if (arg_is_const(op->args[2])) { - and_const: - affected = arg_info(op->args[1])->z_mask & ~z_mask; - } - z_mask = arg_info(op->args[1])->z_mask & z_mask; - break; - - case INDEX_op_ext_i32_i64: - if ((arg_info(op->args[1])->z_mask & 0x80000000) != 0) { - break; - } - QEMU_FALLTHROUGH; - case INDEX_op_extu_i32_i64: - /* We do not compute affected as it is a size changing op. */ - z_mask = (uint32_t)arg_info(op->args[1])->z_mask; - break; - - CASE_OP_32_64(andc): - /* Known-zeros does not imply known-ones. Therefore unless - op->args[2] is constant, we can't infer anything from it. */ - if (arg_is_const(op->args[2])) { - z_mask = ~arg_info(op->args[2])->z_mask; - goto and_const; - } - /* But we certainly know nothing outside args[1] may be set. */ - z_mask = arg_info(op->args[1])->z_mask; - break; - - case INDEX_op_sar_i32: - if (arg_is_const(op->args[2])) { - tmp = arg_info(op->args[2])->val & 31; - z_mask = (int32_t)arg_info(op->args[1])->z_mask >> tmp; - } - break; - case INDEX_op_sar_i64: - if (arg_is_const(op->args[2])) { - tmp = arg_info(op->args[2])->val & 63; - z_mask = (int64_t)arg_info(op->args[1])->z_mask >> tmp; - } - break; - - case INDEX_op_shr_i32: - if (arg_is_const(op->args[2])) { - tmp = arg_info(op->args[2])->val & 31; - z_mask = (uint32_t)arg_info(op->args[1])->z_mask >> tmp; - } - break; - case INDEX_op_shr_i64: - if (arg_is_const(op->args[2])) { - tmp = arg_info(op->args[2])->val & 63; - z_mask = (uint64_t)arg_info(op->args[1])->z_mask >> tmp; - } - break; - - case INDEX_op_extrl_i64_i32: - z_mask = (uint32_t)arg_info(op->args[1])->z_mask; - break; - case INDEX_op_extrh_i64_i32: - z_mask = (uint64_t)arg_info(op->args[1])->z_mask >> 32; - break; - - CASE_OP_32_64(shl): - if (arg_is_const(op->args[2])) { - tmp = arg_info(op->args[2])->val & (TCG_TARGET_REG_BITS - 1); - z_mask = arg_info(op->args[1])->z_mask << tmp; - } - break; - - CASE_OP_32_64(neg): - /* Set to 1 all bits to the left of the rightmost. */ - z_mask = -(arg_info(op->args[1])->z_mask - & -arg_info(op->args[1])->z_mask); - break; - - CASE_OP_32_64(deposit): - z_mask = deposit64(arg_info(op->args[1])->z_mask, - op->args[3], op->args[4], - arg_info(op->args[2])->z_mask); - break; - - CASE_OP_32_64(extract): - z_mask = extract64(arg_info(op->args[1])->z_mask, - op->args[2], op->args[3]); - if (op->args[2] == 0) { - affected = arg_info(op->args[1])->z_mask & ~z_mask; - } - break; - CASE_OP_32_64(sextract): - z_mask = sextract64(arg_info(op->args[1])->z_mask, - op->args[2], op->args[3]); - if (op->args[2] == 0 && (tcg_target_long)z_mask >= 0) { - affected = arg_info(op->args[1])->z_mask & ~z_mask; - } - break; - - CASE_OP_32_64(or): - CASE_OP_32_64(xor): - z_mask = arg_info(op->args[1])->z_mask - | arg_info(op->args[2])->z_mask; - break; - - case INDEX_op_clz_i32: - case INDEX_op_ctz_i32: - z_mask = arg_info(op->args[2])->z_mask | 31; - break; - - case INDEX_op_clz_i64: - case INDEX_op_ctz_i64: - z_mask = arg_info(op->args[2])->z_mask | 63; - break; - - case INDEX_op_ctpop_i32: - z_mask = 32 | 31; - break; - case INDEX_op_ctpop_i64: - z_mask = 64 | 63; - break; - - CASE_OP_32_64(setcond): - case INDEX_op_setcond2_i32: - z_mask = 1; - break; - - CASE_OP_32_64(movcond): - z_mask = arg_info(op->args[3])->z_mask - | arg_info(op->args[4])->z_mask; - break; - - CASE_OP_32_64(ld8u): - z_mask = 0xff; - break; - CASE_OP_32_64(ld16u): - z_mask = 0xffff; - break; - case INDEX_op_ld32u_i64: - z_mask = 0xffffffffu; - break; - - CASE_OP_32_64(qemu_ld): - { - MemOpIdx oi = op->args[def->nb_oargs + def->nb_iargs]; - MemOp mop = get_memop(oi); - if (!(mop & MO_SIGN)) { - z_mask = (2ULL << ((8 << (mop & MO_SIZE)) - 1)) - 1; - } - } - break; - - CASE_OP_32_64(bswap16): - z_mask = arg_info(op->args[1])->z_mask; - if (z_mask <= 0xffff) { - op->args[2] |= TCG_BSWAP_IZ; - } - z_mask = bswap16(z_mask); - switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { - case TCG_BSWAP_OZ: - break; - case TCG_BSWAP_OS: - z_mask = (int16_t)z_mask; - break; - default: /* undefined high bits */ - z_mask |= MAKE_64BIT_MASK(16, 48); - break; - } - break; - - case INDEX_op_bswap32_i64: - z_mask = arg_info(op->args[1])->z_mask; - if (z_mask <= 0xffffffffu) { - op->args[2] |= TCG_BSWAP_IZ; - } - z_mask = bswap32(z_mask); - switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { - case TCG_BSWAP_OZ: - break; - case TCG_BSWAP_OS: - z_mask = (int32_t)z_mask; - break; - default: /* undefined high bits */ - z_mask |= MAKE_64BIT_MASK(32, 32); - break; - } - break; - - default: - break; - } - - /* 32-bit ops generate 32-bit results. For the result is zero test - below, we can ignore high bits, but for further optimizations we - need to record that the high bits contain garbage. */ - partmask = z_mask; - if (ctx.type == TCG_TYPE_I32) { - z_mask |= ~(tcg_target_ulong)0xffffffffu; - partmask &= 0xffffffffu; - affected &= 0xffffffffu; - } - ctx.z_mask = z_mask; - - if (partmask == 0) { - tcg_opt_gen_movi(&ctx, op, op->args[0], 0); - continue; - } - if (affected == 0) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - continue; - } + /* Assume all bits affected, and no bits known zero. */ + ctx.a_mask = -1; + ctx.z_mask = -1; /* * Process each opcode. @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_extrh_i64_i32: done = fold_extu(&ctx, op); break; + CASE_OP_32_64(ld8u): + CASE_OP_32_64(ld16u): + case INDEX_op_ld32u_i64: + done = fold_tcg_ld(&ctx, op); + break; case INDEX_op_mb: done = fold_mb(&ctx, op); break; -- 2.25.1 Rename to fold_multiply2, and handle muls2_i32, mulu2_i64, and muls2_i64. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) return false; } -static bool fold_mulu2_i32(OptContext *ctx, TCGOp *op) +static bool fold_multiply2(OptContext *ctx, TCGOp *op) { if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { - uint32_t a = arg_info(op->args[2])->val; - uint32_t b = arg_info(op->args[3])->val; - uint64_t r = (uint64_t)a * b; + uint64_t a = arg_info(op->args[2])->val; + uint64_t b = arg_info(op->args[3])->val; + uint64_t h, l; TCGArg rl, rh; - TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_mov_i32); + TCGOp *op2; + + switch (op->opc) { + case INDEX_op_mulu2_i32: + l = (uint64_t)(uint32_t)a * (uint32_t)b; + h = (int32_t)(l >> 32); + l = (int32_t)l; + break; + case INDEX_op_muls2_i32: + l = (int64_t)(int32_t)a * (int32_t)b; + h = l >> 32; + l = (int32_t)l; + break; + case INDEX_op_mulu2_i64: + mulu64(&l, &h, a, b); + break; + case INDEX_op_muls2_i64: + muls64(&l, &h, a, b); + break; + default: + g_assert_not_reached(); + } rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(ctx, op, rl, (int32_t)r); - tcg_opt_gen_movi(ctx, op2, rh, (int32_t)(r >> 32)); + + /* The proper opcode is supplied by tcg_opt_gen_mov. */ + op2 = tcg_op_insert_before(ctx->tcg, op, 0); + + tcg_opt_gen_movi(ctx, op, rl, l); + tcg_opt_gen_movi(ctx, op2, rh, h); return true; } return false; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(muluh): done = fold_mul_highpart(&ctx, op); break; - case INDEX_op_mulu2_i32: - done = fold_mulu2_i32(&ctx, op); + CASE_OP_32_64(muls2): + CASE_OP_32_64(mulu2): + done = fold_multiply2(&ctx, op); break; CASE_OP_32_64(nand): done = fold_nand(&ctx, op); -- 2.25.1 Rename to fold_addsub2. Use Int128 to implement the wider operation. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 65 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ */ #include "qemu/osdep.h" +#include "qemu/int128.h" #include "tcg/tcg-op.h" #include "tcg-internal.h" @@ -XXX,XX +XXX,XX @@ static bool fold_add(OptContext *ctx, TCGOp *op) return false; } -static bool fold_addsub2_i32(OptContext *ctx, TCGOp *op, bool add) +static bool fold_addsub2(OptContext *ctx, TCGOp *op, bool add) { if (arg_is_const(op->args[2]) && arg_is_const(op->args[3]) && arg_is_const(op->args[4]) && arg_is_const(op->args[5])) { - uint32_t al = arg_info(op->args[2])->val; - uint32_t ah = arg_info(op->args[3])->val; - uint32_t bl = arg_info(op->args[4])->val; - uint32_t bh = arg_info(op->args[5])->val; - uint64_t a = ((uint64_t)ah << 32) | al; - uint64_t b = ((uint64_t)bh << 32) | bl; + uint64_t al = arg_info(op->args[2])->val; + uint64_t ah = arg_info(op->args[3])->val; + uint64_t bl = arg_info(op->args[4])->val; + uint64_t bh = arg_info(op->args[5])->val; TCGArg rl, rh; - TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_mov_i32); + TCGOp *op2; - if (add) { - a += b; + if (ctx->type == TCG_TYPE_I32) { + uint64_t a = deposit64(al, 32, 32, ah); + uint64_t b = deposit64(bl, 32, 32, bh); + + if (add) { + a += b; + } else { + a -= b; + } + + al = sextract64(a, 0, 32); + ah = sextract64(a, 32, 32); } else { - a -= b; + Int128 a = int128_make128(al, ah); + Int128 b = int128_make128(bl, bh); + + if (add) { + a = int128_add(a, b); + } else { + a = int128_sub(a, b); + } + + al = int128_getlo(a); + ah = int128_gethi(a); } rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(ctx, op, rl, (int32_t)a); - tcg_opt_gen_movi(ctx, op2, rh, (int32_t)(a >> 32)); + + /* The proper opcode is supplied by tcg_opt_gen_mov. */ + op2 = tcg_op_insert_before(ctx->tcg, op, 0); + + tcg_opt_gen_movi(ctx, op, rl, al); + tcg_opt_gen_movi(ctx, op2, rh, ah); return true; } return false; } -static bool fold_add2_i32(OptContext *ctx, TCGOp *op) +static bool fold_add2(OptContext *ctx, TCGOp *op) { - return fold_addsub2_i32(ctx, op, true); + return fold_addsub2(ctx, op, true); } static bool fold_and(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_sub(OptContext *ctx, TCGOp *op) return false; } -static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) +static bool fold_sub2(OptContext *ctx, TCGOp *op) { - return fold_addsub2_i32(ctx, op, false); + return fold_addsub2(ctx, op, false); } static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(add): done = fold_add(&ctx, op); break; - case INDEX_op_add2_i32: - done = fold_add2_i32(&ctx, op); + CASE_OP_32_64(add2): + done = fold_add2(&ctx, op); break; CASE_OP_32_64_VEC(and): done = fold_and(&ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(sub): done = fold_sub(&ctx, op); break; - case INDEX_op_sub2_i32: - done = fold_sub2_i32(&ctx, op); + CASE_OP_32_64(sub2): + done = fold_sub2(&ctx, op); break; CASE_OP_32_64_VEC(xor): done = fold_xor(&ctx, op); -- 2.25.1 Most of these are handled by creating a fold_const2_commutative to handle all of the binary operators. The rest were already handled on a case-by-case basis in the switch, and have their own fold function in which to place the call. We now have only one major switch on TCGOpcode. Introduce NO_DEST and a block comment for swap_commutative in order to make the handling of brcond and movcond opcodes cleaner. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 142 ++++++++++++++++++++++++------------------------- 1 file changed, 70 insertions(+), 72 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static int do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c) return -1; } +/** + * swap_commutative: + * @dest: TCGArg of the destination argument, or NO_DEST. + * @p1: first paired argument + * @p2: second paired argument + * + * If *@p1 is a constant and *@p2 is not, swap. + * If *@p2 matches @dest, swap. + * Return true if a swap was performed. + */ + +#define NO_DEST temp_arg(NULL) + static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2) { TCGArg a1 = *p1, a2 = *p2; @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) return false; } +static bool fold_const2_commutative(OptContext *ctx, TCGOp *op) +{ + swap_commutative(op->args[0], &op->args[1], &op->args[2]); + return fold_const2(ctx, op); +} + static bool fold_masks(OptContext *ctx, TCGOp *op) { uint64_t a_mask = ctx->a_mask; @@ -XXX,XX +XXX,XX @@ static bool fold_xx_to_x(OptContext *ctx, TCGOp *op) static bool fold_add(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_x(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_addsub2(OptContext *ctx, TCGOp *op, bool add) static bool fold_add2(OptContext *ctx, TCGOp *op) { + /* Note that the high and low parts may be independently swapped. */ + swap_commutative(op->args[0], &op->args[2], &op->args[4]); + swap_commutative(op->args[1], &op->args[3], &op->args[5]); + return fold_addsub2(ctx, op, true); } @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) { uint64_t z1, z2; - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, -1) || fold_xx_to_x(ctx, op)) { @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) static bool fold_brcond(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[2]; - int i = do_constant_folding_cond(ctx->type, op->args[0], op->args[1], cond); + int i; + if (swap_commutative(NO_DEST, &op->args[0], &op->args[1])) { + op->args[2] = cond = tcg_swap_cond(cond); + } + + i = do_constant_folding_cond(ctx->type, op->args[0], op->args[1], cond); if (i == 0) { tcg_op_remove(ctx->tcg, op); return true; @@ -XXX,XX +XXX,XX @@ static bool fold_brcond(OptContext *ctx, TCGOp *op) static bool fold_brcond2(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[4]; - int i = do_constant_folding_cond2(&op->args[0], &op->args[2], cond); TCGArg label = op->args[5]; - int inv = 0; + int i, inv = 0; + if (swap_commutative2(&op->args[0], &op->args[2])) { + op->args[4] = cond = tcg_swap_cond(cond); + } + + i = do_constant_folding_cond2(&op->args[0], &op->args[2], cond); if (i >= 0) { goto do_brcond_const; } @@ -XXX,XX +XXX,XX @@ static bool fold_dup2(OptContext *ctx, TCGOp *op) static bool fold_eqv(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_x(ctx, op, -1) || fold_xi_to_not(ctx, op, 0)) { return true; @@ -XXX,XX +XXX,XX @@ static bool fold_mov(OptContext *ctx, TCGOp *op) static bool fold_movcond(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[5]; - int i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); + int i; + if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) { + op->args[5] = cond = tcg_swap_cond(cond); + } + /* + * Canonicalize the "false" input reg to match the destination reg so + * that the tcg backend can implement a "move if true" operation. + */ + if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) { + op->args[5] = cond = tcg_invert_cond(cond); + } + + i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); if (i >= 0) { return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]); } @@ -XXX,XX +XXX,XX @@ static bool fold_mul(OptContext *ctx, TCGOp *op) static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_i(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) static bool fold_multiply2(OptContext *ctx, TCGOp *op) { + swap_commutative(op->args[0], &op->args[2], &op->args[3]); + if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { uint64_t a = arg_info(op->args[2])->val; uint64_t b = arg_info(op->args[3])->val; @@ -XXX,XX +XXX,XX @@ static bool fold_multiply2(OptContext *ctx, TCGOp *op) static bool fold_nand(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_not(ctx, op, -1)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_neg(OptContext *ctx, TCGOp *op) static bool fold_nor(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_not(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_not(OptContext *ctx, TCGOp *op) static bool fold_or(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_x(ctx, op, 0) || fold_xx_to_x(ctx, op)) { return true; @@ -XXX,XX +XXX,XX @@ static bool fold_remainder(OptContext *ctx, TCGOp *op) static bool fold_setcond(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[3]; - int i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); + int i; + if (swap_commutative(op->args[0], &op->args[1], &op->args[2])) { + op->args[3] = cond = tcg_swap_cond(cond); + } + + i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); if (i >= 0) { return tcg_opt_gen_movi(ctx, op, op->args[0], i); } @@ -XXX,XX +XXX,XX @@ static bool fold_setcond(OptContext *ctx, TCGOp *op) static bool fold_setcond2(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[5]; - int i = do_constant_folding_cond2(&op->args[1], &op->args[3], cond); - int inv = 0; + int i, inv = 0; + if (swap_commutative2(&op->args[1], &op->args[3])) { + op->args[5] = cond = tcg_swap_cond(cond); + } + + i = do_constant_folding_cond2(&op->args[1], &op->args[3], cond); if (i >= 0) { goto do_setcond_const; } @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) static bool fold_xor(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xx_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, 0) || fold_xi_to_not(ctx, op, -1)) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) ctx.type = TCG_TYPE_I32; } - /* For commutative operations make constant second argument */ - switch (opc) { - CASE_OP_32_64_VEC(add): - CASE_OP_32_64_VEC(mul): - CASE_OP_32_64_VEC(and): - CASE_OP_32_64_VEC(or): - CASE_OP_32_64_VEC(xor): - CASE_OP_32_64(eqv): - CASE_OP_32_64(nand): - CASE_OP_32_64(nor): - CASE_OP_32_64(muluh): - CASE_OP_32_64(mulsh): - swap_commutative(op->args[0], &op->args[1], &op->args[2]); - break; - CASE_OP_32_64(brcond): - if (swap_commutative(-1, &op->args[0], &op->args[1])) { - op->args[2] = tcg_swap_cond(op->args[2]); - } - break; - CASE_OP_32_64(setcond): - if (swap_commutative(op->args[0], &op->args[1], &op->args[2])) { - op->args[3] = tcg_swap_cond(op->args[3]); - } - break; - CASE_OP_32_64(movcond): - if (swap_commutative(-1, &op->args[1], &op->args[2])) { - op->args[5] = tcg_swap_cond(op->args[5]); - } - /* For movcond, we canonicalize the "false" input reg to match - the destination reg so that the tcg backend can implement - a "move if true" operation. */ - if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) { - op->args[5] = tcg_invert_cond(op->args[5]); - } - break; - CASE_OP_32_64(add2): - swap_commutative(op->args[0], &op->args[2], &op->args[4]); - swap_commutative(op->args[1], &op->args[3], &op->args[5]); - break; - CASE_OP_32_64(mulu2): - CASE_OP_32_64(muls2): - swap_commutative(op->args[0], &op->args[2], &op->args[3]); - break; - case INDEX_op_brcond2_i32: - if (swap_commutative2(&op->args[0], &op->args[2])) { - op->args[4] = tcg_swap_cond(op->args[4]); - } - break; - case INDEX_op_setcond2_i32: - if (swap_commutative2(&op->args[1], &op->args[3])) { - op->args[5] = tcg_swap_cond(op->args[5]); - } - break; - default: - break; - } - /* Assume all bits affected, and no bits known zero. */ ctx.a_mask = -1; ctx.z_mask = -1; -- 2.25.1 This "garbage" setting pre-dates the addition of the type changing opcodes INDEX_op_ext_i32_i64, INDEX_op_extu_i32_i64, and INDEX_op_extr{l,h}_i64_i32. So now we have a definitive points at which to adjust z_mask to eliminate such bits from the 32-bit operands. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void init_ts_info(OptContext *ctx, TCGTemp *ts) ti->is_const = true; ti->val = ts->val; ti->z_mask = ts->val; - if (TCG_TARGET_REG_BITS > 32 && ts->type == TCG_TYPE_I32) { - /* High bits of a 32-bit quantity are garbage. */ - ti->z_mask |= ~0xffffffffull; - } } else { ti->is_const = false; ti->z_mask = -1; @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) TCGTemp *src_ts = arg_temp(src); TempOptInfo *di; TempOptInfo *si; - uint64_t z_mask; TCGOpcode new_op; if (ts_are_copies(dst_ts, src_ts)) { @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) op->args[0] = dst; op->args[1] = src; - z_mask = si->z_mask; - if (TCG_TARGET_REG_BITS > 32 && new_op == INDEX_op_mov_i32) { - /* High bits of the destination are now garbage. */ - z_mask |= ~0xffffffffull; - } - di->z_mask = z_mask; + di->z_mask = si->z_mask; if (src_ts->type == dst_ts->type) { TempOptInfo *ni = ts_info(si->next_copy); @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) static bool tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, TCGArg dst, uint64_t val) { - /* Convert movi to mov with constant temp. */ - TCGTemp *tv = tcg_constant_internal(ctx->type, val); + TCGTemp *tv; + if (ctx->type == TCG_TYPE_I32) { + val = (int32_t)val; + } + + /* Convert movi to mov with constant temp. */ + tv = tcg_constant_internal(ctx->type, val); init_ts_info(ctx, tv); return tcg_opt_gen_mov(ctx, op, dst, temp_arg(tv)); } @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) uint64_t z_mask = ctx->z_mask; /* - * 32-bit ops generate 32-bit results. For the result is zero test - * below, we can ignore high bits, but for further optimizations we - * need to record that the high bits contain garbage. + * 32-bit ops generate 32-bit results, which for the purpose of + * simplifying tcg are sign-extended. Certainly that's how we + * represent our constants elsewhere. Note that the bits will + * be reset properly for a 64-bit value when encountering the + * type changing opcodes. */ if (ctx->type == TCG_TYPE_I32) { - ctx->z_mask |= MAKE_64BIT_MASK(32, 32); - a_mask &= MAKE_64BIT_MASK(0, 32); - z_mask &= MAKE_64BIT_MASK(0, 32); + a_mask = (int32_t)a_mask; + z_mask = (int32_t)z_mask; + ctx->z_mask = z_mask; } if (z_mask == 0) { -- 2.25.1 Recognize the constant function for or-complement. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_or(OptContext *ctx, TCGOp *op) static bool fold_orc(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_xx_to_i(ctx, op, -1) || fold_xi_to_x(ctx, op, -1) || fold_ix_to_not(ctx, op, 0)) { return true; -- 2.25.1 Recognize the identity function for low-part multiply. Suggested-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) static bool fold_mul(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || - fold_xi_to_i(ctx, op, 0)) { + fold_xi_to_i(ctx, op, 0) || + fold_xi_to_x(ctx, op, 1)) { return true; } return false; -- 2.25.1 Recognize the identity function for division. Suggested-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) static bool fold_divide(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, 1)) { + return true; + } + return false; } static bool fold_dup(OptContext *ctx, TCGOp *op) -- 2.25.1 Recognize the constant function for remainder. Suggested-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_qemu_st(OptContext *ctx, TCGOp *op) static bool fold_remainder(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_i(ctx, op, 0)) { + return true; + } + return false; } static bool fold_setcond(OptContext *ctx, TCGOp *op) -- 2.25.1 Certain targets, like riscv, produce signed 32-bit results. This can lead to lots of redundant extensions as values are manipulated. Begin by tracking only the obvious sign-extensions, and converting them to simple copies when possible. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 123 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 102 insertions(+), 21 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { TCGTemp *next_copy; uint64_t val; uint64_t z_mask; /* mask bit is 0 if and only if value bit is 0 */ + uint64_t s_mask; /* a left-aligned mask of clrsb(value) bits. */ } TempOptInfo; typedef struct OptContext { @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { /* In flight values from optimization. */ uint64_t a_mask; /* mask bit is 0 iff value identical to first input */ uint64_t z_mask; /* mask bit is 0 iff value bit is 0 */ + uint64_t s_mask; /* mask of clrsb(value) bits */ TCGType type; } OptContext; +/* Calculate the smask for a specific value. */ +static uint64_t smask_from_value(uint64_t value) +{ + int rep = clrsb64(value); + return ~(~0ull >> rep); +} + +/* + * Calculate the smask for a given set of known-zeros. + * If there are lots of zeros on the left, we can consider the remainder + * an unsigned field, and thus the corresponding signed field is one bit + * larger. + */ +static uint64_t smask_from_zmask(uint64_t zmask) +{ + /* + * Only the 0 bits are significant for zmask, thus the msb itself + * must be zero, else we have no sign information. + */ + int rep = clz64(zmask); + if (rep == 0) { + return 0; + } + rep -= 1; + return ~(~0ull >> rep); +} + static inline TempOptInfo *ts_info(TCGTemp *ts) { return ts->state_ptr; @@ -XXX,XX +XXX,XX @@ static void reset_ts(TCGTemp *ts) ti->prev_copy = ts; ti->is_const = false; ti->z_mask = -1; + ti->s_mask = 0; } static void reset_temp(TCGArg arg) @@ -XXX,XX +XXX,XX @@ static void init_ts_info(OptContext *ctx, TCGTemp *ts) ti->is_const = true; ti->val = ts->val; ti->z_mask = ts->val; + ti->s_mask = smask_from_value(ts->val); } else { ti->is_const = false; ti->z_mask = -1; + ti->s_mask = 0; } } @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) op->args[1] = src; di->z_mask = si->z_mask; + di->s_mask = si->s_mask; if (src_ts->type == dst_ts->type) { TempOptInfo *ni = ts_info(si->next_copy); @@ -XXX,XX +XXX,XX @@ static void finish_folding(OptContext *ctx, TCGOp *op) nb_oargs = def->nb_oargs; for (i = 0; i < nb_oargs; i++) { - reset_temp(op->args[i]); + TCGTemp *ts = arg_temp(op->args[i]); + reset_ts(ts); /* - * Save the corresponding known-zero bits mask for the + * Save the corresponding known-zero/sign bits mask for the * first output argument (only one supported so far). */ if (i == 0) { - arg_info(op->args[i])->z_mask = ctx->z_mask; + ts_info(ts)->z_mask = ctx->z_mask; + ts_info(ts)->s_mask = ctx->s_mask; } } } @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) { uint64_t a_mask = ctx->a_mask; uint64_t z_mask = ctx->z_mask; + uint64_t s_mask = ctx->s_mask; /* * 32-bit ops generate 32-bit results, which for the purpose of @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) if (ctx->type == TCG_TYPE_I32) { a_mask = (int32_t)a_mask; z_mask = (int32_t)z_mask; + s_mask |= MAKE_64BIT_MASK(32, 32); ctx->z_mask = z_mask; + ctx->s_mask = s_mask; } if (z_mask == 0) { @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) static bool fold_bswap(OptContext *ctx, TCGOp *op) { - uint64_t z_mask, sign; + uint64_t z_mask, s_mask, sign; if (arg_is_const(op->args[1])) { uint64_t t = arg_info(op->args[1])->val; @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) } z_mask = arg_info(op->args[1])->z_mask; + switch (op->opc) { case INDEX_op_bswap16_i32: case INDEX_op_bswap16_i64: @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) default: g_assert_not_reached(); } + s_mask = smask_from_zmask(z_mask); switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { case TCG_BSWAP_OZ: @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) /* If the sign bit may be 1, force all the bits above to 1. */ if (z_mask & sign) { z_mask |= sign; + s_mask = sign << 1; } break; default: /* The high bits are undefined: force all bits above the sign to 1. */ z_mask |= sign << 1; + s_mask = 0; break; } ctx->z_mask = z_mask; + ctx->s_mask = s_mask; return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) static bool fold_extract(OptContext *ctx, TCGOp *op) { uint64_t z_mask_old, z_mask; + int pos = op->args[2]; + int len = op->args[3]; if (arg_is_const(op->args[1])) { uint64_t t; t = arg_info(op->args[1])->val; - t = extract64(t, op->args[2], op->args[3]); + t = extract64(t, pos, len); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } z_mask_old = arg_info(op->args[1])->z_mask; - z_mask = extract64(z_mask_old, op->args[2], op->args[3]); - if (op->args[2] == 0) { + z_mask = extract64(z_mask_old, pos, len); + if (pos == 0) { ctx->a_mask = z_mask_old ^ z_mask; } ctx->z_mask = z_mask; + ctx->s_mask = smask_from_zmask(z_mask); return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_extract2(OptContext *ctx, TCGOp *op) static bool fold_exts(OptContext *ctx, TCGOp *op) { - uint64_t z_mask_old, z_mask, sign; + uint64_t s_mask_old, s_mask, z_mask, sign; bool type_change = false; if (fold_const1(ctx, op)) { return true; } - z_mask_old = z_mask = arg_info(op->args[1])->z_mask; + z_mask = arg_info(op->args[1])->z_mask; + s_mask = arg_info(op->args[1])->s_mask; + s_mask_old = s_mask; switch (op->opc) { CASE_OP_32_64(ext8s): @@ -XXX,XX +XXX,XX @@ static bool fold_exts(OptContext *ctx, TCGOp *op) if (z_mask & sign) { z_mask |= sign; - } else if (!type_change) { - ctx->a_mask = z_mask_old ^ z_mask; } + s_mask |= sign << 1; + ctx->z_mask = z_mask; + ctx->s_mask = s_mask; + if (!type_change) { + ctx->a_mask = s_mask & ~s_mask_old; + } return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_extu(OptContext *ctx, TCGOp *op) } ctx->z_mask = z_mask; + ctx->s_mask = smask_from_zmask(z_mask); if (!type_change) { ctx->a_mask = z_mask_old ^ z_mask; } @@ -XXX,XX +XXX,XX @@ static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) MemOp mop = get_memop(oi); int width = 8 * memop_size(mop); - if (!(mop & MO_SIGN) && width < 64) { - ctx->z_mask = MAKE_64BIT_MASK(0, width); + if (width < 64) { + ctx->s_mask = MAKE_64BIT_MASK(width, 64 - width); + if (!(mop & MO_SIGN)) { + ctx->z_mask = MAKE_64BIT_MASK(0, width); + ctx->s_mask <<= 1; + } } /* Opcodes that touch guest memory stop the mb optimization. */ @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) static bool fold_sextract(OptContext *ctx, TCGOp *op) { - int64_t z_mask_old, z_mask; + uint64_t z_mask, s_mask, s_mask_old; + int pos = op->args[2]; + int len = op->args[3]; if (arg_is_const(op->args[1])) { uint64_t t; t = arg_info(op->args[1])->val; - t = sextract64(t, op->args[2], op->args[3]); + t = sextract64(t, pos, len); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } - z_mask_old = arg_info(op->args[1])->z_mask; - z_mask = sextract64(z_mask_old, op->args[2], op->args[3]); - if (op->args[2] == 0 && z_mask >= 0) { - ctx->a_mask = z_mask_old ^ z_mask; - } + z_mask = arg_info(op->args[1])->z_mask; + z_mask = sextract64(z_mask, pos, len); ctx->z_mask = z_mask; + s_mask_old = arg_info(op->args[1])->s_mask; + s_mask = sextract64(s_mask_old, pos, len); + s_mask |= MAKE_64BIT_MASK(len, 64 - len); + ctx->s_mask = s_mask; + + if (pos == 0) { + ctx->a_mask = s_mask & ~s_mask_old; + } + return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) { /* We can't do any folding with a load, but we can record bits. */ switch (op->opc) { + CASE_OP_32_64(ld8s): + ctx->s_mask = MAKE_64BIT_MASK(8, 56); + break; CASE_OP_32_64(ld8u): ctx->z_mask = MAKE_64BIT_MASK(0, 8); + ctx->s_mask = MAKE_64BIT_MASK(9, 55); + break; + CASE_OP_32_64(ld16s): + ctx->s_mask = MAKE_64BIT_MASK(16, 48); break; CASE_OP_32_64(ld16u): ctx->z_mask = MAKE_64BIT_MASK(0, 16); + ctx->s_mask = MAKE_64BIT_MASK(17, 47); + break; + case INDEX_op_ld32s_i64: + ctx->s_mask = MAKE_64BIT_MASK(32, 32); break; case INDEX_op_ld32u_i64: ctx->z_mask = MAKE_64BIT_MASK(0, 32); + ctx->s_mask = MAKE_64BIT_MASK(33, 31); break; default: g_assert_not_reached(); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) ctx.type = TCG_TYPE_I32; } - /* Assume all bits affected, and no bits known zero. */ + /* Assume all bits affected, no bits known zero, no sign reps. */ ctx.a_mask = -1; ctx.z_mask = -1; + ctx.s_mask = 0; /* * Process each opcode. @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_extrh_i64_i32: done = fold_extu(&ctx, op); break; + CASE_OP_32_64(ld8s): CASE_OP_32_64(ld8u): + CASE_OP_32_64(ld16s): CASE_OP_32_64(ld16u): + case INDEX_op_ld32s_i64: case INDEX_op_ld32u_i64: done = fold_tcg_ld(&ctx, op); break; -- 2.25.1 Sign repetitions are perforce all identical, whether they are 1 or 0. Bitwise operations preserve the relative quantity of the repetitions. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) z2 = arg_info(op->args[2])->z_mask; ctx->z_mask = z1 & z2; + /* + * Sign repetitions are perforce all identical, whether they are 1 or 0. + * Bitwise operations preserve the relative quantity of the repetitions. + */ + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; + /* * Known-zeros does not imply known-ones. Therefore unless * arg2 is constant, we can't infer affected bits from it. @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) } ctx->z_mask = z1; + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) fold_xi_to_not(ctx, op, 0)) { return true; } + + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) ctx->z_mask = arg_info(op->args[3])->z_mask | arg_info(op->args[4])->z_mask; + ctx->s_mask = arg_info(op->args[3])->s_mask + & arg_info(op->args[4])->s_mask; if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { uint64_t tv = arg_info(op->args[3])->val; @@ -XXX,XX +XXX,XX @@ static bool fold_nand(OptContext *ctx, TCGOp *op) fold_xi_to_not(ctx, op, -1)) { return true; } + + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_nor(OptContext *ctx, TCGOp *op) fold_xi_to_not(ctx, op, 0)) { return true; } + + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_not(OptContext *ctx, TCGOp *op) return true; } + ctx->s_mask = arg_info(op->args[1])->s_mask; + /* Because of fold_to_not, we want to always return true, via finish. */ finish_folding(ctx, op); return true; @@ -XXX,XX +XXX,XX @@ static bool fold_or(OptContext *ctx, TCGOp *op) ctx->z_mask = arg_info(op->args[1])->z_mask | arg_info(op->args[2])->z_mask; + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_orc(OptContext *ctx, TCGOp *op) fold_ix_to_not(ctx, op, 0)) { return true; } + + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_xor(OptContext *ctx, TCGOp *op) ctx->z_mask = arg_info(op->args[1])->z_mask | arg_info(op->args[2])->z_mask; + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return fold_masks(ctx, op); } -- 2.25.1 The result is either 0 or 1, which means that we have a 2 bit signed result, and thus 62 bits of sign. For clarity, use the smask_from_zmask function. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_setcond(OptContext *ctx, TCGOp *op) } ctx->z_mask = 1; + ctx->s_mask = smask_from_zmask(1); return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) } ctx->z_mask = 1; + ctx->s_mask = smask_from_zmask(1); return false; do_setcond_const: -- 2.25.1 The results are generally 6 bit unsigned values, though the count leading and trailing bits may produce any value for a zero input. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_count_zeros(OptContext *ctx, TCGOp *op) g_assert_not_reached(); } ctx->z_mask = arg_info(op->args[2])->z_mask | z_mask; - + ctx->s_mask = smask_from_zmask(ctx->z_mask); return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_ctpop(OptContext *ctx, TCGOp *op) default: g_assert_not_reached(); } + ctx->s_mask = smask_from_zmask(ctx->z_mask); return false; } -- 2.25.1 For constant shifts, we can simply shift the s_mask. For variable shifts, we know that sar does not reduce the s_mask, which helps for sequences like ext32s_i64 t, in sar_i64 t, t, v ext32s_i64 out, t allowing the final extend to be eliminated. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static uint64_t smask_from_zmask(uint64_t zmask) return ~(~0ull >> rep); } +/* + * Recreate a properly left-aligned smask after manipulation. + * Some bit-shuffling, particularly shifts and rotates, may + * retain sign bits on the left, but may scatter disconnected + * sign bits on the right. Retain only what remains to the left. + */ +static uint64_t smask_from_smask(int64_t smask) +{ + /* Only the 1 bits are significant for smask */ + return smask_from_zmask(~smask); +} + static inline TempOptInfo *ts_info(TCGTemp *ts) { return ts->state_ptr; @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) static bool fold_shift(OptContext *ctx, TCGOp *op) { + uint64_t s_mask, z_mask, sign; + if (fold_const2(ctx, op) || fold_ix_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, 0)) { return true; } + s_mask = arg_info(op->args[1])->s_mask; + z_mask = arg_info(op->args[1])->z_mask; + if (arg_is_const(op->args[2])) { - ctx->z_mask = do_constant_folding(op->opc, ctx->type, - arg_info(op->args[1])->z_mask, - arg_info(op->args[2])->val); + int sh = arg_info(op->args[2])->val; + + ctx->z_mask = do_constant_folding(op->opc, ctx->type, z_mask, sh); + + s_mask = do_constant_folding(op->opc, ctx->type, s_mask, sh); + ctx->s_mask = smask_from_smask(s_mask); + return fold_masks(ctx, op); } + + switch (op->opc) { + CASE_OP_32_64(sar): + /* + * Arithmetic right shift will not reduce the number of + * input sign repetitions. + */ + ctx->s_mask = s_mask; + break; + CASE_OP_32_64(shr): + /* + * If the sign bit is known zero, then logical right shift + * will not reduced the number of input sign repetitions. + */ + sign = (s_mask & -s_mask) >> 1; + if (!(z_mask & sign)) { + ctx->s_mask = s_mask; + } + break; + default: + break; + } + return false; } -- 2.25.1 The following changes since commit a9fe9e191b4305b88c356a1ed9ac3baf89eb18aa: Merge tag 'pull-riscv-to-apply-20230505-1' of https://github.com/alistair23/qemu into staging (2023-05-05 09:25:13 +0100) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20230505 for you to fetch changes up to 35a0bd63b458f30389b6bc6b7471c1665fe7b9d8: tcg: Widen helper_*_st[bw]_mmu val arguments (2023-05-05 17:21:03 +0100) ---------------------------------------------------------------- softfloat: Fix the incorrect computation in float32_exp2 tcg: Remove compatability helpers for qemu ld/st target/alpha: Remove TARGET_ALIGNED_ONLY target/hppa: Remove TARGET_ALIGNED_ONLY target/sparc: Remove TARGET_ALIGNED_ONLY tcg: Cleanups preparing to unify calls to qemu_ld/st helpers ---------------------------------------------------------------- Richard Henderson (41): target/avr: Finish conversion to tcg_gen_qemu_{ld,st}_* target/cris: Finish conversion to tcg_gen_qemu_{ld,st}_* target/Hexagon: Finish conversion to tcg_gen_qemu_{ld, st}_* target/m68k: Finish conversion to tcg_gen_qemu_{ld,st}_* target/mips: Finish conversion to tcg_gen_qemu_{ld,st}_* target/s390x: Finish conversion to tcg_gen_qemu_{ld, st}_* target/sparc: Finish conversion to tcg_gen_qemu_{ld, st}_* target/xtensa: Finish conversion to tcg_gen_qemu_{ld, st}_* tcg: Remove compatability helpers for qemu ld/st target/alpha: Use MO_ALIGN for system UNALIGN() target/alpha: Use MO_ALIGN where required target/alpha: Remove TARGET_ALIGNED_ONLY target/hppa: Use MO_ALIGN for system UNALIGN() target/hppa: Remove TARGET_ALIGNED_ONLY target/sparc: Use MO_ALIGN where required target/sparc: Use cpu_ld*_code_mmu target/sparc: Remove TARGET_ALIGNED_ONLY tcg/i386: Rationalize args to tcg_out_qemu_{ld,st} tcg/i386: Generalize multi-part load overlap test tcg/i386: Introduce HostAddress tcg/i386: Drop r0+r1 local variables from tcg_out_tlb_load tcg/i386: Introduce tcg_out_testi tcg/aarch64: Rationalize args to tcg_out_qemu_{ld,st} tcg/aarch64: Introduce HostAddress tcg/arm: Rationalize args to tcg_out_qemu_{ld,st} tcg/arm: Introduce HostAddress tcg/loongarch64: Rationalize args to tcg_out_qemu_{ld,st} tcg/loongarch64: Introduce HostAddress tcg/mips: Rationalize args to tcg_out_qemu_{ld,st} tcg/ppc: Rationalize args to tcg_out_qemu_{ld,st} tcg/ppc: Introduce HostAddress tcg/riscv: Require TCG_TARGET_REG_BITS == 64 tcg/riscv: Rationalize args to tcg_out_qemu_{ld,st} tcg/s390x: Pass TCGType to tcg_out_qemu_{ld,st} tcg/s390x: Introduce HostAddress tcg/sparc64: Drop is_64 test from tcg_out_qemu_ld data return tcg/sparc64: Pass TCGType to tcg_out_qemu_{ld,st} tcg: Move TCGLabelQemuLdst to tcg.c tcg: Replace REG_P with arg_loc_reg_p tcg: Introduce arg_slot_stk_ofs tcg: Widen helper_*_st[bw]_mmu val arguments Shivaprasad G Bhat (1): softfloat: Fix the incorrect computation in float32_exp2 configs/targets/alpha-linux-user.mak | 1 - configs/targets/alpha-softmmu.mak | 1 - configs/targets/hppa-linux-user.mak | 1 - configs/targets/hppa-softmmu.mak | 1 - configs/targets/sparc-linux-user.mak | 1 - configs/targets/sparc-softmmu.mak | 1 - configs/targets/sparc32plus-linux-user.mak | 1 - configs/targets/sparc64-linux-user.mak | 1 - configs/targets/sparc64-softmmu.mak | 1 - include/tcg/tcg-ldst.h | 10 +- include/tcg/tcg-op.h | 55 ----- target/hexagon/macros.h | 14 +- tcg/riscv/tcg-target-con-set.h | 8 - tcg/riscv/tcg-target.h | 22 +- tcg/tcg-internal.h | 4 - accel/tcg/cputlb.c | 6 +- fpu/softfloat.c | 2 +- target/alpha/translate.c | 38 +-- target/avr/translate.c | 16 +- target/hexagon/genptr.c | 8 +- target/hexagon/idef-parser/parser-helpers.c | 28 +-- target/hexagon/translate.c | 32 +-- target/hppa/translate.c | 2 +- target/m68k/translate.c | 76 ++---- target/mips/tcg/translate.c | 8 +- target/s390x/tcg/translate.c | 152 ++++++------ target/sparc/ldst_helper.c | 10 +- target/sparc/translate.c | 85 ++++--- target/xtensa/translate.c | 4 +- tcg/tcg.c | 58 +++-- target/cris/translate_v10.c.inc | 18 +- target/mips/tcg/nanomips_translate.c.inc | 2 +- tcg/aarch64/tcg-target.c.inc | 108 ++++++--- tcg/arm/tcg-target.c.inc | 357 +++++++++++++--------------- tcg/i386/tcg-target.c.inc | 345 ++++++++++++++------------- tcg/loongarch64/tcg-target.c.inc | 135 +++++------ tcg/mips/tcg-target.c.inc | 186 ++++++++------- tcg/ppc/tcg-target.c.inc | 192 ++++++++------- tcg/riscv/tcg-target.c.inc | 268 ++++++--------------- tcg/s390x/tcg-target.c.inc | 131 +++++----- tcg/sparc64/tcg-target.c.inc | 8 +- tcg/tcg-ldst.c.inc | 14 -- 42 files changed, 1120 insertions(+), 1291 deletions(-) From: Shivaprasad G Bhat <sbhat@linux.ibm.com> The float32_exp2 function is computing wrong exponent of 2. For example, with the following set of values {0.1, 2.0, 2.0, -1.0}, the expected output would be {1.071773, 4.000000, 4.000000, 0.500000}. Instead, the function is computing {1.119102, 3.382044, 3.382044, -0.191022} Looking at the code, the float32_exp2() attempts to do this 2 3 4 5 n x x x x x x x e = 1 + --- + --- + --- + --- + --- + ... + --- + ... 1! 2! 3! 4! 5! n! But because of the typo it ends up doing x x x x x x x e = 1 + --- + --- + --- + --- + --- + ... + --- + ... 1! 2! 3! 4! 5! n! This is because instead of the xnp which holds the numerator, parts_muladd is using the xp which is just 'x'. Commit '572c4d862ff2' refactored this function, and mistakenly used xp instead of xnp. Cc: qemu-stable@nongnu.org Fixes: 572c4d862ff2 "softfloat: Convert float32_exp2 to FloatParts" Partially-Resolves: https://gitlab.com/qemu-project/qemu/-/issues/1623 Reported-By: Luca Barbato (https://gitlab.com/lu-zero) Signed-off-by: Shivaprasad G Bhat <sbhat@linux.ibm.com> Signed-off-by: Vaibhav Jain <vaibhav@linux.ibm.com> Message-Id: <168304110865.537992.13059030916325018670.stgit@localhost.localdomain> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ float32 float32_exp2(float32 a, float_status *status) float64_unpack_canonical(&rp, float64_one, status); for (i = 0 ; i < 15 ; i++) { float64_unpack_canonical(&tp, float32_exp2_coefficients[i], status); - rp = *parts_muladd(&tp, &xp, &rp, 0, status); + rp = *parts_muladd(&tp, &xnp, &rp, 0, status); xnp = *parts_mul(&xnp, &xp, status); } -- 2.34.1 Convert away from the old interface with the implicit MemOp argument. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Anton Johansson <anjo@rev.ng> Message-Id: <20230502135741.1158035-2-richard.henderson@linaro.org> --- target/avr/translate.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/target/avr/translate.c b/target/avr/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/avr/translate.c +++ b/target/avr/translate.c @@ -XXX,XX +XXX,XX @@ static void gen_data_store(DisasContext *ctx, TCGv data, TCGv addr) if (ctx->base.tb->flags & TB_FLAGS_FULL_ACCESS) { gen_helper_fullwr(cpu_env, data, addr); } else { - tcg_gen_qemu_st8(data, addr, MMU_DATA_IDX); /* mem[addr] = data */ + tcg_gen_qemu_st_tl(data, addr, MMU_DATA_IDX, MO_UB); } } @@ -XXX,XX +XXX,XX @@ static void gen_data_load(DisasContext *ctx, TCGv data, TCGv addr) if (ctx->base.tb->flags & TB_FLAGS_FULL_ACCESS) { gen_helper_fullrd(data, cpu_env, addr); } else { - tcg_gen_qemu_ld8u(data, addr, MMU_DATA_IDX); /* data = mem[addr] */ + tcg_gen_qemu_ld_tl(data, addr, MMU_DATA_IDX, MO_UB); } } @@ -XXX,XX +XXX,XX @@ static bool trans_LPM1(DisasContext *ctx, arg_LPM1 *a) tcg_gen_shli_tl(addr, H, 8); /* addr = H:L */ tcg_gen_or_tl(addr, addr, L); - tcg_gen_qemu_ld8u(Rd, addr, MMU_CODE_IDX); /* Rd = mem[addr] */ + tcg_gen_qemu_ld_tl(Rd, addr, MMU_CODE_IDX, MO_UB); return true; } @@ -XXX,XX +XXX,XX @@ static bool trans_LPM2(DisasContext *ctx, arg_LPM2 *a) tcg_gen_shli_tl(addr, H, 8); /* addr = H:L */ tcg_gen_or_tl(addr, addr, L); - tcg_gen_qemu_ld8u(Rd, addr, MMU_CODE_IDX); /* Rd = mem[addr] */ + tcg_gen_qemu_ld_tl(Rd, addr, MMU_CODE_IDX, MO_UB); return true; } @@ -XXX,XX +XXX,XX @@ static bool trans_LPMX(DisasContext *ctx, arg_LPMX *a) tcg_gen_shli_tl(addr, H, 8); /* addr = H:L */ tcg_gen_or_tl(addr, addr, L); - tcg_gen_qemu_ld8u(Rd, addr, MMU_CODE_IDX); /* Rd = mem[addr] */ + tcg_gen_qemu_ld_tl(Rd, addr, MMU_CODE_IDX, MO_UB); tcg_gen_addi_tl(addr, addr, 1); /* addr = addr + 1 */ tcg_gen_andi_tl(L, addr, 0xff); tcg_gen_shri_tl(addr, addr, 8); @@ -XXX,XX +XXX,XX @@ static bool trans_ELPM1(DisasContext *ctx, arg_ELPM1 *a) TCGv Rd = cpu_r[0]; TCGv addr = gen_get_zaddr(); - tcg_gen_qemu_ld8u(Rd, addr, MMU_CODE_IDX); /* Rd = mem[addr] */ + tcg_gen_qemu_ld_tl(Rd, addr, MMU_CODE_IDX, MO_UB); return true; } @@ -XXX,XX +XXX,XX @@ static bool trans_ELPM2(DisasContext *ctx, arg_ELPM2 *a) TCGv Rd = cpu_r[a->rd]; TCGv addr = gen_get_zaddr(); - tcg_gen_qemu_ld8u(Rd, addr, MMU_CODE_IDX); /* Rd = mem[addr] */ + tcg_gen_qemu_ld_tl(Rd, addr, MMU_CODE_IDX, MO_UB); return true; } @@ -XXX,XX +XXX,XX @@ static bool trans_ELPMX(DisasContext *ctx, arg_ELPMX *a) TCGv Rd = cpu_r[a->rd]; TCGv addr = gen_get_zaddr(); - tcg_gen_qemu_ld8u(Rd, addr, MMU_CODE_IDX); /* Rd = mem[addr] */ + tcg_gen_qemu_ld_tl(Rd, addr, MMU_CODE_IDX, MO_UB); tcg_gen_addi_tl(addr, addr, 1); /* addr = addr + 1 */ gen_set_zaddr(addr); return true; -- 2.34.1 Convert away from the old interface with the implicit MemOp argument. In this case we can fold the calls using the size bits of MemOp. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Anton Johansson <anjo@rev.ng> Message-Id: <20230502135741.1158035-3-richard.henderson@linaro.org> --- target/cris/translate_v10.c.inc | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/target/cris/translate_v10.c.inc b/target/cris/translate_v10.c.inc index XXXXXXX..XXXXXXX 100644 --- a/target/cris/translate_v10.c.inc +++ b/target/cris/translate_v10.c.inc @@ -XXX,XX +XXX,XX @@ static void gen_store_v10_conditional(DisasContext *dc, TCGv addr, TCGv val, /* Store only if F flag isn't set */ tcg_gen_andi_tl(t1, cpu_PR[PR_CCS], F_FLAG_V10); tcg_gen_brcondi_tl(TCG_COND_NE, t1, 0, l1); - if (size == 1) { - tcg_gen_qemu_st8(tval, taddr, mem_index); - } else if (size == 2) { - tcg_gen_qemu_st16(tval, taddr, mem_index); - } else { - tcg_gen_qemu_st32(tval, taddr, mem_index); - } + + tcg_gen_qemu_st_tl(tval, taddr, mem_index, ctz32(size) | MO_TE); + gen_set_label(l1); tcg_gen_shri_tl(t1, t1, 1); /* shift F to P position */ tcg_gen_or_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], t1); /*P=F*/ @@ -XXX,XX +XXX,XX @@ static void gen_store_v10(DisasContext *dc, TCGv addr, TCGv val, return; } - if (size == 1) { - tcg_gen_qemu_st8(val, addr, mem_index); - } else if (size == 2) { - tcg_gen_qemu_st16(val, addr, mem_index); - } else { - tcg_gen_qemu_st32(val, addr, mem_index); - } + tcg_gen_qemu_st_tl(val, addr, mem_index, ctz32(size) | MO_TE); } -- 2.34.1 Convert away from the old interface with the implicit MemOp argument. Importantly, this removes some incorrect casts generated by idef-parser's gen_load(). Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Tested-by: Taylor Simpson <tsimpson@quicinc.com> Reviewed-by: Taylor Simpson <tsimpson@quicinc.com> Reviewed-by: Anton Johansson <anjo@rev.ng> Message-Id: <20230502135741.1158035-4-richard.henderson@linaro.org> --- target/hexagon/macros.h | 14 ++++----- target/hexagon/genptr.c | 8 +++--- target/hexagon/idef-parser/parser-helpers.c | 28 +++++++++--------- target/hexagon/translate.c | 32 ++++++++++----------- 4 files changed, 40 insertions(+), 42 deletions(-) diff --git a/target/hexagon/macros.h b/target/hexagon/macros.h index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/macros.h +++ b/target/hexagon/macros.h @@ -XXX,XX +XXX,XX @@ #define MEM_LOAD1s(DST, VA) \ do { \ CHECK_NOSHUF(VA, 1); \ - tcg_gen_qemu_ld8s(DST, VA, ctx->mem_idx); \ + tcg_gen_qemu_ld_tl(DST, VA, ctx->mem_idx, MO_SB); \ } while (0) #define MEM_LOAD1u(DST, VA) \ do { \ CHECK_NOSHUF(VA, 1); \ - tcg_gen_qemu_ld8u(DST, VA, ctx->mem_idx); \ + tcg_gen_qemu_ld_tl(DST, VA, ctx->mem_idx, MO_UB); \ } while (0) #define MEM_LOAD2s(DST, VA) \ do { \ CHECK_NOSHUF(VA, 2); \ - tcg_gen_qemu_ld16s(DST, VA, ctx->mem_idx); \ + tcg_gen_qemu_ld_tl(DST, VA, ctx->mem_idx, MO_TESW); \ } while (0) #define MEM_LOAD2u(DST, VA) \ do { \ CHECK_NOSHUF(VA, 2); \ - tcg_gen_qemu_ld16u(DST, VA, ctx->mem_idx); \ + tcg_gen_qemu_ld_tl(DST, VA, ctx->mem_idx, MO_TEUW); \ } while (0) #define MEM_LOAD4s(DST, VA) \ do { \ CHECK_NOSHUF(VA, 4); \ - tcg_gen_qemu_ld32s(DST, VA, ctx->mem_idx); \ + tcg_gen_qemu_ld_tl(DST, VA, ctx->mem_idx, MO_TESL); \ } while (0) #define MEM_LOAD4u(DST, VA) \ do { \ CHECK_NOSHUF(VA, 4); \ - tcg_gen_qemu_ld32s(DST, VA, ctx->mem_idx); \ + tcg_gen_qemu_ld_tl(DST, VA, ctx->mem_idx, MO_TEUL); \ } while (0) #define MEM_LOAD8u(DST, VA) \ do { \ CHECK_NOSHUF(VA, 8); \ - tcg_gen_qemu_ld64(DST, VA, ctx->mem_idx); \ + tcg_gen_qemu_ld_i64(DST, VA, ctx->mem_idx, MO_TEUQ); \ } while (0) #define MEM_STORE1_FUNC(X) \ diff --git a/target/hexagon/genptr.c b/target/hexagon/genptr.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/genptr.c +++ b/target/hexagon/genptr.c @@ -XXX,XX +XXX,XX @@ void gen_set_byte_i64(int N, TCGv_i64 result, TCGv src) static inline void gen_load_locked4u(TCGv dest, TCGv vaddr, int mem_index) { - tcg_gen_qemu_ld32u(dest, vaddr, mem_index); + tcg_gen_qemu_ld_tl(dest, vaddr, mem_index, MO_TEUL); tcg_gen_mov_tl(hex_llsc_addr, vaddr); tcg_gen_mov_tl(hex_llsc_val, dest); } static inline void gen_load_locked8u(TCGv_i64 dest, TCGv vaddr, int mem_index) { - tcg_gen_qemu_ld64(dest, vaddr, mem_index); + tcg_gen_qemu_ld_i64(dest, vaddr, mem_index, MO_TEUQ); tcg_gen_mov_tl(hex_llsc_addr, vaddr); tcg_gen_mov_i64(hex_llsc_val_i64, dest); } @@ -XXX,XX +XXX,XX @@ static void gen_load_frame(DisasContext *ctx, TCGv_i64 frame, TCGv EA) { Insn *insn = ctx->insn; /* Needed for CHECK_NOSHUF */ CHECK_NOSHUF(EA, 8); - tcg_gen_qemu_ld64(frame, EA, ctx->mem_idx); + tcg_gen_qemu_ld_i64(frame, EA, ctx->mem_idx, MO_TEUQ); } static void gen_return(DisasContext *ctx, TCGv_i64 dst, TCGv src) @@ -XXX,XX +XXX,XX @@ static void gen_vreg_load(DisasContext *ctx, intptr_t dstoff, TCGv src, tcg_gen_andi_tl(src, src, ~((int32_t)sizeof(MMVector) - 1)); } for (int i = 0; i < sizeof(MMVector) / 8; i++) { - tcg_gen_qemu_ld64(tmp, src, ctx->mem_idx); + tcg_gen_qemu_ld_i64(tmp, src, ctx->mem_idx, MO_TEUQ); tcg_gen_addi_tl(src, src, 8); tcg_gen_st_i64(tmp, cpu_env, dstoff + i * 8); } diff --git a/target/hexagon/idef-parser/parser-helpers.c b/target/hexagon/idef-parser/parser-helpers.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/idef-parser/parser-helpers.c +++ b/target/hexagon/idef-parser/parser-helpers.c @@ -XXX,XX +XXX,XX @@ void gen_load_cancel(Context *c, YYLTYPE *locp) void gen_load(Context *c, YYLTYPE *locp, HexValue *width, HexSignedness signedness, HexValue *ea, HexValue *dst) { - char size_suffix[4] = {0}; - const char *sign_suffix; + unsigned dst_bit_width; + unsigned src_bit_width; + /* Memop width is specified in the load macro */ assert_signedness(c, locp, signedness); - sign_suffix = (width->imm.value > 4) - ? "" - : ((signedness == UNSIGNED) ? "u" : "s"); + /* If dst is a variable, assert that is declared and load the type info */ if (dst->type == VARID) { find_variable(c, locp, dst, dst); } - snprintf(size_suffix, 4, "%" PRIu64, width->imm.value * 8); + src_bit_width = width->imm.value * 8; + dst_bit_width = MAX(dst->bit_width, 32); + /* Lookup the effective address EA */ find_variable(c, locp, ea, ea); OUT(c, locp, "if (insn->slot == 0 && pkt->pkt_has_store_s1) {\n"); OUT(c, locp, "probe_noshuf_load(", ea, ", ", width, ", ctx->mem_idx);\n"); OUT(c, locp, "process_store(ctx, 1);\n"); OUT(c, locp, "}\n"); - OUT(c, locp, "tcg_gen_qemu_ld", size_suffix, sign_suffix); + + OUT(c, locp, "tcg_gen_qemu_ld_i", &dst_bit_width); OUT(c, locp, "("); - if (dst->bit_width > width->imm.value * 8) { - /* - * Cast to the correct TCG type if necessary, to avoid implict cast - * warnings. This is needed when the width of the destination var is - * larger than the size of the requested load. - */ - OUT(c, locp, "(TCGv) "); + OUT(c, locp, dst, ", ", ea, ", ctx->mem_idx, MO_", &src_bit_width); + if (signedness == SIGNED) { + OUT(c, locp, " | MO_SIGN"); } - OUT(c, locp, dst, ", ", ea, ", ctx->mem_idx);\n"); + OUT(c, locp, " | MO_TE);\n"); } void gen_store(Context *c, YYLTYPE *locp, HexValue *width, HexValue *ea, diff --git a/target/hexagon/translate.c b/target/hexagon/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/hexagon/translate.c +++ b/target/hexagon/translate.c @@ -XXX,XX +XXX,XX @@ void process_store(DisasContext *ctx, int slot_num) switch (ctx->store_width[slot_num]) { case 1: gen_check_store_width(ctx, slot_num); - tcg_gen_qemu_st8(hex_store_val32[slot_num], - hex_store_addr[slot_num], - ctx->mem_idx); + tcg_gen_qemu_st_tl(hex_store_val32[slot_num], + hex_store_addr[slot_num], + ctx->mem_idx, MO_UB); break; case 2: gen_check_store_width(ctx, slot_num); - tcg_gen_qemu_st16(hex_store_val32[slot_num], - hex_store_addr[slot_num], - ctx->mem_idx); + tcg_gen_qemu_st_tl(hex_store_val32[slot_num], + hex_store_addr[slot_num], + ctx->mem_idx, MO_TEUW); break; case 4: gen_check_store_width(ctx, slot_num); - tcg_gen_qemu_st32(hex_store_val32[slot_num], - hex_store_addr[slot_num], - ctx->mem_idx); + tcg_gen_qemu_st_tl(hex_store_val32[slot_num], + hex_store_addr[slot_num], + ctx->mem_idx, MO_TEUL); break; case 8: gen_check_store_width(ctx, slot_num); - tcg_gen_qemu_st64(hex_store_val64[slot_num], - hex_store_addr[slot_num], - ctx->mem_idx); + tcg_gen_qemu_st_i64(hex_store_val64[slot_num], + hex_store_addr[slot_num], + ctx->mem_idx, MO_TEUQ); break; default: { @@ -XXX,XX +XXX,XX @@ static void process_dczeroa(DisasContext *ctx) TCGv_i64 zero = tcg_constant_i64(0); tcg_gen_andi_tl(addr, hex_dczero_addr, ~0x1f); - tcg_gen_qemu_st64(zero, addr, ctx->mem_idx); + tcg_gen_qemu_st_i64(zero, addr, ctx->mem_idx, MO_UQ); tcg_gen_addi_tl(addr, addr, 8); - tcg_gen_qemu_st64(zero, addr, ctx->mem_idx); + tcg_gen_qemu_st_i64(zero, addr, ctx->mem_idx, MO_UQ); tcg_gen_addi_tl(addr, addr, 8); - tcg_gen_qemu_st64(zero, addr, ctx->mem_idx); + tcg_gen_qemu_st_i64(zero, addr, ctx->mem_idx, MO_UQ); tcg_gen_addi_tl(addr, addr, 8); - tcg_gen_qemu_st64(zero, addr, ctx->mem_idx); + tcg_gen_qemu_st_i64(zero, addr, ctx->mem_idx, MO_UQ); } } -- 2.34.1 Convert away from the old interface with the implicit MemOp argument. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Anton Johansson <anjo@rev.ng> Message-Id: <20230502135741.1158035-5-richard.henderson@linaro.org> --- target/m68k/translate.c | 76 ++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 51 deletions(-) diff --git a/target/m68k/translate.c b/target/m68k/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/m68k/translate.c +++ b/target/m68k/translate.c @@ -XXX,XX +XXX,XX @@ static inline void gen_addr_fault(DisasContext *s) static inline TCGv gen_load(DisasContext *s, int opsize, TCGv addr, int sign, int index) { - TCGv tmp; - tmp = tcg_temp_new_i32(); - switch(opsize) { + TCGv tmp = tcg_temp_new_i32(); + + switch (opsize) { case OS_BYTE: - if (sign) - tcg_gen_qemu_ld8s(tmp, addr, index); - else - tcg_gen_qemu_ld8u(tmp, addr, index); - break; case OS_WORD: - if (sign) - tcg_gen_qemu_ld16s(tmp, addr, index); - else - tcg_gen_qemu_ld16u(tmp, addr, index); - break; case OS_LONG: - tcg_gen_qemu_ld32u(tmp, addr, index); + tcg_gen_qemu_ld_tl(tmp, addr, index, + opsize | (sign ? MO_SIGN : 0) | MO_TE); break; default: g_assert_not_reached(); @@ -XXX,XX +XXX,XX @@ static inline TCGv gen_load(DisasContext *s, int opsize, TCGv addr, static inline void gen_store(DisasContext *s, int opsize, TCGv addr, TCGv val, int index) { - switch(opsize) { + switch (opsize) { case OS_BYTE: - tcg_gen_qemu_st8(val, addr, index); - break; case OS_WORD: - tcg_gen_qemu_st16(val, addr, index); - break; case OS_LONG: - tcg_gen_qemu_st32(val, addr, index); + tcg_gen_qemu_st_tl(val, addr, index, opsize | MO_TE); break; default: g_assert_not_reached(); @@ -XXX,XX +XXX,XX @@ static void gen_load_fp(DisasContext *s, int opsize, TCGv addr, TCGv_ptr fp, tmp = tcg_temp_new(); switch (opsize) { case OS_BYTE: - tcg_gen_qemu_ld8s(tmp, addr, index); - gen_helper_exts32(cpu_env, fp, tmp); - break; case OS_WORD: - tcg_gen_qemu_ld16s(tmp, addr, index); - gen_helper_exts32(cpu_env, fp, tmp); - break; - case OS_LONG: - tcg_gen_qemu_ld32u(tmp, addr, index); + tcg_gen_qemu_ld_tl(tmp, addr, index, opsize | MO_SIGN | MO_TE); gen_helper_exts32(cpu_env, fp, tmp); break; case OS_SINGLE: - tcg_gen_qemu_ld32u(tmp, addr, index); + tcg_gen_qemu_ld_tl(tmp, addr, index, MO_TEUL); gen_helper_extf32(cpu_env, fp, tmp); break; case OS_DOUBLE: - tcg_gen_qemu_ld64(t64, addr, index); + tcg_gen_qemu_ld_i64(t64, addr, index, MO_TEUQ); gen_helper_extf64(cpu_env, fp, t64); break; case OS_EXTENDED: @@ -XXX,XX +XXX,XX @@ static void gen_load_fp(DisasContext *s, int opsize, TCGv addr, TCGv_ptr fp, gen_exception(s, s->base.pc_next, EXCP_FP_UNIMP); break; } - tcg_gen_qemu_ld32u(tmp, addr, index); + tcg_gen_qemu_ld_i32(tmp, addr, index, MO_TEUL); tcg_gen_shri_i32(tmp, tmp, 16); tcg_gen_st16_i32(tmp, fp, offsetof(FPReg, l.upper)); tcg_gen_addi_i32(tmp, addr, 4); - tcg_gen_qemu_ld64(t64, tmp, index); + tcg_gen_qemu_ld_i64(t64, tmp, index, MO_TEUQ); tcg_gen_st_i64(t64, fp, offsetof(FPReg, l.lower)); break; case OS_PACKED: @@ -XXX,XX +XXX,XX @@ static void gen_store_fp(DisasContext *s, int opsize, TCGv addr, TCGv_ptr fp, tmp = tcg_temp_new(); switch (opsize) { case OS_BYTE: - gen_helper_reds32(tmp, cpu_env, fp); - tcg_gen_qemu_st8(tmp, addr, index); - break; case OS_WORD: - gen_helper_reds32(tmp, cpu_env, fp); - tcg_gen_qemu_st16(tmp, addr, index); - break; case OS_LONG: gen_helper_reds32(tmp, cpu_env, fp); - tcg_gen_qemu_st32(tmp, addr, index); + tcg_gen_qemu_st_tl(tmp, addr, index, opsize | MO_TE); break; case OS_SINGLE: gen_helper_redf32(tmp, cpu_env, fp); - tcg_gen_qemu_st32(tmp, addr, index); + tcg_gen_qemu_st_tl(tmp, addr, index, MO_TEUL); break; case OS_DOUBLE: gen_helper_redf64(t64, cpu_env, fp); - tcg_gen_qemu_st64(t64, addr, index); + tcg_gen_qemu_st_i64(t64, addr, index, MO_TEUQ); break; case OS_EXTENDED: if (m68k_feature(s->env, M68K_FEATURE_CF_FPU)) { @@ -XXX,XX +XXX,XX @@ static void gen_store_fp(DisasContext *s, int opsize, TCGv addr, TCGv_ptr fp, } tcg_gen_ld16u_i32(tmp, fp, offsetof(FPReg, l.upper)); tcg_gen_shli_i32(tmp, tmp, 16); - tcg_gen_qemu_st32(tmp, addr, index); + tcg_gen_qemu_st_i32(tmp, addr, index, MO_TEUL); tcg_gen_addi_i32(tmp, addr, 4); tcg_gen_ld_i64(t64, fp, offsetof(FPReg, l.lower)); - tcg_gen_qemu_st64(t64, tmp, index); + tcg_gen_qemu_st_i64(t64, tmp, index, MO_TEUQ); break; case OS_PACKED: /* @@ -XXX,XX +XXX,XX @@ DISAS_INSN(movep) if (insn & 0x80) { for ( ; i > 0 ; i--) { tcg_gen_shri_i32(dbuf, reg, (i - 1) * 8); - tcg_gen_qemu_st8(dbuf, abuf, IS_USER(s)); + tcg_gen_qemu_st_i32(dbuf, abuf, IS_USER(s), MO_UB); if (i > 1) { tcg_gen_addi_i32(abuf, abuf, 2); } } } else { for ( ; i > 0 ; i--) { - tcg_gen_qemu_ld8u(dbuf, abuf, IS_USER(s)); + tcg_gen_qemu_ld_tl(dbuf, abuf, IS_USER(s), MO_UB); tcg_gen_deposit_i32(reg, reg, dbuf, (i - 1) * 8, 8); if (i > 1) { tcg_gen_addi_i32(abuf, abuf, 2); @@ -XXX,XX +XXX,XX @@ static void m68k_copy_line(TCGv dst, TCGv src, int index) t1 = tcg_temp_new_i64(); tcg_gen_andi_i32(addr, src, ~15); - tcg_gen_qemu_ld64(t0, addr, index); + tcg_gen_qemu_ld_i64(t0, addr, index, MO_TEUQ); tcg_gen_addi_i32(addr, addr, 8); - tcg_gen_qemu_ld64(t1, addr, index); + tcg_gen_qemu_ld_i64(t1, addr, index, MO_TEUQ); tcg_gen_andi_i32(addr, dst, ~15); - tcg_gen_qemu_st64(t0, addr, index); + tcg_gen_qemu_st_i64(t0, addr, index, MO_TEUQ); tcg_gen_addi_i32(addr, addr, 8); - tcg_gen_qemu_st64(t1, addr, index); + tcg_gen_qemu_st_i64(t1, addr, index, MO_TEUQ); } DISAS_INSN(move16_reg) @@ -XXX,XX +XXX,XX @@ static void gen_qemu_store_fcr(DisasContext *s, TCGv addr, int reg) tmp = tcg_temp_new(); gen_load_fcr(s, tmp, reg); - tcg_gen_qemu_st32(tmp, addr, index); + tcg_gen_qemu_st_tl(tmp, addr, index, MO_TEUL); } static void gen_qemu_load_fcr(DisasContext *s, TCGv addr, int reg) @@ -XXX,XX +XXX,XX @@ static void gen_qemu_load_fcr(DisasContext *s, TCGv addr, int reg) TCGv tmp; tmp = tcg_temp_new(); - tcg_gen_qemu_ld32u(tmp, addr, index); + tcg_gen_qemu_ld_tl(tmp, addr, index, MO_TEUL); gen_store_fcr(s, tmp, reg); } -- 2.34.1 Convert away from the old interface with the implicit MemOp argument. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Anton Johansson <anjo@rev.ng> Message-Id: <20230502135741.1158035-6-richard.henderson@linaro.org> --- target/mips/tcg/translate.c | 8 ++++---- target/mips/tcg/nanomips_translate.c.inc | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/target/mips/tcg/translate.c b/target/mips/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/mips/tcg/translate.c +++ b/target/mips/tcg/translate.c @@ -XXX,XX +XXX,XX @@ FOP_CONDNS(s, FMT_S, 32, gen_store_fpr32(ctx, fp0, fd)) /* load/store instructions. */ #ifdef CONFIG_USER_ONLY -#define OP_LD_ATOMIC(insn, fname) \ +#define OP_LD_ATOMIC(insn, memop) \ static inline void op_ld_##insn(TCGv ret, TCGv arg1, int mem_idx, \ DisasContext *ctx) \ { \ TCGv t0 = tcg_temp_new(); \ tcg_gen_mov_tl(t0, arg1); \ - tcg_gen_qemu_##fname(ret, arg1, ctx->mem_idx); \ + tcg_gen_qemu_ld_tl(ret, arg1, ctx->mem_idx, memop); \ tcg_gen_st_tl(t0, cpu_env, offsetof(CPUMIPSState, lladdr)); \ tcg_gen_st_tl(ret, cpu_env, offsetof(CPUMIPSState, llval)); \ } @@ -XXX,XX +XXX,XX @@ static inline void op_ld_##insn(TCGv ret, TCGv arg1, int mem_idx, \ gen_helper_##insn(ret, cpu_env, arg1, tcg_constant_i32(mem_idx)); \ } #endif -OP_LD_ATOMIC(ll, ld32s); +OP_LD_ATOMIC(ll, MO_TESL); #if defined(TARGET_MIPS64) -OP_LD_ATOMIC(lld, ld64); +OP_LD_ATOMIC(lld, MO_TEUQ); #endif #undef OP_LD_ATOMIC diff --git a/target/mips/tcg/nanomips_translate.c.inc b/target/mips/tcg/nanomips_translate.c.inc index XXXXXXX..XXXXXXX 100644 --- a/target/mips/tcg/nanomips_translate.c.inc +++ b/target/mips/tcg/nanomips_translate.c.inc @@ -XXX,XX +XXX,XX @@ static void gen_llwp(DisasContext *ctx, uint32_t base, int16_t offset, TCGv tmp2 = tcg_temp_new(); gen_base_offset_addr(ctx, taddr, base, offset); - tcg_gen_qemu_ld64(tval, taddr, ctx->mem_idx); + tcg_gen_qemu_ld_i64(tval, taddr, ctx->mem_idx, MO_TEUQ); if (cpu_is_bigendian(ctx)) { tcg_gen_extr_i64_tl(tmp2, tmp1, tval); } else { -- 2.34.1 Convert away from the old interface with the implicit MemOp argument. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: David Hildenbrand <david@redhat.com> Reviewed-by: Ilya Leoshkevich <iii@linux.ibm.com> Message-Id: <20230502135741.1158035-7-richard.henderson@linaro.org> --- target/s390x/tcg/translate.c | 152 ++++++++++++++++------------------- 1 file changed, 71 insertions(+), 81 deletions(-) diff --git a/target/s390x/tcg/translate.c b/target/s390x/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/tcg/translate.c +++ b/target/s390x/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_clc(DisasContext *s, DisasOps *o) { int l = get_field(s, l1); TCGv_i32 vl; + MemOp mop; switch (l + 1) { case 1: - tcg_gen_qemu_ld8u(cc_src, o->addr1, get_mem_index(s)); - tcg_gen_qemu_ld8u(cc_dst, o->in2, get_mem_index(s)); - break; case 2: - tcg_gen_qemu_ld16u(cc_src, o->addr1, get_mem_index(s)); - tcg_gen_qemu_ld16u(cc_dst, o->in2, get_mem_index(s)); - break; case 4: - tcg_gen_qemu_ld32u(cc_src, o->addr1, get_mem_index(s)); - tcg_gen_qemu_ld32u(cc_dst, o->in2, get_mem_index(s)); - break; case 8: - tcg_gen_qemu_ld64(cc_src, o->addr1, get_mem_index(s)); - tcg_gen_qemu_ld64(cc_dst, o->in2, get_mem_index(s)); - break; + mop = ctz32(l + 1) | MO_TE; + tcg_gen_qemu_ld_tl(cc_src, o->addr1, get_mem_index(s), mop); + tcg_gen_qemu_ld_tl(cc_dst, o->in2, get_mem_index(s), mop); + gen_op_update2_cc_i64(s, CC_OP_LTUGTU_64, cc_src, cc_dst); + return DISAS_NEXT; default: vl = tcg_constant_i32(l); gen_helper_clc(cc_op, cpu_env, vl, o->addr1, o->in2); set_cc_static(s); return DISAS_NEXT; } - gen_op_update2_cc_i64(s, CC_OP_LTUGTU_64, cc_src, cc_dst); - return DISAS_NEXT; } static DisasJumpType op_clcl(DisasContext *s, DisasOps *o) @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_cvd(DisasContext *s, DisasOps *o) TCGv_i32 t2 = tcg_temp_new_i32(); tcg_gen_extrl_i64_i32(t2, o->in1); gen_helper_cvd(t1, t2); - tcg_gen_qemu_st64(t1, o->in2, get_mem_index(s)); + tcg_gen_qemu_st_i64(t1, o->in2, get_mem_index(s), MO_TEUQ); return DISAS_NEXT; } @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_icm(DisasContext *s, DisasOps *o) switch (m3) { case 0xf: /* Effectively a 32-bit load. */ - tcg_gen_qemu_ld32u(tmp, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(tmp, o->in2, get_mem_index(s), MO_TEUL); len = 32; goto one_insert; @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_icm(DisasContext *s, DisasOps *o) case 0x6: case 0x3: /* Effectively a 16-bit load. */ - tcg_gen_qemu_ld16u(tmp, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(tmp, o->in2, get_mem_index(s), MO_TEUW); len = 16; goto one_insert; @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_icm(DisasContext *s, DisasOps *o) case 0x2: case 0x1: /* Effectively an 8-bit load. */ - tcg_gen_qemu_ld8u(tmp, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(tmp, o->in2, get_mem_index(s), MO_UB); len = 8; goto one_insert; @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_icm(DisasContext *s, DisasOps *o) ccm = 0; while (m3) { if (m3 & 0x8) { - tcg_gen_qemu_ld8u(tmp, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(tmp, o->in2, get_mem_index(s), MO_UB); tcg_gen_addi_i64(o->in2, o->in2, 1); tcg_gen_deposit_i64(o->out, o->out, tmp, pos, 8); ccm |= 0xffull << pos; @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_llgt(DisasContext *s, DisasOps *o) static DisasJumpType op_ld8s(DisasContext *s, DisasOps *o) { - tcg_gen_qemu_ld8s(o->out, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->out, o->in2, get_mem_index(s), MO_SB); return DISAS_NEXT; } static DisasJumpType op_ld8u(DisasContext *s, DisasOps *o) { - tcg_gen_qemu_ld8u(o->out, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->out, o->in2, get_mem_index(s), MO_UB); return DISAS_NEXT; } static DisasJumpType op_ld16s(DisasContext *s, DisasOps *o) { - tcg_gen_qemu_ld16s(o->out, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->out, o->in2, get_mem_index(s), MO_TESW); return DISAS_NEXT; } static DisasJumpType op_ld16u(DisasContext *s, DisasOps *o) { - tcg_gen_qemu_ld16u(o->out, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->out, o->in2, get_mem_index(s), MO_TEUW); return DISAS_NEXT; } @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_lat(DisasContext *s, DisasOps *o) static DisasJumpType op_lgat(DisasContext *s, DisasOps *o) { TCGLabel *lab = gen_new_label(); - tcg_gen_qemu_ld64(o->out, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->out, o->in2, get_mem_index(s), MO_TEUQ); /* The value is stored even in case of trap. */ tcg_gen_brcondi_i64(TCG_COND_NE, o->out, 0, lab); gen_trap(s); @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_lfhat(DisasContext *s, DisasOps *o) static DisasJumpType op_llgfat(DisasContext *s, DisasOps *o) { TCGLabel *lab = gen_new_label(); - tcg_gen_qemu_ld32u(o->out, o->in2, get_mem_index(s)); + + tcg_gen_qemu_ld_i64(o->out, o->in2, get_mem_index(s), MO_TEUL); /* The value is stored even in case of trap. */ tcg_gen_brcondi_i64(TCG_COND_NE, o->out, 0, lab); gen_trap(s); @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_lpswe(DisasContext *s, DisasOps *o) tcg_gen_qemu_ld_i64(t1, o->in2, get_mem_index(s), MO_TEUQ | MO_ALIGN_8); tcg_gen_addi_i64(o->in2, o->in2, 8); - tcg_gen_qemu_ld64(t2, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(t2, o->in2, get_mem_index(s), MO_TEUQ); gen_helper_load_psw(cpu_env, t1, t2); return DISAS_NORETURN; } @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_lm32(DisasContext *s, DisasOps *o) /* Only one register to read. */ t1 = tcg_temp_new_i64(); if (unlikely(r1 == r3)) { - tcg_gen_qemu_ld32u(t1, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(t1, o->in2, get_mem_index(s), MO_TEUL); store_reg32_i64(r1, t1); return DISAS_NEXT; } @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_lm32(DisasContext *s, DisasOps *o) /* First load the values of the first and last registers to trigger possible page faults. */ t2 = tcg_temp_new_i64(); - tcg_gen_qemu_ld32u(t1, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(t1, o->in2, get_mem_index(s), MO_TEUL); tcg_gen_addi_i64(t2, o->in2, 4 * ((r3 - r1) & 15)); - tcg_gen_qemu_ld32u(t2, t2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(t2, t2, get_mem_index(s), MO_TEUL); store_reg32_i64(r1, t1); store_reg32_i64(r3, t2); @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_lm32(DisasContext *s, DisasOps *o) while (r1 != r3) { r1 = (r1 + 1) & 15; tcg_gen_add_i64(o->in2, o->in2, t2); - tcg_gen_qemu_ld32u(t1, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(t1, o->in2, get_mem_index(s), MO_TEUL); store_reg32_i64(r1, t1); } return DISAS_NEXT; @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_lmh(DisasContext *s, DisasOps *o) /* Only one register to read. */ t1 = tcg_temp_new_i64(); if (unlikely(r1 == r3)) { - tcg_gen_qemu_ld32u(t1, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(t1, o->in2, get_mem_index(s), MO_TEUL); store_reg32h_i64(r1, t1); return DISAS_NEXT; } @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_lmh(DisasContext *s, DisasOps *o) /* First load the values of the first and last registers to trigger possible page faults. */ t2 = tcg_temp_new_i64(); - tcg_gen_qemu_ld32u(t1, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(t1, o->in2, get_mem_index(s), MO_TEUL); tcg_gen_addi_i64(t2, o->in2, 4 * ((r3 - r1) & 15)); - tcg_gen_qemu_ld32u(t2, t2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(t2, t2, get_mem_index(s), MO_TEUL); store_reg32h_i64(r1, t1); store_reg32h_i64(r3, t2); @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_lmh(DisasContext *s, DisasOps *o) while (r1 != r3) { r1 = (r1 + 1) & 15; tcg_gen_add_i64(o->in2, o->in2, t2); - tcg_gen_qemu_ld32u(t1, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(t1, o->in2, get_mem_index(s), MO_TEUL); store_reg32h_i64(r1, t1); } return DISAS_NEXT; @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_lm64(DisasContext *s, DisasOps *o) /* Only one register to read. */ if (unlikely(r1 == r3)) { - tcg_gen_qemu_ld64(regs[r1], o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(regs[r1], o->in2, get_mem_index(s), MO_TEUQ); return DISAS_NEXT; } @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_lm64(DisasContext *s, DisasOps *o) possible page faults. */ t1 = tcg_temp_new_i64(); t2 = tcg_temp_new_i64(); - tcg_gen_qemu_ld64(t1, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(t1, o->in2, get_mem_index(s), MO_TEUQ); tcg_gen_addi_i64(t2, o->in2, 8 * ((r3 - r1) & 15)); - tcg_gen_qemu_ld64(regs[r3], t2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(regs[r3], t2, get_mem_index(s), MO_TEUQ); tcg_gen_mov_i64(regs[r1], t1); /* Only two registers to read. */ @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_lm64(DisasContext *s, DisasOps *o) while (r1 != r3) { r1 = (r1 + 1) & 15; tcg_gen_add_i64(o->in2, o->in2, t1); - tcg_gen_qemu_ld64(regs[r1], o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(regs[r1], o->in2, get_mem_index(s), MO_TEUQ); } return DISAS_NEXT; } @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_soc(DisasContext *s, DisasOps *o) a = get_address(s, 0, get_field(s, b2), get_field(s, d2)); switch (s->insn->data) { case 1: /* STOCG */ - tcg_gen_qemu_st64(regs[r1], a, get_mem_index(s)); + tcg_gen_qemu_st_i64(regs[r1], a, get_mem_index(s), MO_TEUQ); break; case 0: /* STOC */ - tcg_gen_qemu_st32(regs[r1], a, get_mem_index(s)); + tcg_gen_qemu_st_i64(regs[r1], a, get_mem_index(s), MO_TEUL); break; case 2: /* STOCFH */ h = tcg_temp_new_i64(); tcg_gen_shri_i64(h, regs[r1], 32); - tcg_gen_qemu_st32(h, a, get_mem_index(s)); + tcg_gen_qemu_st_i64(h, a, get_mem_index(s), MO_TEUL); break; default: g_assert_not_reached(); @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_ectg(DisasContext *s, DisasOps *o) gen_addi_and_wrap_i64(s, o->addr1, regs[r3], 0); /* load the third operand into r3 before modifying anything */ - tcg_gen_qemu_ld64(regs[r3], o->addr1, get_mem_index(s)); + tcg_gen_qemu_ld_i64(regs[r3], o->addr1, get_mem_index(s), MO_TEUQ); /* subtract CPU timer from first operand and store in GR0 */ gen_helper_stpt(tmp, cpu_env); @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_stcke(DisasContext *s, DisasOps *o) tcg_gen_shri_i64(c1, c1, 8); tcg_gen_ori_i64(c2, c2, 0x10000); tcg_gen_or_i64(c2, c2, todpr); - tcg_gen_qemu_st64(c1, o->in2, get_mem_index(s)); + tcg_gen_qemu_st_i64(c1, o->in2, get_mem_index(s), MO_TEUQ); tcg_gen_addi_i64(o->in2, o->in2, 8); - tcg_gen_qemu_st64(c2, o->in2, get_mem_index(s)); + tcg_gen_qemu_st_i64(c2, o->in2, get_mem_index(s), MO_TEUQ); /* ??? We don't implement clock states. */ gen_op_movi_cc(s, 0); return DISAS_NEXT; @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_stnosm(DisasContext *s, DisasOps *o) restart, we'll have the wrong SYSTEM MASK in place. */ t = tcg_temp_new_i64(); tcg_gen_shri_i64(t, psw_mask, 56); - tcg_gen_qemu_st8(t, o->addr1, get_mem_index(s)); + tcg_gen_qemu_st_i64(t, o->addr1, get_mem_index(s), MO_UB); if (s->fields.op == 0xac) { tcg_gen_andi_i64(psw_mask, psw_mask, @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_stfle(DisasContext *s, DisasOps *o) static DisasJumpType op_st8(DisasContext *s, DisasOps *o) { - tcg_gen_qemu_st8(o->in1, o->in2, get_mem_index(s)); + tcg_gen_qemu_st_i64(o->in1, o->in2, get_mem_index(s), MO_UB); return DISAS_NEXT; } static DisasJumpType op_st16(DisasContext *s, DisasOps *o) { - tcg_gen_qemu_st16(o->in1, o->in2, get_mem_index(s)); + tcg_gen_qemu_st_i64(o->in1, o->in2, get_mem_index(s), MO_TEUW); return DISAS_NEXT; } @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_stcm(DisasContext *s, DisasOps *o) case 0xf: /* Effectively a 32-bit store. */ tcg_gen_shri_i64(tmp, o->in1, pos); - tcg_gen_qemu_st32(tmp, o->in2, get_mem_index(s)); + tcg_gen_qemu_st_i64(tmp, o->in2, get_mem_index(s), MO_TEUL); break; case 0xc: @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_stcm(DisasContext *s, DisasOps *o) case 0x3: /* Effectively a 16-bit store. */ tcg_gen_shri_i64(tmp, o->in1, pos); - tcg_gen_qemu_st16(tmp, o->in2, get_mem_index(s)); + tcg_gen_qemu_st_i64(tmp, o->in2, get_mem_index(s), MO_TEUW); break; case 0x8: @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_stcm(DisasContext *s, DisasOps *o) case 0x1: /* Effectively an 8-bit store. */ tcg_gen_shri_i64(tmp, o->in1, pos); - tcg_gen_qemu_st8(tmp, o->in2, get_mem_index(s)); + tcg_gen_qemu_st_i64(tmp, o->in2, get_mem_index(s), MO_UB); break; default: @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_stcm(DisasContext *s, DisasOps *o) while (m3) { if (m3 & 0x8) { tcg_gen_shri_i64(tmp, o->in1, pos); - tcg_gen_qemu_st8(tmp, o->in2, get_mem_index(s)); + tcg_gen_qemu_st_i64(tmp, o->in2, get_mem_index(s), MO_UB); tcg_gen_addi_i64(o->in2, o->in2, 1); } m3 = (m3 << 1) & 0xf; @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_stm(DisasContext *s, DisasOps *o) TCGv_i64 tsize = tcg_constant_i64(size); while (1) { - if (size == 8) { - tcg_gen_qemu_st64(regs[r1], o->in2, get_mem_index(s)); - } else { - tcg_gen_qemu_st32(regs[r1], o->in2, get_mem_index(s)); - } + tcg_gen_qemu_st_i64(regs[r1], o->in2, get_mem_index(s), + size == 8 ? MO_TEUQ : MO_TEUL); if (r1 == r3) { break; } @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_stmh(DisasContext *s, DisasOps *o) while (1) { tcg_gen_shl_i64(t, regs[r1], t32); - tcg_gen_qemu_st32(t, o->in2, get_mem_index(s)); + tcg_gen_qemu_st_i64(t, o->in2, get_mem_index(s), MO_TEUL); if (r1 == r3) { break; } @@ -XXX,XX +XXX,XX @@ static DisasJumpType op_xc(DisasContext *s, DisasOps *o) l++; while (l >= 8) { - tcg_gen_qemu_st64(o->in2, o->addr1, get_mem_index(s)); + tcg_gen_qemu_st_i64(o->in2, o->addr1, get_mem_index(s), MO_UQ); l -= 8; if (l > 0) { tcg_gen_addi_i64(o->addr1, o->addr1, 8); } } if (l >= 4) { - tcg_gen_qemu_st32(o->in2, o->addr1, get_mem_index(s)); + tcg_gen_qemu_st_i64(o->in2, o->addr1, get_mem_index(s), MO_UL); l -= 4; if (l > 0) { tcg_gen_addi_i64(o->addr1, o->addr1, 4); } } if (l >= 2) { - tcg_gen_qemu_st16(o->in2, o->addr1, get_mem_index(s)); + tcg_gen_qemu_st_i64(o->in2, o->addr1, get_mem_index(s), MO_UW); l -= 2; if (l > 0) { tcg_gen_addi_i64(o->addr1, o->addr1, 2); } } if (l) { - tcg_gen_qemu_st8(o->in2, o->addr1, get_mem_index(s)); + tcg_gen_qemu_st_i64(o->in2, o->addr1, get_mem_index(s), MO_UB); } gen_op_movi_cc(s, 0); return DISAS_NEXT; @@ -XXX,XX +XXX,XX @@ static void wout_cond_e1e2(DisasContext *s, DisasOps *o) static void wout_m1_8(DisasContext *s, DisasOps *o) { - tcg_gen_qemu_st8(o->out, o->addr1, get_mem_index(s)); + tcg_gen_qemu_st_i64(o->out, o->addr1, get_mem_index(s), MO_UB); } #define SPEC_wout_m1_8 0 static void wout_m1_16(DisasContext *s, DisasOps *o) { - tcg_gen_qemu_st16(o->out, o->addr1, get_mem_index(s)); + tcg_gen_qemu_st_i64(o->out, o->addr1, get_mem_index(s), MO_TEUW); } #define SPEC_wout_m1_16 0 @@ -XXX,XX +XXX,XX @@ static void wout_m1_16a(DisasContext *s, DisasOps *o) static void wout_m1_32(DisasContext *s, DisasOps *o) { - tcg_gen_qemu_st32(o->out, o->addr1, get_mem_index(s)); + tcg_gen_qemu_st_i64(o->out, o->addr1, get_mem_index(s), MO_TEUL); } #define SPEC_wout_m1_32 0 @@ -XXX,XX +XXX,XX @@ static void wout_m1_32a(DisasContext *s, DisasOps *o) static void wout_m1_64(DisasContext *s, DisasOps *o) { - tcg_gen_qemu_st64(o->out, o->addr1, get_mem_index(s)); + tcg_gen_qemu_st_i64(o->out, o->addr1, get_mem_index(s), MO_TEUQ); } #define SPEC_wout_m1_64 0 @@ -XXX,XX +XXX,XX @@ static void wout_m1_64a(DisasContext *s, DisasOps *o) static void wout_m2_32(DisasContext *s, DisasOps *o) { - tcg_gen_qemu_st32(o->out, o->in2, get_mem_index(s)); + tcg_gen_qemu_st_i64(o->out, o->in2, get_mem_index(s), MO_TEUL); } #define SPEC_wout_m2_32 0 @@ -XXX,XX +XXX,XX @@ static void in1_m1_8u(DisasContext *s, DisasOps *o) { in1_la1(s, o); o->in1 = tcg_temp_new_i64(); - tcg_gen_qemu_ld8u(o->in1, o->addr1, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->in1, o->addr1, get_mem_index(s), MO_UB); } #define SPEC_in1_m1_8u 0 @@ -XXX,XX +XXX,XX @@ static void in1_m1_16s(DisasContext *s, DisasOps *o) { in1_la1(s, o); o->in1 = tcg_temp_new_i64(); - tcg_gen_qemu_ld16s(o->in1, o->addr1, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->in1, o->addr1, get_mem_index(s), MO_TESW); } #define SPEC_in1_m1_16s 0 @@ -XXX,XX +XXX,XX @@ static void in1_m1_16u(DisasContext *s, DisasOps *o) { in1_la1(s, o); o->in1 = tcg_temp_new_i64(); - tcg_gen_qemu_ld16u(o->in1, o->addr1, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->in1, o->addr1, get_mem_index(s), MO_TEUW); } #define SPEC_in1_m1_16u 0 @@ -XXX,XX +XXX,XX @@ static void in1_m1_32s(DisasContext *s, DisasOps *o) { in1_la1(s, o); o->in1 = tcg_temp_new_i64(); - tcg_gen_qemu_ld32s(o->in1, o->addr1, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->in1, o->addr1, get_mem_index(s), MO_TESL); } #define SPEC_in1_m1_32s 0 @@ -XXX,XX +XXX,XX @@ static void in1_m1_32u(DisasContext *s, DisasOps *o) { in1_la1(s, o); o->in1 = tcg_temp_new_i64(); - tcg_gen_qemu_ld32u(o->in1, o->addr1, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->in1, o->addr1, get_mem_index(s), MO_TEUL); } #define SPEC_in1_m1_32u 0 @@ -XXX,XX +XXX,XX @@ static void in1_m1_64(DisasContext *s, DisasOps *o) { in1_la1(s, o); o->in1 = tcg_temp_new_i64(); - tcg_gen_qemu_ld64(o->in1, o->addr1, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->in1, o->addr1, get_mem_index(s), MO_TEUQ); } #define SPEC_in1_m1_64 0 @@ -XXX,XX +XXX,XX @@ static void in2_sh(DisasContext *s, DisasOps *o) static void in2_m2_8u(DisasContext *s, DisasOps *o) { in2_a2(s, o); - tcg_gen_qemu_ld8u(o->in2, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->in2, o->in2, get_mem_index(s), MO_UB); } #define SPEC_in2_m2_8u 0 static void in2_m2_16s(DisasContext *s, DisasOps *o) { in2_a2(s, o); - tcg_gen_qemu_ld16s(o->in2, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->in2, o->in2, get_mem_index(s), MO_TESW); } #define SPEC_in2_m2_16s 0 static void in2_m2_16u(DisasContext *s, DisasOps *o) { in2_a2(s, o); - tcg_gen_qemu_ld16u(o->in2, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->in2, o->in2, get_mem_index(s), MO_TEUW); } #define SPEC_in2_m2_16u 0 static void in2_m2_32s(DisasContext *s, DisasOps *o) { in2_a2(s, o); - tcg_gen_qemu_ld32s(o->in2, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->in2, o->in2, get_mem_index(s), MO_TESL); } #define SPEC_in2_m2_32s 0 static void in2_m2_32u(DisasContext *s, DisasOps *o) { in2_a2(s, o); - tcg_gen_qemu_ld32u(o->in2, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->in2, o->in2, get_mem_index(s), MO_TEUL); } #define SPEC_in2_m2_32u 0 @@ -XXX,XX +XXX,XX @@ static void in2_m2_32ua(DisasContext *s, DisasOps *o) static void in2_m2_64(DisasContext *s, DisasOps *o) { in2_a2(s, o); - tcg_gen_qemu_ld64(o->in2, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->in2, o->in2, get_mem_index(s), MO_TEUQ); } #define SPEC_in2_m2_64 0 static void in2_m2_64w(DisasContext *s, DisasOps *o) { in2_a2(s, o); - tcg_gen_qemu_ld64(o->in2, o->in2, get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->in2, o->in2, get_mem_index(s), MO_TEUQ); gen_addi_and_wrap_i64(s, o->in2, o->in2, 0); } #define SPEC_in2_m2_64w 0 @@ -XXX,XX +XXX,XX @@ static void in2_m2_64a(DisasContext *s, DisasOps *o) static void in2_mri2_16s(DisasContext *s, DisasOps *o) { o->in2 = tcg_temp_new_i64(); - tcg_gen_qemu_ld16s(o->in2, gen_ri2(s), get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->in2, gen_ri2(s), get_mem_index(s), MO_TESW); } #define SPEC_in2_mri2_16s 0 static void in2_mri2_16u(DisasContext *s, DisasOps *o) { o->in2 = tcg_temp_new_i64(); - tcg_gen_qemu_ld16u(o->in2, gen_ri2(s), get_mem_index(s)); + tcg_gen_qemu_ld_i64(o->in2, gen_ri2(s), get_mem_index(s), MO_TEUW); } #define SPEC_in2_mri2_16u 0 -- 2.34.1 Convert away from the old interface with the implicit MemOp argument. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Anton Johansson <anjo@rev.ng> Message-Id: <20230502135741.1158035-8-richard.henderson@linaro.org> --- target/sparc/translate.c | 43 ++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/target/sparc/translate.c b/target/sparc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/translate.c +++ b/target/sparc/translate.c @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) switch (xop) { case 0x0: /* ld, V9 lduw, load unsigned word */ gen_address_mask(dc, cpu_addr); - tcg_gen_qemu_ld32u(cpu_val, cpu_addr, dc->mem_idx); + tcg_gen_qemu_ld_tl(cpu_val, cpu_addr, + dc->mem_idx, MO_TEUL); break; case 0x1: /* ldub, load unsigned byte */ gen_address_mask(dc, cpu_addr); - tcg_gen_qemu_ld8u(cpu_val, cpu_addr, dc->mem_idx); + tcg_gen_qemu_ld_tl(cpu_val, cpu_addr, + dc->mem_idx, MO_UB); break; case 0x2: /* lduh, load unsigned halfword */ gen_address_mask(dc, cpu_addr); - tcg_gen_qemu_ld16u(cpu_val, cpu_addr, dc->mem_idx); + tcg_gen_qemu_ld_tl(cpu_val, cpu_addr, + dc->mem_idx, MO_TEUW); break; case 0x3: /* ldd, load double word */ if (rd & 1) @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) gen_address_mask(dc, cpu_addr); t64 = tcg_temp_new_i64(); - tcg_gen_qemu_ld64(t64, cpu_addr, dc->mem_idx); + tcg_gen_qemu_ld_i64(t64, cpu_addr, + dc->mem_idx, MO_TEUQ); tcg_gen_trunc_i64_tl(cpu_val, t64); tcg_gen_ext32u_tl(cpu_val, cpu_val); gen_store_gpr(dc, rd + 1, cpu_val); @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) break; case 0x9: /* ldsb, load signed byte */ gen_address_mask(dc, cpu_addr); - tcg_gen_qemu_ld8s(cpu_val, cpu_addr, dc->mem_idx); + tcg_gen_qemu_ld_tl(cpu_val, cpu_addr, dc->mem_idx, MO_SB); break; case 0xa: /* ldsh, load signed halfword */ gen_address_mask(dc, cpu_addr); - tcg_gen_qemu_ld16s(cpu_val, cpu_addr, dc->mem_idx); + tcg_gen_qemu_ld_tl(cpu_val, cpu_addr, + dc->mem_idx, MO_TESW); break; case 0xd: /* ldstub */ gen_ldstub(dc, cpu_val, cpu_addr, dc->mem_idx); @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) #ifdef TARGET_SPARC64 case 0x08: /* V9 ldsw */ gen_address_mask(dc, cpu_addr); - tcg_gen_qemu_ld32s(cpu_val, cpu_addr, dc->mem_idx); + tcg_gen_qemu_ld_tl(cpu_val, cpu_addr, + dc->mem_idx, MO_TESL); break; case 0x0b: /* V9 ldx */ gen_address_mask(dc, cpu_addr); - tcg_gen_qemu_ld64(cpu_val, cpu_addr, dc->mem_idx); + tcg_gen_qemu_ld_tl(cpu_val, cpu_addr, + dc->mem_idx, MO_TEUQ); break; case 0x18: /* V9 ldswa */ gen_ld_asi(dc, cpu_val, cpu_addr, insn, MO_TESL); @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) switch (xop) { case 0x4: /* st, store word */ gen_address_mask(dc, cpu_addr); - tcg_gen_qemu_st32(cpu_val, cpu_addr, dc->mem_idx); + tcg_gen_qemu_st_tl(cpu_val, cpu_addr, + dc->mem_idx, MO_TEUL); break; case 0x5: /* stb, store byte */ gen_address_mask(dc, cpu_addr); - tcg_gen_qemu_st8(cpu_val, cpu_addr, dc->mem_idx); + tcg_gen_qemu_st_tl(cpu_val, cpu_addr, dc->mem_idx, MO_UB); break; case 0x6: /* sth, store halfword */ gen_address_mask(dc, cpu_addr); - tcg_gen_qemu_st16(cpu_val, cpu_addr, dc->mem_idx); + tcg_gen_qemu_st_tl(cpu_val, cpu_addr, + dc->mem_idx, MO_TEUW); break; case 0x7: /* std, store double word */ if (rd & 1) @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) lo = gen_load_gpr(dc, rd + 1); t64 = tcg_temp_new_i64(); tcg_gen_concat_tl_i64(t64, lo, cpu_val); - tcg_gen_qemu_st64(t64, cpu_addr, dc->mem_idx); + tcg_gen_qemu_st_i64(t64, cpu_addr, + dc->mem_idx, MO_TEUQ); } break; #if !defined(CONFIG_USER_ONLY) || defined(TARGET_SPARC64) @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) #ifdef TARGET_SPARC64 case 0x0e: /* V9 stx */ gen_address_mask(dc, cpu_addr); - tcg_gen_qemu_st64(cpu_val, cpu_addr, dc->mem_idx); + tcg_gen_qemu_st_tl(cpu_val, cpu_addr, + dc->mem_idx, MO_TEUQ); break; case 0x1e: /* V9 stxa */ gen_st_asi(dc, cpu_val, cpu_addr, insn, MO_TEUQ); @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) #ifdef TARGET_SPARC64 gen_address_mask(dc, cpu_addr); if (rd == 1) { - tcg_gen_qemu_st64(cpu_fsr, cpu_addr, dc->mem_idx); + tcg_gen_qemu_st_tl(cpu_fsr, cpu_addr, + dc->mem_idx, MO_TEUQ); break; } #endif - tcg_gen_qemu_st32(cpu_fsr, cpu_addr, dc->mem_idx); + tcg_gen_qemu_st_tl(cpu_fsr, cpu_addr, + dc->mem_idx, MO_TEUL); } break; case 0x26: -- 2.34.1 Convert away from the old interface with the implicit MemOp argument. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Max Filippov <jcmvbkbc@gmail.com> Message-Id: <20230502135741.1158035-9-richard.henderson@linaro.org> --- target/xtensa/translate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/target/xtensa/translate.c b/target/xtensa/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/xtensa/translate.c +++ b/target/xtensa/translate.c @@ -XXX,XX +XXX,XX @@ static void translate_dcache(DisasContext *dc, const OpcodeArg arg[], TCGv_i32 res = tcg_temp_new_i32(); tcg_gen_addi_i32(addr, arg[0].in, arg[1].imm); - tcg_gen_qemu_ld8u(res, addr, dc->cring); + tcg_gen_qemu_ld_i32(res, addr, dc->cring, MO_UB); } static void translate_depbits(DisasContext *dc, const OpcodeArg arg[], @@ -XXX,XX +XXX,XX @@ static void translate_l32r(DisasContext *dc, const OpcodeArg arg[], } else { tmp = tcg_constant_i32(arg[1].imm); } - tcg_gen_qemu_ld32u(arg[0].out, tmp, dc->cring); + tcg_gen_qemu_ld_i32(arg[0].out, tmp, dc->cring, MO_TEUL); } static void translate_loop(DisasContext *dc, const OpcodeArg arg[], -- 2.34.1 Remove the old interfaces with the implicit MemOp argument. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Acked-by: David Hildenbrand <david@redhat.com> Message-Id: <20230502135741.1158035-10-richard.henderson@linaro.org> --- include/tcg/tcg-op.h | 55 -------------------------------------------- 1 file changed, 55 deletions(-) diff --git a/include/tcg/tcg-op.h b/include/tcg/tcg-op.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg-op.h +++ b/include/tcg/tcg-op.h @@ -XXX,XX +XXX,XX @@ void tcg_gen_qemu_st_i64(TCGv_i64, TCGv, TCGArg, MemOp); void tcg_gen_qemu_ld_i128(TCGv_i128, TCGv, TCGArg, MemOp); void tcg_gen_qemu_st_i128(TCGv_i128, TCGv, TCGArg, MemOp); -static inline void tcg_gen_qemu_ld8u(TCGv ret, TCGv addr, int mem_index) -{ - tcg_gen_qemu_ld_tl(ret, addr, mem_index, MO_UB); -} - -static inline void tcg_gen_qemu_ld8s(TCGv ret, TCGv addr, int mem_index) -{ - tcg_gen_qemu_ld_tl(ret, addr, mem_index, MO_SB); -} - -static inline void tcg_gen_qemu_ld16u(TCGv ret, TCGv addr, int mem_index) -{ - tcg_gen_qemu_ld_tl(ret, addr, mem_index, MO_TEUW); -} - -static inline void tcg_gen_qemu_ld16s(TCGv ret, TCGv addr, int mem_index) -{ - tcg_gen_qemu_ld_tl(ret, addr, mem_index, MO_TESW); -} - -static inline void tcg_gen_qemu_ld32u(TCGv ret, TCGv addr, int mem_index) -{ - tcg_gen_qemu_ld_tl(ret, addr, mem_index, MO_TEUL); -} - -static inline void tcg_gen_qemu_ld32s(TCGv ret, TCGv addr, int mem_index) -{ - tcg_gen_qemu_ld_tl(ret, addr, mem_index, MO_TESL); -} - -static inline void tcg_gen_qemu_ld64(TCGv_i64 ret, TCGv addr, int mem_index) -{ - tcg_gen_qemu_ld_i64(ret, addr, mem_index, MO_TEUQ); -} - -static inline void tcg_gen_qemu_st8(TCGv arg, TCGv addr, int mem_index) -{ - tcg_gen_qemu_st_tl(arg, addr, mem_index, MO_UB); -} - -static inline void tcg_gen_qemu_st16(TCGv arg, TCGv addr, int mem_index) -{ - tcg_gen_qemu_st_tl(arg, addr, mem_index, MO_TEUW); -} - -static inline void tcg_gen_qemu_st32(TCGv arg, TCGv addr, int mem_index) -{ - tcg_gen_qemu_st_tl(arg, addr, mem_index, MO_TEUL); -} - -static inline void tcg_gen_qemu_st64(TCGv_i64 arg, TCGv addr, int mem_index) -{ - tcg_gen_qemu_st_i64(arg, addr, mem_index, MO_TEUQ); -} - void tcg_gen_atomic_cmpxchg_i32(TCGv_i32, TCGv, TCGv_i32, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_cmpxchg_i64(TCGv_i64, TCGv, TCGv_i64, TCGv_i64, -- 2.34.1 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/alpha/translate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/alpha/translate.c b/target/alpha/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/alpha/translate.c +++ b/target/alpha/translate.c @@ -XXX,XX +XXX,XX @@ struct DisasContext { #ifdef CONFIG_USER_ONLY #define UNALIGN(C) (C)->unalign #else -#define UNALIGN(C) 0 +#define UNALIGN(C) MO_ALIGN #endif /* Target-specific return values from translate_one, indicating the -- 2.34.1 Mark all memory operations that are not already marked with UNALIGN. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/alpha/translate.c | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/target/alpha/translate.c b/target/alpha/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/alpha/translate.c +++ b/target/alpha/translate.c @@ -XXX,XX +XXX,XX @@ static DisasJumpType translate_one(DisasContext *ctx, uint32_t insn) switch ((insn >> 12) & 0xF) { case 0x0: /* Longword physical access (hw_ldl/p) */ - tcg_gen_qemu_ld_i64(va, addr, MMU_PHYS_IDX, MO_LESL); + tcg_gen_qemu_ld_i64(va, addr, MMU_PHYS_IDX, MO_LESL | MO_ALIGN); break; case 0x1: /* Quadword physical access (hw_ldq/p) */ - tcg_gen_qemu_ld_i64(va, addr, MMU_PHYS_IDX, MO_LEUQ); + tcg_gen_qemu_ld_i64(va, addr, MMU_PHYS_IDX, MO_LEUQ | MO_ALIGN); break; case 0x2: /* Longword physical access with lock (hw_ldl_l/p) */ - tcg_gen_qemu_ld_i64(va, addr, MMU_PHYS_IDX, MO_LESL); + tcg_gen_qemu_ld_i64(va, addr, MMU_PHYS_IDX, MO_LESL | MO_ALIGN); tcg_gen_mov_i64(cpu_lock_addr, addr); tcg_gen_mov_i64(cpu_lock_value, va); break; case 0x3: /* Quadword physical access with lock (hw_ldq_l/p) */ - tcg_gen_qemu_ld_i64(va, addr, MMU_PHYS_IDX, MO_LEUQ); + tcg_gen_qemu_ld_i64(va, addr, MMU_PHYS_IDX, MO_LEUQ | MO_ALIGN); tcg_gen_mov_i64(cpu_lock_addr, addr); tcg_gen_mov_i64(cpu_lock_value, va); break; @@ -XXX,XX +XXX,XX @@ static DisasJumpType translate_one(DisasContext *ctx, uint32_t insn) goto invalid_opc; case 0xA: /* Longword virtual access with protection check (hw_ldl/w) */ - tcg_gen_qemu_ld_i64(va, addr, MMU_KERNEL_IDX, MO_LESL); + tcg_gen_qemu_ld_i64(va, addr, MMU_KERNEL_IDX, + MO_LESL | MO_ALIGN); break; case 0xB: /* Quadword virtual access with protection check (hw_ldq/w) */ - tcg_gen_qemu_ld_i64(va, addr, MMU_KERNEL_IDX, MO_LEUQ); + tcg_gen_qemu_ld_i64(va, addr, MMU_KERNEL_IDX, + MO_LEUQ | MO_ALIGN); break; case 0xC: /* Longword virtual access with alt access mode (hw_ldl/a)*/ @@ -XXX,XX +XXX,XX @@ static DisasJumpType translate_one(DisasContext *ctx, uint32_t insn) case 0xE: /* Longword virtual access with alternate access mode and protection checks (hw_ldl/wa) */ - tcg_gen_qemu_ld_i64(va, addr, MMU_USER_IDX, MO_LESL); + tcg_gen_qemu_ld_i64(va, addr, MMU_USER_IDX, + MO_LESL | MO_ALIGN); break; case 0xF: /* Quadword virtual access with alternate access mode and protection checks (hw_ldq/wa) */ - tcg_gen_qemu_ld_i64(va, addr, MMU_USER_IDX, MO_LEUQ); + tcg_gen_qemu_ld_i64(va, addr, MMU_USER_IDX, + MO_LEUQ | MO_ALIGN); break; } break; @@ -XXX,XX +XXX,XX @@ static DisasJumpType translate_one(DisasContext *ctx, uint32_t insn) vb = load_gpr(ctx, rb); tmp = tcg_temp_new(); tcg_gen_addi_i64(tmp, vb, disp12); - tcg_gen_qemu_st_i64(va, tmp, MMU_PHYS_IDX, MO_LESL); + tcg_gen_qemu_st_i64(va, tmp, MMU_PHYS_IDX, MO_LESL | MO_ALIGN); break; case 0x1: /* Quadword physical access */ @@ -XXX,XX +XXX,XX @@ static DisasJumpType translate_one(DisasContext *ctx, uint32_t insn) vb = load_gpr(ctx, rb); tmp = tcg_temp_new(); tcg_gen_addi_i64(tmp, vb, disp12); - tcg_gen_qemu_st_i64(va, tmp, MMU_PHYS_IDX, MO_LEUQ); + tcg_gen_qemu_st_i64(va, tmp, MMU_PHYS_IDX, MO_LEUQ | MO_ALIGN); break; case 0x2: /* Longword physical access with lock */ ret = gen_store_conditional(ctx, ra, rb, disp12, - MMU_PHYS_IDX, MO_LESL); + MMU_PHYS_IDX, MO_LESL | MO_ALIGN); break; case 0x3: /* Quadword physical access with lock */ ret = gen_store_conditional(ctx, ra, rb, disp12, - MMU_PHYS_IDX, MO_LEUQ); + MMU_PHYS_IDX, MO_LEUQ | MO_ALIGN); break; case 0x4: /* Longword virtual access */ @@ -XXX,XX +XXX,XX @@ static DisasJumpType translate_one(DisasContext *ctx, uint32_t insn) break; case 0x2A: /* LDL_L */ - gen_load_int(ctx, ra, rb, disp16, MO_LESL, 0, 1); + gen_load_int(ctx, ra, rb, disp16, MO_LESL | MO_ALIGN, 0, 1); break; case 0x2B: /* LDQ_L */ - gen_load_int(ctx, ra, rb, disp16, MO_LEUQ, 0, 1); + gen_load_int(ctx, ra, rb, disp16, MO_LEUQ | MO_ALIGN, 0, 1); break; case 0x2C: /* STL */ @@ -XXX,XX +XXX,XX @@ static DisasJumpType translate_one(DisasContext *ctx, uint32_t insn) case 0x2E: /* STL_C */ ret = gen_store_conditional(ctx, ra, rb, disp16, - ctx->mem_idx, MO_LESL); + ctx->mem_idx, MO_LESL | MO_ALIGN); break; case 0x2F: /* STQ_C */ ret = gen_store_conditional(ctx, ra, rb, disp16, - ctx->mem_idx, MO_LEUQ); + ctx->mem_idx, MO_LEUQ | MO_ALIGN); break; case 0x30: /* BR */ -- 2.34.1 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- configs/targets/alpha-linux-user.mak | 1 - configs/targets/alpha-softmmu.mak | 1 - 2 files changed, 2 deletions(-) diff --git a/configs/targets/alpha-linux-user.mak b/configs/targets/alpha-linux-user.mak index XXXXXXX..XXXXXXX 100644 --- a/configs/targets/alpha-linux-user.mak +++ b/configs/targets/alpha-linux-user.mak @@ -XXX,XX +XXX,XX @@ TARGET_ARCH=alpha TARGET_SYSTBL_ABI=common TARGET_SYSTBL=syscall.tbl -TARGET_ALIGNED_ONLY=y diff --git a/configs/targets/alpha-softmmu.mak b/configs/targets/alpha-softmmu.mak index XXXXXXX..XXXXXXX 100644 --- a/configs/targets/alpha-softmmu.mak +++ b/configs/targets/alpha-softmmu.mak @@ -XXX,XX +XXX,XX @@ TARGET_ARCH=alpha -TARGET_ALIGNED_ONLY=y TARGET_SUPPORTS_MTTCG=y -- 2.34.1 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/hppa/translate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/hppa/translate.c b/target/hppa/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/hppa/translate.c +++ b/target/hppa/translate.c @@ -XXX,XX +XXX,XX @@ typedef struct DisasContext { #ifdef CONFIG_USER_ONLY #define UNALIGN(C) (C)->unalign #else -#define UNALIGN(C) 0 +#define UNALIGN(C) MO_ALIGN #endif /* Note that ssm/rsm instructions number PSW_W and PSW_E differently. */ -- 2.34.1 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- configs/targets/hppa-linux-user.mak | 1 - configs/targets/hppa-softmmu.mak | 1 - 2 files changed, 2 deletions(-) diff --git a/configs/targets/hppa-linux-user.mak b/configs/targets/hppa-linux-user.mak index XXXXXXX..XXXXXXX 100644 --- a/configs/targets/hppa-linux-user.mak +++ b/configs/targets/hppa-linux-user.mak @@ -XXX,XX +XXX,XX @@ TARGET_ARCH=hppa TARGET_SYSTBL_ABI=common,32 TARGET_SYSTBL=syscall.tbl -TARGET_ALIGNED_ONLY=y TARGET_BIG_ENDIAN=y diff --git a/configs/targets/hppa-softmmu.mak b/configs/targets/hppa-softmmu.mak index XXXXXXX..XXXXXXX 100644 --- a/configs/targets/hppa-softmmu.mak +++ b/configs/targets/hppa-softmmu.mak @@ -XXX,XX +XXX,XX @@ TARGET_ARCH=hppa -TARGET_ALIGNED_ONLY=y TARGET_BIG_ENDIAN=y TARGET_SUPPORTS_MTTCG=y -- 2.34.1 Acked-by: Mark Cave-Ayland <mark.cave-ayland@ilande.co.uk> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/sparc/translate.c | 66 +++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/target/sparc/translate.c b/target/sparc/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/translate.c +++ b/target/sparc/translate.c @@ -XXX,XX +XXX,XX @@ static void gen_swap(DisasContext *dc, TCGv dst, TCGv src, TCGv addr, int mmu_idx, MemOp memop) { gen_address_mask(dc, addr); - tcg_gen_atomic_xchg_tl(dst, addr, src, mmu_idx, memop); + tcg_gen_atomic_xchg_tl(dst, addr, src, mmu_idx, memop | MO_ALIGN); } static void gen_ldstub(DisasContext *dc, TCGv dst, TCGv addr, int mmu_idx) @@ -XXX,XX +XXX,XX @@ static void gen_ld_asi(DisasContext *dc, TCGv dst, TCGv addr, break; case GET_ASI_DIRECT: gen_address_mask(dc, addr); - tcg_gen_qemu_ld_tl(dst, addr, da.mem_idx, da.memop); + tcg_gen_qemu_ld_tl(dst, addr, da.mem_idx, da.memop | MO_ALIGN); break; default: { TCGv_i32 r_asi = tcg_constant_i32(da.asi); - TCGv_i32 r_mop = tcg_constant_i32(memop); + TCGv_i32 r_mop = tcg_constant_i32(memop | MO_ALIGN); save_state(dc); #ifdef TARGET_SPARC64 @@ -XXX,XX +XXX,XX @@ static void gen_st_asi(DisasContext *dc, TCGv src, TCGv addr, /* fall through */ case GET_ASI_DIRECT: gen_address_mask(dc, addr); - tcg_gen_qemu_st_tl(src, addr, da.mem_idx, da.memop); + tcg_gen_qemu_st_tl(src, addr, da.mem_idx, da.memop | MO_ALIGN); break; #if !defined(TARGET_SPARC64) && !defined(CONFIG_USER_ONLY) case GET_ASI_BCOPY: @@ -XXX,XX +XXX,XX @@ static void gen_st_asi(DisasContext *dc, TCGv src, TCGv addr, default: { TCGv_i32 r_asi = tcg_constant_i32(da.asi); - TCGv_i32 r_mop = tcg_constant_i32(memop & MO_SIZE); + TCGv_i32 r_mop = tcg_constant_i32(memop | MO_ALIGN); save_state(dc); #ifdef TARGET_SPARC64 @@ -XXX,XX +XXX,XX @@ static void gen_cas_asi(DisasContext *dc, TCGv addr, TCGv cmpv, case GET_ASI_DIRECT: oldv = tcg_temp_new(); tcg_gen_atomic_cmpxchg_tl(oldv, addr, cmpv, gen_load_gpr(dc, rd), - da.mem_idx, da.memop); + da.mem_idx, da.memop | MO_ALIGN); gen_store_gpr(dc, rd, oldv); break; default: @@ -XXX,XX +XXX,XX @@ static void gen_ldf_asi(DisasContext *dc, TCGv addr, switch (size) { case 4: d32 = gen_dest_fpr_F(dc); - tcg_gen_qemu_ld_i32(d32, addr, da.mem_idx, da.memop); + tcg_gen_qemu_ld_i32(d32, addr, da.mem_idx, da.memop | MO_ALIGN); gen_store_fpr_F(dc, rd, d32); break; case 8: @@ -XXX,XX +XXX,XX @@ static void gen_ldf_asi(DisasContext *dc, TCGv addr, /* Valid for lddfa only. */ if (size == 8) { gen_address_mask(dc, addr); - tcg_gen_qemu_ld_i64(cpu_fpr[rd / 2], addr, da.mem_idx, da.memop); + tcg_gen_qemu_ld_i64(cpu_fpr[rd / 2], addr, da.mem_idx, + da.memop | MO_ALIGN); } else { gen_exception(dc, TT_ILL_INSN); } @@ -XXX,XX +XXX,XX @@ static void gen_ldf_asi(DisasContext *dc, TCGv addr, default: { TCGv_i32 r_asi = tcg_constant_i32(da.asi); - TCGv_i32 r_mop = tcg_constant_i32(da.memop); + TCGv_i32 r_mop = tcg_constant_i32(da.memop | MO_ALIGN); save_state(dc); /* According to the table in the UA2011 manual, the only @@ -XXX,XX +XXX,XX @@ static void gen_stf_asi(DisasContext *dc, TCGv addr, switch (size) { case 4: d32 = gen_load_fpr_F(dc, rd); - tcg_gen_qemu_st_i32(d32, addr, da.mem_idx, da.memop); + tcg_gen_qemu_st_i32(d32, addr, da.mem_idx, da.memop | MO_ALIGN); break; case 8: tcg_gen_qemu_st_i64(cpu_fpr[rd / 2], addr, da.mem_idx, @@ -XXX,XX +XXX,XX @@ static void gen_stf_asi(DisasContext *dc, TCGv addr, /* Valid for stdfa only. */ if (size == 8) { gen_address_mask(dc, addr); - tcg_gen_qemu_st_i64(cpu_fpr[rd / 2], addr, da.mem_idx, da.memop); + tcg_gen_qemu_st_i64(cpu_fpr[rd / 2], addr, da.mem_idx, + da.memop | MO_ALIGN); } else { gen_exception(dc, TT_ILL_INSN); } @@ -XXX,XX +XXX,XX @@ static void gen_ldda_asi(DisasContext *dc, TCGv addr, int insn, int rd) TCGv_i64 tmp = tcg_temp_new_i64(); gen_address_mask(dc, addr); - tcg_gen_qemu_ld_i64(tmp, addr, da.mem_idx, da.memop); + tcg_gen_qemu_ld_i64(tmp, addr, da.mem_idx, da.memop | MO_ALIGN); /* Note that LE ldda acts as if each 32-bit register result is byte swapped. Having just performed one @@ -XXX,XX +XXX,XX @@ static void gen_stda_asi(DisasContext *dc, TCGv hi, TCGv addr, tcg_gen_concat32_i64(t64, hi, lo); } gen_address_mask(dc, addr); - tcg_gen_qemu_st_i64(t64, addr, da.mem_idx, da.memop); + tcg_gen_qemu_st_i64(t64, addr, da.mem_idx, da.memop | MO_ALIGN); } break; @@ -XXX,XX +XXX,XX @@ static void gen_casx_asi(DisasContext *dc, TCGv addr, TCGv cmpv, case GET_ASI_DIRECT: oldv = tcg_temp_new(); tcg_gen_atomic_cmpxchg_tl(oldv, addr, cmpv, gen_load_gpr(dc, rd), - da.mem_idx, da.memop); + da.mem_idx, da.memop | MO_ALIGN); gen_store_gpr(dc, rd, oldv); break; default: @@ -XXX,XX +XXX,XX @@ static void gen_ldda_asi(DisasContext *dc, TCGv addr, int insn, int rd) return; case GET_ASI_DIRECT: gen_address_mask(dc, addr); - tcg_gen_qemu_ld_i64(t64, addr, da.mem_idx, da.memop); + tcg_gen_qemu_ld_i64(t64, addr, da.mem_idx, da.memop | MO_ALIGN); break; default: { @@ -XXX,XX +XXX,XX @@ static void gen_stda_asi(DisasContext *dc, TCGv hi, TCGv addr, break; case GET_ASI_DIRECT: gen_address_mask(dc, addr); - tcg_gen_qemu_st_i64(t64, addr, da.mem_idx, da.memop); + tcg_gen_qemu_st_i64(t64, addr, da.mem_idx, da.memop | MO_ALIGN); break; case GET_ASI_BFILL: /* Store 32 bytes of T64 to ADDR. */ @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) case 0x0: /* ld, V9 lduw, load unsigned word */ gen_address_mask(dc, cpu_addr); tcg_gen_qemu_ld_tl(cpu_val, cpu_addr, - dc->mem_idx, MO_TEUL); + dc->mem_idx, MO_TEUL | MO_ALIGN); break; case 0x1: /* ldub, load unsigned byte */ gen_address_mask(dc, cpu_addr); @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) case 0x2: /* lduh, load unsigned halfword */ gen_address_mask(dc, cpu_addr); tcg_gen_qemu_ld_tl(cpu_val, cpu_addr, - dc->mem_idx, MO_TEUW); + dc->mem_idx, MO_TEUW | MO_ALIGN); break; case 0x3: /* ldd, load double word */ if (rd & 1) @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) gen_address_mask(dc, cpu_addr); t64 = tcg_temp_new_i64(); tcg_gen_qemu_ld_i64(t64, cpu_addr, - dc->mem_idx, MO_TEUQ); + dc->mem_idx, MO_TEUQ | MO_ALIGN); tcg_gen_trunc_i64_tl(cpu_val, t64); tcg_gen_ext32u_tl(cpu_val, cpu_val); gen_store_gpr(dc, rd + 1, cpu_val); @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) case 0xa: /* ldsh, load signed halfword */ gen_address_mask(dc, cpu_addr); tcg_gen_qemu_ld_tl(cpu_val, cpu_addr, - dc->mem_idx, MO_TESW); + dc->mem_idx, MO_TESW | MO_ALIGN); break; case 0xd: /* ldstub */ gen_ldstub(dc, cpu_val, cpu_addr, dc->mem_idx); @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) case 0x08: /* V9 ldsw */ gen_address_mask(dc, cpu_addr); tcg_gen_qemu_ld_tl(cpu_val, cpu_addr, - dc->mem_idx, MO_TESL); + dc->mem_idx, MO_TESL | MO_ALIGN); break; case 0x0b: /* V9 ldx */ gen_address_mask(dc, cpu_addr); tcg_gen_qemu_ld_tl(cpu_val, cpu_addr, - dc->mem_idx, MO_TEUQ); + dc->mem_idx, MO_TEUQ | MO_ALIGN); break; case 0x18: /* V9 ldswa */ gen_ld_asi(dc, cpu_val, cpu_addr, insn, MO_TESL); @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) gen_address_mask(dc, cpu_addr); cpu_dst_32 = gen_dest_fpr_F(dc); tcg_gen_qemu_ld_i32(cpu_dst_32, cpu_addr, - dc->mem_idx, MO_TEUL); + dc->mem_idx, MO_TEUL | MO_ALIGN); gen_store_fpr_F(dc, rd, cpu_dst_32); break; case 0x21: /* ldfsr, V9 ldxfsr */ @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) if (rd == 1) { TCGv_i64 t64 = tcg_temp_new_i64(); tcg_gen_qemu_ld_i64(t64, cpu_addr, - dc->mem_idx, MO_TEUQ); + dc->mem_idx, MO_TEUQ | MO_ALIGN); gen_helper_ldxfsr(cpu_fsr, cpu_env, cpu_fsr, t64); break; } #endif cpu_dst_32 = tcg_temp_new_i32(); tcg_gen_qemu_ld_i32(cpu_dst_32, cpu_addr, - dc->mem_idx, MO_TEUL); + dc->mem_idx, MO_TEUL | MO_ALIGN); gen_helper_ldfsr(cpu_fsr, cpu_env, cpu_fsr, cpu_dst_32); break; case 0x22: /* ldqf, load quad fpreg */ @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) case 0x4: /* st, store word */ gen_address_mask(dc, cpu_addr); tcg_gen_qemu_st_tl(cpu_val, cpu_addr, - dc->mem_idx, MO_TEUL); + dc->mem_idx, MO_TEUL | MO_ALIGN); break; case 0x5: /* stb, store byte */ gen_address_mask(dc, cpu_addr); @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) case 0x6: /* sth, store halfword */ gen_address_mask(dc, cpu_addr); tcg_gen_qemu_st_tl(cpu_val, cpu_addr, - dc->mem_idx, MO_TEUW); + dc->mem_idx, MO_TEUW | MO_ALIGN); break; case 0x7: /* std, store double word */ if (rd & 1) @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) t64 = tcg_temp_new_i64(); tcg_gen_concat_tl_i64(t64, lo, cpu_val); tcg_gen_qemu_st_i64(t64, cpu_addr, - dc->mem_idx, MO_TEUQ); + dc->mem_idx, MO_TEUQ | MO_ALIGN); } break; #if !defined(CONFIG_USER_ONLY) || defined(TARGET_SPARC64) @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) case 0x0e: /* V9 stx */ gen_address_mask(dc, cpu_addr); tcg_gen_qemu_st_tl(cpu_val, cpu_addr, - dc->mem_idx, MO_TEUQ); + dc->mem_idx, MO_TEUQ | MO_ALIGN); break; case 0x1e: /* V9 stxa */ gen_st_asi(dc, cpu_val, cpu_addr, insn, MO_TEUQ); @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) gen_address_mask(dc, cpu_addr); cpu_src1_32 = gen_load_fpr_F(dc, rd); tcg_gen_qemu_st_i32(cpu_src1_32, cpu_addr, - dc->mem_idx, MO_TEUL); + dc->mem_idx, MO_TEUL | MO_ALIGN); break; case 0x25: /* stfsr, V9 stxfsr */ { @@ -XXX,XX +XXX,XX @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) gen_address_mask(dc, cpu_addr); if (rd == 1) { tcg_gen_qemu_st_tl(cpu_fsr, cpu_addr, - dc->mem_idx, MO_TEUQ); + dc->mem_idx, MO_TEUQ | MO_ALIGN); break; } #endif tcg_gen_qemu_st_tl(cpu_fsr, cpu_addr, - dc->mem_idx, MO_TEUL); + dc->mem_idx, MO_TEUL | MO_ALIGN); } break; case 0x26: -- 2.34.1 This passes on the memop as given as argument to helper_ld_asi to the ultimate load primitive. Reviewed-by: Mark Cave-Ayland <mark.cave-ayland@ilande.co.uk> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/sparc/ldst_helper.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/target/sparc/ldst_helper.c b/target/sparc/ldst_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/sparc/ldst_helper.c +++ b/target/sparc/ldst_helper.c @@ -XXX,XX +XXX,XX @@ uint64_t helper_ld_asi(CPUSPARCState *env, target_ulong addr, #if defined(DEBUG_MXCC) || defined(DEBUG_ASI) uint32_t last_addr = addr; #endif + MemOpIdx oi; do_check_align(env, addr, size - 1, GETPC()); switch (asi) { @@ -XXX,XX +XXX,XX @@ uint64_t helper_ld_asi(CPUSPARCState *env, target_ulong addr, case ASI_M_IODIAG: /* Turbosparc IOTLB Diagnostic */ break; case ASI_KERNELTXT: /* Supervisor code access */ + oi = make_memop_idx(memop, cpu_mmu_index(env, true)); switch (size) { case 1: - ret = cpu_ldub_code(env, addr); + ret = cpu_ldb_code_mmu(env, addr, oi, GETPC()); break; case 2: - ret = cpu_lduw_code(env, addr); + ret = cpu_ldw_code_mmu(env, addr, oi, GETPC()); break; default: case 4: - ret = cpu_ldl_code(env, addr); + ret = cpu_ldl_code_mmu(env, addr, oi, GETPC()); break; case 8: - ret = cpu_ldq_code(env, addr); + ret = cpu_ldq_code_mmu(env, addr, oi, GETPC()); break; } break; -- 2.34.1 Reviewed-by: Mark Cave-Ayland <mark.cave-ayland@ilande.co.uk> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- configs/targets/sparc-linux-user.mak | 1 - configs/targets/sparc-softmmu.mak | 1 - configs/targets/sparc32plus-linux-user.mak | 1 - configs/targets/sparc64-linux-user.mak | 1 - configs/targets/sparc64-softmmu.mak | 1 - 5 files changed, 5 deletions(-) diff --git a/configs/targets/sparc-linux-user.mak b/configs/targets/sparc-linux-user.mak index XXXXXXX..XXXXXXX 100644 --- a/configs/targets/sparc-linux-user.mak +++ b/configs/targets/sparc-linux-user.mak @@ -XXX,XX +XXX,XX @@ TARGET_ARCH=sparc TARGET_SYSTBL_ABI=common,32 TARGET_SYSTBL=syscall.tbl -TARGET_ALIGNED_ONLY=y TARGET_BIG_ENDIAN=y diff --git a/configs/targets/sparc-softmmu.mak b/configs/targets/sparc-softmmu.mak index XXXXXXX..XXXXXXX 100644 --- a/configs/targets/sparc-softmmu.mak +++ b/configs/targets/sparc-softmmu.mak @@ -XXX,XX +XXX,XX @@ TARGET_ARCH=sparc -TARGET_ALIGNED_ONLY=y TARGET_BIG_ENDIAN=y diff --git a/configs/targets/sparc32plus-linux-user.mak b/configs/targets/sparc32plus-linux-user.mak index XXXXXXX..XXXXXXX 100644 --- a/configs/targets/sparc32plus-linux-user.mak +++ b/configs/targets/sparc32plus-linux-user.mak @@ -XXX,XX +XXX,XX @@ TARGET_BASE_ARCH=sparc TARGET_ABI_DIR=sparc TARGET_SYSTBL_ABI=common,32 TARGET_SYSTBL=syscall.tbl -TARGET_ALIGNED_ONLY=y TARGET_BIG_ENDIAN=y diff --git a/configs/targets/sparc64-linux-user.mak b/configs/targets/sparc64-linux-user.mak index XXXXXXX..XXXXXXX 100644 --- a/configs/targets/sparc64-linux-user.mak +++ b/configs/targets/sparc64-linux-user.mak @@ -XXX,XX +XXX,XX @@ TARGET_BASE_ARCH=sparc TARGET_ABI_DIR=sparc TARGET_SYSTBL_ABI=common,64 TARGET_SYSTBL=syscall.tbl -TARGET_ALIGNED_ONLY=y TARGET_BIG_ENDIAN=y diff --git a/configs/targets/sparc64-softmmu.mak b/configs/targets/sparc64-softmmu.mak index XXXXXXX..XXXXXXX 100644 --- a/configs/targets/sparc64-softmmu.mak +++ b/configs/targets/sparc64-softmmu.mak @@ -XXX,XX +XXX,XX @@ TARGET_ARCH=sparc64 TARGET_BASE_ARCH=sparc -TARGET_ALIGNED_ONLY=y TARGET_BIG_ENDIAN=y -- 2.34.1 Interpret the variable argument placement in the caller. Pass data_type instead of is64 -- there are several places where we already convert back from bool to type. Clean things up by using type throughout. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 111 +++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 61 deletions(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, * Record the context of a call to the out of line helper code for the slow path * for a load or store, so that we can later generate the correct helper code */ -static void add_qemu_ldst_label(TCGContext *s, bool is_ld, bool is_64, - MemOpIdx oi, +static void add_qemu_ldst_label(TCGContext *s, bool is_ld, + TCGType type, MemOpIdx oi, TCGReg datalo, TCGReg datahi, TCGReg addrlo, TCGReg addrhi, tcg_insn_unit *raddr, @@ -XXX,XX +XXX,XX @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, bool is_64, label->is_ld = is_ld; label->oi = oi; - label->type = is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32; + label->type = type; label->datalo_reg = datalo; label->datahi_reg = datahi; label->addrlo_reg = addrlo; @@ -XXX,XX +XXX,XX @@ static inline int setup_guest_base_seg(void) static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, TCGReg base, int index, intptr_t ofs, - int seg, bool is64, MemOp memop) + int seg, TCGType type, MemOp memop) { - TCGType type = is64 ? TCG_TYPE_I64 : TCG_TYPE_I32; bool use_movbe = false; - int rexw = is64 * P_REXW; + int rexw = (type == TCG_TYPE_I32 ? 0 : P_REXW); int movop = OPC_MOVL_GvEv; /* Do big-endian loads with movbe. */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, } } -/* XXX: qemu_ld and qemu_st could be modified to clobber only EDX and - EAX. It will be useful once fixed registers globals are less - common. */ -static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64) +static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + MemOpIdx oi, TCGType data_type) { - TCGReg datalo, datahi, addrlo; - TCGReg addrhi __attribute__((unused)); - MemOpIdx oi; - MemOp opc; + MemOp opc = get_memop(oi); + #if defined(CONFIG_SOFTMMU) - int mem_index; tcg_insn_unit *label_ptr[2]; -#else - unsigned a_bits; -#endif - datalo = *args++; - datahi = (TCG_TARGET_REG_BITS == 32 && is64 ? *args++ : 0); - addrlo = *args++; - addrhi = (TARGET_LONG_BITS > TCG_TARGET_REG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); - -#if defined(CONFIG_SOFTMMU) - mem_index = get_mmuidx(oi); - - tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc, + tcg_out_tlb_load(s, addrlo, addrhi, get_mmuidx(oi), opc, label_ptr, offsetof(CPUTLBEntry, addr_read)); /* TLB Hit. */ - tcg_out_qemu_ld_direct(s, datalo, datahi, TCG_REG_L1, -1, 0, 0, is64, opc); + tcg_out_qemu_ld_direct(s, datalo, datahi, TCG_REG_L1, + -1, 0, 0, data_type, opc); /* Record the current context of a load into ldst label */ - add_qemu_ldst_label(s, true, is64, oi, datalo, datahi, addrlo, addrhi, - s->code_ptr, label_ptr); + add_qemu_ldst_label(s, true, data_type, oi, datalo, datahi, + addrlo, addrhi, s->code_ptr, label_ptr); #else - a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); } tcg_out_qemu_ld_direct(s, datalo, datahi, addrlo, x86_guest_base_index, x86_guest_base_offset, x86_guest_base_seg, - is64, opc); + data_type, opc); #endif } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, } } -static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) +static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + MemOpIdx oi, TCGType data_type) { - TCGReg datalo, datahi, addrlo; - TCGReg addrhi __attribute__((unused)); - MemOpIdx oi; - MemOp opc; + MemOp opc = get_memop(oi); + #if defined(CONFIG_SOFTMMU) - int mem_index; tcg_insn_unit *label_ptr[2]; -#else - unsigned a_bits; -#endif - datalo = *args++; - datahi = (TCG_TARGET_REG_BITS == 32 && is64 ? *args++ : 0); - addrlo = *args++; - addrhi = (TARGET_LONG_BITS > TCG_TARGET_REG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); - -#if defined(CONFIG_SOFTMMU) - mem_index = get_mmuidx(oi); - - tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc, + tcg_out_tlb_load(s, addrlo, addrhi, get_mmuidx(oi), opc, label_ptr, offsetof(CPUTLBEntry, addr_write)); /* TLB Hit. */ tcg_out_qemu_st_direct(s, datalo, datahi, TCG_REG_L1, -1, 0, 0, opc); /* Record the current context of a store into ldst label */ - add_qemu_ldst_label(s, false, is64, oi, datalo, datahi, addrlo, addrhi, - s->code_ptr, label_ptr); + add_qemu_ldst_label(s, false, data_type, oi, datalo, datahi, + addrlo, addrhi, s->code_ptr, label_ptr); #else - a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, false, addrlo, addrhi, a_bits); } @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, args, 0); + if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { + tcg_out_qemu_ld(s, a0, -1, a1, -1, a2, TCG_TYPE_I32); + } else { + tcg_out_qemu_ld(s, a0, -1, a1, a2, args[3], TCG_TYPE_I32); + } break; case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args, 1); + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_ld(s, a0, -1, a1, -1, a2, TCG_TYPE_I64); + } else if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_ld(s, a0, a1, a2, -1, args[3], TCG_TYPE_I64); + } else { + tcg_out_qemu_ld(s, a0, a1, a2, args[3], args[4], TCG_TYPE_I64); + } break; case INDEX_op_qemu_st_i32: case INDEX_op_qemu_st8_i32: - tcg_out_qemu_st(s, args, 0); + if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { + tcg_out_qemu_st(s, a0, -1, a1, -1, a2, TCG_TYPE_I32); + } else { + tcg_out_qemu_st(s, a0, -1, a1, a2, args[3], TCG_TYPE_I32); + } break; case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args, 1); + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_st(s, a0, -1, a1, -1, a2, TCG_TYPE_I64); + } else if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_st(s, a0, a1, a2, -1, args[3], TCG_TYPE_I64); + } else { + tcg_out_qemu_st(s, a0, a1, a2, args[3], args[4], TCG_TYPE_I64); + } break; OP_32_64(mulu2): -- 2.34.1 Test for both base and index; use datahi as a temporary, overwritten by the final load. Always perform the loads in ascending order, so that any (user-only) fault sees the correct address. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, if (TCG_TARGET_REG_BITS == 64) { tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo, base, index, 0, ofs); + break; + } + if (use_movbe) { + TCGReg t = datalo; + datalo = datahi; + datahi = t; + } + if (base == datalo || index == datalo) { + tcg_out_modrm_sib_offset(s, OPC_LEA, datahi, base, index, 0, ofs); + tcg_out_modrm_offset(s, movop + seg, datalo, datahi, 0); + tcg_out_modrm_offset(s, movop + seg, datahi, datahi, 4); } else { - if (use_movbe) { - TCGReg t = datalo; - datalo = datahi; - datahi = t; - } - if (base != datalo) { - tcg_out_modrm_sib_offset(s, movop + seg, datalo, - base, index, 0, ofs); - tcg_out_modrm_sib_offset(s, movop + seg, datahi, - base, index, 0, ofs + 4); - } else { - tcg_out_modrm_sib_offset(s, movop + seg, datahi, - base, index, 0, ofs + 4); - tcg_out_modrm_sib_offset(s, movop + seg, datalo, - base, index, 0, ofs); - } + tcg_out_modrm_sib_offset(s, movop + seg, datalo, + base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, movop + seg, datahi, + base, index, 0, ofs + 4); } break; default: -- 2.34.1 Collect the 4 potential parts of the host address into a struct. Reorg tcg_out_qemu_{ld,st}_direct to use it. Reorg guest_base handling to use it. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 165 +++++++++++++++++++++----------------- 1 file changed, 90 insertions(+), 75 deletions(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_nopn(TCGContext *s, int n) tcg_out8(s, 0x90); } +typedef struct { + TCGReg base; + int index; + int ofs; + int seg; +} HostAddress; + #if defined(CONFIG_SOFTMMU) /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, * int mmu_idx, uintptr_t ra) @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) return tcg_out_fail_alignment(s, l); } -#if TCG_TARGET_REG_BITS == 32 -# define x86_guest_base_seg 0 -# define x86_guest_base_index -1 -# define x86_guest_base_offset guest_base -#else -static int x86_guest_base_seg; -static int x86_guest_base_index = -1; -static int32_t x86_guest_base_offset; -# if defined(__x86_64__) && defined(__linux__) -# include <asm/prctl.h> -# include <sys/prctl.h> +static HostAddress x86_guest_base = { + .index = -1 +}; + +#if defined(__x86_64__) && defined(__linux__) +# include <asm/prctl.h> +# include <sys/prctl.h> int arch_prctl(int code, unsigned long addr); static inline int setup_guest_base_seg(void) { @@ -XXX,XX +XXX,XX @@ static inline int setup_guest_base_seg(void) } return 0; } -# elif defined (__FreeBSD__) || defined (__FreeBSD_kernel__) -# include <machine/sysarch.h> +#elif defined(__x86_64__) && \ + (defined (__FreeBSD__) || defined (__FreeBSD_kernel__)) +# include <machine/sysarch.h> static inline int setup_guest_base_seg(void) { if (sysarch(AMD64_SET_GSBASE, &guest_base) == 0) { @@ -XXX,XX +XXX,XX @@ static inline int setup_guest_base_seg(void) } return 0; } -# else +#else static inline int setup_guest_base_seg(void) { return 0; } -# endif -#endif +#endif /* setup_guest_base_seg */ #endif /* SOFTMMU */ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, - TCGReg base, int index, intptr_t ofs, - int seg, TCGType type, MemOp memop) + HostAddress h, TCGType type, MemOp memop) { bool use_movbe = false; int rexw = (type == TCG_TYPE_I32 ? 0 : P_REXW); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, switch (memop & MO_SSIZE) { case MO_UB: - tcg_out_modrm_sib_offset(s, OPC_MOVZBL + seg, datalo, - base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVZBL + h.seg, datalo, + h.base, h.index, 0, h.ofs); break; case MO_SB: - tcg_out_modrm_sib_offset(s, OPC_MOVSBL + rexw + seg, datalo, - base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVSBL + rexw + h.seg, datalo, + h.base, h.index, 0, h.ofs); break; case MO_UW: if (use_movbe) { /* There is no extending movbe; only low 16-bits are modified. */ - if (datalo != base && datalo != index) { + if (datalo != h.base && datalo != h.index) { /* XOR breaks dependency chains. */ tgen_arithr(s, ARITH_XOR, datalo, datalo); - tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg, - datalo, base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + h.seg, + datalo, h.base, h.index, 0, h.ofs); } else { - tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg, - datalo, base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + h.seg, + datalo, h.base, h.index, 0, h.ofs); tcg_out_ext16u(s, datalo, datalo); } } else { - tcg_out_modrm_sib_offset(s, OPC_MOVZWL + seg, datalo, - base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVZWL + h.seg, datalo, + h.base, h.index, 0, h.ofs); } break; case MO_SW: if (use_movbe) { - tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg, - datalo, base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + h.seg, + datalo, h.base, h.index, 0, h.ofs); tcg_out_ext16s(s, type, datalo, datalo); } else { - tcg_out_modrm_sib_offset(s, OPC_MOVSWL + rexw + seg, - datalo, base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVSWL + rexw + h.seg, + datalo, h.base, h.index, 0, h.ofs); } break; case MO_UL: - tcg_out_modrm_sib_offset(s, movop + seg, datalo, base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, movop + h.seg, datalo, + h.base, h.index, 0, h.ofs); break; #if TCG_TARGET_REG_BITS == 64 case MO_SL: if (use_movbe) { - tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + seg, datalo, - base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + h.seg, datalo, + h.base, h.index, 0, h.ofs); tcg_out_ext32s(s, datalo, datalo); } else { - tcg_out_modrm_sib_offset(s, OPC_MOVSLQ + seg, datalo, - base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVSLQ + h.seg, datalo, + h.base, h.index, 0, h.ofs); } break; #endif case MO_UQ: if (TCG_TARGET_REG_BITS == 64) { - tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo, - base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, movop + P_REXW + h.seg, datalo, + h.base, h.index, 0, h.ofs); break; } if (use_movbe) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, datalo = datahi; datahi = t; } - if (base == datalo || index == datalo) { - tcg_out_modrm_sib_offset(s, OPC_LEA, datahi, base, index, 0, ofs); - tcg_out_modrm_offset(s, movop + seg, datalo, datahi, 0); - tcg_out_modrm_offset(s, movop + seg, datahi, datahi, 4); + if (h.base == datalo || h.index == datalo) { + tcg_out_modrm_sib_offset(s, OPC_LEA, datahi, + h.base, h.index, 0, h.ofs); + tcg_out_modrm_offset(s, movop + h.seg, datalo, datahi, 0); + tcg_out_modrm_offset(s, movop + h.seg, datahi, datahi, 4); } else { - tcg_out_modrm_sib_offset(s, movop + seg, datalo, - base, index, 0, ofs); - tcg_out_modrm_sib_offset(s, movop + seg, datahi, - base, index, 0, ofs + 4); + tcg_out_modrm_sib_offset(s, movop + h.seg, datalo, + h.base, h.index, 0, h.ofs); + tcg_out_modrm_sib_offset(s, movop + h.seg, datahi, + h.base, h.index, 0, h.ofs + 4); } break; default: @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, MemOpIdx oi, TCGType data_type) { MemOp opc = get_memop(oi); + HostAddress h; #if defined(CONFIG_SOFTMMU) tcg_insn_unit *label_ptr[2]; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, label_ptr, offsetof(CPUTLBEntry, addr_read)); /* TLB Hit. */ - tcg_out_qemu_ld_direct(s, datalo, datahi, TCG_REG_L1, - -1, 0, 0, data_type, opc); + h.base = TCG_REG_L1; + h.index = -1; + h.ofs = 0; + h.seg = 0; + tcg_out_qemu_ld_direct(s, datalo, datahi, h, data_type, opc); /* Record the current context of a load into ldst label */ add_qemu_ldst_label(s, true, data_type, oi, datalo, datahi, @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); } - tcg_out_qemu_ld_direct(s, datalo, datahi, addrlo, x86_guest_base_index, - x86_guest_base_offset, x86_guest_base_seg, - data_type, opc); + h = x86_guest_base; + h.base = addrlo; + tcg_out_qemu_ld_direct(s, datalo, datahi, h, data_type, opc); #endif } static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, - TCGReg base, int index, intptr_t ofs, - int seg, MemOp memop) + HostAddress h, MemOp memop) { bool use_movbe = false; int movop = OPC_MOVL_EvGv; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, case MO_8: /* This is handled with constraints on INDEX_op_qemu_st8_i32. */ tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || datalo < 4); - tcg_out_modrm_sib_offset(s, OPC_MOVB_EvGv + P_REXB_R + seg, - datalo, base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVB_EvGv + P_REXB_R + h.seg, + datalo, h.base, h.index, 0, h.ofs); break; case MO_16: - tcg_out_modrm_sib_offset(s, movop + P_DATA16 + seg, datalo, - base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, movop + P_DATA16 + h.seg, datalo, + h.base, h.index, 0, h.ofs); break; case MO_32: - tcg_out_modrm_sib_offset(s, movop + seg, datalo, base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, movop + h.seg, datalo, + h.base, h.index, 0, h.ofs); break; case MO_64: if (TCG_TARGET_REG_BITS == 64) { - tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo, - base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, movop + P_REXW + h.seg, datalo, + h.base, h.index, 0, h.ofs); } else { if (use_movbe) { TCGReg t = datalo; datalo = datahi; datahi = t; } - tcg_out_modrm_sib_offset(s, movop + seg, datalo, - base, index, 0, ofs); - tcg_out_modrm_sib_offset(s, movop + seg, datahi, - base, index, 0, ofs + 4); + tcg_out_modrm_sib_offset(s, movop + h.seg, datalo, + h.base, h.index, 0, h.ofs); + tcg_out_modrm_sib_offset(s, movop + h.seg, datahi, + h.base, h.index, 0, h.ofs + 4); } break; default: @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, MemOpIdx oi, TCGType data_type) { MemOp opc = get_memop(oi); + HostAddress h; #if defined(CONFIG_SOFTMMU) tcg_insn_unit *label_ptr[2]; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, label_ptr, offsetof(CPUTLBEntry, addr_write)); /* TLB Hit. */ - tcg_out_qemu_st_direct(s, datalo, datahi, TCG_REG_L1, -1, 0, 0, opc); + h.base = TCG_REG_L1; + h.index = -1; + h.ofs = 0; + h.seg = 0; + tcg_out_qemu_st_direct(s, datalo, datahi, h, opc); /* Record the current context of a store into ldst label */ add_qemu_ldst_label(s, false, data_type, oi, datalo, datahi, @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, tcg_out_test_alignment(s, false, addrlo, addrhi, a_bits); } - tcg_out_qemu_st_direct(s, datalo, datahi, addrlo, x86_guest_base_index, - x86_guest_base_offset, x86_guest_base_seg, opc); + h = x86_guest_base; + h.base = addrlo; + + tcg_out_qemu_st_direct(s, datalo, datahi, h, opc); #endif } @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) (ARRAY_SIZE(tcg_target_callee_save_regs) + 2) * 4 + stack_addend); #else -# if !defined(CONFIG_SOFTMMU) && TCG_TARGET_REG_BITS == 64 +# if !defined(CONFIG_SOFTMMU) if (guest_base) { int seg = setup_guest_base_seg(); if (seg != 0) { - x86_guest_base_seg = seg; + x86_guest_base.seg = seg; } else if (guest_base == (int32_t)guest_base) { - x86_guest_base_offset = guest_base; + x86_guest_base.ofs = guest_base; } else { /* Choose R12 because, as a base, it requires a SIB byte. */ - x86_guest_base_index = TCG_REG_R12; - tcg_out_movi(s, TCG_TYPE_PTR, x86_guest_base_index, guest_base); - tcg_regset_set_reg(s->reserved_regs, x86_guest_base_index); + x86_guest_base.index = TCG_REG_R12; + tcg_out_movi(s, TCG_TYPE_PTR, x86_guest_base.index, guest_base); + tcg_regset_set_reg(s->reserved_regs, x86_guest_base.index); } } # endif -- 2.34.1 Use TCG_REG_L[01] constants directly. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, int mem_index, MemOp opc, tcg_insn_unit **label_ptr, int which) { - const TCGReg r0 = TCG_REG_L0; - const TCGReg r1 = TCG_REG_L1; TCGType ttype = TCG_TYPE_I32; TCGType tlbtype = TCG_TYPE_I32; int trexw = 0, hrexw = 0, tlbrexw = 0; @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, } } - tcg_out_mov(s, tlbtype, r0, addrlo); - tcg_out_shifti(s, SHIFT_SHR + tlbrexw, r0, + tcg_out_mov(s, tlbtype, TCG_REG_L0, addrlo); + tcg_out_shifti(s, SHIFT_SHR + tlbrexw, TCG_REG_L0, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); - tcg_out_modrm_offset(s, OPC_AND_GvEv + trexw, r0, TCG_AREG0, + tcg_out_modrm_offset(s, OPC_AND_GvEv + trexw, TCG_REG_L0, TCG_AREG0, TLB_MASK_TABLE_OFS(mem_index) + offsetof(CPUTLBDescFast, mask)); - tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, r0, TCG_AREG0, + tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, TCG_REG_L0, TCG_AREG0, TLB_MASK_TABLE_OFS(mem_index) + offsetof(CPUTLBDescFast, table)); @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, copy the address and mask. For lesser alignments, check that we don't cross pages for the complete access. */ if (a_bits >= s_bits) { - tcg_out_mov(s, ttype, r1, addrlo); + tcg_out_mov(s, ttype, TCG_REG_L1, addrlo); } else { - tcg_out_modrm_offset(s, OPC_LEA + trexw, r1, addrlo, s_mask - a_mask); + tcg_out_modrm_offset(s, OPC_LEA + trexw, TCG_REG_L1, + addrlo, s_mask - a_mask); } tlb_mask = (target_ulong)TARGET_PAGE_MASK | a_mask; - tgen_arithi(s, ARITH_AND + trexw, r1, tlb_mask, 0); + tgen_arithi(s, ARITH_AND + trexw, TCG_REG_L1, tlb_mask, 0); - /* cmp 0(r0), r1 */ - tcg_out_modrm_offset(s, OPC_CMP_GvEv + trexw, r1, r0, which); + /* cmp 0(TCG_REG_L0), TCG_REG_L1 */ + tcg_out_modrm_offset(s, OPC_CMP_GvEv + trexw, + TCG_REG_L1, TCG_REG_L0, which); /* Prepare for both the fast path add of the tlb addend, and the slow path function argument setup. */ - tcg_out_mov(s, ttype, r1, addrlo); + tcg_out_mov(s, ttype, TCG_REG_L1, addrlo); /* jne slow_path */ tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0); @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, s->code_ptr += 4; if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) { - /* cmp 4(r0), addrhi */ - tcg_out_modrm_offset(s, OPC_CMP_GvEv, addrhi, r0, which + 4); + /* cmp 4(TCG_REG_L0), addrhi */ + tcg_out_modrm_offset(s, OPC_CMP_GvEv, addrhi, TCG_REG_L0, which + 4); /* jne slow_path */ tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0); @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, /* TLB Hit. */ - /* add addend(r0), r1 */ - tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, r1, r0, + /* add addend(TCG_REG_L0), TCG_REG_L1 */ + tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, TCG_REG_L1, TCG_REG_L0, offsetof(CPUTLBEntry, addend)); } -- 2.34.1 Split out a helper for choosing testb vs testl. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.c.inc | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_nopn(TCGContext *s, int n) tcg_out8(s, 0x90); } +/* Test register R vs immediate bits I, setting Z flag for EQ/NE. */ +static void __attribute__((unused)) +tcg_out_testi(TCGContext *s, TCGReg r, uint32_t i) +{ + /* + * This is used for testing alignment, so we can usually use testb. + * For i686, we have to use testl for %esi/%edi. + */ + if (i <= 0xff && (TCG_TARGET_REG_BITS == 64 || r < 4)) { + tcg_out_modrm(s, OPC_GRP3_Eb | P_REXB_RM, EXT3_TESTi, r); + tcg_out8(s, i); + } else { + tcg_out_modrm(s, OPC_GRP3_Ev, EXT3_TESTi, r); + tcg_out32(s, i); + } +} + typedef struct { TCGReg base; int index; @@ -XXX,XX +XXX,XX @@ static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addrlo, unsigned a_mask = (1 << a_bits) - 1; TCGLabelQemuLdst *label; - /* - * We are expecting a_bits to max out at 7, so we can usually use testb. - * For i686, we have to use testl for %esi/%edi. - */ - if (a_mask <= 0xff && (TCG_TARGET_REG_BITS == 64 || addrlo < 4)) { - tcg_out_modrm(s, OPC_GRP3_Eb | P_REXB_RM, EXT3_TESTi, addrlo); - tcg_out8(s, a_mask); - } else { - tcg_out_modrm(s, OPC_GRP3_Ev, EXT3_TESTi, addrlo); - tcg_out32(s, a_mask); - } - + tcg_out_testi(s, addrlo, a_mask); /* jne slow_path */ tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0); -- 2.34.1 Rename the 'ext' parameter 'data_type' to make the use clearer; pass it to tcg_out_qemu_st as well to even out the interfaces. Rename the 'otype' local 'addr_type' to make the use clearer. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target.c.inc | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp memop, } static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, - MemOpIdx oi, TCGType ext) + MemOpIdx oi, TCGType data_type) { MemOp memop = get_memop(oi); - const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32; + TCGType addr_type = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32; /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((memop & MO_BSWAP) == 0); #ifdef CONFIG_SOFTMMU - unsigned mem_index = get_mmuidx(oi); tcg_insn_unit *label_ptr; - tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 1); - tcg_out_qemu_ld_direct(s, memop, ext, data_reg, - TCG_REG_X1, otype, addr_reg); - add_qemu_ldst_label(s, true, oi, ext, data_reg, addr_reg, + tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, get_mmuidx(oi), 1); + tcg_out_qemu_ld_direct(s, memop, data_type, data_reg, + TCG_REG_X1, addr_type, addr_reg); + add_qemu_ldst_label(s, true, oi, data_type, data_reg, addr_reg, s->code_ptr, label_ptr); #else /* !CONFIG_SOFTMMU */ unsigned a_bits = get_alignment_bits(memop); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, tcg_out_test_alignment(s, true, addr_reg, a_bits); } if (USE_GUEST_BASE) { - tcg_out_qemu_ld_direct(s, memop, ext, data_reg, - TCG_REG_GUEST_BASE, otype, addr_reg); + tcg_out_qemu_ld_direct(s, memop, data_type, data_reg, + TCG_REG_GUEST_BASE, addr_type, addr_reg); } else { - tcg_out_qemu_ld_direct(s, memop, ext, data_reg, + tcg_out_qemu_ld_direct(s, memop, data_type, data_reg, addr_reg, TCG_TYPE_I64, TCG_REG_XZR); } #endif /* CONFIG_SOFTMMU */ } static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, - MemOpIdx oi) + MemOpIdx oi, TCGType data_type) { MemOp memop = get_memop(oi); - const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32; + TCGType addr_type = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32; /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((memop & MO_BSWAP) == 0); #ifdef CONFIG_SOFTMMU - unsigned mem_index = get_mmuidx(oi); tcg_insn_unit *label_ptr; - tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 0); + tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, get_mmuidx(oi), 0); tcg_out_qemu_st_direct(s, memop, data_reg, - TCG_REG_X1, otype, addr_reg); - add_qemu_ldst_label(s, false, oi, (memop & MO_SIZE)== MO_64, - data_reg, addr_reg, s->code_ptr, label_ptr); + TCG_REG_X1, addr_type, addr_reg); + add_qemu_ldst_label(s, false, oi, data_type, data_reg, addr_reg, + s->code_ptr, label_ptr); #else /* !CONFIG_SOFTMMU */ unsigned a_bits = get_alignment_bits(memop); if (a_bits) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, } if (USE_GUEST_BASE) { tcg_out_qemu_st_direct(s, memop, data_reg, - TCG_REG_GUEST_BASE, otype, addr_reg); + TCG_REG_GUEST_BASE, addr_type, addr_reg); } else { tcg_out_qemu_st_direct(s, memop, data_reg, addr_reg, TCG_TYPE_I64, TCG_REG_XZR); @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_qemu_st_i32: case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, REG0(0), a1, a2); + tcg_out_qemu_st(s, REG0(0), a1, a2, ext); break; case INDEX_op_bswap64_i64: -- 2.34.1 Collect the 3 potential parts of the host address into a struct. Reorg tcg_out_qemu_{ld,st}_direct to use it. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target.c.inc | 86 +++++++++++++++++++++++++----------- 1 file changed, 59 insertions(+), 27 deletions(-) diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_adr(TCGContext *s, TCGReg rd, const void *target) tcg_out_insn(s, 3406, ADR, rd, offset); } +typedef struct { + TCGReg base; + TCGReg index; + TCGType index_ext; +} HostAddress; + #ifdef CONFIG_SOFTMMU /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, * MemOpIdx oi, uintptr_t ra) @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) #endif /* CONFIG_SOFTMMU */ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp memop, TCGType ext, - TCGReg data_r, TCGReg addr_r, - TCGType otype, TCGReg off_r) + TCGReg data_r, HostAddress h) { switch (memop & MO_SSIZE) { case MO_UB: - tcg_out_ldst_r(s, I3312_LDRB, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_LDRB, data_r, h.base, h.index_ext, h.index); break; case MO_SB: tcg_out_ldst_r(s, ext ? I3312_LDRSBX : I3312_LDRSBW, - data_r, addr_r, otype, off_r); + data_r, h.base, h.index_ext, h.index); break; case MO_UW: - tcg_out_ldst_r(s, I3312_LDRH, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_LDRH, data_r, h.base, h.index_ext, h.index); break; case MO_SW: tcg_out_ldst_r(s, (ext ? I3312_LDRSHX : I3312_LDRSHW), - data_r, addr_r, otype, off_r); + data_r, h.base, h.index_ext, h.index); break; case MO_UL: - tcg_out_ldst_r(s, I3312_LDRW, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_LDRW, data_r, h.base, h.index_ext, h.index); break; case MO_SL: - tcg_out_ldst_r(s, I3312_LDRSWX, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_LDRSWX, data_r, h.base, h.index_ext, h.index); break; case MO_UQ: - tcg_out_ldst_r(s, I3312_LDRX, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_LDRX, data_r, h.base, h.index_ext, h.index); break; default: g_assert_not_reached(); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp memop, TCGType ext, } static void tcg_out_qemu_st_direct(TCGContext *s, MemOp memop, - TCGReg data_r, TCGReg addr_r, - TCGType otype, TCGReg off_r) + TCGReg data_r, HostAddress h) { switch (memop & MO_SIZE) { case MO_8: - tcg_out_ldst_r(s, I3312_STRB, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_STRB, data_r, h.base, h.index_ext, h.index); break; case MO_16: - tcg_out_ldst_r(s, I3312_STRH, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_STRH, data_r, h.base, h.index_ext, h.index); break; case MO_32: - tcg_out_ldst_r(s, I3312_STRW, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_STRW, data_r, h.base, h.index_ext, h.index); break; case MO_64: - tcg_out_ldst_r(s, I3312_STRX, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_STRX, data_r, h.base, h.index_ext, h.index); break; default: g_assert_not_reached(); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, { MemOp memop = get_memop(oi); TCGType addr_type = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32; + HostAddress h; /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((memop & MO_BSWAP) == 0); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, tcg_insn_unit *label_ptr; tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, get_mmuidx(oi), 1); - tcg_out_qemu_ld_direct(s, memop, data_type, data_reg, - TCG_REG_X1, addr_type, addr_reg); + + h = (HostAddress){ + .base = TCG_REG_X1, + .index = addr_reg, + .index_ext = addr_type + }; + tcg_out_qemu_ld_direct(s, memop, data_type, data_reg, h); + add_qemu_ldst_label(s, true, oi, data_type, data_reg, addr_reg, s->code_ptr, label_ptr); #else /* !CONFIG_SOFTMMU */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, tcg_out_test_alignment(s, true, addr_reg, a_bits); } if (USE_GUEST_BASE) { - tcg_out_qemu_ld_direct(s, memop, data_type, data_reg, - TCG_REG_GUEST_BASE, addr_type, addr_reg); + h = (HostAddress){ + .base = TCG_REG_GUEST_BASE, + .index = addr_reg, + .index_ext = addr_type + }; } else { - tcg_out_qemu_ld_direct(s, memop, data_type, data_reg, - addr_reg, TCG_TYPE_I64, TCG_REG_XZR); + h = (HostAddress){ + .base = addr_reg, + .index = TCG_REG_XZR, + .index_ext = TCG_TYPE_I64 + }; } + tcg_out_qemu_ld_direct(s, memop, data_type, data_reg, h); #endif /* CONFIG_SOFTMMU */ } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, { MemOp memop = get_memop(oi); TCGType addr_type = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32; + HostAddress h; /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((memop & MO_BSWAP) == 0); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, tcg_insn_unit *label_ptr; tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, get_mmuidx(oi), 0); - tcg_out_qemu_st_direct(s, memop, data_reg, - TCG_REG_X1, addr_type, addr_reg); + + h = (HostAddress){ + .base = TCG_REG_X1, + .index = addr_reg, + .index_ext = addr_type + }; + tcg_out_qemu_st_direct(s, memop, data_reg, h); + add_qemu_ldst_label(s, false, oi, data_type, data_reg, addr_reg, s->code_ptr, label_ptr); #else /* !CONFIG_SOFTMMU */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, tcg_out_test_alignment(s, false, addr_reg, a_bits); } if (USE_GUEST_BASE) { - tcg_out_qemu_st_direct(s, memop, data_reg, - TCG_REG_GUEST_BASE, addr_type, addr_reg); + h = (HostAddress){ + .base = TCG_REG_GUEST_BASE, + .index = addr_reg, + .index_ext = addr_type + }; } else { - tcg_out_qemu_st_direct(s, memop, data_reg, - addr_reg, TCG_TYPE_I64, TCG_REG_XZR); + h = (HostAddress){ + .base = addr_reg, + .index = TCG_REG_XZR, + .index_ext = TCG_TYPE_I64 + }; } + tcg_out_qemu_st_direct(s, memop, data_reg, h); #endif /* CONFIG_SOFTMMU */ } -- 2.34.1 Interpret the variable argument placement in the caller. Pass data_type instead of is_64. We need to set this in TCGLabelQemuLdst, so plumb this all the way through from tcg_out_op. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/arm/tcg-target.c.inc | 113 +++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 57 deletions(-) diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi, /* Record the context of a call to the out of line helper code for the slow path for a load or store, so that we can later generate the correct helper code. */ -static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi, - TCGReg datalo, TCGReg datahi, TCGReg addrlo, - TCGReg addrhi, tcg_insn_unit *raddr, +static void add_qemu_ldst_label(TCGContext *s, bool is_ld, + MemOpIdx oi, TCGType type, + TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + tcg_insn_unit *raddr, tcg_insn_unit *label_ptr) { TCGLabelQemuLdst *label = new_ldst_label(s); label->is_ld = is_ld; label->oi = oi; + label->type = type; label->datalo_reg = datalo; label->datahi_reg = datahi; label->addrlo_reg = addrlo; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg datalo, } #endif -static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64) +static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + MemOpIdx oi, TCGType data_type) { - TCGReg addrlo, datalo, datahi, addrhi __attribute__((unused)); - MemOpIdx oi; - MemOp opc; -#ifdef CONFIG_SOFTMMU - int mem_index; - TCGReg addend; - tcg_insn_unit *label_ptr; -#else - unsigned a_bits; -#endif - - datalo = *args++; - datahi = (is64 ? *args++ : 0); - addrlo = *args++; - addrhi = (TARGET_LONG_BITS == 64 ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); + MemOp opc = get_memop(oi); #ifdef CONFIG_SOFTMMU - mem_index = get_mmuidx(oi); - addend = tcg_out_tlb_read(s, addrlo, addrhi, opc, mem_index, 1); + TCGReg addend= tcg_out_tlb_read(s, addrlo, addrhi, opc, get_mmuidx(oi), 1); - /* This a conditional BL only to load a pointer within this opcode into LR - for the slow path. We will not be using the value for a tail call. */ - label_ptr = s->code_ptr; + /* + * This a conditional BL only to load a pointer within this opcode into + * LR for the slow path. We will not be using the value for a tail call. + */ + tcg_insn_unit *label_ptr = s->code_ptr; tcg_out_bl_imm(s, COND_NE, 0); tcg_out_qemu_ld_index(s, opc, datalo, datahi, addrlo, addend, true); - add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi, - s->code_ptr, label_ptr); + add_qemu_ldst_label(s, true, oi, data_type, datalo, datahi, + addrlo, addrhi, s->code_ptr, label_ptr); #else /* !CONFIG_SOFTMMU */ - a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg datalo, } #endif -static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) +static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + MemOpIdx oi, TCGType data_type) { - TCGReg addrlo, datalo, datahi, addrhi __attribute__((unused)); - MemOpIdx oi; - MemOp opc; -#ifdef CONFIG_SOFTMMU - int mem_index; - TCGReg addend; - tcg_insn_unit *label_ptr; -#else - unsigned a_bits; -#endif - - datalo = *args++; - datahi = (is64 ? *args++ : 0); - addrlo = *args++; - addrhi = (TARGET_LONG_BITS == 64 ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); + MemOp opc = get_memop(oi); #ifdef CONFIG_SOFTMMU - mem_index = get_mmuidx(oi); - addend = tcg_out_tlb_read(s, addrlo, addrhi, opc, mem_index, 0); + TCGReg addend = tcg_out_tlb_read(s, addrlo, addrhi, opc, get_mmuidx(oi), 0); tcg_out_qemu_st_index(s, COND_EQ, opc, datalo, datahi, addrlo, addend, true); /* The conditional call must come last, as we're going to return here. */ - label_ptr = s->code_ptr; + tcg_insn_unit *label_ptr = s->code_ptr; tcg_out_bl_imm(s, COND_NE, 0); - add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi, - s->code_ptr, label_ptr); + add_qemu_ldst_label(s, false, oi, data_type, datalo, datahi, + addrlo, addrhi, s->code_ptr, label_ptr); #else /* !CONFIG_SOFTMMU */ - a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, false, addrlo, addrhi, a_bits); } @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, args, 0); + if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_ld(s, args[0], -1, args[1], -1, + args[2], TCG_TYPE_I32); + } else { + tcg_out_qemu_ld(s, args[0], -1, args[1], args[2], + args[3], TCG_TYPE_I32); + } break; case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args, 1); + if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_ld(s, args[0], args[1], args[2], -1, + args[3], TCG_TYPE_I64); + } else { + tcg_out_qemu_ld(s, args[0], args[1], args[2], args[3], + args[4], TCG_TYPE_I64); + } break; case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, args, 0); + if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_st(s, args[0], -1, args[1], -1, + args[2], TCG_TYPE_I32); + } else { + tcg_out_qemu_st(s, args[0], -1, args[1], args[2], + args[3], TCG_TYPE_I32); + } break; case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args, 1); + if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_st(s, args[0], args[1], args[2], -1, + args[3], TCG_TYPE_I64); + } else { + tcg_out_qemu_st(s, args[0], args[1], args[2], args[3], + args[4], TCG_TYPE_I64); + } break; case INDEX_op_bswap16_i32: -- 2.34.1 Collect the parts of the host address, and condition, into a struct. Merge tcg_out_qemu_*_{index,direct} and use it. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/arm/tcg-target.c.inc | 248 ++++++++++++++++++--------------------- 1 file changed, 115 insertions(+), 133 deletions(-) diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_vldst(TCGContext *s, ARMInsn insn, tcg_out32(s, insn | (rn << 16) | encode_vd(rd) | 0xf); } +typedef struct { + ARMCond cond; + TCGReg base; + int index; + bool index_scratch; +} HostAddress; + #ifdef CONFIG_SOFTMMU /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, * int mmu_idx, uintptr_t ra) @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) } #endif /* SOFTMMU */ -static void tcg_out_qemu_ld_index(TCGContext *s, MemOp opc, - TCGReg datalo, TCGReg datahi, - TCGReg addrlo, TCGReg addend, - bool scratch_addend) +static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg datalo, + TCGReg datahi, HostAddress h) { + TCGReg base; + /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((opc & MO_BSWAP) == 0); switch (opc & MO_SSIZE) { case MO_UB: - tcg_out_ld8_r(s, COND_AL, datalo, addrlo, addend); + if (h.index < 0) { + tcg_out_ld8_12(s, h.cond, datalo, h.base, 0); + } else { + tcg_out_ld8_r(s, h.cond, datalo, h.base, h.index); + } break; case MO_SB: - tcg_out_ld8s_r(s, COND_AL, datalo, addrlo, addend); + if (h.index < 0) { + tcg_out_ld8s_8(s, h.cond, datalo, h.base, 0); + } else { + tcg_out_ld8s_r(s, h.cond, datalo, h.base, h.index); + } break; case MO_UW: - tcg_out_ld16u_r(s, COND_AL, datalo, addrlo, addend); + if (h.index < 0) { + tcg_out_ld16u_8(s, h.cond, datalo, h.base, 0); + } else { + tcg_out_ld16u_r(s, h.cond, datalo, h.base, h.index); + } break; case MO_SW: - tcg_out_ld16s_r(s, COND_AL, datalo, addrlo, addend); + if (h.index < 0) { + tcg_out_ld16s_8(s, h.cond, datalo, h.base, 0); + } else { + tcg_out_ld16s_r(s, h.cond, datalo, h.base, h.index); + } break; case MO_UL: - tcg_out_ld32_r(s, COND_AL, datalo, addrlo, addend); + if (h.index < 0) { + tcg_out_ld32_12(s, h.cond, datalo, h.base, 0); + } else { + tcg_out_ld32_r(s, h.cond, datalo, h.base, h.index); + } break; case MO_UQ: /* We used pair allocation for datalo, so already should be aligned. */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_index(TCGContext *s, MemOp opc, tcg_debug_assert(datahi == datalo + 1); /* LDRD requires alignment; double-check that. */ if (get_alignment_bits(opc) >= MO_64) { + if (h.index < 0) { + tcg_out_ldrd_8(s, h.cond, datalo, h.base, 0); + break; + } /* * Rm (the second address op) must not overlap Rt or Rt + 1. * Since datalo is aligned, we can simplify the test via alignment. * Flip the two address arguments if that works. */ - if ((addend & ~1) != datalo) { - tcg_out_ldrd_r(s, COND_AL, datalo, addrlo, addend); + if ((h.index & ~1) != datalo) { + tcg_out_ldrd_r(s, h.cond, datalo, h.base, h.index); break; } - if ((addrlo & ~1) != datalo) { - tcg_out_ldrd_r(s, COND_AL, datalo, addend, addrlo); + if ((h.base & ~1) != datalo) { + tcg_out_ldrd_r(s, h.cond, datalo, h.index, h.base); break; } } - if (scratch_addend) { - tcg_out_ld32_rwb(s, COND_AL, datalo, addend, addrlo); - tcg_out_ld32_12(s, COND_AL, datahi, addend, 4); + if (h.index < 0) { + base = h.base; + if (datalo == h.base) { + tcg_out_mov_reg(s, h.cond, TCG_REG_TMP, base); + base = TCG_REG_TMP; + } + } else if (h.index_scratch) { + tcg_out_ld32_rwb(s, h.cond, datalo, h.index, h.base); + tcg_out_ld32_12(s, h.cond, datahi, h.index, 4); + break; } else { - tcg_out_dat_reg(s, COND_AL, ARITH_ADD, TCG_REG_TMP, - addend, addrlo, SHIFT_IMM_LSL(0)); - tcg_out_ld32_12(s, COND_AL, datalo, TCG_REG_TMP, 0); - tcg_out_ld32_12(s, COND_AL, datahi, TCG_REG_TMP, 4); + tcg_out_dat_reg(s, h.cond, ARITH_ADD, TCG_REG_TMP, + h.base, h.index, SHIFT_IMM_LSL(0)); + base = TCG_REG_TMP; } + tcg_out_ld32_12(s, h.cond, datalo, base, 0); + tcg_out_ld32_12(s, h.cond, datahi, base, 4); break; default: g_assert_not_reached(); } } -#ifndef CONFIG_SOFTMMU -static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg datalo, - TCGReg datahi, TCGReg addrlo) -{ - /* Byte swapping is left to middle-end expansion. */ - tcg_debug_assert((opc & MO_BSWAP) == 0); - - switch (opc & MO_SSIZE) { - case MO_UB: - tcg_out_ld8_12(s, COND_AL, datalo, addrlo, 0); - break; - case MO_SB: - tcg_out_ld8s_8(s, COND_AL, datalo, addrlo, 0); - break; - case MO_UW: - tcg_out_ld16u_8(s, COND_AL, datalo, addrlo, 0); - break; - case MO_SW: - tcg_out_ld16s_8(s, COND_AL, datalo, addrlo, 0); - break; - case MO_UL: - tcg_out_ld32_12(s, COND_AL, datalo, addrlo, 0); - break; - case MO_UQ: - /* We used pair allocation for datalo, so already should be aligned. */ - tcg_debug_assert((datalo & 1) == 0); - tcg_debug_assert(datahi == datalo + 1); - /* LDRD requires alignment; double-check that. */ - if (get_alignment_bits(opc) >= MO_64) { - tcg_out_ldrd_8(s, COND_AL, datalo, addrlo, 0); - } else if (datalo == addrlo) { - tcg_out_ld32_12(s, COND_AL, datahi, addrlo, 4); - tcg_out_ld32_12(s, COND_AL, datalo, addrlo, 0); - } else { - tcg_out_ld32_12(s, COND_AL, datalo, addrlo, 0); - tcg_out_ld32_12(s, COND_AL, datahi, addrlo, 4); - } - break; - default: - g_assert_not_reached(); - } -} -#endif - static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, TCGReg addrlo, TCGReg addrhi, MemOpIdx oi, TCGType data_type) { MemOp opc = get_memop(oi); + HostAddress h; #ifdef CONFIG_SOFTMMU - TCGReg addend= tcg_out_tlb_read(s, addrlo, addrhi, opc, get_mmuidx(oi), 1); + h.cond = COND_AL; + h.base = addrlo; + h.index_scratch = true; + h.index = tcg_out_tlb_read(s, addrlo, addrhi, opc, get_mmuidx(oi), 1); /* * This a conditional BL only to load a pointer within this opcode into @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, tcg_insn_unit *label_ptr = s->code_ptr; tcg_out_bl_imm(s, COND_NE, 0); - tcg_out_qemu_ld_index(s, opc, datalo, datahi, addrlo, addend, true); + tcg_out_qemu_ld_direct(s, opc, datalo, datahi, h); add_qemu_ldst_label(s, true, oi, data_type, datalo, datahi, addrlo, addrhi, s->code_ptr, label_ptr); -#else /* !CONFIG_SOFTMMU */ +#else unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); } - if (guest_base) { - tcg_out_qemu_ld_index(s, opc, datalo, datahi, - addrlo, TCG_REG_GUEST_BASE, false); - } else { - tcg_out_qemu_ld_direct(s, opc, datalo, datahi, addrlo); - } + + h.cond = COND_AL; + h.base = addrlo; + h.index = guest_base ? TCG_REG_GUEST_BASE : -1; + h.index_scratch = false; + tcg_out_qemu_ld_direct(s, opc, datalo, datahi, h); #endif } -static void tcg_out_qemu_st_index(TCGContext *s, ARMCond cond, MemOp opc, - TCGReg datalo, TCGReg datahi, - TCGReg addrlo, TCGReg addend, - bool scratch_addend) -{ - /* Byte swapping is left to middle-end expansion. */ - tcg_debug_assert((opc & MO_BSWAP) == 0); - - switch (opc & MO_SIZE) { - case MO_8: - tcg_out_st8_r(s, cond, datalo, addrlo, addend); - break; - case MO_16: - tcg_out_st16_r(s, cond, datalo, addrlo, addend); - break; - case MO_32: - tcg_out_st32_r(s, cond, datalo, addrlo, addend); - break; - case MO_64: - /* We used pair allocation for datalo, so already should be aligned. */ - tcg_debug_assert((datalo & 1) == 0); - tcg_debug_assert(datahi == datalo + 1); - /* STRD requires alignment; double-check that. */ - if (get_alignment_bits(opc) >= MO_64) { - tcg_out_strd_r(s, cond, datalo, addrlo, addend); - } else if (scratch_addend) { - tcg_out_st32_rwb(s, cond, datalo, addend, addrlo); - tcg_out_st32_12(s, cond, datahi, addend, 4); - } else { - tcg_out_dat_reg(s, cond, ARITH_ADD, TCG_REG_TMP, - addend, addrlo, SHIFT_IMM_LSL(0)); - tcg_out_st32_12(s, cond, datalo, TCG_REG_TMP, 0); - tcg_out_st32_12(s, cond, datahi, TCG_REG_TMP, 4); - } - break; - default: - g_assert_not_reached(); - } -} - -#ifndef CONFIG_SOFTMMU static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg datalo, - TCGReg datahi, TCGReg addrlo) + TCGReg datahi, HostAddress h) { /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((opc & MO_BSWAP) == 0); switch (opc & MO_SIZE) { case MO_8: - tcg_out_st8_12(s, COND_AL, datalo, addrlo, 0); + if (h.index < 0) { + tcg_out_st8_12(s, h.cond, datalo, h.base, 0); + } else { + tcg_out_st8_r(s, h.cond, datalo, h.base, h.index); + } break; case MO_16: - tcg_out_st16_8(s, COND_AL, datalo, addrlo, 0); + if (h.index < 0) { + tcg_out_st16_8(s, h.cond, datalo, h.base, 0); + } else { + tcg_out_st16_r(s, h.cond, datalo, h.base, h.index); + } break; case MO_32: - tcg_out_st32_12(s, COND_AL, datalo, addrlo, 0); + if (h.index < 0) { + tcg_out_st32_12(s, h.cond, datalo, h.base, 0); + } else { + tcg_out_st32_r(s, h.cond, datalo, h.base, h.index); + } break; case MO_64: /* We used pair allocation for datalo, so already should be aligned. */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg datalo, tcg_debug_assert(datahi == datalo + 1); /* STRD requires alignment; double-check that. */ if (get_alignment_bits(opc) >= MO_64) { - tcg_out_strd_8(s, COND_AL, datalo, addrlo, 0); + if (h.index < 0) { + tcg_out_strd_8(s, h.cond, datalo, h.base, 0); + } else { + tcg_out_strd_r(s, h.cond, datalo, h.base, h.index); + } + } else if (h.index_scratch) { + tcg_out_st32_rwb(s, h.cond, datalo, h.index, h.base); + tcg_out_st32_12(s, h.cond, datahi, h.index, 4); } else { - tcg_out_st32_12(s, COND_AL, datalo, addrlo, 0); - tcg_out_st32_12(s, COND_AL, datahi, addrlo, 4); + tcg_out_dat_reg(s, h.cond, ARITH_ADD, TCG_REG_TMP, + h.base, h.index, SHIFT_IMM_LSL(0)); + tcg_out_st32_12(s, h.cond, datalo, TCG_REG_TMP, 0); + tcg_out_st32_12(s, h.cond, datahi, TCG_REG_TMP, 4); } break; default: g_assert_not_reached(); } } -#endif static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, TCGReg addrlo, TCGReg addrhi, MemOpIdx oi, TCGType data_type) { MemOp opc = get_memop(oi); + HostAddress h; #ifdef CONFIG_SOFTMMU - TCGReg addend = tcg_out_tlb_read(s, addrlo, addrhi, opc, get_mmuidx(oi), 0); - - tcg_out_qemu_st_index(s, COND_EQ, opc, datalo, datahi, - addrlo, addend, true); + h.cond = COND_EQ; + h.base = addrlo; + h.index_scratch = true; + h.index = tcg_out_tlb_read(s, addrlo, addrhi, opc, get_mmuidx(oi), 0); + tcg_out_qemu_st_direct(s, opc, datalo, datahi, h); /* The conditional call must come last, as we're going to return here. */ tcg_insn_unit *label_ptr = s->code_ptr; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, add_qemu_ldst_label(s, false, oi, data_type, datalo, datahi, addrlo, addrhi, s->code_ptr, label_ptr); -#else /* !CONFIG_SOFTMMU */ +#else unsigned a_bits = get_alignment_bits(opc); + + h.cond = COND_AL; if (a_bits) { tcg_out_test_alignment(s, false, addrlo, addrhi, a_bits); + h.cond = COND_EQ; } - if (guest_base) { - tcg_out_qemu_st_index(s, COND_AL, opc, datalo, datahi, - addrlo, TCG_REG_GUEST_BASE, false); - } else { - tcg_out_qemu_st_direct(s, opc, datalo, datahi, addrlo); - } + + h.base = addrlo; + h.index = guest_base ? TCG_REG_GUEST_BASE : -1; + h.index_scratch = false; + tcg_out_qemu_st_direct(s, opc, datalo, datahi, h); #endif } -- 2.34.1 Interpret the variable argument placement in the caller. Shift some code around slightly to share more between softmmu and user-only. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/loongarch64/tcg-target.c.inc | 100 +++++++++++++------------------ 1 file changed, 42 insertions(+), 58 deletions(-) diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/loongarch64/tcg-target.c.inc +++ b/tcg/loongarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_indexed(TCGContext *s, TCGReg rd, TCGReg rj, } } -static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, TCGType type) +static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, + MemOpIdx oi, TCGType data_type) { - TCGReg addr_regl; - TCGReg data_regl; - MemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) + MemOp opc = get_memop(oi); + TCGReg base, index; + +#ifdef CONFIG_SOFTMMU tcg_insn_unit *label_ptr[1]; -#else - unsigned a_bits; -#endif - TCGReg base; - data_regl = *args++; - addr_regl = *args++; - oi = *args++; - opc = get_memop(oi); - -#if defined(CONFIG_SOFTMMU) - tcg_out_tlb_load(s, addr_regl, oi, label_ptr, 1); - base = tcg_out_zext_addr_if_32_bit(s, addr_regl, TCG_REG_TMP0); - tcg_out_qemu_ld_indexed(s, data_regl, base, TCG_REG_TMP2, opc, type); - add_qemu_ldst_label(s, 1, oi, type, - data_regl, addr_regl, - s->code_ptr, label_ptr); + tcg_out_tlb_load(s, addr_reg, oi, label_ptr, 1); + index = TCG_REG_TMP2; #else - a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); if (a_bits) { - tcg_out_test_alignment(s, true, addr_regl, a_bits); + tcg_out_test_alignment(s, true, addr_reg, a_bits); } - base = tcg_out_zext_addr_if_32_bit(s, addr_regl, TCG_REG_TMP0); - TCGReg guest_base_reg = USE_GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_ZERO; - tcg_out_qemu_ld_indexed(s, data_regl, base, guest_base_reg, opc, type); + index = USE_GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_ZERO; +#endif + + base = tcg_out_zext_addr_if_32_bit(s, addr_reg, TCG_REG_TMP0); + tcg_out_qemu_ld_indexed(s, data_reg, base, index, opc, data_type); + +#ifdef CONFIG_SOFTMMU + add_qemu_ldst_label(s, true, oi, data_type, data_reg, addr_reg, + s->code_ptr, label_ptr); #endif } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_indexed(TCGContext *s, TCGReg data, } } -static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, TCGType type) +static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, + MemOpIdx oi, TCGType data_type) { - TCGReg addr_regl; - TCGReg data_regl; - MemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) + MemOp opc = get_memop(oi); + TCGReg base, index; + +#ifdef CONFIG_SOFTMMU tcg_insn_unit *label_ptr[1]; -#else - unsigned a_bits; -#endif - TCGReg base; - data_regl = *args++; - addr_regl = *args++; - oi = *args++; - opc = get_memop(oi); - -#if defined(CONFIG_SOFTMMU) - tcg_out_tlb_load(s, addr_regl, oi, label_ptr, 0); - base = tcg_out_zext_addr_if_32_bit(s, addr_regl, TCG_REG_TMP0); - tcg_out_qemu_st_indexed(s, data_regl, base, TCG_REG_TMP2, opc); - add_qemu_ldst_label(s, 0, oi, type, - data_regl, addr_regl, - s->code_ptr, label_ptr); + tcg_out_tlb_load(s, addr_reg, oi, label_ptr, 0); + index = TCG_REG_TMP2; #else - a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); if (a_bits) { - tcg_out_test_alignment(s, false, addr_regl, a_bits); + tcg_out_test_alignment(s, false, addr_reg, a_bits); } - base = tcg_out_zext_addr_if_32_bit(s, addr_regl, TCG_REG_TMP0); - TCGReg guest_base_reg = USE_GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_ZERO; - tcg_out_qemu_st_indexed(s, data_regl, base, guest_base_reg, opc); + index = USE_GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_ZERO; +#endif + + base = tcg_out_zext_addr_if_32_bit(s, addr_reg, TCG_REG_TMP0); + tcg_out_qemu_st_indexed(s, data_reg, base, index, opc); + +#ifdef CONFIG_SOFTMMU + add_qemu_ldst_label(s, false, oi, data_type, data_reg, addr_reg, + s->code_ptr, label_ptr); #endif } @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, args, TCG_TYPE_I32); + tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I32); break; case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args, TCG_TYPE_I64); + tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I64); break; case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, args, TCG_TYPE_I32); + tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32); break; case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args, TCG_TYPE_I64); + tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I64); break; case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ -- 2.34.1 Collect the 2 parts of the host address into a struct. Reorg tcg_out_qemu_{ld,st}_direct to use it. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/loongarch64/tcg-target.c.inc | 55 +++++++++++++++++--------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/loongarch64/tcg-target.c.inc +++ b/tcg/loongarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_zext_addr_if_32_bit(TCGContext *s, return addr; } -static void tcg_out_qemu_ld_indexed(TCGContext *s, TCGReg rd, TCGReg rj, - TCGReg rk, MemOp opc, TCGType type) +typedef struct { + TCGReg base; + TCGReg index; +} HostAddress; + +static void tcg_out_qemu_ld_indexed(TCGContext *s, MemOp opc, TCGType type, + TCGReg rd, HostAddress h) { /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((opc & MO_BSWAP) == 0); switch (opc & MO_SSIZE) { case MO_UB: - tcg_out_opc_ldx_bu(s, rd, rj, rk); + tcg_out_opc_ldx_bu(s, rd, h.base, h.index); break; case MO_SB: - tcg_out_opc_ldx_b(s, rd, rj, rk); + tcg_out_opc_ldx_b(s, rd, h.base, h.index); break; case MO_UW: - tcg_out_opc_ldx_hu(s, rd, rj, rk); + tcg_out_opc_ldx_hu(s, rd, h.base, h.index); break; case MO_SW: - tcg_out_opc_ldx_h(s, rd, rj, rk); + tcg_out_opc_ldx_h(s, rd, h.base, h.index); break; case MO_UL: if (type == TCG_TYPE_I64) { - tcg_out_opc_ldx_wu(s, rd, rj, rk); + tcg_out_opc_ldx_wu(s, rd, h.base, h.index); break; } /* fallthrough */ case MO_SL: - tcg_out_opc_ldx_w(s, rd, rj, rk); + tcg_out_opc_ldx_w(s, rd, h.base, h.index); break; case MO_UQ: - tcg_out_opc_ldx_d(s, rd, rj, rk); + tcg_out_opc_ldx_d(s, rd, h.base, h.index); break; default: g_assert_not_reached(); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, MemOpIdx oi, TCGType data_type) { MemOp opc = get_memop(oi); - TCGReg base, index; + HostAddress h; #ifdef CONFIG_SOFTMMU tcg_insn_unit *label_ptr[1]; tcg_out_tlb_load(s, addr_reg, oi, label_ptr, 1); - index = TCG_REG_TMP2; + h.index = TCG_REG_TMP2; #else unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, true, addr_reg, a_bits); } - index = USE_GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_ZERO; + h.index = USE_GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_ZERO; #endif - base = tcg_out_zext_addr_if_32_bit(s, addr_reg, TCG_REG_TMP0); - tcg_out_qemu_ld_indexed(s, data_reg, base, index, opc, data_type); + h.base = tcg_out_zext_addr_if_32_bit(s, addr_reg, TCG_REG_TMP0); + tcg_out_qemu_ld_indexed(s, opc, data_type, data_reg, h); #ifdef CONFIG_SOFTMMU add_qemu_ldst_label(s, true, oi, data_type, data_reg, addr_reg, @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, #endif } -static void tcg_out_qemu_st_indexed(TCGContext *s, TCGReg data, - TCGReg rj, TCGReg rk, MemOp opc) +static void tcg_out_qemu_st_indexed(TCGContext *s, MemOp opc, + TCGReg rd, HostAddress h) { /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((opc & MO_BSWAP) == 0); switch (opc & MO_SIZE) { case MO_8: - tcg_out_opc_stx_b(s, data, rj, rk); + tcg_out_opc_stx_b(s, rd, h.base, h.index); break; case MO_16: - tcg_out_opc_stx_h(s, data, rj, rk); + tcg_out_opc_stx_h(s, rd, h.base, h.index); break; case MO_32: - tcg_out_opc_stx_w(s, data, rj, rk); + tcg_out_opc_stx_w(s, rd, h.base, h.index); break; case MO_64: - tcg_out_opc_stx_d(s, data, rj, rk); + tcg_out_opc_stx_d(s, rd, h.base, h.index); break; default: g_assert_not_reached(); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, MemOpIdx oi, TCGType data_type) { MemOp opc = get_memop(oi); - TCGReg base, index; + HostAddress h; #ifdef CONFIG_SOFTMMU tcg_insn_unit *label_ptr[1]; tcg_out_tlb_load(s, addr_reg, oi, label_ptr, 0); - index = TCG_REG_TMP2; + h.index = TCG_REG_TMP2; #else unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, false, addr_reg, a_bits); } - index = USE_GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_ZERO; + h.index = USE_GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_ZERO; #endif - base = tcg_out_zext_addr_if_32_bit(s, addr_reg, TCG_REG_TMP0); - tcg_out_qemu_st_indexed(s, data_reg, base, index, opc); + h.base = tcg_out_zext_addr_if_32_bit(s, addr_reg, TCG_REG_TMP0); + tcg_out_qemu_st_indexed(s, opc, data_reg, h); #ifdef CONFIG_SOFTMMU add_qemu_ldst_label(s, false, oi, data_type, data_reg, addr_reg, -- 2.34.1 Interpret the variable argument placement in the caller. There are several places where we already convert back from bool to type. Clean things up by using type throughout. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target.c.inc | 186 +++++++++++++++++++------------------- 1 file changed, 95 insertions(+), 91 deletions(-) diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) #endif /* SOFTMMU */ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, - TCGReg base, MemOp opc, bool is_64) + TCGReg base, MemOp opc, TCGType type) { switch (opc & (MO_SSIZE | MO_BSWAP)) { case MO_UB: @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, tcg_out_opc_imm(s, OPC_LH, lo, base, 0); break; case MO_UL | MO_BSWAP: - if (TCG_TARGET_REG_BITS == 64 && is_64) { + if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I64) { if (use_mips32r2_instructions) { tcg_out_opc_imm(s, OPC_LWU, lo, base, 0); tcg_out_bswap32(s, lo, lo, TCG_BSWAP_IZ | TCG_BSWAP_OZ); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, } break; case MO_UL: - if (TCG_TARGET_REG_BITS == 64 && is_64) { + if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I64) { tcg_out_opc_imm(s, OPC_LWU, lo, base, 0); break; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, } static void tcg_out_qemu_ld_unalign(TCGContext *s, TCGReg lo, TCGReg hi, - TCGReg base, MemOp opc, bool is_64) + TCGReg base, MemOp opc, TCGType type) { const MIPSInsn lw1 = MIPS_BE ? OPC_LWL : OPC_LWR; const MIPSInsn lw2 = MIPS_BE ? OPC_LWR : OPC_LWL; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_unalign(TCGContext *s, TCGReg lo, TCGReg hi, case MO_UL: tcg_out_opc_imm(s, lw1, lo, base, 0); tcg_out_opc_imm(s, lw2, lo, base, 3); - if (TCG_TARGET_REG_BITS == 64 && is_64 && !sgn) { + if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I64 && !sgn) { tcg_out_ext32u(s, lo, lo); } break; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_unalign(TCGContext *s, TCGReg lo, TCGReg hi, tcg_out_opc_imm(s, lw1, lo, base, 0); tcg_out_opc_imm(s, lw2, lo, base, 3); tcg_out_bswap32(s, lo, lo, - TCG_TARGET_REG_BITS == 64 && is_64 + TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I64 ? (sgn ? TCG_BSWAP_OS : TCG_BSWAP_OZ) : 0); } else { const tcg_insn_unit *subr = - (TCG_TARGET_REG_BITS == 64 && is_64 && !sgn + (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I64 && !sgn ? bswap32u_addr : bswap32_addr); tcg_out_opc_imm(s, lw1, TCG_TMP0, base, 0); tcg_out_bswap_subr(s, subr); /* delay slot */ tcg_out_opc_imm(s, lw2, TCG_TMP0, base, 3); - tcg_out_mov(s, is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32, lo, TCG_TMP3); + tcg_out_mov(s, type, lo, TCG_TMP3); } break; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_unalign(TCGContext *s, TCGReg lo, TCGReg hi, } } -static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) +static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + MemOpIdx oi, TCGType data_type) { - TCGReg addr_regl, addr_regh __attribute__((unused)); - TCGReg data_regl, data_regh; - MemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) - tcg_insn_unit *label_ptr[2]; -#else -#endif - unsigned a_bits, s_bits; - TCGReg base = TCG_REG_A0; - - data_regl = *args++; - data_regh = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); - addr_regl = *args++; - addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); - a_bits = get_alignment_bits(opc); - s_bits = opc & MO_SIZE; + MemOp opc = get_memop(oi); + unsigned a_bits = get_alignment_bits(opc); + unsigned s_bits = opc & MO_SIZE; + TCGReg base; /* * R6 removes the left/right instructions but requires the * system to support misaligned memory accesses. */ #if defined(CONFIG_SOFTMMU) - tcg_out_tlb_load(s, base, addr_regl, addr_regh, oi, label_ptr, 1); + tcg_insn_unit *label_ptr[2]; + + base = TCG_REG_A0; + tcg_out_tlb_load(s, base, addrlo, addrhi, oi, label_ptr, 1); if (use_mips32r6_instructions || a_bits >= s_bits) { - tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); + tcg_out_qemu_ld_direct(s, datalo, datahi, base, opc, data_type); } else { - tcg_out_qemu_ld_unalign(s, data_regl, data_regh, base, opc, is_64); + tcg_out_qemu_ld_unalign(s, datalo, datahi, base, opc, data_type); } - add_qemu_ldst_label(s, 1, oi, - (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), - data_regl, data_regh, addr_regl, addr_regh, - s->code_ptr, label_ptr); + add_qemu_ldst_label(s, true, oi, data_type, datalo, datahi, + addrlo, addrhi, s->code_ptr, label_ptr); #else + base = addrlo; if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { - tcg_out_ext32u(s, base, addr_regl); - addr_regl = base; + tcg_out_ext32u(s, TCG_REG_A0, base); + base = TCG_REG_A0; } - if (guest_base == 0 && data_regl != addr_regl) { - base = addr_regl; - } else if (guest_base == (int16_t)guest_base) { - tcg_out_opc_imm(s, ALIAS_PADDI, base, addr_regl, guest_base); - } else { - tcg_out_opc_reg(s, ALIAS_PADD, base, TCG_GUEST_BASE_REG, addr_regl); + if (guest_base) { + if (guest_base == (int16_t)guest_base) { + tcg_out_opc_imm(s, ALIAS_PADDI, TCG_REG_A0, base, guest_base); + } else { + tcg_out_opc_reg(s, ALIAS_PADD, TCG_REG_A0, base, + TCG_GUEST_BASE_REG); + } + base = TCG_REG_A0; } if (use_mips32r6_instructions) { if (a_bits) { - tcg_out_test_alignment(s, true, addr_regl, addr_regh, a_bits); + tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); } - tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); + tcg_out_qemu_ld_direct(s, datalo, datahi, base, opc, data_type); } else { if (a_bits && a_bits != s_bits) { - tcg_out_test_alignment(s, true, addr_regl, addr_regh, a_bits); + tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); } if (a_bits >= s_bits) { - tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); + tcg_out_qemu_ld_direct(s, datalo, datahi, base, opc, data_type); } else { - tcg_out_qemu_ld_unalign(s, data_regl, data_regh, base, opc, is_64); + tcg_out_qemu_ld_unalign(s, datalo, datahi, base, opc, data_type); } } #endif @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_unalign(TCGContext *s, TCGReg lo, TCGReg hi, g_assert_not_reached(); } } -static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) -{ - TCGReg addr_regl, addr_regh __attribute__((unused)); - TCGReg data_regl, data_regh; - MemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) - tcg_insn_unit *label_ptr[2]; -#endif - unsigned a_bits, s_bits; - TCGReg base = TCG_REG_A0; - data_regl = *args++; - data_regh = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); - addr_regl = *args++; - addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); - a_bits = get_alignment_bits(opc); - s_bits = opc & MO_SIZE; +static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + MemOpIdx oi, TCGType data_type) +{ + MemOp opc = get_memop(oi); + unsigned a_bits = get_alignment_bits(opc); + unsigned s_bits = opc & MO_SIZE; + TCGReg base; /* * R6 removes the left/right instructions but requires the * system to support misaligned memory accesses. */ #if defined(CONFIG_SOFTMMU) - tcg_out_tlb_load(s, base, addr_regl, addr_regh, oi, label_ptr, 0); + tcg_insn_unit *label_ptr[2]; + + base = TCG_REG_A0; + tcg_out_tlb_load(s, base, addrlo, addrhi, oi, label_ptr, 0); if (use_mips32r6_instructions || a_bits >= s_bits) { - tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); + tcg_out_qemu_st_direct(s, datalo, datahi, base, opc); } else { - tcg_out_qemu_st_unalign(s, data_regl, data_regh, base, opc); + tcg_out_qemu_st_unalign(s, datalo, datahi, base, opc); } - add_qemu_ldst_label(s, 0, oi, - (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), - data_regl, data_regh, addr_regl, addr_regh, - s->code_ptr, label_ptr); + add_qemu_ldst_label(s, false, oi, data_type, datalo, datahi, + addrlo, addrhi, s->code_ptr, label_ptr); #else + base = addrlo; if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { - tcg_out_ext32u(s, base, addr_regl); - addr_regl = base; + tcg_out_ext32u(s, TCG_REG_A0, base); + base = TCG_REG_A0; } - if (guest_base == 0) { - base = addr_regl; - } else if (guest_base == (int16_t)guest_base) { - tcg_out_opc_imm(s, ALIAS_PADDI, base, addr_regl, guest_base); - } else { - tcg_out_opc_reg(s, ALIAS_PADD, base, TCG_GUEST_BASE_REG, addr_regl); + if (guest_base) { + if (guest_base == (int16_t)guest_base) { + tcg_out_opc_imm(s, ALIAS_PADDI, TCG_REG_A0, base, guest_base); + } else { + tcg_out_opc_reg(s, ALIAS_PADD, TCG_REG_A0, base, + TCG_GUEST_BASE_REG); + } + base = TCG_REG_A0; } if (use_mips32r6_instructions) { if (a_bits) { - tcg_out_test_alignment(s, true, addr_regl, addr_regh, a_bits); + tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); } - tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); + tcg_out_qemu_st_direct(s, datalo, datahi, base, opc); } else { if (a_bits && a_bits != s_bits) { - tcg_out_test_alignment(s, true, addr_regl, addr_regh, a_bits); + tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); } if (a_bits >= s_bits) { - tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); + tcg_out_qemu_st_direct(s, datalo, datahi, base, opc); } else { - tcg_out_qemu_st_unalign(s, data_regl, data_regh, base, opc); + tcg_out_qemu_st_unalign(s, datalo, datahi, base, opc); } } #endif @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, args, false); + if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { + tcg_out_qemu_ld(s, a0, 0, a1, 0, a2, TCG_TYPE_I32); + } else { + tcg_out_qemu_ld(s, a0, 0, a1, a2, args[3], TCG_TYPE_I32); + } break; case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args, true); + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_ld(s, a0, 0, a1, 0, a2, TCG_TYPE_I64); + } else if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_ld(s, a0, a1, a2, 0, args[3], TCG_TYPE_I64); + } else { + tcg_out_qemu_ld(s, a0, a1, a2, args[3], args[4], TCG_TYPE_I64); + } break; case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, args, false); + if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { + tcg_out_qemu_st(s, a0, 0, a1, 0, a2, TCG_TYPE_I32); + } else { + tcg_out_qemu_st(s, a0, 0, a1, a2, args[3], TCG_TYPE_I32); + } break; case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args, true); + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_st(s, a0, 0, a1, 0, a2, TCG_TYPE_I64); + } else if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_st(s, a0, a1, a2, 0, args[3], TCG_TYPE_I64); + } else { + tcg_out_qemu_st(s, a0, a1, a2, args[3], args[4], TCG_TYPE_I64); + } break; case INDEX_op_add2_i32: -- 2.34.1 Interpret the variable argument placement in the caller. Pass data_type instead of is64 -- there are several places where we already convert back from bool to type. Clean things up by using type throughout. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/ppc/tcg-target.c.inc | 110 +++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 51 deletions(-) diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_read(TCGContext *s, MemOp opc, /* Record the context of a call to the out of line helper code for the slow path for a load or store, so that we can later generate the correct helper code. */ -static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi, +static void add_qemu_ldst_label(TCGContext *s, bool is_ld, + TCGType type, MemOpIdx oi, TCGReg datalo_reg, TCGReg datahi_reg, TCGReg addrlo_reg, TCGReg addrhi_reg, tcg_insn_unit *raddr, tcg_insn_unit *lptr) @@ -XXX,XX +XXX,XX @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi, TCGLabelQemuLdst *label = new_ldst_label(s); label->is_ld = is_ld; + label->type = type; label->oi = oi; label->datalo_reg = datalo_reg; label->datahi_reg = datahi_reg; @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) #endif /* SOFTMMU */ -static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) +static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + MemOpIdx oi, TCGType data_type) { - TCGReg datalo, datahi, addrlo, rbase; - TCGReg addrhi __attribute__((unused)); - MemOpIdx oi; - MemOp opc, s_bits; + MemOp opc = get_memop(oi); + MemOp s_bits = opc & MO_SIZE; + TCGReg rbase; + #ifdef CONFIG_SOFTMMU - int mem_index; tcg_insn_unit *label_ptr; -#else - unsigned a_bits; -#endif - datalo = *args++; - datahi = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); - addrlo = *args++; - addrhi = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); - s_bits = opc & MO_SIZE; - -#ifdef CONFIG_SOFTMMU - mem_index = get_mmuidx(oi); - addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, true); + addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, get_mmuidx(oi), true); /* Load a pointer into the current opcode w/conditional branch-link. */ label_ptr = s->code_ptr; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) rbase = TCG_REG_R3; #else /* !CONFIG_SOFTMMU */ - a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) } #ifdef CONFIG_SOFTMMU - add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi, - s->code_ptr, label_ptr); + add_qemu_ldst_label(s, true, data_type, oi, datalo, datahi, + addrlo, addrhi, s->code_ptr, label_ptr); #endif } -static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) +static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + MemOpIdx oi, TCGType data_type) { - TCGReg datalo, datahi, addrlo, rbase; - TCGReg addrhi __attribute__((unused)); - MemOpIdx oi; - MemOp opc, s_bits; + MemOp opc = get_memop(oi); + MemOp s_bits = opc & MO_SIZE; + TCGReg rbase; + #ifdef CONFIG_SOFTMMU - int mem_index; tcg_insn_unit *label_ptr; -#else - unsigned a_bits; -#endif - datalo = *args++; - datahi = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); - addrlo = *args++; - addrhi = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); - s_bits = opc & MO_SIZE; - -#ifdef CONFIG_SOFTMMU - mem_index = get_mmuidx(oi); - addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, false); + addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, get_mmuidx(oi), false); /* Load a pointer into the current opcode w/conditional branch-link. */ label_ptr = s->code_ptr; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) rbase = TCG_REG_R3; #else /* !CONFIG_SOFTMMU */ - a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, false, addrlo, addrhi, a_bits); } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) } #ifdef CONFIG_SOFTMMU - add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi, - s->code_ptr, label_ptr); + add_qemu_ldst_label(s, false, data_type, oi, datalo, datahi, + addrlo, addrhi, s->code_ptr, label_ptr); #endif } @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, args, false); + if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { + tcg_out_qemu_ld(s, args[0], -1, args[1], -1, + args[2], TCG_TYPE_I32); + } else { + tcg_out_qemu_ld(s, args[0], -1, args[1], args[2], + args[3], TCG_TYPE_I32); + } break; case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args, true); + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_ld(s, args[0], -1, args[1], -1, + args[2], TCG_TYPE_I64); + } else if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_ld(s, args[0], args[1], args[2], -1, + args[3], TCG_TYPE_I64); + } else { + tcg_out_qemu_ld(s, args[0], args[1], args[2], args[3], + args[4], TCG_TYPE_I64); + } break; case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, args, false); + if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { + tcg_out_qemu_st(s, args[0], -1, args[1], -1, + args[2], TCG_TYPE_I32); + } else { + tcg_out_qemu_st(s, args[0], -1, args[1], args[2], + args[3], TCG_TYPE_I32); + } break; case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args, true); + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_st(s, args[0], -1, args[1], -1, + args[2], TCG_TYPE_I64); + } else if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_st(s, args[0], args[1], args[2], -1, + args[3], TCG_TYPE_I64); + } else { + tcg_out_qemu_st(s, args[0], args[1], args[2], args[3], + args[4], TCG_TYPE_I64); + } break; case INDEX_op_setcond_i32: -- 2.34.1 Collect the parts of the host address into a struct. Reorg tcg_out_qemu_{ld,st} to use it. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/ppc/tcg-target.c.inc | 90 +++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 43 deletions(-) diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) { return tcg_out_fail_alignment(s, l); } - #endif /* SOFTMMU */ +typedef struct { + TCGReg base; + TCGReg index; +} HostAddress; + static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, TCGReg addrlo, TCGReg addrhi, MemOpIdx oi, TCGType data_type) { MemOp opc = get_memop(oi); MemOp s_bits = opc & MO_SIZE; - TCGReg rbase; + HostAddress h; #ifdef CONFIG_SOFTMMU tcg_insn_unit *label_ptr; - addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, get_mmuidx(oi), true); + h.index = tcg_out_tlb_read(s, opc, addrlo, addrhi, get_mmuidx(oi), true); + h.base = TCG_REG_R3; /* Load a pointer into the current opcode w/conditional branch-link. */ label_ptr = s->code_ptr; tcg_out32(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK); - - rbase = TCG_REG_R3; #else /* !CONFIG_SOFTMMU */ unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); } - rbase = guest_base ? TCG_GUEST_BASE_REG : 0; + h.base = guest_base ? TCG_GUEST_BASE_REG : 0; + h.index = addrlo; if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { tcg_out_ext32u(s, TCG_REG_TMP1, addrlo); - addrlo = TCG_REG_TMP1; + h.index = TCG_REG_TMP1; } #endif if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) { if (opc & MO_BSWAP) { - tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4)); - tcg_out32(s, LWBRX | TAB(datalo, rbase, addrlo)); - tcg_out32(s, LWBRX | TAB(datahi, rbase, TCG_REG_R0)); - } else if (rbase != 0) { - tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4)); - tcg_out32(s, LWZX | TAB(datahi, rbase, addrlo)); - tcg_out32(s, LWZX | TAB(datalo, rbase, TCG_REG_R0)); - } else if (addrlo == datahi) { - tcg_out32(s, LWZ | TAI(datalo, addrlo, 4)); - tcg_out32(s, LWZ | TAI(datahi, addrlo, 0)); + tcg_out32(s, ADDI | TAI(TCG_REG_R0, h.index, 4)); + tcg_out32(s, LWBRX | TAB(datalo, h.base, h.index)); + tcg_out32(s, LWBRX | TAB(datahi, h.base, TCG_REG_R0)); + } else if (h.base != 0) { + tcg_out32(s, ADDI | TAI(TCG_REG_R0, h.index, 4)); + tcg_out32(s, LWZX | TAB(datahi, h.base, h.index)); + tcg_out32(s, LWZX | TAB(datalo, h.base, TCG_REG_R0)); + } else if (h.index == datahi) { + tcg_out32(s, LWZ | TAI(datalo, h.index, 4)); + tcg_out32(s, LWZ | TAI(datahi, h.index, 0)); } else { - tcg_out32(s, LWZ | TAI(datahi, addrlo, 0)); - tcg_out32(s, LWZ | TAI(datalo, addrlo, 4)); + tcg_out32(s, LWZ | TAI(datahi, h.index, 0)); + tcg_out32(s, LWZ | TAI(datalo, h.index, 4)); } } else { uint32_t insn = qemu_ldx_opc[opc & (MO_BSWAP | MO_SSIZE)]; if (!have_isa_2_06 && insn == LDBRX) { - tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4)); - tcg_out32(s, LWBRX | TAB(datalo, rbase, addrlo)); - tcg_out32(s, LWBRX | TAB(TCG_REG_R0, rbase, TCG_REG_R0)); + tcg_out32(s, ADDI | TAI(TCG_REG_R0, h.index, 4)); + tcg_out32(s, LWBRX | TAB(datalo, h.base, h.index)); + tcg_out32(s, LWBRX | TAB(TCG_REG_R0, h.base, TCG_REG_R0)); tcg_out_rld(s, RLDIMI, datalo, TCG_REG_R0, 32, 0); } else if (insn) { - tcg_out32(s, insn | TAB(datalo, rbase, addrlo)); + tcg_out32(s, insn | TAB(datalo, h.base, h.index)); } else { insn = qemu_ldx_opc[opc & (MO_SIZE | MO_BSWAP)]; - tcg_out32(s, insn | TAB(datalo, rbase, addrlo)); + tcg_out32(s, insn | TAB(datalo, h.base, h.index)); tcg_out_movext(s, TCG_TYPE_REG, datalo, TCG_TYPE_REG, opc & MO_SSIZE, datalo); } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, { MemOp opc = get_memop(oi); MemOp s_bits = opc & MO_SIZE; - TCGReg rbase; + HostAddress h; #ifdef CONFIG_SOFTMMU tcg_insn_unit *label_ptr; - addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, get_mmuidx(oi), false); + h.index = tcg_out_tlb_read(s, opc, addrlo, addrhi, get_mmuidx(oi), false); + h.base = TCG_REG_R3; /* Load a pointer into the current opcode w/conditional branch-link. */ label_ptr = s->code_ptr; tcg_out32(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK); - - rbase = TCG_REG_R3; #else /* !CONFIG_SOFTMMU */ unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, false, addrlo, addrhi, a_bits); } - rbase = guest_base ? TCG_GUEST_BASE_REG : 0; + h.base = guest_base ? TCG_GUEST_BASE_REG : 0; + h.index = addrlo; if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { tcg_out_ext32u(s, TCG_REG_TMP1, addrlo); - addrlo = TCG_REG_TMP1; + h.index = TCG_REG_TMP1; } #endif if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) { if (opc & MO_BSWAP) { - tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4)); - tcg_out32(s, STWBRX | SAB(datalo, rbase, addrlo)); - tcg_out32(s, STWBRX | SAB(datahi, rbase, TCG_REG_R0)); - } else if (rbase != 0) { - tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4)); - tcg_out32(s, STWX | SAB(datahi, rbase, addrlo)); - tcg_out32(s, STWX | SAB(datalo, rbase, TCG_REG_R0)); + tcg_out32(s, ADDI | TAI(TCG_REG_R0, h.index, 4)); + tcg_out32(s, STWBRX | SAB(datalo, h.base, h.index)); + tcg_out32(s, STWBRX | SAB(datahi, h.base, TCG_REG_R0)); + } else if (h.base != 0) { + tcg_out32(s, ADDI | TAI(TCG_REG_R0, h.index, 4)); + tcg_out32(s, STWX | SAB(datahi, h.base, h.index)); + tcg_out32(s, STWX | SAB(datalo, h.base, TCG_REG_R0)); } else { - tcg_out32(s, STW | TAI(datahi, addrlo, 0)); - tcg_out32(s, STW | TAI(datalo, addrlo, 4)); + tcg_out32(s, STW | TAI(datahi, h.index, 0)); + tcg_out32(s, STW | TAI(datalo, h.index, 4)); } } else { uint32_t insn = qemu_stx_opc[opc & (MO_BSWAP | MO_SIZE)]; if (!have_isa_2_06 && insn == STDBRX) { - tcg_out32(s, STWBRX | SAB(datalo, rbase, addrlo)); - tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, addrlo, 4)); + tcg_out32(s, STWBRX | SAB(datalo, h.base, h.index)); + tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, h.index, 4)); tcg_out_shri64(s, TCG_REG_R0, datalo, 32); - tcg_out32(s, STWBRX | SAB(TCG_REG_R0, rbase, TCG_REG_TMP1)); + tcg_out32(s, STWBRX | SAB(TCG_REG_R0, h.base, TCG_REG_TMP1)); } else { - tcg_out32(s, insn | SAB(datalo, rbase, addrlo)); + tcg_out32(s, insn | SAB(datalo, h.base, h.index)); } } -- 2.34.1 The port currently does not support "oversize" guests, which means riscv32 can only target 32-bit guests. We will soon be building TCG once for all guests. This implies that we can only support riscv64. Since all Linux distributions target riscv64 not riscv32, this is not much of a restriction and simplifies the code. The brcond2 and setcond2 opcodes are exclusive to 32-bit hosts, so we can and should remove the stubs. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target-con-set.h | 8 -- tcg/riscv/tcg-target.h | 22 ++-- tcg/riscv/tcg-target.c.inc | 232 +++++++++------------------------ 3 files changed, 72 insertions(+), 190 deletions(-) diff --git a/tcg/riscv/tcg-target-con-set.h b/tcg/riscv/tcg-target-con-set.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target-con-set.h +++ b/tcg/riscv/tcg-target-con-set.h @@ -XXX,XX +XXX,XX @@ C_O0_I1(r) C_O0_I2(LZ, L) C_O0_I2(rZ, r) C_O0_I2(rZ, rZ) -C_O0_I3(LZ, L, L) -C_O0_I3(LZ, LZ, L) -C_O0_I4(LZ, LZ, L, L) -C_O0_I4(rZ, rZ, rZ, rZ) C_O1_I1(r, L) C_O1_I1(r, r) -C_O1_I2(r, L, L) C_O1_I2(r, r, ri) C_O1_I2(r, r, rI) C_O1_I2(r, rZ, rN) C_O1_I2(r, rZ, rZ) -C_O1_I4(r, rZ, rZ, rZ, rZ) -C_O2_I1(r, r, L) -C_O2_I2(r, r, L, L) C_O2_I4(r, r, rZ, rZ, rM, rM) diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ #ifndef RISCV_TCG_TARGET_H #define RISCV_TCG_TARGET_H -#if __riscv_xlen == 32 -# define TCG_TARGET_REG_BITS 32 -#elif __riscv_xlen == 64 -# define TCG_TARGET_REG_BITS 64 +/* + * We don't support oversize guests. + * Since we will only build tcg once, this in turn requires a 64-bit host. + */ +#if __riscv_xlen != 64 +#error "unsupported code generation mode" #endif +#define TCG_TARGET_REG_BITS 64 #define TCG_TARGET_INSN_UNIT_SIZE 4 #define TCG_TARGET_TLB_DISPLACEMENT_BITS 20 @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_STACK_ALIGN 16 #define TCG_TARGET_CALL_STACK_OFFSET 0 #define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_NORMAL -#if TCG_TARGET_REG_BITS == 32 -#define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_EVEN -#define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_EVEN -#else #define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL #define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_NORMAL -#endif #define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_NORMAL /* optional instructions */ @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_HAS_sub2_i32 1 #define TCG_TARGET_HAS_mulu2_i32 0 #define TCG_TARGET_HAS_muls2_i32 0 -#define TCG_TARGET_HAS_muluh_i32 (TCG_TARGET_REG_BITS == 32) -#define TCG_TARGET_HAS_mulsh_i32 (TCG_TARGET_REG_BITS == 32) +#define TCG_TARGET_HAS_muluh_i32 0 +#define TCG_TARGET_HAS_mulsh_i32 0 #define TCG_TARGET_HAS_ext8s_i32 1 #define TCG_TARGET_HAS_ext16s_i32 1 #define TCG_TARGET_HAS_ext8u_i32 1 @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_HAS_setcond2 1 #define TCG_TARGET_HAS_qemu_st8_i32 0 -#if TCG_TARGET_REG_BITS == 64 #define TCG_TARGET_HAS_movcond_i64 0 #define TCG_TARGET_HAS_div_i64 1 #define TCG_TARGET_HAS_rem_i64 1 @@ -XXX,XX +XXX,XX @@ typedef enum { #define TCG_TARGET_HAS_muls2_i64 0 #define TCG_TARGET_HAS_muluh_i64 1 #define TCG_TARGET_HAS_mulsh_i64 1 -#endif #define TCG_TARGET_DEFAULT_MO (0) diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) #define SOFTMMU_RESERVE_REGS 0 #endif - -static inline tcg_target_long sextreg(tcg_target_long val, int pos, int len) -{ - if (TCG_TARGET_REG_BITS == 32) { - return sextract32(val, pos, len); - } else { - return sextract64(val, pos, len); - } -} +#define sextreg sextract64 /* test if a constant matches the constraint */ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) @@ -XXX,XX +XXX,XX @@ typedef enum { OPC_XOR = 0x4033, OPC_XORI = 0x4013, -#if TCG_TARGET_REG_BITS == 64 OPC_ADDIW = 0x1b, OPC_ADDW = 0x3b, OPC_DIVUW = 0x200503b, @@ -XXX,XX +XXX,XX @@ typedef enum { OPC_SRLIW = 0x501b, OPC_SRLW = 0x503b, OPC_SUBW = 0x4000003b, -#else - /* Simplify code throughout by defining aliases for RV32. */ - OPC_ADDIW = OPC_ADDI, - OPC_ADDW = OPC_ADD, - OPC_DIVUW = OPC_DIVU, - OPC_DIVW = OPC_DIV, - OPC_MULW = OPC_MUL, - OPC_REMUW = OPC_REMU, - OPC_REMW = OPC_REM, - OPC_SLLIW = OPC_SLLI, - OPC_SLLW = OPC_SLL, - OPC_SRAIW = OPC_SRAI, - OPC_SRAW = OPC_SRA, - OPC_SRLIW = OPC_SRLI, - OPC_SRLW = OPC_SRL, - OPC_SUBW = OPC_SUB, -#endif OPC_FENCE = 0x0000000f, OPC_NOP = OPC_ADDI, /* nop = addi r0,r0,0 */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, tcg_target_long lo, hi, tmp; int shift, ret; - if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) { + if (type == TCG_TYPE_I32) { val = (int32_t)val; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, } hi = val - lo; - if (TCG_TARGET_REG_BITS == 32 || val == (int32_t)val) { + if (val == (int32_t)val) { tcg_out_opc_upper(s, OPC_LUI, rd, hi); if (lo != 0) { tcg_out_opc_imm(s, OPC_ADDIW, rd, rd, lo); @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, return; } - /* We can only be here if TCG_TARGET_REG_BITS != 32 */ tmp = tcg_pcrel_diff(s, (void *)val); if (tmp == (int32_t)tmp) { tcg_out_opc_upper(s, OPC_AUIPC, rd, 0); @@ -XXX,XX +XXX,XX @@ static void tcg_out_ldst(TCGContext *s, RISCVInsn opc, TCGReg data, static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, intptr_t arg2) { - bool is32bit = (TCG_TARGET_REG_BITS == 32 || type == TCG_TYPE_I32); - tcg_out_ldst(s, is32bit ? OPC_LW : OPC_LD, arg, arg1, arg2); + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_LW : OPC_LD; + tcg_out_ldst(s, insn, arg, arg1, arg2); } static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, intptr_t arg2) { - bool is32bit = (TCG_TARGET_REG_BITS == 32 || type == TCG_TYPE_I32); - tcg_out_ldst(s, is32bit ? OPC_SW : OPC_SD, arg, arg1, arg2); + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SW : OPC_SD; + tcg_out_ldst(s, insn, arg, arg1, arg2); } static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, @@ -XXX,XX +XXX,XX @@ static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret, } } -static void tcg_out_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, - TCGReg bl, TCGReg bh, TCGLabel *l) -{ - /* todo */ - g_assert_not_reached(); -} - -static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret, - TCGReg al, TCGReg ah, TCGReg bl, TCGReg bh) -{ - /* todo */ - g_assert_not_reached(); -} - static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) { TCGReg link = tail ? TCG_REG_ZERO : TCG_REG_RA; @@ -XXX,XX +XXX,XX @@ static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) if (offset == sextreg(offset, 0, 20)) { /* short jump: -2097150 to 2097152 */ tcg_out_opc_jump(s, OPC_JAL, link, offset); - } else if (TCG_TARGET_REG_BITS == 32 || offset == (int32_t)offset) { + } else if (offset == (int32_t)offset) { /* long jump: -2147483646 to 2147483648 */ tcg_out_opc_upper(s, OPC_AUIPC, TCG_REG_TMP0, 0); tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, 0); ret = reloc_call(s->code_ptr - 2, arg); tcg_debug_assert(ret == true); - } else if (TCG_TARGET_REG_BITS == 64) { + } else { /* far jump: 64-bit */ tcg_target_long imm = sextreg((tcg_target_long)arg, 0, 12); tcg_target_long base = (tcg_target_long)arg - imm; tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP0, base); tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, imm); - } else { - g_assert_not_reached(); } } @@ -XXX,XX +XXX,XX @@ static void * const qemu_st_helpers[MO_SIZE + 1] = { #endif }; -/* We don't support oversize guests */ -QEMU_BUILD_BUG_ON(TCG_TARGET_REG_BITS < TARGET_LONG_BITS); - /* We expect to use a 12-bit negative offset from ENV. */ QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 11)); @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target) tcg_debug_assert(ok); } -static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addrl, - TCGReg addrh, MemOpIdx oi, +static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addr, MemOpIdx oi, tcg_insn_unit **label_ptr, bool is_load) { MemOp opc = get_memop(oi); @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addrl, tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, mask_base, mask_ofs); tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, table_base, table_ofs); - tcg_out_opc_imm(s, OPC_SRLI, TCG_REG_TMP2, addrl, + tcg_out_opc_imm(s, OPC_SRLI, TCG_REG_TMP2, addr, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP0); tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP1); @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addrl, /* Clear the non-page, non-alignment bits from the address. */ compare_mask = (tcg_target_long)TARGET_PAGE_MASK | ((1 << a_bits) - 1); if (compare_mask == sextreg(compare_mask, 0, 12)) { - tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addrl, compare_mask); + tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addr, compare_mask); } else { tcg_out_movi(s, TCG_TYPE_TL, TCG_REG_TMP1, compare_mask); - tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP1, TCG_REG_TMP1, addrl); + tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP1, TCG_REG_TMP1, addr); } /* Compare masked address with the TLB entry. */ @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addrl, tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP0, TCG_REG_TMP1, 0); /* TLB Hit - translate address using addend. */ - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { - tcg_out_ext32u(s, TCG_REG_TMP0, addrl); - addrl = TCG_REG_TMP0; + if (TARGET_LONG_BITS == 32) { + tcg_out_ext32u(s, TCG_REG_TMP0, addr); + addr = TCG_REG_TMP0; } - tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_REG_TMP2, addrl); + tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_REG_TMP2, addr); return TCG_REG_TMP0; } static void add_qemu_ldst_label(TCGContext *s, int is_ld, MemOpIdx oi, - TCGType ext, - TCGReg datalo, TCGReg datahi, - TCGReg addrlo, TCGReg addrhi, - void *raddr, tcg_insn_unit **label_ptr) + TCGType data_type, TCGReg data_reg, + TCGReg addr_reg, void *raddr, + tcg_insn_unit **label_ptr) { TCGLabelQemuLdst *label = new_ldst_label(s); label->is_ld = is_ld; label->oi = oi; - label->type = ext; - label->datalo_reg = datalo; - label->datahi_reg = datahi; - label->addrlo_reg = addrlo; - label->addrhi_reg = addrhi; + label->type = data_type; + label->datalo_reg = data_reg; + label->addrlo_reg = addr_reg; label->raddr = tcg_splitwx_to_rx(raddr); label->label_ptr[0] = label_ptr[0]; } @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) TCGReg a2 = tcg_target_call_iarg_regs[2]; TCGReg a3 = tcg_target_call_iarg_regs[3]; - /* We don't support oversize guests */ - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { - g_assert_not_reached(); - } - /* resolve label address */ if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { return false; @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) TCGReg a3 = tcg_target_call_iarg_regs[3]; TCGReg a4 = tcg_target_call_iarg_regs[4]; - /* We don't support oversize guests */ - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { - g_assert_not_reached(); - } - /* resolve label address */ if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { return false; @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) #endif /* CONFIG_SOFTMMU */ -static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, +static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg val, TCGReg base, MemOp opc, bool is_64) { /* Byte swapping is left to middle-end expansion. */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, switch (opc & (MO_SSIZE)) { case MO_UB: - tcg_out_opc_imm(s, OPC_LBU, lo, base, 0); + tcg_out_opc_imm(s, OPC_LBU, val, base, 0); break; case MO_SB: - tcg_out_opc_imm(s, OPC_LB, lo, base, 0); + tcg_out_opc_imm(s, OPC_LB, val, base, 0); break; case MO_UW: - tcg_out_opc_imm(s, OPC_LHU, lo, base, 0); + tcg_out_opc_imm(s, OPC_LHU, val, base, 0); break; case MO_SW: - tcg_out_opc_imm(s, OPC_LH, lo, base, 0); + tcg_out_opc_imm(s, OPC_LH, val, base, 0); break; case MO_UL: - if (TCG_TARGET_REG_BITS == 64 && is_64) { - tcg_out_opc_imm(s, OPC_LWU, lo, base, 0); + if (is_64) { + tcg_out_opc_imm(s, OPC_LWU, val, base, 0); break; } /* FALLTHRU */ case MO_SL: - tcg_out_opc_imm(s, OPC_LW, lo, base, 0); + tcg_out_opc_imm(s, OPC_LW, val, base, 0); break; case MO_UQ: - /* Prefer to load from offset 0 first, but allow for overlap. */ - if (TCG_TARGET_REG_BITS == 64) { - tcg_out_opc_imm(s, OPC_LD, lo, base, 0); - } else if (lo != base) { - tcg_out_opc_imm(s, OPC_LW, lo, base, 0); - tcg_out_opc_imm(s, OPC_LW, hi, base, 4); - } else { - tcg_out_opc_imm(s, OPC_LW, hi, base, 4); - tcg_out_opc_imm(s, OPC_LW, lo, base, 0); - } + tcg_out_opc_imm(s, OPC_LD, val, base, 0); break; default: g_assert_not_reached(); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) { - TCGReg addr_regl, addr_regh __attribute__((unused)); - TCGReg data_regl, data_regh; + TCGReg addr_reg, data_reg; MemOpIdx oi; MemOp opc; #if defined(CONFIG_SOFTMMU) @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) #endif TCGReg base; - data_regl = *args++; - data_regh = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); - addr_regl = *args++; - addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); + data_reg = *args++; + addr_reg = *args++; oi = *args++; opc = get_memop(oi); #if defined(CONFIG_SOFTMMU) - base = tcg_out_tlb_load(s, addr_regl, addr_regh, oi, label_ptr, 1); - tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); - add_qemu_ldst_label(s, 1, oi, - (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), - data_regl, data_regh, addr_regl, addr_regh, - s->code_ptr, label_ptr); + base = tcg_out_tlb_load(s, addr_reg, oi, label_ptr, 1); + tcg_out_qemu_ld_direct(s, data_reg, base, opc, is_64); + add_qemu_ldst_label(s, 1, oi, (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), + data_reg, addr_reg, s->code_ptr, label_ptr); #else a_bits = get_alignment_bits(opc); if (a_bits) { - tcg_out_test_alignment(s, true, addr_regl, a_bits); + tcg_out_test_alignment(s, true, addr_reg, a_bits); } - base = addr_regl; - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { + base = addr_reg; + if (TARGET_LONG_BITS == 32) { tcg_out_ext32u(s, TCG_REG_TMP0, base); base = TCG_REG_TMP0; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_GUEST_BASE_REG, base); base = TCG_REG_TMP0; } - tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); + tcg_out_qemu_ld_direct(s, data_reg, base, opc, is_64); #endif } -static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi, +static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg val, TCGReg base, MemOp opc) { /* Byte swapping is left to middle-end expansion. */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi, switch (opc & (MO_SSIZE)) { case MO_8: - tcg_out_opc_store(s, OPC_SB, base, lo, 0); + tcg_out_opc_store(s, OPC_SB, base, val, 0); break; case MO_16: - tcg_out_opc_store(s, OPC_SH, base, lo, 0); + tcg_out_opc_store(s, OPC_SH, base, val, 0); break; case MO_32: - tcg_out_opc_store(s, OPC_SW, base, lo, 0); + tcg_out_opc_store(s, OPC_SW, base, val, 0); break; case MO_64: - if (TCG_TARGET_REG_BITS == 64) { - tcg_out_opc_store(s, OPC_SD, base, lo, 0); - } else { - tcg_out_opc_store(s, OPC_SW, base, lo, 0); - tcg_out_opc_store(s, OPC_SW, base, hi, 4); - } + tcg_out_opc_store(s, OPC_SD, base, val, 0); break; default: g_assert_not_reached(); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi, static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) { - TCGReg addr_regl, addr_regh __attribute__((unused)); - TCGReg data_regl, data_regh; + TCGReg addr_reg, data_reg; MemOpIdx oi; MemOp opc; #if defined(CONFIG_SOFTMMU) @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) #endif TCGReg base; - data_regl = *args++; - data_regh = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); - addr_regl = *args++; - addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); + data_reg = *args++; + addr_reg = *args++; oi = *args++; opc = get_memop(oi); #if defined(CONFIG_SOFTMMU) - base = tcg_out_tlb_load(s, addr_regl, addr_regh, oi, label_ptr, 0); - tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); - add_qemu_ldst_label(s, 0, oi, - (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), - data_regl, data_regh, addr_regl, addr_regh, - s->code_ptr, label_ptr); + base = tcg_out_tlb_load(s, addr_reg, oi, label_ptr, 0); + tcg_out_qemu_st_direct(s, data_reg, base, opc); + add_qemu_ldst_label(s, 0, oi, (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), + data_reg, addr_reg, s->code_ptr, label_ptr); #else a_bits = get_alignment_bits(opc); if (a_bits) { - tcg_out_test_alignment(s, false, addr_regl, a_bits); + tcg_out_test_alignment(s, false, addr_reg, a_bits); } - base = addr_regl; - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { + base = addr_reg; + if (TARGET_LONG_BITS == 32) { tcg_out_ext32u(s, TCG_REG_TMP0, base); base = TCG_REG_TMP0; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_GUEST_BASE_REG, base); base = TCG_REG_TMP0; } - tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); + tcg_out_qemu_st_direct(s, data_reg, base, opc); #endif } @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_brcond_i64: tcg_out_brcond(s, a2, a0, a1, arg_label(args[3])); break; - case INDEX_op_brcond2_i32: - tcg_out_brcond2(s, args[4], a0, a1, a2, args[3], arg_label(args[5])); - break; case INDEX_op_setcond_i32: case INDEX_op_setcond_i64: tcg_out_setcond(s, args[3], a0, a1, a2); break; - case INDEX_op_setcond2_i32: - tcg_out_setcond2(s, args[5], a0, a1, a2, args[3], args[4]); - break; case INDEX_op_qemu_ld_i32: tcg_out_qemu_ld(s, args, false); @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_sub2_i64: return C_O2_I4(r, r, rZ, rZ, rM, rM); - case INDEX_op_brcond2_i32: - return C_O0_I4(rZ, rZ, rZ, rZ); - - case INDEX_op_setcond2_i32: - return C_O1_I4(r, rZ, rZ, rZ, rZ); - case INDEX_op_qemu_ld_i32: - return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS - ? C_O1_I1(r, L) : C_O1_I2(r, L, L)); - case INDEX_op_qemu_st_i32: - return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS - ? C_O0_I2(LZ, L) : C_O0_I3(LZ, L, L)); case INDEX_op_qemu_ld_i64: - return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) - : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O2_I1(r, r, L) - : C_O2_I2(r, r, L, L)); + return C_O1_I1(r, L); + case INDEX_op_qemu_st_i32: case INDEX_op_qemu_st_i64: - return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(LZ, L) - : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O0_I3(LZ, LZ, L) - : C_O0_I4(LZ, LZ, L, L)); + return C_O0_I2(LZ, L); default: g_assert_not_reached(); @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) static void tcg_target_init(TCGContext *s) { tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffff; - if (TCG_TARGET_REG_BITS == 64) { - tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffff; - } + tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffff; tcg_target_call_clobber_regs = -1u; tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S0); -- 2.34.1 Interpret the variable argument placement in the caller. Pass data_type instead of is64 -- there are several places where we already convert back from bool to type. Clean things up by using type throughout. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target.c.inc | 66 ++++++++++++++------------------------ 1 file changed, 24 insertions(+), 42 deletions(-) diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) #endif /* CONFIG_SOFTMMU */ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg val, - TCGReg base, MemOp opc, bool is_64) + TCGReg base, MemOp opc, TCGType type) { /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((opc & MO_BSWAP) == 0); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg val, tcg_out_opc_imm(s, OPC_LH, val, base, 0); break; case MO_UL: - if (is_64) { + if (type == TCG_TYPE_I64) { tcg_out_opc_imm(s, OPC_LWU, val, base, 0); break; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg val, } } -static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) +static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, + MemOpIdx oi, TCGType data_type) { - TCGReg addr_reg, data_reg; - MemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) - tcg_insn_unit *label_ptr[1]; -#else - unsigned a_bits; -#endif + MemOp opc = get_memop(oi); TCGReg base; - data_reg = *args++; - addr_reg = *args++; - oi = *args++; - opc = get_memop(oi); - #if defined(CONFIG_SOFTMMU) + tcg_insn_unit *label_ptr[1]; + base = tcg_out_tlb_load(s, addr_reg, oi, label_ptr, 1); - tcg_out_qemu_ld_direct(s, data_reg, base, opc, is_64); - add_qemu_ldst_label(s, 1, oi, (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), - data_reg, addr_reg, s->code_ptr, label_ptr); + tcg_out_qemu_ld_direct(s, data_reg, base, opc, data_type); + add_qemu_ldst_label(s, true, oi, data_type, data_reg, addr_reg, + s->code_ptr, label_ptr); #else - a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, true, addr_reg, a_bits); } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_GUEST_BASE_REG, base); base = TCG_REG_TMP0; } - tcg_out_qemu_ld_direct(s, data_reg, base, opc, is_64); + tcg_out_qemu_ld_direct(s, data_reg, base, opc, data_type); #endif } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg val, } } -static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) +static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, + MemOpIdx oi, TCGType data_type) { - TCGReg addr_reg, data_reg; - MemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) - tcg_insn_unit *label_ptr[1]; -#else - unsigned a_bits; -#endif + MemOp opc = get_memop(oi); TCGReg base; - data_reg = *args++; - addr_reg = *args++; - oi = *args++; - opc = get_memop(oi); - #if defined(CONFIG_SOFTMMU) + tcg_insn_unit *label_ptr[1]; + base = tcg_out_tlb_load(s, addr_reg, oi, label_ptr, 0); tcg_out_qemu_st_direct(s, data_reg, base, opc); - add_qemu_ldst_label(s, 0, oi, (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), - data_reg, addr_reg, s->code_ptr, label_ptr); + add_qemu_ldst_label(s, false, oi, data_type, data_reg, addr_reg, + s->code_ptr, label_ptr); #else - a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, false, addr_reg, a_bits); } @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, args, false); + tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I32); break; case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args, true); + tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I64); break; case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, args, false); + tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32); break; case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args, true); + tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I64); break; case INDEX_op_extrh_i64_i32: -- 2.34.1 We need to set this in TCGLabelQemuLdst, so plumb this all the way through from tcg_out_op. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/s390x/tcg-target.c.inc | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, MemOp opc, } static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi, - TCGReg data, TCGReg addr, + TCGType type, TCGReg data, TCGReg addr, tcg_insn_unit *raddr, tcg_insn_unit *label_ptr) { TCGLabelQemuLdst *label = new_ldst_label(s); label->is_ld = is_ld; label->oi = oi; + label->type = type; label->datalo_reg = data; label->addrlo_reg = addr; label->raddr = tcg_splitwx_to_rx(raddr); @@ -XXX,XX +XXX,XX @@ static void tcg_prepare_user_ldst(TCGContext *s, TCGReg *addr_reg, #endif /* CONFIG_SOFTMMU */ static void tcg_out_qemu_ld(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, - MemOpIdx oi) + MemOpIdx oi, TCGType data_type) { MemOp opc = get_memop(oi); #ifdef CONFIG_SOFTMMU @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, tcg_out_qemu_ld_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0); - add_qemu_ldst_label(s, 1, oi, data_reg, addr_reg, s->code_ptr, label_ptr); + add_qemu_ldst_label(s, true, oi, data_type, data_reg, addr_reg, + s->code_ptr, label_ptr); #else TCGReg index_reg; tcg_target_long disp; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, } static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, - MemOpIdx oi) + MemOpIdx oi, TCGType data_type) { MemOp opc = get_memop(oi); #ifdef CONFIG_SOFTMMU @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, tcg_out_qemu_st_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0); - add_qemu_ldst_label(s, 0, oi, data_reg, addr_reg, s->code_ptr, label_ptr); + add_qemu_ldst_label(s, false, oi, data_type, data_reg, addr_reg, + s->code_ptr, label_ptr); #else TCGReg index_reg; tcg_target_long disp; @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_qemu_ld_i32: - /* ??? Technically we can use a non-extending instruction. */ + tcg_out_qemu_ld(s, args[0], args[1], args[2], TCG_TYPE_I32); + break; case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args[0], args[1], args[2]); + tcg_out_qemu_ld(s, args[0], args[1], args[2], TCG_TYPE_I64); break; case INDEX_op_qemu_st_i32: + tcg_out_qemu_st(s, args[0], args[1], args[2], TCG_TYPE_I32); + break; case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args[0], args[1], args[2]); + tcg_out_qemu_st(s, args[0], args[1], args[2], TCG_TYPE_I64); break; case INDEX_op_ld16s_i64: -- 2.34.1 Collect the 3 potential parts of the host address into a struct. Reorg tcg_out_qemu_{ld,st}_direct to use it. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/s390x/tcg-target.c.inc | 109 ++++++++++++++++++++----------------- 1 file changed, 60 insertions(+), 49 deletions(-) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest, tcg_out_call_int(s, dest); } +typedef struct { + TCGReg base; + TCGReg index; + int disp; +} HostAddress; + static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data, - TCGReg base, TCGReg index, int disp) + HostAddress h) { switch (opc & (MO_SSIZE | MO_BSWAP)) { case MO_UB: - tcg_out_insn(s, RXY, LLGC, data, base, index, disp); + tcg_out_insn(s, RXY, LLGC, data, h.base, h.index, h.disp); break; case MO_SB: - tcg_out_insn(s, RXY, LGB, data, base, index, disp); + tcg_out_insn(s, RXY, LGB, data, h.base, h.index, h.disp); break; case MO_UW | MO_BSWAP: /* swapped unsigned halfword load with upper bits zeroed */ - tcg_out_insn(s, RXY, LRVH, data, base, index, disp); + tcg_out_insn(s, RXY, LRVH, data, h.base, h.index, h.disp); tcg_out_ext16u(s, data, data); break; case MO_UW: - tcg_out_insn(s, RXY, LLGH, data, base, index, disp); + tcg_out_insn(s, RXY, LLGH, data, h.base, h.index, h.disp); break; case MO_SW | MO_BSWAP: /* swapped sign-extended halfword load */ - tcg_out_insn(s, RXY, LRVH, data, base, index, disp); + tcg_out_insn(s, RXY, LRVH, data, h.base, h.index, h.disp); tcg_out_ext16s(s, TCG_TYPE_REG, data, data); break; case MO_SW: - tcg_out_insn(s, RXY, LGH, data, base, index, disp); + tcg_out_insn(s, RXY, LGH, data, h.base, h.index, h.disp); break; case MO_UL | MO_BSWAP: /* swapped unsigned int load with upper bits zeroed */ - tcg_out_insn(s, RXY, LRV, data, base, index, disp); + tcg_out_insn(s, RXY, LRV, data, h.base, h.index, h.disp); tcg_out_ext32u(s, data, data); break; case MO_UL: - tcg_out_insn(s, RXY, LLGF, data, base, index, disp); + tcg_out_insn(s, RXY, LLGF, data, h.base, h.index, h.disp); break; case MO_SL | MO_BSWAP: /* swapped sign-extended int load */ - tcg_out_insn(s, RXY, LRV, data, base, index, disp); + tcg_out_insn(s, RXY, LRV, data, h.base, h.index, h.disp); tcg_out_ext32s(s, data, data); break; case MO_SL: - tcg_out_insn(s, RXY, LGF, data, base, index, disp); + tcg_out_insn(s, RXY, LGF, data, h.base, h.index, h.disp); break; case MO_UQ | MO_BSWAP: - tcg_out_insn(s, RXY, LRVG, data, base, index, disp); + tcg_out_insn(s, RXY, LRVG, data, h.base, h.index, h.disp); break; case MO_UQ: - tcg_out_insn(s, RXY, LG, data, base, index, disp); + tcg_out_insn(s, RXY, LG, data, h.base, h.index, h.disp); break; default: @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data, } static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg data, - TCGReg base, TCGReg index, int disp) + HostAddress h) { switch (opc & (MO_SIZE | MO_BSWAP)) { case MO_UB: - if (disp >= 0 && disp < 0x1000) { - tcg_out_insn(s, RX, STC, data, base, index, disp); + if (h.disp >= 0 && h.disp < 0x1000) { + tcg_out_insn(s, RX, STC, data, h.base, h.index, h.disp); } else { - tcg_out_insn(s, RXY, STCY, data, base, index, disp); + tcg_out_insn(s, RXY, STCY, data, h.base, h.index, h.disp); } break; case MO_UW | MO_BSWAP: - tcg_out_insn(s, RXY, STRVH, data, base, index, disp); + tcg_out_insn(s, RXY, STRVH, data, h.base, h.index, h.disp); break; case MO_UW: - if (disp >= 0 && disp < 0x1000) { - tcg_out_insn(s, RX, STH, data, base, index, disp); + if (h.disp >= 0 && h.disp < 0x1000) { + tcg_out_insn(s, RX, STH, data, h.base, h.index, h.disp); } else { - tcg_out_insn(s, RXY, STHY, data, base, index, disp); + tcg_out_insn(s, RXY, STHY, data, h.base, h.index, h.disp); } break; case MO_UL | MO_BSWAP: - tcg_out_insn(s, RXY, STRV, data, base, index, disp); + tcg_out_insn(s, RXY, STRV, data, h.base, h.index, h.disp); break; case MO_UL: - if (disp >= 0 && disp < 0x1000) { - tcg_out_insn(s, RX, ST, data, base, index, disp); + if (h.disp >= 0 && h.disp < 0x1000) { + tcg_out_insn(s, RX, ST, data, h.base, h.index, h.disp); } else { - tcg_out_insn(s, RXY, STY, data, base, index, disp); + tcg_out_insn(s, RXY, STY, data, h.base, h.index, h.disp); } break; case MO_UQ | MO_BSWAP: - tcg_out_insn(s, RXY, STRVG, data, base, index, disp); + tcg_out_insn(s, RXY, STRVG, data, h.base, h.index, h.disp); break; case MO_UQ: - tcg_out_insn(s, RXY, STG, data, base, index, disp); + tcg_out_insn(s, RXY, STG, data, h.base, h.index, h.disp); break; default: @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) return tcg_out_fail_alignment(s, l); } -static void tcg_prepare_user_ldst(TCGContext *s, TCGReg *addr_reg, - TCGReg *index_reg, tcg_target_long *disp) +static HostAddress tcg_prepare_user_ldst(TCGContext *s, TCGReg addr_reg) { + TCGReg index; + int disp; + if (TARGET_LONG_BITS == 32) { - tcg_out_ext32u(s, TCG_TMP0, *addr_reg); - *addr_reg = TCG_TMP0; + tcg_out_ext32u(s, TCG_TMP0, addr_reg); + addr_reg = TCG_TMP0; } if (guest_base < 0x80000) { - *index_reg = TCG_REG_NONE; - *disp = guest_base; + index = TCG_REG_NONE; + disp = guest_base; } else { - *index_reg = TCG_GUEST_BASE_REG; - *disp = 0; + index = TCG_GUEST_BASE_REG; + disp = 0; } + return (HostAddress){ .base = addr_reg, .index = index, .disp = disp }; } #endif /* CONFIG_SOFTMMU */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, MemOpIdx oi, TCGType data_type) { MemOp opc = get_memop(oi); + HostAddress h; + #ifdef CONFIG_SOFTMMU unsigned mem_index = get_mmuidx(oi); tcg_insn_unit *label_ptr; - TCGReg base_reg; - base_reg = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 1); + h.base = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 1); + h.index = TCG_REG_R2; + h.disp = 0; tcg_out16(s, RI_BRC | (S390_CC_NE << 4)); label_ptr = s->code_ptr; s->code_ptr += 1; - tcg_out_qemu_ld_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0); + tcg_out_qemu_ld_direct(s, opc, data_reg, h); add_qemu_ldst_label(s, true, oi, data_type, data_reg, addr_reg, s->code_ptr, label_ptr); #else - TCGReg index_reg; - tcg_target_long disp; unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, true, addr_reg, a_bits); } - tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp); - tcg_out_qemu_ld_direct(s, opc, data_reg, addr_reg, index_reg, disp); + h = tcg_prepare_user_ldst(s, addr_reg); + tcg_out_qemu_ld_direct(s, opc, data_reg, h); #endif } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, MemOpIdx oi, TCGType data_type) { MemOp opc = get_memop(oi); + HostAddress h; + #ifdef CONFIG_SOFTMMU unsigned mem_index = get_mmuidx(oi); tcg_insn_unit *label_ptr; - TCGReg base_reg; - base_reg = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 0); + h.base = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 0); + h.index = TCG_REG_R2; + h.disp = 0; tcg_out16(s, RI_BRC | (S390_CC_NE << 4)); label_ptr = s->code_ptr; s->code_ptr += 1; - tcg_out_qemu_st_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0); + tcg_out_qemu_st_direct(s, opc, data_reg, h); add_qemu_ldst_label(s, false, oi, data_type, data_reg, addr_reg, s->code_ptr, label_ptr); #else - TCGReg index_reg; - tcg_target_long disp; unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, false, addr_reg, a_bits); } - tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp); - tcg_out_qemu_st_direct(s, opc, data_reg, addr_reg, index_reg, disp); + h = tcg_prepare_user_ldst(s, addr_reg); + tcg_out_qemu_st_direct(s, opc, data_reg, h); #endif } -- 2.34.1 In tcg_canonicalize_memop, we remove MO_SIGN from MO_32 operations with TCG_TYPE_I32. Thus this is never set. We already have an identical test just above which does not include is_64 Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc64/tcg-target.c.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc64/tcg-target.c.inc +++ b/tcg/sparc64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_O2, oi); /* We let the helper sign-extend SB and SW, but leave SL for here. */ - if (is_64 && (memop & MO_SSIZE) == MO_SL) { + if ((memop & MO_SSIZE) == MO_SL) { tcg_out_ext32s(s, data, TCG_REG_O0); } else { tcg_out_mov(s, TCG_TYPE_REG, data, TCG_REG_O0); -- 2.34.1 We need to set this in TCGLabelQemuLdst, so plumb this all the way through from tcg_out_op. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc64/tcg-target.c.inc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc64/tcg-target.c.inc +++ b/tcg/sparc64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const int qemu_st_opc[(MO_SIZE | MO_BSWAP) + 1] = { }; static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, - MemOpIdx oi, bool is_64) + MemOpIdx oi, TCGType data_type) { MemOp memop = get_memop(oi); tcg_insn_unit *label_ptr; @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, a0, a1, a2, false); + tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I32); break; case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, a0, a1, a2, true); + tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I64); break; case INDEX_op_qemu_st_i32: tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32); -- 2.34.1 Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 13 +++++++++++++ tcg/tcg-ldst.c.inc | 14 -------------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ typedef struct QEMU_PACKED { DebugFrameFDEHeader fde; } DebugFrameHeader; +typedef struct TCGLabelQemuLdst { + bool is_ld; /* qemu_ld: true, qemu_st: false */ + MemOpIdx oi; + TCGType type; /* result type of a load */ + TCGReg addrlo_reg; /* reg index for low word of guest virtual addr */ + TCGReg addrhi_reg; /* reg index for high word of guest virtual addr */ + TCGReg datalo_reg; /* reg index for low word to be loaded or stored */ + TCGReg datahi_reg; /* reg index for high word to be loaded or stored */ + const tcg_insn_unit *raddr; /* addr of the next IR of qemu_ld/st IR */ + tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */ + QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next; +} TCGLabelQemuLdst; + static void tcg_register_jit_int(const void *buf, size_t size, const void *debug_frame, size_t debug_frame_size) diff --git a/tcg/tcg-ldst.c.inc b/tcg/tcg-ldst.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-ldst.c.inc +++ b/tcg/tcg-ldst.c.inc @@ -XXX,XX +XXX,XX @@ * THE SOFTWARE. */ -typedef struct TCGLabelQemuLdst { - bool is_ld; /* qemu_ld: true, qemu_st: false */ - MemOpIdx oi; - TCGType type; /* result type of a load */ - TCGReg addrlo_reg; /* reg index for low word of guest virtual addr */ - TCGReg addrhi_reg; /* reg index for high word of guest virtual addr */ - TCGReg datalo_reg; /* reg index for low word to be loaded or stored */ - TCGReg datahi_reg; /* reg index for high word to be loaded or stored */ - const tcg_insn_unit *raddr; /* addr of the next IR of qemu_ld/st IR */ - tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */ - QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next; -} TCGLabelQemuLdst; - - /* * Generate TB finalization at the end of block */ -- 2.34.1 An inline function is safer than a macro, and REG_P was rather too generic. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg-internal.h | 4 ---- tcg/tcg.c | 16 +++++++++++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tcg/tcg-internal.h b/tcg/tcg-internal.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg-internal.h +++ b/tcg/tcg-internal.h @@ -XXX,XX +XXX,XX @@ typedef struct TCGCallArgumentLoc { unsigned tmp_subindex : 2; } TCGCallArgumentLoc; -/* Avoid "unsigned < 0 is always false" Werror, when iarg_regs is empty. */ -#define REG_P(L) \ - ((int)(L)->arg_slot < (int)ARRAY_SIZE(tcg_target_call_iarg_regs)) - typedef struct TCGHelperInfo { void *func; const char *name; diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void init_ffi_layouts(void) } #endif /* CONFIG_TCG_INTERPRETER */ +static inline bool arg_slot_reg_p(unsigned arg_slot) +{ + /* + * Split the sizeof away from the comparison to avoid Werror from + * "unsigned < 0 is always false", when iarg_regs is empty. + */ + unsigned nreg = ARRAY_SIZE(tcg_target_call_iarg_regs); + return arg_slot < nreg; +} + typedef struct TCGCumulativeArgs { int arg_idx; /* tcg_gen_callN args[] */ int info_in_idx; /* TCGHelperInfo in[] */ @@ -XXX,XX +XXX,XX @@ liveness_pass_1(TCGContext *s) case TCG_CALL_ARG_NORMAL: case TCG_CALL_ARG_EXTEND_U: case TCG_CALL_ARG_EXTEND_S: - if (REG_P(loc)) { + if (arg_slot_reg_p(loc->arg_slot)) { *la_temp_pref(ts) = 0; break; } @@ -XXX,XX +XXX,XX @@ liveness_pass_1(TCGContext *s) case TCG_CALL_ARG_NORMAL: case TCG_CALL_ARG_EXTEND_U: case TCG_CALL_ARG_EXTEND_S: - if (REG_P(loc)) { + if (arg_slot_reg_p(loc->arg_slot)) { tcg_regset_set_reg(*la_temp_pref(ts), tcg_target_call_iarg_regs[loc->arg_slot]); } @@ -XXX,XX +XXX,XX @@ static void load_arg_stk(TCGContext *s, int stk_slot, TCGTemp *ts, static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l, TCGTemp *ts, TCGRegSet *allocated_regs) { - if (REG_P(l)) { + if (arg_slot_reg_p(l->arg_slot)) { TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot]; load_arg_reg(s, reg, ts, *allocated_regs); tcg_regset_set_reg(*allocated_regs, reg); -- 2.34.1 Unify all computation of argument stack offset in one function. This requires that we adjust ref_slot to be in the same units, by adding max_reg_slots during init_call_layout. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static inline bool arg_slot_reg_p(unsigned arg_slot) return arg_slot < nreg; } +static inline int arg_slot_stk_ofs(unsigned arg_slot) +{ + unsigned max = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long); + unsigned stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs); + + tcg_debug_assert(stk_slot < max); + return TCG_TARGET_CALL_STACK_OFFSET + stk_slot * sizeof(tcg_target_long); +} + typedef struct TCGCumulativeArgs { int arg_idx; /* tcg_gen_callN args[] */ int info_in_idx; /* TCGHelperInfo in[] */ @@ -XXX,XX +XXX,XX @@ static void init_call_layout(TCGHelperInfo *info) } } assert(ref_base + cum.ref_slot <= max_stk_slots); + ref_base += max_reg_slots; if (ref_base != 0) { for (int i = cum.info_in_idx - 1; i >= 0; --i) { @@ -XXX,XX +XXX,XX @@ static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts, } } -static void load_arg_stk(TCGContext *s, int stk_slot, TCGTemp *ts, +static void load_arg_stk(TCGContext *s, unsigned arg_slot, TCGTemp *ts, TCGRegSet allocated_regs) { /* @@ -XXX,XX +XXX,XX @@ static void load_arg_stk(TCGContext *s, int stk_slot, TCGTemp *ts, */ temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0); tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, - TCG_TARGET_CALL_STACK_OFFSET + - stk_slot * sizeof(tcg_target_long)); + arg_slot_stk_ofs(arg_slot)); } static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l, @@ -XXX,XX +XXX,XX @@ static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l, load_arg_reg(s, reg, ts, *allocated_regs); tcg_regset_set_reg(*allocated_regs, reg); } else { - load_arg_stk(s, l->arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs), - ts, *allocated_regs); + load_arg_stk(s, l->arg_slot, ts, *allocated_regs); } } -static void load_arg_ref(TCGContext *s, int arg_slot, TCGReg ref_base, +static void load_arg_ref(TCGContext *s, unsigned arg_slot, TCGReg ref_base, intptr_t ref_off, TCGRegSet *allocated_regs) { TCGReg reg; - int stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs); - if (stk_slot < 0) { + if (arg_slot_reg_p(arg_slot)) { reg = tcg_target_call_iarg_regs[arg_slot]; tcg_reg_free(s, reg, *allocated_regs); tcg_out_addi_ptr(s, reg, ref_base, ref_off); @@ -XXX,XX +XXX,XX @@ static void load_arg_ref(TCGContext *s, int arg_slot, TCGReg ref_base, *allocated_regs, 0, false); tcg_out_addi_ptr(s, reg, ref_base, ref_off); tcg_out_st(s, TCG_TYPE_PTR, reg, TCG_REG_CALL_STACK, - TCG_TARGET_CALL_STACK_OFFSET - + stk_slot * sizeof(tcg_target_long)); + arg_slot_stk_ofs(arg_slot)); } } @@ -XXX,XX +XXX,XX @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) case TCG_CALL_ARG_BY_REF: load_arg_stk(s, loc->ref_slot, ts, allocated_regs); load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK, - TCG_TARGET_CALL_STACK_OFFSET - + loc->ref_slot * sizeof(tcg_target_long), + arg_slot_stk_ofs(loc->ref_slot), &allocated_regs); break; case TCG_CALL_ARG_BY_REF_N: -- 2.34.1 While the old type was correct in the ideal sense, some ABIs require the argument to be zero-extended. Using uint32_t for all such values is a decent compromise. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/tcg/tcg-ldst.h | 10 +++++++--- accel/tcg/cputlb.c | 6 +++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/include/tcg/tcg-ldst.h b/include/tcg/tcg-ldst.h index XXXXXXX..XXXXXXX 100644 --- a/include/tcg/tcg-ldst.h +++ b/include/tcg/tcg-ldst.h @@ -XXX,XX +XXX,XX @@ tcg_target_ulong helper_be_ldsw_mmu(CPUArchState *env, target_ulong addr, tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr, MemOpIdx oi, uintptr_t retaddr); -void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val, +/* + * Value extended to at least uint32_t, so that some ABIs do not require + * zero-extension from uint8_t or uint16_t. + */ +void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint32_t val, MemOpIdx oi, uintptr_t retaddr); -void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val, +void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint32_t val, MemOpIdx oi, uintptr_t retaddr); void helper_le_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val, MemOpIdx oi, uintptr_t retaddr); void helper_le_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val, MemOpIdx oi, uintptr_t retaddr); -void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val, +void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint32_t val, MemOpIdx oi, uintptr_t retaddr); void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val, MemOpIdx oi, uintptr_t retaddr); diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ full_stb_mmu(CPUArchState *env, target_ulong addr, uint64_t val, store_helper(env, addr, val, oi, retaddr, MO_UB); } -void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val, +void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint32_t val, MemOpIdx oi, uintptr_t retaddr) { full_stb_mmu(env, addr, val, oi, retaddr); @@ -XXX,XX +XXX,XX @@ static void full_le_stw_mmu(CPUArchState *env, target_ulong addr, uint64_t val, store_helper(env, addr, val, oi, retaddr, MO_LEUW); } -void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val, +void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint32_t val, MemOpIdx oi, uintptr_t retaddr) { full_le_stw_mmu(env, addr, val, oi, retaddr); @@ -XXX,XX +XXX,XX @@ static void full_be_stw_mmu(CPUArchState *env, target_ulong addr, uint64_t val, store_helper(env, addr, val, oi, retaddr, MO_BEUW); } -void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val, +void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint32_t val, MemOpIdx oi, uintptr_t retaddr) { full_be_stw_mmu(env, addr, val, oi, retaddr); -- 2.34.1