The following changes since commit c52d69e7dbaaed0ffdef8125e79218672c30161d: Merge remote-tracking branch 'remotes/cschoenebeck/tags/pull-9p-20211027' into staging (2021-10-27 11:45:18 -0700) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20211027 for you to fetch changes up to 820c025f0dcacf2f3c12735b1f162893fbfa7bc6: tcg/optimize: Propagate sign info for shifting (2021-10-27 17:11:23 -0700) ---------------------------------------------------------------- Improvements to qemu/int128 Fixes for 128/64 division. Cleanup tcg/optimize.c Optimize redundant sign extensions ---------------------------------------------------------------- Frédéric Pétrot (1): qemu/int128: Add int128_{not,xor} Luis Pires (4): host-utils: move checks out of divu128/divs128 host-utils: move udiv_qrnnd() to host-utils host-utils: add 128-bit quotient support to divu128/divs128 host-utils: add unit tests for divu128/divs128 Richard Henderson (51): tcg/optimize: Rename "mask" to "z_mask" tcg/optimize: Split out OptContext tcg/optimize: Remove do_default label tcg/optimize: Change tcg_opt_gen_{mov,movi} interface tcg/optimize: Move prev_mb into OptContext tcg/optimize: Split out init_arguments tcg/optimize: Split out copy_propagate tcg/optimize: Split out fold_call tcg/optimize: Drop nb_oargs, nb_iargs locals tcg/optimize: Change fail return for do_constant_folding_cond* tcg/optimize: Return true from tcg_opt_gen_{mov,movi} tcg/optimize: Split out finish_folding tcg/optimize: Use a boolean to avoid a mass of continues tcg/optimize: Split out fold_mb, fold_qemu_{ld,st} tcg/optimize: Split out fold_const{1,2} tcg/optimize: Split out fold_setcond2 tcg/optimize: Split out fold_brcond2 tcg/optimize: Split out fold_brcond tcg/optimize: Split out fold_setcond tcg/optimize: Split out fold_mulu2_i32 tcg/optimize: Split out fold_addsub2_i32 tcg/optimize: Split out fold_movcond tcg/optimize: Split out fold_extract2 tcg/optimize: Split out fold_extract, fold_sextract tcg/optimize: Split out fold_deposit tcg/optimize: Split out fold_count_zeros tcg/optimize: Split out fold_bswap tcg/optimize: Split out fold_dup, fold_dup2 tcg/optimize: Split out fold_mov tcg/optimize: Split out fold_xx_to_i tcg/optimize: Split out fold_xx_to_x tcg/optimize: Split out fold_xi_to_i tcg/optimize: Add type to OptContext tcg/optimize: Split out fold_to_not tcg/optimize: Split out fold_sub_to_neg tcg/optimize: Split out fold_xi_to_x tcg/optimize: Split out fold_ix_to_i tcg/optimize: Split out fold_masks tcg/optimize: Expand fold_mulu2_i32 to all 4-arg multiplies tcg/optimize: Expand fold_addsub2_i32 to 64-bit ops tcg/optimize: Sink commutative operand swapping into fold functions tcg/optimize: Stop forcing z_mask to "garbage" for 32-bit values tcg/optimize: Use fold_xx_to_i for orc tcg/optimize: Use fold_xi_to_x for mul tcg/optimize: Use fold_xi_to_x for div tcg/optimize: Use fold_xx_to_i for rem tcg/optimize: Optimize sign extensions tcg/optimize: Propagate sign info for logical operations tcg/optimize: Propagate sign info for setcond tcg/optimize: Propagate sign info for bit counting tcg/optimize: Propagate sign info for shifting include/fpu/softfloat-macros.h | 82 -- include/hw/clock.h | 5 +- include/qemu/host-utils.h | 121 +- include/qemu/int128.h | 20 + target/ppc/int_helper.c | 23 +- tcg/optimize.c | 2644 ++++++++++++++++++++++++---------------- tests/unit/test-div128.c | 197 +++ util/host-utils.c | 147 ++- tests/unit/meson.build | 1 + 9 files changed, 2053 insertions(+), 1187 deletions(-) create mode 100644 tests/unit/test-div128.c From: Frédéric Pétrot <frederic.petrot@univ-grenoble-alpes.fr> Addition of not and xor on 128-bit integers. Signed-off-by: Frédéric Pétrot <frederic.petrot@univ-grenoble-alpes.fr> Co-authored-by: Fabien Portas <fabien.portas@grenoble-inp.org> Message-Id: <20211025122818.168890-3-frederic.petrot@univ-grenoble-alpes.fr> [rth: Split out logical operations.] Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/qemu/int128.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/include/qemu/int128.h b/include/qemu/int128.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/int128.h +++ b/include/qemu/int128.h @@ -XXX,XX +XXX,XX @@ static inline Int128 int128_exts64(int64_t a) return a; } +static inline Int128 int128_not(Int128 a) +{ + return ~a; +} + static inline Int128 int128_and(Int128 a, Int128 b) { return a & b; @@ -XXX,XX +XXX,XX @@ static inline Int128 int128_or(Int128 a, Int128 b) return a | b; } +static inline Int128 int128_xor(Int128 a, Int128 b) +{ + return a ^ b; +} + static inline Int128 int128_rshift(Int128 a, int n) { return a >> n; @@ -XXX,XX +XXX,XX @@ static inline Int128 int128_exts64(int64_t a) return int128_make128(a, (a < 0) ? -1 : 0); } +static inline Int128 int128_not(Int128 a) +{ + return int128_make128(~a.lo, ~a.hi); +} + static inline Int128 int128_and(Int128 a, Int128 b) { return int128_make128(a.lo & b.lo, a.hi & b.hi); @@ -XXX,XX +XXX,XX @@ static inline Int128 int128_or(Int128 a, Int128 b) return int128_make128(a.lo | b.lo, a.hi | b.hi); } +static inline Int128 int128_xor(Int128 a, Int128 b) +{ + return int128_make128(a.lo ^ b.lo, a.hi ^ b.hi); +} + static inline Int128 int128_rshift(Int128 a, int n) { int64_t h; -- 2.25.1 From: Luis Pires <luis.pires@eldorado.org.br> In preparation for changing the divu128/divs128 implementations to allow for quotients larger than 64 bits, move the div-by-zero and overflow checks to the callers. Signed-off-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20211025191154.350831-2-luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/hw/clock.h | 5 +++-- include/qemu/host-utils.h | 34 ++++++++++++--------------------- target/ppc/int_helper.c | 14 +++++++++----- util/host-utils.c | 40 ++++++++++++++++++--------------------- 4 files changed, 42 insertions(+), 51 deletions(-) diff --git a/include/hw/clock.h b/include/hw/clock.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/clock.h +++ b/include/hw/clock.h @@ -XXX,XX +XXX,XX @@ static inline uint64_t clock_ns_to_ticks(const Clock *clk, uint64_t ns) return 0; } /* - * Ignore divu128() return value as we've caught div-by-zero and don't - * need different behaviour for overflow. + * BUG: when CONFIG_INT128 is not defined, the current implementation of + * divu128 does not return a valid truncated quotient, so the result will + * be wrong. */ divu128(&lo, &hi, clk->period); return lo; diff --git a/include/qemu/host-utils.h b/include/qemu/host-utils.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/host-utils.h +++ b/include/qemu/host-utils.h @@ -XXX,XX +XXX,XX @@ static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c) return (__int128_t)a * b / c; } -static inline int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) +static inline void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) { - if (divisor == 0) { - return 1; - } else { - __uint128_t dividend = ((__uint128_t)*phigh << 64) | *plow; - __uint128_t result = dividend / divisor; - *plow = result; - *phigh = dividend % divisor; - return result > UINT64_MAX; - } + __uint128_t dividend = ((__uint128_t)*phigh << 64) | *plow; + __uint128_t result = dividend / divisor; + *plow = result; + *phigh = dividend % divisor; } -static inline int divs128(int64_t *plow, int64_t *phigh, int64_t divisor) +static inline void divs128(int64_t *plow, int64_t *phigh, int64_t divisor) { - if (divisor == 0) { - return 1; - } else { - __int128_t dividend = ((__int128_t)*phigh << 64) | (uint64_t)*plow; - __int128_t result = dividend / divisor; - *plow = result; - *phigh = dividend % divisor; - return result != *plow; - } + __int128_t dividend = ((__int128_t)*phigh << 64) | (uint64_t)*plow; + __int128_t result = dividend / divisor; + *plow = result; + *phigh = dividend % divisor; } #else void muls64(uint64_t *plow, uint64_t *phigh, int64_t a, int64_t b); void mulu64(uint64_t *plow, uint64_t *phigh, uint64_t a, uint64_t b); -int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor); -int divs128(int64_t *plow, int64_t *phigh, int64_t divisor); +void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor); +void divs128(int64_t *plow, int64_t *phigh, int64_t divisor); static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c) { diff --git a/target/ppc/int_helper.c b/target/ppc/int_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/int_helper.c +++ b/target/ppc/int_helper.c @@ -XXX,XX +XXX,XX @@ uint64_t helper_divdeu(CPUPPCState *env, uint64_t ra, uint64_t rb, uint32_t oe) uint64_t rt = 0; int overflow = 0; - overflow = divu128(&rt, &ra, rb); - - if (unlikely(overflow)) { + if (unlikely(rb == 0 || ra >= rb)) { + overflow = 1; rt = 0; /* Undefined */ + } else { + divu128(&rt, &ra, rb); } if (oe) { @@ -XXX,XX +XXX,XX @@ uint64_t helper_divde(CPUPPCState *env, uint64_t rau, uint64_t rbu, uint32_t oe) int64_t rt = 0; int64_t ra = (int64_t)rau; int64_t rb = (int64_t)rbu; - int overflow = divs128(&rt, &ra, rb); + int overflow = 0; - if (unlikely(overflow)) { + if (unlikely(rb == 0 || uabs64(ra) >= uabs64(rb))) { + overflow = 1; rt = 0; /* Undefined */ + } else { + divs128(&rt, &ra, rb); } if (oe) { diff --git a/util/host-utils.c b/util/host-utils.c index XXXXXXX..XXXXXXX 100644 --- a/util/host-utils.c +++ b/util/host-utils.c @@ -XXX,XX +XXX,XX @@ void muls64 (uint64_t *plow, uint64_t *phigh, int64_t a, int64_t b) *phigh = rh; } -/* Unsigned 128x64 division. Returns 1 if overflow (divide by zero or */ -/* quotient exceeds 64 bits). Otherwise returns quotient via plow and */ -/* remainder via phigh. */ -int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) +/* + * Unsigned 128-by-64 division. Returns quotient via plow and + * remainder via phigh. + * The result must fit in 64 bits (plow) - otherwise, the result + * is undefined. + * This function will cause a division by zero if passed a zero divisor. + */ +void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) { uint64_t dhi = *phigh; uint64_t dlo = *plow; unsigned i; uint64_t carry = 0; - if (divisor == 0) { - return 1; - } else if (dhi == 0) { + if (divisor == 0 || dhi == 0) { *plow = dlo / divisor; *phigh = dlo % divisor; - return 0; - } else if (dhi >= divisor) { - return 1; } else { for (i = 0; i < 64; i++) { @@ -XXX,XX +XXX,XX @@ int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) *plow = dlo; *phigh = dhi; - return 0; } } -int divs128(int64_t *plow, int64_t *phigh, int64_t divisor) +/* + * Signed 128-by-64 division. Returns quotient via plow and + * remainder via phigh. + * The result must fit in 64 bits (plow) - otherwise, the result + * is undefined. + * This function will cause a division by zero if passed a zero divisor. + */ +void divs128(int64_t *plow, int64_t *phigh, int64_t divisor) { int sgn_dvdnd = *phigh < 0; int sgn_divsr = divisor < 0; - int overflow = 0; if (sgn_dvdnd) { *plow = ~(*plow); @@ -XXX,XX +XXX,XX @@ int divs128(int64_t *plow, int64_t *phigh, int64_t divisor) divisor = 0 - divisor; } - overflow = divu128((uint64_t *)plow, (uint64_t *)phigh, (uint64_t)divisor); + divu128((uint64_t *)plow, (uint64_t *)phigh, (uint64_t)divisor); if (sgn_dvdnd ^ sgn_divsr) { *plow = 0 - *plow; } - - if (!overflow) { - if ((*plow < 0) ^ (sgn_dvdnd ^ sgn_divsr)) { - overflow = 1; - } - } - - return overflow; } #endif -- 2.25.1 From: Luis Pires <luis.pires@eldorado.org.br> Move udiv_qrnnd() from include/fpu/softfloat-macros.h to host-utils, so it can be reused by divu128(). Signed-off-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20211025191154.350831-3-luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-macros.h | 82 ---------------------------------- include/qemu/host-utils.h | 81 +++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 82 deletions(-) diff --git a/include/fpu/softfloat-macros.h b/include/fpu/softfloat-macros.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-macros.h +++ b/include/fpu/softfloat-macros.h @@ -XXX,XX +XXX,XX @@ * so some portions are provided under: * the SoftFloat-2a license * the BSD license - * GPL-v2-or-later * * Any future contributions to this file after December 1st 2014 will be * taken to be licensed under the Softfloat-2a license unless specifically @@ -XXX,XX +XXX,XX @@ this code that are retained. * THE POSSIBILITY OF SUCH DAMAGE. */ -/* Portions of this work are licensed under the terms of the GNU GPL, - * version 2 or later. See the COPYING file in the top-level directory. - */ - #ifndef FPU_SOFTFLOAT_MACROS_H #define FPU_SOFTFLOAT_MACROS_H @@ -XXX,XX +XXX,XX @@ static inline uint64_t estimateDiv128To64(uint64_t a0, uint64_t a1, uint64_t b) } -/* From the GNU Multi Precision Library - longlong.h __udiv_qrnnd - * (https://gmplib.org/repo/gmp/file/tip/longlong.h) - * - * Licensed under the GPLv2/LGPLv3 - */ -static inline uint64_t udiv_qrnnd(uint64_t *r, uint64_t n1, - uint64_t n0, uint64_t d) -{ -#if defined(__x86_64__) - uint64_t q; - asm("divq %4" : "=a"(q), "=d"(*r) : "0"(n0), "1"(n1), "rm"(d)); - return q; -#elif defined(__s390x__) && !defined(__clang__) - /* Need to use a TImode type to get an even register pair for DLGR. */ - unsigned __int128 n = (unsigned __int128)n1 << 64 | n0; - asm("dlgr %0, %1" : "+r"(n) : "r"(d)); - *r = n >> 64; - return n; -#elif defined(_ARCH_PPC64) && defined(_ARCH_PWR7) - /* From Power ISA 2.06, programming note for divdeu. */ - uint64_t q1, q2, Q, r1, r2, R; - asm("divdeu %0,%2,%4; divdu %1,%3,%4" - : "=&r"(q1), "=r"(q2) - : "r"(n1), "r"(n0), "r"(d)); - r1 = -(q1 * d); /* low part of (n1<<64) - (q1 * d) */ - r2 = n0 - (q2 * d); - Q = q1 + q2; - R = r1 + r2; - if (R >= d || R < r2) { /* overflow implies R > d */ - Q += 1; - R -= d; - } - *r = R; - return Q; -#else - uint64_t d0, d1, q0, q1, r1, r0, m; - - d0 = (uint32_t)d; - d1 = d >> 32; - - r1 = n1 % d1; - q1 = n1 / d1; - m = q1 * d0; - r1 = (r1 << 32) | (n0 >> 32); - if (r1 < m) { - q1 -= 1; - r1 += d; - if (r1 >= d) { - if (r1 < m) { - q1 -= 1; - r1 += d; - } - } - } - r1 -= m; - - r0 = r1 % d1; - q0 = r1 / d1; - m = q0 * d0; - r0 = (r0 << 32) | (uint32_t)n0; - if (r0 < m) { - q0 -= 1; - r0 += d; - if (r0 >= d) { - if (r0 < m) { - q0 -= 1; - r0 += d; - } - } - } - r0 -= m; - - *r = r0; - return (q1 << 32) | q0; -#endif -} - /*---------------------------------------------------------------------------- | Returns an approximation to the square root of the 32-bit significand given | by `a'. Considered as an integer, `a' must be at least 2^31. If bit 0 of diff --git a/include/qemu/host-utils.h b/include/qemu/host-utils.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/host-utils.h +++ b/include/qemu/host-utils.h @@ -XXX,XX +XXX,XX @@ * THE SOFTWARE. */ +/* Portions of this work are licensed under the terms of the GNU GPL, + * version 2 or later. See the COPYING file in the top-level directory. + */ + #ifndef HOST_UTILS_H #define HOST_UTILS_H @@ -XXX,XX +XXX,XX @@ void urshift(uint64_t *plow, uint64_t *phigh, int32_t shift); */ void ulshift(uint64_t *plow, uint64_t *phigh, int32_t shift, bool *overflow); +/* From the GNU Multi Precision Library - longlong.h __udiv_qrnnd + * (https://gmplib.org/repo/gmp/file/tip/longlong.h) + * + * Licensed under the GPLv2/LGPLv3 + */ +static inline uint64_t udiv_qrnnd(uint64_t *r, uint64_t n1, + uint64_t n0, uint64_t d) +{ +#if defined(__x86_64__) + uint64_t q; + asm("divq %4" : "=a"(q), "=d"(*r) : "0"(n0), "1"(n1), "rm"(d)); + return q; +#elif defined(__s390x__) && !defined(__clang__) + /* Need to use a TImode type to get an even register pair for DLGR. */ + unsigned __int128 n = (unsigned __int128)n1 << 64 | n0; + asm("dlgr %0, %1" : "+r"(n) : "r"(d)); + *r = n >> 64; + return n; +#elif defined(_ARCH_PPC64) && defined(_ARCH_PWR7) + /* From Power ISA 2.06, programming note for divdeu. */ + uint64_t q1, q2, Q, r1, r2, R; + asm("divdeu %0,%2,%4; divdu %1,%3,%4" + : "=&r"(q1), "=r"(q2) + : "r"(n1), "r"(n0), "r"(d)); + r1 = -(q1 * d); /* low part of (n1<<64) - (q1 * d) */ + r2 = n0 - (q2 * d); + Q = q1 + q2; + R = r1 + r2; + if (R >= d || R < r2) { /* overflow implies R > d */ + Q += 1; + R -= d; + } + *r = R; + return Q; +#else + uint64_t d0, d1, q0, q1, r1, r0, m; + + d0 = (uint32_t)d; + d1 = d >> 32; + + r1 = n1 % d1; + q1 = n1 / d1; + m = q1 * d0; + r1 = (r1 << 32) | (n0 >> 32); + if (r1 < m) { + q1 -= 1; + r1 += d; + if (r1 >= d) { + if (r1 < m) { + q1 -= 1; + r1 += d; + } + } + } + r1 -= m; + + r0 = r1 % d1; + q0 = r1 / d1; + m = q0 * d0; + r0 = (r0 << 32) | (uint32_t)n0; + if (r0 < m) { + q0 -= 1; + r0 += d; + if (r0 >= d) { + if (r0 < m) { + q0 -= 1; + r0 += d; + } + } + } + r0 -= m; + + *r = r0; + return (q1 << 32) | q0; +#endif +} + #endif -- 2.25.1 From: Luis Pires <luis.pires@eldorado.org.br> These will be used to implement new decimal floating point instructions from Power ISA 3.1. The remainder is now returned directly by divu128/divs128, freeing up phigh to receive the high 64 bits of the quotient. Signed-off-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20211025191154.350831-4-luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/hw/clock.h | 6 +- include/qemu/host-utils.h | 20 ++++-- target/ppc/int_helper.c | 9 +-- util/host-utils.c | 133 +++++++++++++++++++++++++------------- 4 files changed, 108 insertions(+), 60 deletions(-) diff --git a/include/hw/clock.h b/include/hw/clock.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/clock.h +++ b/include/hw/clock.h @@ -XXX,XX +XXX,XX @@ static inline uint64_t clock_ns_to_ticks(const Clock *clk, uint64_t ns) if (clk->period == 0) { return 0; } - /* - * BUG: when CONFIG_INT128 is not defined, the current implementation of - * divu128 does not return a valid truncated quotient, so the result will - * be wrong. - */ + divu128(&lo, &hi, clk->period); return lo; } diff --git a/include/qemu/host-utils.h b/include/qemu/host-utils.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/host-utils.h +++ b/include/qemu/host-utils.h @@ -XXX,XX +XXX,XX @@ static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c) return (__int128_t)a * b / c; } -static inline void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) +static inline uint64_t divu128(uint64_t *plow, uint64_t *phigh, + uint64_t divisor) { __uint128_t dividend = ((__uint128_t)*phigh << 64) | *plow; __uint128_t result = dividend / divisor; + *plow = result; - *phigh = dividend % divisor; + *phigh = result >> 64; + return dividend % divisor; } -static inline void divs128(int64_t *plow, int64_t *phigh, int64_t divisor) +static inline int64_t divs128(uint64_t *plow, int64_t *phigh, + int64_t divisor) { - __int128_t dividend = ((__int128_t)*phigh << 64) | (uint64_t)*plow; + __int128_t dividend = ((__int128_t)*phigh << 64) | *plow; __int128_t result = dividend / divisor; + *plow = result; - *phigh = dividend % divisor; + *phigh = result >> 64; + return dividend % divisor; } #else void muls64(uint64_t *plow, uint64_t *phigh, int64_t a, int64_t b); void mulu64(uint64_t *plow, uint64_t *phigh, uint64_t a, uint64_t b); -void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor); -void divs128(int64_t *plow, int64_t *phigh, int64_t divisor); +uint64_t divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor); +int64_t divs128(uint64_t *plow, int64_t *phigh, int64_t divisor); static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c) { diff --git a/target/ppc/int_helper.c b/target/ppc/int_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/int_helper.c +++ b/target/ppc/int_helper.c @@ -XXX,XX +XXX,XX @@ uint64_t helper_divdeu(CPUPPCState *env, uint64_t ra, uint64_t rb, uint32_t oe) uint64_t helper_divde(CPUPPCState *env, uint64_t rau, uint64_t rbu, uint32_t oe) { - int64_t rt = 0; + uint64_t rt = 0; int64_t ra = (int64_t)rau; int64_t rb = (int64_t)rbu; int overflow = 0; @@ -XXX,XX +XXX,XX @@ uint32_t helper_bcdcfsq(ppc_avr_t *r, ppc_avr_t *b, uint32_t ps) int cr; uint64_t lo_value; uint64_t hi_value; + uint64_t rem; ppc_avr_t ret = { .u64 = { 0, 0 } }; if (b->VsrSD(0) < 0) { @@ -XXX,XX +XXX,XX @@ uint32_t helper_bcdcfsq(ppc_avr_t *r, ppc_avr_t *b, uint32_t ps) * In that case, we leave r unchanged. */ } else { - divu128(&lo_value, &hi_value, 1000000000000000ULL); + rem = divu128(&lo_value, &hi_value, 1000000000000000ULL); - for (i = 1; i < 16; hi_value /= 10, i++) { - bcd_put_digit(&ret, hi_value % 10, i); + for (i = 1; i < 16; rem /= 10, i++) { + bcd_put_digit(&ret, rem % 10, i); } for (; i < 32; lo_value /= 10, i++) { diff --git a/util/host-utils.c b/util/host-utils.c index XXXXXXX..XXXXXXX 100644 --- a/util/host-utils.c +++ b/util/host-utils.c @@ -XXX,XX +XXX,XX @@ void muls64 (uint64_t *plow, uint64_t *phigh, int64_t a, int64_t b) } /* - * Unsigned 128-by-64 division. Returns quotient via plow and - * remainder via phigh. - * The result must fit in 64 bits (plow) - otherwise, the result - * is undefined. - * This function will cause a division by zero if passed a zero divisor. + * Unsigned 128-by-64 division. + * Returns the remainder. + * Returns quotient via plow and phigh. + * Also returns the remainder via the function return value. */ -void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) +uint64_t divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) { uint64_t dhi = *phigh; uint64_t dlo = *plow; - unsigned i; - uint64_t carry = 0; + uint64_t rem, dhighest; + int sh; if (divisor == 0 || dhi == 0) { *plow = dlo / divisor; - *phigh = dlo % divisor; + *phigh = 0; + return dlo % divisor; } else { + sh = clz64(divisor); - for (i = 0; i < 64; i++) { - carry = dhi >> 63; - dhi = (dhi << 1) | (dlo >> 63); - if (carry || (dhi >= divisor)) { - dhi -= divisor; - carry = 1; - } else { - carry = 0; + if (dhi < divisor) { + if (sh != 0) { + /* normalize the divisor, shifting the dividend accordingly */ + divisor <<= sh; + dhi = (dhi << sh) | (dlo >> (64 - sh)); + dlo <<= sh; } - dlo = (dlo << 1) | carry; + + *phigh = 0; + *plow = udiv_qrnnd(&rem, dhi, dlo, divisor); + } else { + if (sh != 0) { + /* normalize the divisor, shifting the dividend accordingly */ + divisor <<= sh; + dhighest = dhi >> (64 - sh); + dhi = (dhi << sh) | (dlo >> (64 - sh)); + dlo <<= sh; + + *phigh = udiv_qrnnd(&dhi, dhighest, dhi, divisor); + } else { + /** + * dhi >= divisor + * Since the MSB of divisor is set (sh == 0), + * (dhi - divisor) < divisor + * + * Thus, the high part of the quotient is 1, and we can + * calculate the low part with a single call to udiv_qrnnd + * after subtracting divisor from dhi + */ + dhi -= divisor; + *phigh = 1; + } + + *plow = udiv_qrnnd(&rem, dhi, dlo, divisor); } - *plow = dlo; - *phigh = dhi; + /* + * since the dividend/divisor might have been normalized, + * the remainder might also have to be shifted back + */ + return rem >> sh; } } /* - * Signed 128-by-64 division. Returns quotient via plow and - * remainder via phigh. - * The result must fit in 64 bits (plow) - otherwise, the result - * is undefined. - * This function will cause a division by zero if passed a zero divisor. + * Signed 128-by-64 division. + * Returns quotient via plow and phigh. + * Also returns the remainder via the function return value. */ -void divs128(int64_t *plow, int64_t *phigh, int64_t divisor) +int64_t divs128(uint64_t *plow, int64_t *phigh, int64_t divisor) { - int sgn_dvdnd = *phigh < 0; - int sgn_divsr = divisor < 0; + bool neg_quotient = false, neg_remainder = false; + uint64_t unsig_hi = *phigh, unsig_lo = *plow; + uint64_t rem; - if (sgn_dvdnd) { - *plow = ~(*plow); - *phigh = ~(*phigh); - if (*plow == (int64_t)-1) { + if (*phigh < 0) { + neg_quotient = !neg_quotient; + neg_remainder = !neg_remainder; + + if (unsig_lo == 0) { + unsig_hi = -unsig_hi; + } else { + unsig_hi = ~unsig_hi; + unsig_lo = -unsig_lo; + } + } + + if (divisor < 0) { + neg_quotient = !neg_quotient; + + divisor = -divisor; + } + + rem = divu128(&unsig_lo, &unsig_hi, (uint64_t)divisor); + + if (neg_quotient) { + if (unsig_lo == 0) { + *phigh = -unsig_hi; *plow = 0; - (*phigh)++; - } else { - (*plow)++; - } + } else { + *phigh = ~unsig_hi; + *plow = -unsig_lo; + } + } else { + *phigh = unsig_hi; + *plow = unsig_lo; } - if (sgn_divsr) { - divisor = 0 - divisor; - } - - divu128((uint64_t *)plow, (uint64_t *)phigh, (uint64_t)divisor); - - if (sgn_dvdnd ^ sgn_divsr) { - *plow = 0 - *plow; + if (neg_remainder) { + return -rem; + } else { + return rem; } } #endif -- 2.25.1 From: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20211025191154.350831-5-luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tests/unit/test-div128.c | 197 +++++++++++++++++++++++++++++++++++++++ tests/unit/meson.build | 1 + 2 files changed, 198 insertions(+) create mode 100644 tests/unit/test-div128.c diff --git a/tests/unit/test-div128.c b/tests/unit/test-div128.c new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/unit/test-div128.c @@ -XXX,XX +XXX,XX @@ +/* + * Test 128-bit division functions + * + * Copyright (c) 2021 Instituto de Pesquisas Eldorado (eldorado.org.br) + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ + +#include "qemu/osdep.h" +#include "qemu/host-utils.h" + +typedef struct { + uint64_t high; + uint64_t low; + uint64_t rhigh; + uint64_t rlow; + uint64_t divisor; + uint64_t remainder; +} test_data_unsigned; + +typedef struct { + int64_t high; + uint64_t low; + int64_t rhigh; + uint64_t rlow; + int64_t divisor; + int64_t remainder; +} test_data_signed; + +static const test_data_unsigned test_table_unsigned[] = { + /* Dividend fits in 64 bits */ + { 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL}, + { 0x0000000000000000ULL, 0x0000000000000001ULL, + 0x0000000000000000ULL, 0x0000000000000001ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL}, + { 0x0000000000000000ULL, 0x0000000000000003ULL, + 0x0000000000000000ULL, 0x0000000000000001ULL, + 0x0000000000000002ULL, 0x0000000000000001ULL}, + { 0x0000000000000000ULL, 0x8000000000000000ULL, + 0x0000000000000000ULL, 0x8000000000000000ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL}, + { 0x0000000000000000ULL, 0xa000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000002ULL, + 0x4000000000000000ULL, 0x2000000000000000ULL}, + { 0x0000000000000000ULL, 0x8000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000001ULL, + 0x8000000000000000ULL, 0x0000000000000000ULL}, + + /* Dividend > 64 bits, with MSB 0 */ + { 0x123456789abcdefeULL, 0xefedcba987654321ULL, + 0x123456789abcdefeULL, 0xefedcba987654321ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL}, + { 0x123456789abcdefeULL, 0xefedcba987654321ULL, + 0x0000000000000001ULL, 0x000000000000000dULL, + 0x123456789abcdefeULL, 0x03456789abcdf03bULL}, + { 0x123456789abcdefeULL, 0xefedcba987654321ULL, + 0x0123456789abcdefULL, 0xeefedcba98765432ULL, + 0x0000000000000010ULL, 0x0000000000000001ULL}, + + /* Dividend > 64 bits, with MSB 1 */ + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x0000000000000001ULL, 0x0000000000000000ULL}, + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x0000000000000001ULL, 0x0000000000000000ULL, + 0xfeeddccbbaa99887ULL, 0x766554433221100fULL}, + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x0feeddccbbaa9988ULL, 0x7766554433221100ULL, + 0x0000000000000010ULL, 0x000000000000000fULL}, + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x000000000000000eULL, 0x00f0f0f0f0f0f35aULL, + 0x123456789abcdefeULL, 0x0f8922bc55ef90c3ULL}, + + /** + * Divisor == 64 bits, with MSB 1 + * and high 64 bits of dividend >= divisor + * (for testing normalization) + */ + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x0000000000000001ULL, 0x0000000000000000ULL, + 0xfeeddccbbaa99887ULL, 0x766554433221100fULL}, + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x0000000000000001ULL, 0xfddbb9977553310aULL, + 0x8000000000000001ULL, 0x78899aabbccddf05ULL}, + + /* Dividend > 64 bits, divisor almost as big */ + { 0x0000000000000001ULL, 0x23456789abcdef01ULL, + 0x0000000000000000ULL, 0x000000000000000fULL, + 0x123456789abcdefeULL, 0x123456789abcde1fULL}, +}; + +static const test_data_signed test_table_signed[] = { + /* Positive dividend, positive/negative divisors */ + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0x0000000000000000LL, 0x0000000000bc614eULL, + 0x0000000000000001LL, 0x0000000000000000LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0xffffffffffffffffLL, 0x0000000000000000LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0x0000000000000000LL, 0x00000000005e30a7ULL, + 0x0000000000000002LL, 0x0000000000000000LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0xffffffffffffffffLL, 0xffffffffffa1cf59ULL, + 0xfffffffffffffffeLL, 0x0000000000000000LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0x0000000000000000LL, 0x0000000000178c29ULL, + 0x0000000000000008LL, 0x0000000000000006LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0xffffffffffffffffLL, 0xffffffffffe873d7ULL, + 0xfffffffffffffff8LL, 0x0000000000000006LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0x0000000000000000LL, 0x000000000000550dULL, + 0x0000000000000237LL, 0x0000000000000183LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0xffffffffffffffffLL, 0xffffffffffffaaf3ULL, + 0xfffffffffffffdc9LL, 0x0000000000000183LL}, + + /* Negative dividend, positive/negative divisors */ + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0x0000000000000001LL, 0x0000000000000000LL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0x0000000000000000LL, 0x0000000000bc614eULL, + 0xffffffffffffffffLL, 0x0000000000000000LL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0xffffffffffffffffLL, 0xffffffffffa1cf59ULL, + 0x0000000000000002LL, 0x0000000000000000LL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0x0000000000000000LL, 0x00000000005e30a7ULL, + 0xfffffffffffffffeLL, 0x0000000000000000LL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0xffffffffffffffffLL, 0xffffffffffe873d7ULL, + 0x0000000000000008LL, 0xfffffffffffffffaLL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0x0000000000000000LL, 0x0000000000178c29ULL, + 0xfffffffffffffff8LL, 0xfffffffffffffffaLL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0xffffffffffffffffLL, 0xffffffffffffaaf3ULL, + 0x0000000000000237LL, 0xfffffffffffffe7dLL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0x0000000000000000LL, 0x000000000000550dULL, + 0xfffffffffffffdc9LL, 0xfffffffffffffe7dLL}, +}; + +static void test_divu128(void) +{ + int i; + uint64_t rem; + test_data_unsigned tmp; + + for (i = 0; i < ARRAY_SIZE(test_table_unsigned); ++i) { + tmp = test_table_unsigned[i]; + + rem = divu128(&tmp.low, &tmp.high, tmp.divisor); + g_assert_cmpuint(tmp.low, ==, tmp.rlow); + g_assert_cmpuint(tmp.high, ==, tmp.rhigh); + g_assert_cmpuint(rem, ==, tmp.remainder); + } +} + +static void test_divs128(void) +{ + int i; + int64_t rem; + test_data_signed tmp; + + for (i = 0; i < ARRAY_SIZE(test_table_signed); ++i) { + tmp = test_table_signed[i]; + + rem = divs128(&tmp.low, &tmp.high, tmp.divisor); + g_assert_cmpuint(tmp.low, ==, tmp.rlow); + g_assert_cmpuint(tmp.high, ==, tmp.rhigh); + g_assert_cmpuint(rem, ==, tmp.remainder); + } +} + +int main(int argc, char **argv) +{ + g_test_init(&argc, &argv, NULL); + g_test_add_func("/host-utils/test_divu128", test_divu128); + g_test_add_func("/host-utils/test_divs128", test_divs128); + return g_test_run(); +} diff --git a/tests/unit/meson.build b/tests/unit/meson.build index XXXXXXX..XXXXXXX 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -XXX,XX +XXX,XX @@ tests = { # all code tested by test-x86-cpuid is inside topology.h 'test-x86-cpuid': [], 'test-cutils': [], + 'test-div128': [], 'test-shift128': [], 'test-mul64': [], # all code tested by test-int128 is inside int128.h -- 2.25.1 Prepare for tracking different masks by renaming this one. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 142 +++++++++++++++++++++++++------------------------ 1 file changed, 72 insertions(+), 70 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { TCGTemp *prev_copy; TCGTemp *next_copy; uint64_t val; - uint64_t mask; + uint64_t z_mask; /* mask bit is 0 if and only if value bit is 0 */ } TempOptInfo; static inline TempOptInfo *ts_info(TCGTemp *ts) @@ -XXX,XX +XXX,XX @@ static void reset_ts(TCGTemp *ts) ti->next_copy = ts; ti->prev_copy = ts; ti->is_const = false; - ti->mask = -1; + ti->z_mask = -1; } static void reset_temp(TCGArg arg) @@ -XXX,XX +XXX,XX @@ static void init_ts_info(TCGTempSet *temps_used, TCGTemp *ts) if (ts->kind == TEMP_CONST) { ti->is_const = true; ti->val = ts->val; - ti->mask = ts->val; + ti->z_mask = ts->val; if (TCG_TARGET_REG_BITS > 32 && ts->type == TCG_TYPE_I32) { /* High bits of a 32-bit quantity are garbage. */ - ti->mask |= ~0xffffffffull; + ti->z_mask |= ~0xffffffffull; } } else { ti->is_const = false; - ti->mask = -1; + ti->z_mask = -1; } } @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) const TCGOpDef *def; TempOptInfo *di; TempOptInfo *si; - uint64_t mask; + uint64_t z_mask; TCGOpcode new_op; if (ts_are_copies(dst_ts, src_ts)) { @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) op->args[0] = dst; op->args[1] = src; - mask = si->mask; + z_mask = si->z_mask; if (TCG_TARGET_REG_BITS > 32 && new_op == INDEX_op_mov_i32) { /* High bits of the destination are now garbage. */ - mask |= ~0xffffffffull; + z_mask |= ~0xffffffffull; } - di->mask = mask; + di->z_mask = z_mask; if (src_ts->type == dst_ts->type) { TempOptInfo *ni = ts_info(si->next_copy); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { - uint64_t mask, partmask, affected, tmp; + uint64_t z_mask, partmask, affected, tmp; int nb_oargs, nb_iargs; TCGOpcode opc = op->opc; const TCGOpDef *def = &tcg_op_defs[opc]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* Simplify using known-zero bits. Currently only ops with a single output argument is supported. */ - mask = -1; + z_mask = -1; affected = -1; switch (opc) { CASE_OP_32_64(ext8s): - if ((arg_info(op->args[1])->mask & 0x80) != 0) { + if ((arg_info(op->args[1])->z_mask & 0x80) != 0) { break; } QEMU_FALLTHROUGH; CASE_OP_32_64(ext8u): - mask = 0xff; + z_mask = 0xff; goto and_const; CASE_OP_32_64(ext16s): - if ((arg_info(op->args[1])->mask & 0x8000) != 0) { + if ((arg_info(op->args[1])->z_mask & 0x8000) != 0) { break; } QEMU_FALLTHROUGH; CASE_OP_32_64(ext16u): - mask = 0xffff; + z_mask = 0xffff; goto and_const; case INDEX_op_ext32s_i64: - if ((arg_info(op->args[1])->mask & 0x80000000) != 0) { + if ((arg_info(op->args[1])->z_mask & 0x80000000) != 0) { break; } QEMU_FALLTHROUGH; case INDEX_op_ext32u_i64: - mask = 0xffffffffU; + z_mask = 0xffffffffU; goto and_const; CASE_OP_32_64(and): - mask = arg_info(op->args[2])->mask; + z_mask = arg_info(op->args[2])->z_mask; if (arg_is_const(op->args[2])) { and_const: - affected = arg_info(op->args[1])->mask & ~mask; + affected = arg_info(op->args[1])->z_mask & ~z_mask; } - mask = arg_info(op->args[1])->mask & mask; + z_mask = arg_info(op->args[1])->z_mask & z_mask; break; case INDEX_op_ext_i32_i64: - if ((arg_info(op->args[1])->mask & 0x80000000) != 0) { + if ((arg_info(op->args[1])->z_mask & 0x80000000) != 0) { break; } QEMU_FALLTHROUGH; case INDEX_op_extu_i32_i64: /* We do not compute affected as it is a size changing op. */ - mask = (uint32_t)arg_info(op->args[1])->mask; + z_mask = (uint32_t)arg_info(op->args[1])->z_mask; break; CASE_OP_32_64(andc): /* Known-zeros does not imply known-ones. Therefore unless op->args[2] is constant, we can't infer anything from it. */ if (arg_is_const(op->args[2])) { - mask = ~arg_info(op->args[2])->mask; + z_mask = ~arg_info(op->args[2])->z_mask; goto and_const; } /* But we certainly know nothing outside args[1] may be set. */ - mask = arg_info(op->args[1])->mask; + z_mask = arg_info(op->args[1])->z_mask; break; case INDEX_op_sar_i32: if (arg_is_const(op->args[2])) { tmp = arg_info(op->args[2])->val & 31; - mask = (int32_t)arg_info(op->args[1])->mask >> tmp; + z_mask = (int32_t)arg_info(op->args[1])->z_mask >> tmp; } break; case INDEX_op_sar_i64: if (arg_is_const(op->args[2])) { tmp = arg_info(op->args[2])->val & 63; - mask = (int64_t)arg_info(op->args[1])->mask >> tmp; + z_mask = (int64_t)arg_info(op->args[1])->z_mask >> tmp; } break; case INDEX_op_shr_i32: if (arg_is_const(op->args[2])) { tmp = arg_info(op->args[2])->val & 31; - mask = (uint32_t)arg_info(op->args[1])->mask >> tmp; + z_mask = (uint32_t)arg_info(op->args[1])->z_mask >> tmp; } break; case INDEX_op_shr_i64: if (arg_is_const(op->args[2])) { tmp = arg_info(op->args[2])->val & 63; - mask = (uint64_t)arg_info(op->args[1])->mask >> tmp; + z_mask = (uint64_t)arg_info(op->args[1])->z_mask >> tmp; } break; case INDEX_op_extrl_i64_i32: - mask = (uint32_t)arg_info(op->args[1])->mask; + z_mask = (uint32_t)arg_info(op->args[1])->z_mask; break; case INDEX_op_extrh_i64_i32: - mask = (uint64_t)arg_info(op->args[1])->mask >> 32; + z_mask = (uint64_t)arg_info(op->args[1])->z_mask >> 32; break; CASE_OP_32_64(shl): if (arg_is_const(op->args[2])) { tmp = arg_info(op->args[2])->val & (TCG_TARGET_REG_BITS - 1); - mask = arg_info(op->args[1])->mask << tmp; + z_mask = arg_info(op->args[1])->z_mask << tmp; } break; CASE_OP_32_64(neg): /* Set to 1 all bits to the left of the rightmost. */ - mask = -(arg_info(op->args[1])->mask - & -arg_info(op->args[1])->mask); + z_mask = -(arg_info(op->args[1])->z_mask + & -arg_info(op->args[1])->z_mask); break; CASE_OP_32_64(deposit): - mask = deposit64(arg_info(op->args[1])->mask, - op->args[3], op->args[4], - arg_info(op->args[2])->mask); + z_mask = deposit64(arg_info(op->args[1])->z_mask, + op->args[3], op->args[4], + arg_info(op->args[2])->z_mask); break; CASE_OP_32_64(extract): - mask = extract64(arg_info(op->args[1])->mask, - op->args[2], op->args[3]); + z_mask = extract64(arg_info(op->args[1])->z_mask, + op->args[2], op->args[3]); if (op->args[2] == 0) { - affected = arg_info(op->args[1])->mask & ~mask; + affected = arg_info(op->args[1])->z_mask & ~z_mask; } break; CASE_OP_32_64(sextract): - mask = sextract64(arg_info(op->args[1])->mask, - op->args[2], op->args[3]); - if (op->args[2] == 0 && (tcg_target_long)mask >= 0) { - affected = arg_info(op->args[1])->mask & ~mask; + z_mask = sextract64(arg_info(op->args[1])->z_mask, + op->args[2], op->args[3]); + if (op->args[2] == 0 && (tcg_target_long)z_mask >= 0) { + affected = arg_info(op->args[1])->z_mask & ~z_mask; } break; CASE_OP_32_64(or): CASE_OP_32_64(xor): - mask = arg_info(op->args[1])->mask | arg_info(op->args[2])->mask; + z_mask = arg_info(op->args[1])->z_mask + | arg_info(op->args[2])->z_mask; break; case INDEX_op_clz_i32: case INDEX_op_ctz_i32: - mask = arg_info(op->args[2])->mask | 31; + z_mask = arg_info(op->args[2])->z_mask | 31; break; case INDEX_op_clz_i64: case INDEX_op_ctz_i64: - mask = arg_info(op->args[2])->mask | 63; + z_mask = arg_info(op->args[2])->z_mask | 63; break; case INDEX_op_ctpop_i32: - mask = 32 | 31; + z_mask = 32 | 31; break; case INDEX_op_ctpop_i64: - mask = 64 | 63; + z_mask = 64 | 63; break; CASE_OP_32_64(setcond): case INDEX_op_setcond2_i32: - mask = 1; + z_mask = 1; break; CASE_OP_32_64(movcond): - mask = arg_info(op->args[3])->mask | arg_info(op->args[4])->mask; + z_mask = arg_info(op->args[3])->z_mask + | arg_info(op->args[4])->z_mask; break; CASE_OP_32_64(ld8u): - mask = 0xff; + z_mask = 0xff; break; CASE_OP_32_64(ld16u): - mask = 0xffff; + z_mask = 0xffff; break; case INDEX_op_ld32u_i64: - mask = 0xffffffffu; + z_mask = 0xffffffffu; break; CASE_OP_32_64(qemu_ld): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) MemOpIdx oi = op->args[nb_oargs + nb_iargs]; MemOp mop = get_memop(oi); if (!(mop & MO_SIGN)) { - mask = (2ULL << ((8 << (mop & MO_SIZE)) - 1)) - 1; + z_mask = (2ULL << ((8 << (mop & MO_SIZE)) - 1)) - 1; } } break; CASE_OP_32_64(bswap16): - mask = arg_info(op->args[1])->mask; - if (mask <= 0xffff) { + z_mask = arg_info(op->args[1])->z_mask; + if (z_mask <= 0xffff) { op->args[2] |= TCG_BSWAP_IZ; } - mask = bswap16(mask); + z_mask = bswap16(z_mask); switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { case TCG_BSWAP_OZ: break; case TCG_BSWAP_OS: - mask = (int16_t)mask; + z_mask = (int16_t)z_mask; break; default: /* undefined high bits */ - mask |= MAKE_64BIT_MASK(16, 48); + z_mask |= MAKE_64BIT_MASK(16, 48); break; } break; case INDEX_op_bswap32_i64: - mask = arg_info(op->args[1])->mask; - if (mask <= 0xffffffffu) { + z_mask = arg_info(op->args[1])->z_mask; + if (z_mask <= 0xffffffffu) { op->args[2] |= TCG_BSWAP_IZ; } - mask = bswap32(mask); + z_mask = bswap32(z_mask); switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { case TCG_BSWAP_OZ: break; case TCG_BSWAP_OS: - mask = (int32_t)mask; + z_mask = (int32_t)z_mask; break; default: /* undefined high bits */ - mask |= MAKE_64BIT_MASK(32, 32); + z_mask |= MAKE_64BIT_MASK(32, 32); break; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* 32-bit ops generate 32-bit results. For the result is zero test below, we can ignore high bits, but for further optimizations we need to record that the high bits contain garbage. */ - partmask = mask; + partmask = z_mask; if (!(def->flags & TCG_OPF_64BIT)) { - mask |= ~(tcg_target_ulong)0xffffffffu; + z_mask |= ~(tcg_target_ulong)0xffffffffu; partmask &= 0xffffffffu; affected &= 0xffffffffu; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) vs the high word of the input. */ do_setcond_high: reset_temp(op->args[0]); - arg_info(op->args[0])->mask = 1; + arg_info(op->args[0])->z_mask = 1; op->opc = INDEX_op_setcond_i32; op->args[1] = op->args[2]; op->args[2] = op->args[4]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } do_setcond_low: reset_temp(op->args[0]); - arg_info(op->args[0])->mask = 1; + arg_info(op->args[0])->z_mask = 1; op->opc = INDEX_op_setcond_i32; op->args[2] = op->args[3]; op->args[3] = op->args[5]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* Default case: we know nothing about operation (or were unable to compute the operation result) so no propagation is done. We trash everything if the operation is the end of a basic - block, otherwise we only trash the output args. "mask" is + block, otherwise we only trash the output args. "z_mask" is the non-zero bits mask for the first output arg. */ if (def->flags & TCG_OPF_BB_END) { memset(&temps_used, 0, sizeof(temps_used)); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* Save the corresponding known-zero bits mask for the first output argument (only one supported so far). */ if (i == 0) { - arg_info(op->args[i])->mask = mask; + arg_info(op->args[i])->z_mask = z_mask; } } } -- 2.25.1 Provide what will become a larger context for splitting the very large tcg_optimize function. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 77 ++++++++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { uint64_t z_mask; /* mask bit is 0 if and only if value bit is 0 */ } TempOptInfo; +typedef struct OptContext { + TCGTempSet temps_used; +} OptContext; + static inline TempOptInfo *ts_info(TCGTemp *ts) { return ts->state_ptr; @@ -XXX,XX +XXX,XX @@ static void reset_temp(TCGArg arg) } /* Initialize and activate a temporary. */ -static void init_ts_info(TCGTempSet *temps_used, TCGTemp *ts) +static void init_ts_info(OptContext *ctx, TCGTemp *ts) { size_t idx = temp_idx(ts); TempOptInfo *ti; - if (test_bit(idx, temps_used->l)) { + if (test_bit(idx, ctx->temps_used.l)) { return; } - set_bit(idx, temps_used->l); + set_bit(idx, ctx->temps_used.l); ti = ts->state_ptr; if (ti == NULL) { @@ -XXX,XX +XXX,XX @@ static void init_ts_info(TCGTempSet *temps_used, TCGTemp *ts) } } -static void init_arg_info(TCGTempSet *temps_used, TCGArg arg) +static void init_arg_info(OptContext *ctx, TCGArg arg) { - init_ts_info(temps_used, arg_temp(arg)); + init_ts_info(ctx, arg_temp(arg)); } static TCGTemp *find_better_copy(TCGContext *s, TCGTemp *ts) @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) } } -static void tcg_opt_gen_movi(TCGContext *s, TCGTempSet *temps_used, +static void tcg_opt_gen_movi(TCGContext *s, OptContext *ctx, TCGOp *op, TCGArg dst, uint64_t val) { const TCGOpDef *def = &tcg_op_defs[op->opc]; @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_movi(TCGContext *s, TCGTempSet *temps_used, /* Convert movi to mov with constant temp. */ tv = tcg_constant_internal(type, val); - init_ts_info(temps_used, tv); + init_ts_info(ctx, tv); tcg_opt_gen_mov(s, op, dst, temp_arg(tv)); } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) { int nb_temps, nb_globals, i; TCGOp *op, *op_next, *prev_mb = NULL; - TCGTempSet temps_used; + OptContext ctx = {}; /* Array VALS has an element for each temp. If this temp holds a constant then its value is kept in VALS' element. @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) nb_temps = s->nb_temps; nb_globals = s->nb_globals; - memset(&temps_used, 0, sizeof(temps_used)); for (i = 0; i < nb_temps; ++i) { s->temps[i].state_ptr = NULL; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) for (i = 0; i < nb_oargs + nb_iargs; i++) { TCGTemp *ts = arg_temp(op->args[i]); if (ts) { - init_ts_info(&temps_used, ts); + init_ts_info(&ctx, ts); } } } else { nb_oargs = def->nb_oargs; nb_iargs = def->nb_iargs; for (i = 0; i < nb_oargs + nb_iargs; i++) { - init_arg_info(&temps_used, op->args[i]); + init_arg_info(&ctx, op->args[i]); } } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(rotr): if (arg_is_const(op->args[1]) && arg_info(op->args[1])->val == 0) { - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], 0); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (partmask == 0) { tcg_debug_assert(nb_oargs == 1); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], 0); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); continue; } if (affected == 0) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(mulsh): if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == 0) { - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], 0); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(sub): CASE_OP_32_64_VEC(xor): if (args_are_copies(op->args[1], op->args[2])) { - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], 0); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = arg_info(op->args[1])->val; tmp = dup_const(TCGOP_VECE(op), tmp); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_dup2_vec: assert(TCG_TARGET_REG_BITS == 32); if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], + tcg_opt_gen_movi(s, &ctx, op, op->args[0], deposit64(arg_info(op->args[1])->val, 32, 32, arg_info(op->args[2])->val)); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_extrh_i64_i32: if (arg_is_const(op->args[1])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, 0); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, op->args[2]); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, arg_info(op->args[2])->val); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) TCGArg v = arg_info(op->args[1])->val; if (v != 0) { tmp = do_constant_folding(opc, v, 0); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); } else { tcg_opt_gen_mov(s, op, op->args[0], op->args[2]); } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = deposit64(arg_info(op->args[1])->val, op->args[3], op->args[4], arg_info(op->args[2])->val); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = extract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = sextract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = (int32_t)(((uint32_t)v1 >> shr) | ((uint32_t)v2 << (32 - shr))); } - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[3]); if (tmp != 2) { - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[1], op->args[2]); if (tmp != 2) { if (tmp) { - memset(&temps_used, 0, sizeof(temps_used)); + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); op->opc = INDEX_op_br; op->args[0] = op->args[3]; } else { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(s, &temps_used, op, rl, (int32_t)a); - tcg_opt_gen_movi(s, &temps_used, op2, rh, (int32_t)(a >> 32)); + tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)a); + tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(a >> 32)); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(s, &temps_used, op, rl, (int32_t)r); - tcg_opt_gen_movi(s, &temps_used, op2, rh, (int32_t)(r >> 32)); + tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)r); + tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(r >> 32)); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (tmp != 2) { if (tmp) { do_brcond_true: - memset(&temps_used, 0, sizeof(temps_used)); + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); op->opc = INDEX_op_br; op->args[0] = op->args[5]; } else { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* Simplify LT/GE comparisons vs zero to a single compare vs the high word of the input. */ do_brcond_high: - memset(&temps_used, 0, sizeof(temps_used)); + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); op->opc = INDEX_op_brcond_i32; op->args[0] = op->args[1]; op->args[1] = op->args[3]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) goto do_default; } do_brcond_low: - memset(&temps_used, 0, sizeof(temps_used)); + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); op->opc = INDEX_op_brcond_i32; op->args[1] = op->args[2]; op->args[2] = op->args[4]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[5]); if (tmp != 2) { do_setcond_const: - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); } else if ((op->args[5] == TCG_COND_LT || op->args[5] == TCG_COND_GE) && arg_is_const(op->args[3]) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (!(tcg_call_flags(op) & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) { for (i = 0; i < nb_globals; i++) { - if (test_bit(i, temps_used.l)) { + if (test_bit(i, ctx.temps_used.l)) { reset_ts(&s->temps[i]); } } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) block, otherwise we only trash the output args. "z_mask" is the non-zero bits mask for the first output arg. */ if (def->flags & TCG_OPF_BB_END) { - memset(&temps_used, 0, sizeof(temps_used)); + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); } else { do_reset_output: for (i = 0; i < nb_oargs; i++) { -- 2.25.1 Break the final cleanup clause out of the main switch statement. When fully folding an opcode to mov/movi, use "continue" to process the next opcode, else break to fall into the final cleanup. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 190 ++++++++++++++++++++++++------------------------- 1 file changed, 94 insertions(+), 96 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) switch (opc) { CASE_OP_32_64_VEC(mov): tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); - break; + continue; case INDEX_op_dup_vec: if (arg_is_const(op->args[1])) { tmp = arg_info(op->args[1])->val; tmp = dup_const(TCGOP_VECE(op), tmp); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; case INDEX_op_dup2_vec: assert(TCG_TARGET_REG_BITS == 32); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tcg_opt_gen_movi(s, &ctx, op, op->args[0], deposit64(arg_info(op->args[1])->val, 32, 32, arg_info(op->args[2])->val)); - break; + continue; } else if (args_are_copies(op->args[1], op->args[2])) { op->opc = INDEX_op_dup_vec; TCGOP_VECE(op) = MO_32; nb_iargs = 1; } - goto do_default; + break; CASE_OP_32_64(not): CASE_OP_32_64(neg): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, 0); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(bswap16): CASE_OP_32_64(bswap32): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = do_constant_folding(opc, arg_info(op->args[1])->val, op->args[2]); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(add): CASE_OP_32_64(sub): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = do_constant_folding(opc, arg_info(op->args[1])->val, arg_info(op->args[2])->val); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(clz): CASE_OP_32_64(ctz): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } else { tcg_opt_gen_mov(s, op, op->args[0], op->args[2]); } - break; + continue; } - goto do_default; + break; CASE_OP_32_64(deposit): if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[3], op->args[4], arg_info(op->args[2])->val); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(extract): if (arg_is_const(op->args[1])) { tmp = extract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(sextract): if (arg_is_const(op->args[1])) { tmp = sextract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(extract2): if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) ((uint32_t)v2 << (32 - shr))); } tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(setcond): tmp = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[3]); if (tmp != 2) { tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(brcond): tmp = do_constant_folding_cond(opc, op->args[0], op->args[1], op->args[2]); - if (tmp != 2) { - if (tmp) { - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - op->opc = INDEX_op_br; - op->args[0] = op->args[3]; - } else { - tcg_op_remove(s, op); - } + switch (tmp) { + case 0: + tcg_op_remove(s, op); + continue; + case 1: + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); + op->opc = opc = INDEX_op_br; + op->args[0] = op->args[3]; break; } - goto do_default; + break; CASE_OP_32_64(movcond): tmp = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[5]); if (tmp != 2) { tcg_opt_gen_mov(s, op, op->args[0], op->args[4-tmp]); - break; + continue; } if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { uint64_t tv = arg_info(op->args[3])->val; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (fv == 1 && tv == 0) { cond = tcg_invert_cond(cond); } else if (!(tv == 1 && fv == 0)) { - goto do_default; + break; } op->args[3] = cond; op->opc = opc = (opc == INDEX_op_movcond_i32 @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) : INDEX_op_setcond_i64); nb_iargs = 2; } - goto do_default; + break; case INDEX_op_add2_i32: case INDEX_op_sub2_i32: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rh = op->args[1]; tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)a); tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(a >> 32)); - break; + continue; } - goto do_default; + break; case INDEX_op_mulu2_i32: if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rh = op->args[1]; tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)r); tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(r >> 32)); - break; + continue; } - goto do_default; + break; case INDEX_op_brcond2_i32: tmp = do_constant_folding_cond2(&op->args[0], &op->args[2], op->args[4]); - if (tmp != 2) { - if (tmp) { - do_brcond_true: - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - op->opc = INDEX_op_br; - op->args[0] = op->args[5]; - } else { + if (tmp == 0) { do_brcond_false: - tcg_op_remove(s, op); - } - } else if ((op->args[4] == TCG_COND_LT - || op->args[4] == TCG_COND_GE) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == 0 - && arg_is_const(op->args[3]) - && arg_info(op->args[3])->val == 0) { + tcg_op_remove(s, op); + continue; + } + if (tmp == 1) { + do_brcond_true: + op->opc = opc = INDEX_op_br; + op->args[0] = op->args[5]; + break; + } + if ((op->args[4] == TCG_COND_LT || op->args[4] == TCG_COND_GE) + && arg_is_const(op->args[2]) + && arg_info(op->args[2])->val == 0 + && arg_is_const(op->args[3]) + && arg_info(op->args[3])->val == 0) { /* Simplify LT/GE comparisons vs zero to a single compare vs the high word of the input. */ do_brcond_high: - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - op->opc = INDEX_op_brcond_i32; + op->opc = opc = INDEX_op_brcond_i32; op->args[0] = op->args[1]; op->args[1] = op->args[3]; op->args[2] = op->args[4]; op->args[3] = op->args[5]; - } else if (op->args[4] == TCG_COND_EQ) { + break; + } + if (op->args[4] == TCG_COND_EQ) { /* Simplify EQ comparisons where one of the pairs can be simplified. */ tmp = do_constant_folding_cond(INDEX_op_brcond_i32, @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (tmp == 0) { goto do_brcond_false; } else if (tmp != 1) { - goto do_default; + break; } do_brcond_low: memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[1] = op->args[2]; op->args[2] = op->args[4]; op->args[3] = op->args[5]; - } else if (op->args[4] == TCG_COND_NE) { + break; + } + if (op->args[4] == TCG_COND_NE) { /* Simplify NE comparisons where one of the pairs can be simplified. */ tmp = do_constant_folding_cond(INDEX_op_brcond_i32, @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } else if (tmp == 1) { goto do_brcond_true; } - goto do_default; - } else { - goto do_default; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (tmp != 2) { do_setcond_const: tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - } else if ((op->args[5] == TCG_COND_LT - || op->args[5] == TCG_COND_GE) - && arg_is_const(op->args[3]) - && arg_info(op->args[3])->val == 0 - && arg_is_const(op->args[4]) - && arg_info(op->args[4])->val == 0) { + continue; + } + if ((op->args[5] == TCG_COND_LT || op->args[5] == TCG_COND_GE) + && arg_is_const(op->args[3]) + && arg_info(op->args[3])->val == 0 + && arg_is_const(op->args[4]) + && arg_info(op->args[4])->val == 0) { /* Simplify LT/GE comparisons vs zero to a single compare vs the high word of the input. */ do_setcond_high: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[1] = op->args[2]; op->args[2] = op->args[4]; op->args[3] = op->args[5]; - } else if (op->args[5] == TCG_COND_EQ) { + break; + } + if (op->args[5] == TCG_COND_EQ) { /* Simplify EQ comparisons where one of the pairs can be simplified. */ tmp = do_constant_folding_cond(INDEX_op_setcond_i32, @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (tmp == 0) { goto do_setcond_high; } else if (tmp != 1) { - goto do_default; + break; } do_setcond_low: reset_temp(op->args[0]); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->opc = INDEX_op_setcond_i32; op->args[2] = op->args[3]; op->args[3] = op->args[5]; - } else if (op->args[5] == TCG_COND_NE) { + break; + } + if (op->args[5] == TCG_COND_NE) { /* Simplify NE comparisons where one of the pairs can be simplified. */ tmp = do_constant_folding_cond(INDEX_op_setcond_i32, @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } else if (tmp == 1) { goto do_setcond_const; } - goto do_default; - } else { - goto do_default; } break; - case INDEX_op_call: - if (!(tcg_call_flags(op) + default: + break; + } + + /* Some of the folding above can change opc. */ + opc = op->opc; + def = &tcg_op_defs[opc]; + if (def->flags & TCG_OPF_BB_END) { + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); + } else { + if (opc == INDEX_op_call && + !(tcg_call_flags(op) & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) { for (i = 0; i < nb_globals; i++) { if (test_bit(i, ctx.temps_used.l)) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } } } - goto do_reset_output; - default: - do_default: - /* Default case: we know nothing about operation (or were unable - to compute the operation result) so no propagation is done. - We trash everything if the operation is the end of a basic - block, otherwise we only trash the output args. "z_mask" is - the non-zero bits mask for the first output arg. */ - if (def->flags & TCG_OPF_BB_END) { - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - } else { - do_reset_output: - for (i = 0; i < nb_oargs; i++) { - reset_temp(op->args[i]); - /* Save the corresponding known-zero bits mask for the - first output argument (only one supported so far). */ - if (i == 0) { - arg_info(op->args[i])->z_mask = z_mask; - } + for (i = 0; i < nb_oargs; i++) { + reset_temp(op->args[i]); + /* Save the corresponding known-zero bits mask for the + first output argument (only one supported so far). */ + if (i == 0) { + arg_info(op->args[i])->z_mask = z_mask; } } - break; } /* Eliminate duplicate and redundant fence instructions. */ -- 2.25.1 Adjust the interface to take the OptContext parameter instead of TCGContext or both. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 67 +++++++++++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { } TempOptInfo; typedef struct OptContext { + TCGContext *tcg; TCGTempSet temps_used; } OptContext; @@ -XXX,XX +XXX,XX @@ static bool args_are_copies(TCGArg arg1, TCGArg arg2) return ts_are_copies(arg_temp(arg1), arg_temp(arg2)); } -static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) +static void tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) { TCGTemp *dst_ts = arg_temp(dst); TCGTemp *src_ts = arg_temp(src); @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) TCGOpcode new_op; if (ts_are_copies(dst_ts, src_ts)) { - tcg_op_remove(s, op); + tcg_op_remove(ctx->tcg, op); return; } @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) } } -static void tcg_opt_gen_movi(TCGContext *s, OptContext *ctx, - TCGOp *op, TCGArg dst, uint64_t val) +static void tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, + TCGArg dst, uint64_t val) { const TCGOpDef *def = &tcg_op_defs[op->opc]; TCGType type; @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_movi(TCGContext *s, OptContext *ctx, /* Convert movi to mov with constant temp. */ tv = tcg_constant_internal(type, val); init_ts_info(ctx, tv); - tcg_opt_gen_mov(s, op, dst, temp_arg(tv)); + tcg_opt_gen_mov(ctx, op, dst, temp_arg(tv)); } static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) { int nb_temps, nb_globals, i; TCGOp *op, *op_next, *prev_mb = NULL; - OptContext ctx = {}; + OptContext ctx = { .tcg = s }; /* Array VALS has an element for each temp. If this temp holds a constant then its value is kept in VALS' element. @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(rotr): if (arg_is_const(op->args[1]) && arg_info(op->args[1])->val == 0) { - tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); + tcg_opt_gen_movi(&ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (!arg_is_const(op->args[1]) && arg_is_const(op->args[2]) && arg_info(op->args[2])->val == 0) { - tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (!arg_is_const(op->args[1]) && arg_is_const(op->args[2]) && arg_info(op->args[2])->val == -1) { - tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (partmask == 0) { tcg_debug_assert(nb_oargs == 1); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); + tcg_opt_gen_movi(&ctx, op, op->args[0], 0); continue; } if (affected == 0) { tcg_debug_assert(nb_oargs == 1); - tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(mulsh): if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == 0) { - tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); + tcg_opt_gen_movi(&ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(or): CASE_OP_32_64_VEC(and): if (args_are_copies(op->args[1], op->args[2])) { - tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(sub): CASE_OP_32_64_VEC(xor): if (args_are_copies(op->args[1], op->args[2])) { - tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); + tcg_opt_gen_movi(&ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) allocator where needed and possible. Also detect copies. */ switch (opc) { CASE_OP_32_64_VEC(mov): - tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; case INDEX_op_dup_vec: if (arg_is_const(op->args[1])) { tmp = arg_info(op->args[1])->val; tmp = dup_const(TCGOP_VECE(op), tmp); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_dup2_vec: assert(TCG_TARGET_REG_BITS == 32); if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - tcg_opt_gen_movi(s, &ctx, op, op->args[0], + tcg_opt_gen_movi(&ctx, op, op->args[0], deposit64(arg_info(op->args[1])->val, 32, 32, arg_info(op->args[2])->val)); continue; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_extrh_i64_i32: if (arg_is_const(op->args[1])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, 0); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, op->args[2]); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, arg_info(op->args[2])->val); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) TCGArg v = arg_info(op->args[1])->val; if (v != 0) { tmp = do_constant_folding(opc, v, 0); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); } else { - tcg_opt_gen_mov(s, op, op->args[0], op->args[2]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[2]); } continue; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = deposit64(arg_info(op->args[1])->val, op->args[3], op->args[4], arg_info(op->args[2])->val); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = extract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = sextract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = (int32_t)(((uint32_t)v1 >> shr) | ((uint32_t)v2 << (32 - shr))); } - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[3]); if (tmp != 2) { - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[5]); if (tmp != 2) { - tcg_opt_gen_mov(s, op, op->args[0], op->args[4-tmp]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[4-tmp]); continue; } if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)a); - tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(a >> 32)); + tcg_opt_gen_movi(&ctx, op, rl, (int32_t)a); + tcg_opt_gen_movi(&ctx, op2, rh, (int32_t)(a >> 32)); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)r); - tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(r >> 32)); + tcg_opt_gen_movi(&ctx, op, rl, (int32_t)r); + tcg_opt_gen_movi(&ctx, op2, rh, (int32_t)(r >> 32)); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[5]); if (tmp != 2) { do_setcond_const: - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } if ((op->args[5] == TCG_COND_LT || op->args[5] == TCG_COND_GE) -- 2.25.1 This will expose the variable to subroutines that will be broken out of tcg_optimize. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { typedef struct OptContext { TCGContext *tcg; + TCGOp *prev_mb; TCGTempSet temps_used; } OptContext; @@ -XXX,XX +XXX,XX @@ static bool swap_commutative2(TCGArg *p1, TCGArg *p2) void tcg_optimize(TCGContext *s) { int nb_temps, nb_globals, i; - TCGOp *op, *op_next, *prev_mb = NULL; + TCGOp *op, *op_next; OptContext ctx = { .tcg = s }; /* Array VALS has an element for each temp. @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } /* Eliminate duplicate and redundant fence instructions. */ - if (prev_mb) { + if (ctx.prev_mb) { switch (opc) { case INDEX_op_mb: /* Merge two barriers of the same type into one, @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) * barrier. This is stricter than specified but for * the purposes of TCG is better than not optimizing. */ - prev_mb->args[0] |= op->args[0]; + ctx.prev_mb->args[0] |= op->args[0]; tcg_op_remove(s, op); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_qemu_st_i64: case INDEX_op_call: /* Opcodes that touch guest memory stop the optimization. */ - prev_mb = NULL; + ctx.prev_mb = NULL; break; } } else if (opc == INDEX_op_mb) { - prev_mb = op; + ctx.prev_mb = op; } } } -- 2.25.1 There was no real reason for calls to have separate code here. Unify init for calls vs non-calls using the call path, which handles TCG_CALL_DUMMY_ARG. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void init_ts_info(OptContext *ctx, TCGTemp *ts) } } -static void init_arg_info(OptContext *ctx, TCGArg arg) -{ - init_ts_info(ctx, arg_temp(arg)); -} - static TCGTemp *find_better_copy(TCGContext *s, TCGTemp *ts) { TCGTemp *i, *g, *l; @@ -XXX,XX +XXX,XX @@ static bool swap_commutative2(TCGArg *p1, TCGArg *p2) return false; } +static void init_arguments(OptContext *ctx, TCGOp *op, int nb_args) +{ + for (int i = 0; i < nb_args; i++) { + TCGTemp *ts = arg_temp(op->args[i]); + if (ts) { + init_ts_info(ctx, ts); + } + } +} + /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (opc == INDEX_op_call) { nb_oargs = TCGOP_CALLO(op); nb_iargs = TCGOP_CALLI(op); - for (i = 0; i < nb_oargs + nb_iargs; i++) { - TCGTemp *ts = arg_temp(op->args[i]); - if (ts) { - init_ts_info(&ctx, ts); - } - } } else { nb_oargs = def->nb_oargs; nb_iargs = def->nb_iargs; - for (i = 0; i < nb_oargs + nb_iargs; i++) { - init_arg_info(&ctx, op->args[i]); - } } + init_arguments(&ctx, op, nb_oargs + nb_iargs); /* Do copy propagation */ for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { -- 2.25.1 Continue splitting tcg_optimize. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void init_arguments(OptContext *ctx, TCGOp *op, int nb_args) } } +static void copy_propagate(OptContext *ctx, TCGOp *op, + int nb_oargs, int nb_iargs) +{ + TCGContext *s = ctx->tcg; + + for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) { + TCGTemp *ts = arg_temp(op->args[i]); + if (ts && ts_is_copy(ts)) { + op->args[i] = temp_arg(find_better_copy(s, ts)); + } + } +} + /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) nb_iargs = def->nb_iargs; } init_arguments(&ctx, op, nb_oargs + nb_iargs); - - /* Do copy propagation */ - for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { - TCGTemp *ts = arg_temp(op->args[i]); - if (ts && ts_is_copy(ts)) { - op->args[i] = temp_arg(find_better_copy(s, ts)); - } - } + copy_propagate(&ctx, op, nb_oargs, nb_iargs); /* For commutative operations make constant second argument */ switch (opc) { -- 2.25.1 Calls are special in that they have a variable number of arguments, and need to be able to clobber globals. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 63 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void copy_propagate(OptContext *ctx, TCGOp *op, } } +static bool fold_call(OptContext *ctx, TCGOp *op) +{ + TCGContext *s = ctx->tcg; + int nb_oargs = TCGOP_CALLO(op); + int nb_iargs = TCGOP_CALLI(op); + int flags, i; + + init_arguments(ctx, op, nb_oargs + nb_iargs); + copy_propagate(ctx, op, nb_oargs, nb_iargs); + + /* If the function reads or writes globals, reset temp data. */ + flags = tcg_call_flags(op); + if (!(flags & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) { + int nb_globals = s->nb_globals; + + for (i = 0; i < nb_globals; i++) { + if (test_bit(i, ctx->temps_used.l)) { + reset_ts(&ctx->tcg->temps[i]); + } + } + } + + /* Reset temp data for outputs. */ + for (i = 0; i < nb_oargs; i++) { + reset_temp(op->args[i]); + } + + /* Stop optimizing MB across calls. */ + ctx->prev_mb = NULL; + return true; +} + /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { - int nb_temps, nb_globals, i; + int nb_temps, i; TCGOp *op, *op_next; OptContext ctx = { .tcg = s }; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) available through the doubly linked circular list. */ nb_temps = s->nb_temps; - nb_globals = s->nb_globals; - for (i = 0; i < nb_temps; ++i) { s->temps[i].state_ptr = NULL; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) uint64_t z_mask, partmask, affected, tmp; int nb_oargs, nb_iargs; TCGOpcode opc = op->opc; - const TCGOpDef *def = &tcg_op_defs[opc]; + const TCGOpDef *def; - /* Count the arguments, and initialize the temps that are - going to be used */ + /* Calls are special. */ if (opc == INDEX_op_call) { - nb_oargs = TCGOP_CALLO(op); - nb_iargs = TCGOP_CALLI(op); - } else { - nb_oargs = def->nb_oargs; - nb_iargs = def->nb_iargs; + fold_call(&ctx, op); + continue; } + + def = &tcg_op_defs[opc]; + nb_oargs = def->nb_oargs; + nb_iargs = def->nb_iargs; init_arguments(&ctx, op, nb_oargs + nb_iargs); copy_propagate(&ctx, op, nb_oargs, nb_iargs); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (def->flags & TCG_OPF_BB_END) { memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); } else { - if (opc == INDEX_op_call && - !(tcg_call_flags(op) - & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) { - for (i = 0; i < nb_globals; i++) { - if (test_bit(i, ctx.temps_used.l)) { - reset_ts(&s->temps[i]); - } - } - } - for (i = 0; i < nb_oargs; i++) { reset_temp(op->args[i]); /* Save the corresponding known-zero bits mask for the @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_qemu_st_i32: case INDEX_op_qemu_st8_i32: case INDEX_op_qemu_st_i64: - case INDEX_op_call: /* Opcodes that touch guest memory stop the optimization. */ ctx.prev_mb = NULL; break; -- 2.25.1 Rather than try to keep these up-to-date across folding, re-read nb_oargs at the end, after re-reading the opcode. A couple of asserts need dropping, but that will take care of itself as we split the function further. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { uint64_t z_mask, partmask, affected, tmp; - int nb_oargs, nb_iargs; TCGOpcode opc = op->opc; const TCGOpDef *def; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } def = &tcg_op_defs[opc]; - nb_oargs = def->nb_oargs; - nb_iargs = def->nb_iargs; - init_arguments(&ctx, op, nb_oargs + nb_iargs); - copy_propagate(&ctx, op, nb_oargs, nb_iargs); + init_arguments(&ctx, op, def->nb_oargs + def->nb_iargs); + copy_propagate(&ctx, op, def->nb_oargs, def->nb_iargs); /* For commutative operations make constant second argument */ switch (opc) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(qemu_ld): { - MemOpIdx oi = op->args[nb_oargs + nb_iargs]; + MemOpIdx oi = op->args[def->nb_oargs + def->nb_iargs]; MemOp mop = get_memop(oi); if (!(mop & MO_SIGN)) { z_mask = (2ULL << ((8 << (mop & MO_SIZE)) - 1)) - 1; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } if (partmask == 0) { - tcg_debug_assert(nb_oargs == 1); tcg_opt_gen_movi(&ctx, op, op->args[0], 0); continue; } if (affected == 0) { - tcg_debug_assert(nb_oargs == 1); tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } else if (args_are_copies(op->args[1], op->args[2])) { op->opc = INDEX_op_dup_vec; TCGOP_VECE(op) = MO_32; - nb_iargs = 1; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->opc = opc = (opc == INDEX_op_movcond_i32 ? INDEX_op_setcond_i32 : INDEX_op_setcond_i64); - nb_iargs = 2; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (def->flags & TCG_OPF_BB_END) { memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); } else { + int nb_oargs = def->nb_oargs; for (i = 0; i < nb_oargs; i++) { reset_temp(op->args[i]); /* Save the corresponding known-zero bits mask for the -- 2.25.1 Return -1 instead of 2 for failure, so that we can use comparisons against 0 for all cases. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 145 +++++++++++++++++++++++++------------------------ 1 file changed, 74 insertions(+), 71 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool do_constant_folding_cond_eq(TCGCond c) } } -/* Return 2 if the condition can't be simplified, and the result - of the condition (0 or 1) if it can */ -static TCGArg do_constant_folding_cond(TCGOpcode op, TCGArg x, - TCGArg y, TCGCond c) +/* + * Return -1 if the condition can't be simplified, + * and the result of the condition (0 or 1) if it can. + */ +static int do_constant_folding_cond(TCGOpcode op, TCGArg x, + TCGArg y, TCGCond c) { uint64_t xv = arg_info(x)->val; uint64_t yv = arg_info(y)->val; @@ -XXX,XX +XXX,XX @@ static TCGArg do_constant_folding_cond(TCGOpcode op, TCGArg x, case TCG_COND_GEU: return 1; default: - return 2; + return -1; } } - return 2; + return -1; } -/* Return 2 if the condition can't be simplified, and the result - of the condition (0 or 1) if it can */ -static TCGArg do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c) +/* + * Return -1 if the condition can't be simplified, + * and the result of the condition (0 or 1) if it can. + */ +static int do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c) { TCGArg al = p1[0], ah = p1[1]; TCGArg bl = p2[0], bh = p2[1]; @@ -XXX,XX +XXX,XX @@ static TCGArg do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c) if (args_are_copies(al, bl) && args_are_copies(ah, bh)) { return do_constant_folding_cond_eq(c); } - return 2; + return -1; } static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; CASE_OP_32_64(setcond): - tmp = do_constant_folding_cond(opc, op->args[1], - op->args[2], op->args[3]); - if (tmp != 2) { - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); + i = do_constant_folding_cond(opc, op->args[1], + op->args[2], op->args[3]); + if (i >= 0) { + tcg_opt_gen_movi(&ctx, op, op->args[0], i); continue; } break; CASE_OP_32_64(brcond): - tmp = do_constant_folding_cond(opc, op->args[0], - op->args[1], op->args[2]); - switch (tmp) { - case 0: + i = do_constant_folding_cond(opc, op->args[0], + op->args[1], op->args[2]); + if (i == 0) { tcg_op_remove(s, op); continue; - case 1: + } else if (i > 0) { memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); op->opc = opc = INDEX_op_br; op->args[0] = op->args[3]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; CASE_OP_32_64(movcond): - tmp = do_constant_folding_cond(opc, op->args[1], - op->args[2], op->args[5]); - if (tmp != 2) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[4-tmp]); + i = do_constant_folding_cond(opc, op->args[1], + op->args[2], op->args[5]); + if (i >= 0) { + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[4 - i]); continue; } if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; case INDEX_op_brcond2_i32: - tmp = do_constant_folding_cond2(&op->args[0], &op->args[2], - op->args[4]); - if (tmp == 0) { + i = do_constant_folding_cond2(&op->args[0], &op->args[2], + op->args[4]); + if (i == 0) { do_brcond_false: tcg_op_remove(s, op); continue; } - if (tmp == 1) { + if (i > 0) { do_brcond_true: op->opc = opc = INDEX_op_br; op->args[0] = op->args[5]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (op->args[4] == TCG_COND_EQ) { /* Simplify EQ comparisons where one of the pairs can be simplified. */ - tmp = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[0], op->args[2], - TCG_COND_EQ); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_brcond_i32, + op->args[0], op->args[2], + TCG_COND_EQ); + if (i == 0) { goto do_brcond_false; - } else if (tmp == 1) { + } else if (i > 0) { goto do_brcond_high; } - tmp = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[1], op->args[3], - TCG_COND_EQ); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_brcond_i32, + op->args[1], op->args[3], + TCG_COND_EQ); + if (i == 0) { goto do_brcond_false; - } else if (tmp != 1) { + } else if (i < 0) { break; } do_brcond_low: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (op->args[4] == TCG_COND_NE) { /* Simplify NE comparisons where one of the pairs can be simplified. */ - tmp = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[0], op->args[2], - TCG_COND_NE); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_brcond_i32, + op->args[0], op->args[2], + TCG_COND_NE); + if (i == 0) { goto do_brcond_high; - } else if (tmp == 1) { + } else if (i > 0) { goto do_brcond_true; } - tmp = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[1], op->args[3], - TCG_COND_NE); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_brcond_i32, + op->args[1], op->args[3], + TCG_COND_NE); + if (i == 0) { goto do_brcond_low; - } else if (tmp == 1) { + } else if (i > 0) { goto do_brcond_true; } } break; case INDEX_op_setcond2_i32: - tmp = do_constant_folding_cond2(&op->args[1], &op->args[3], - op->args[5]); - if (tmp != 2) { + i = do_constant_folding_cond2(&op->args[1], &op->args[3], + op->args[5]); + if (i >= 0) { do_setcond_const: - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], i); continue; } if ((op->args[5] == TCG_COND_LT || op->args[5] == TCG_COND_GE) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (op->args[5] == TCG_COND_EQ) { /* Simplify EQ comparisons where one of the pairs can be simplified. */ - tmp = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[1], op->args[3], - TCG_COND_EQ); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_setcond_i32, + op->args[1], op->args[3], + TCG_COND_EQ); + if (i == 0) { goto do_setcond_const; - } else if (tmp == 1) { + } else if (i > 0) { goto do_setcond_high; } - tmp = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[2], op->args[4], - TCG_COND_EQ); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_setcond_i32, + op->args[2], op->args[4], + TCG_COND_EQ); + if (i == 0) { goto do_setcond_high; - } else if (tmp != 1) { + } else if (i < 0) { break; } do_setcond_low: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (op->args[5] == TCG_COND_NE) { /* Simplify NE comparisons where one of the pairs can be simplified. */ - tmp = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[1], op->args[3], - TCG_COND_NE); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_setcond_i32, + op->args[1], op->args[3], + TCG_COND_NE); + if (i == 0) { goto do_setcond_high; - } else if (tmp == 1) { + } else if (i > 0) { goto do_setcond_const; } - tmp = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[2], op->args[4], - TCG_COND_NE); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_setcond_i32, + op->args[2], op->args[4], + TCG_COND_NE); + if (i == 0) { goto do_setcond_low; - } else if (tmp == 1) { + } else if (i > 0) { goto do_setcond_const; } } -- 2.25.1 This will allow callers to tail call to these functions and return true indicating processing complete. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool args_are_copies(TCGArg arg1, TCGArg arg2) return ts_are_copies(arg_temp(arg1), arg_temp(arg2)); } -static void tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) +static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) { TCGTemp *dst_ts = arg_temp(dst); TCGTemp *src_ts = arg_temp(src); @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) if (ts_are_copies(dst_ts, src_ts)) { tcg_op_remove(ctx->tcg, op); - return; + return true; } reset_ts(dst_ts); @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) di->is_const = si->is_const; di->val = si->val; } + return true; } -static void tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, +static bool tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, TCGArg dst, uint64_t val) { const TCGOpDef *def = &tcg_op_defs[op->opc]; @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, /* Convert movi to mov with constant temp. */ tv = tcg_constant_internal(type, val); init_ts_info(ctx, tv); - tcg_opt_gen_mov(ctx, op, dst, temp_arg(tv)); + return tcg_opt_gen_mov(ctx, op, dst, temp_arg(tv)); } static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y) -- 2.25.1 Copy z_mask into OptContext, for writeback to the first output within the new function. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 49 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { TCGContext *tcg; TCGOp *prev_mb; TCGTempSet temps_used; + + /* In flight values from optimization. */ + uint64_t z_mask; } OptContext; static inline TempOptInfo *ts_info(TCGTemp *ts) @@ -XXX,XX +XXX,XX @@ static void copy_propagate(OptContext *ctx, TCGOp *op, } } +static void finish_folding(OptContext *ctx, TCGOp *op) +{ + const TCGOpDef *def = &tcg_op_defs[op->opc]; + int i, nb_oargs; + + /* + * For an opcode that ends a BB, reset all temp data. + * We do no cross-BB optimization. + */ + if (def->flags & TCG_OPF_BB_END) { + memset(&ctx->temps_used, 0, sizeof(ctx->temps_used)); + ctx->prev_mb = NULL; + return; + } + + nb_oargs = def->nb_oargs; + for (i = 0; i < nb_oargs; i++) { + reset_temp(op->args[i]); + /* + * Save the corresponding known-zero bits mask for the + * first output argument (only one supported so far). + */ + if (i == 0) { + arg_info(op->args[i])->z_mask = ctx->z_mask; + } + } +} + static bool fold_call(OptContext *ctx, TCGOp *op) { TCGContext *s = ctx->tcg; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) partmask &= 0xffffffffu; affected &= 0xffffffffu; } + ctx.z_mask = z_mask; if (partmask == 0) { tcg_opt_gen_movi(&ctx, op, op->args[0], 0); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Some of the folding above can change opc. */ - opc = op->opc; - def = &tcg_op_defs[opc]; - if (def->flags & TCG_OPF_BB_END) { - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - } else { - int nb_oargs = def->nb_oargs; - for (i = 0; i < nb_oargs; i++) { - reset_temp(op->args[i]); - /* Save the corresponding known-zero bits mask for the - first output argument (only one supported so far). */ - if (i == 0) { - arg_info(op->args[i])->z_mask = z_mask; - } - } - } + finish_folding(&ctx, op); /* Eliminate duplicate and redundant fence instructions. */ if (ctx.prev_mb) { -- 2.25.1 Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) uint64_t z_mask, partmask, affected, tmp; TCGOpcode opc = op->opc; const TCGOpDef *def; + bool done = false; /* Calls are special. */ if (opc == INDEX_op_call) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) allocator where needed and possible. Also detect copies. */ switch (opc) { CASE_OP_32_64_VEC(mov): - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - continue; + done = tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); + break; case INDEX_op_dup_vec: if (arg_is_const(op->args[1])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - finish_folding(&ctx, op); + if (!done) { + finish_folding(&ctx, op); + } /* Eliminate duplicate and redundant fence instructions. */ if (ctx.prev_mb) { -- 2.25.1 This puts the separate mb optimization into the same framework as the others. While fold_qemu_{ld,st} are currently identical, that won't last as more code gets moved. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 89 +++++++++++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_call(OptContext *ctx, TCGOp *op) return true; } +static bool fold_mb(OptContext *ctx, TCGOp *op) +{ + /* Eliminate duplicate and redundant fence instructions. */ + if (ctx->prev_mb) { + /* + * Merge two barriers of the same type into one, + * or a weaker barrier into a stronger one, + * or two weaker barriers into a stronger one. + * mb X; mb Y => mb X|Y + * mb; strl => mb; st + * ldaq; mb => ld; mb + * ldaq; strl => ld; mb; st + * Other combinations are also merged into a strong + * barrier. This is stricter than specified but for + * the purposes of TCG is better than not optimizing. + */ + ctx->prev_mb->args[0] |= op->args[0]; + tcg_op_remove(ctx->tcg, op); + } else { + ctx->prev_mb = op; + } + return true; +} + +static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) +{ + /* Opcodes that touch guest memory stop the mb optimization. */ + ctx->prev_mb = NULL; + return false; +} + +static bool fold_qemu_st(OptContext *ctx, TCGOp *op) +{ + /* Opcodes that touch guest memory stop the mb optimization. */ + ctx->prev_mb = NULL; + return false; +} + /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; + case INDEX_op_mb: + done = fold_mb(&ctx, op); + break; + case INDEX_op_qemu_ld_i32: + case INDEX_op_qemu_ld_i64: + done = fold_qemu_ld(&ctx, op); + break; + case INDEX_op_qemu_st_i32: + case INDEX_op_qemu_st8_i32: + case INDEX_op_qemu_st_i64: + done = fold_qemu_st(&ctx, op); + break; + default: break; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (!done) { finish_folding(&ctx, op); } - - /* Eliminate duplicate and redundant fence instructions. */ - if (ctx.prev_mb) { - switch (opc) { - case INDEX_op_mb: - /* Merge two barriers of the same type into one, - * or a weaker barrier into a stronger one, - * or two weaker barriers into a stronger one. - * mb X; mb Y => mb X|Y - * mb; strl => mb; st - * ldaq; mb => ld; mb - * ldaq; strl => ld; mb; st - * Other combinations are also merged into a strong - * barrier. This is stricter than specified but for - * the purposes of TCG is better than not optimizing. - */ - ctx.prev_mb->args[0] |= op->args[0]; - tcg_op_remove(s, op); - break; - - default: - /* Opcodes that end the block stop the optimization. */ - if ((def->flags & TCG_OPF_BB_END) == 0) { - break; - } - /* fallthru */ - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_ld_i64: - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st8_i32: - case INDEX_op_qemu_st_i64: - /* Opcodes that touch guest memory stop the optimization. */ - ctx.prev_mb = NULL; - break; - } - } else if (opc == INDEX_op_mb) { - ctx.prev_mb = op; - } } } -- 2.25.1 Split out a whole bunch of placeholder functions, which are currently identical. That won't last as more code gets moved. Use CASE_32_64_VEC for some logical operators that previously missed the addition of vectors. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 271 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 219 insertions(+), 52 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void finish_folding(OptContext *ctx, TCGOp *op) } } +/* + * The fold_* functions return true when processing is complete, + * usually by folding the operation to a constant or to a copy, + * and calling tcg_opt_gen_{mov,movi}. They may do other things, + * like collect information about the value produced, for use in + * optimizing a subsequent operation. + * + * These first fold_* functions are all helpers, used by other + * folders for more specific operations. + */ + +static bool fold_const1(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t; + + t = arg_info(op->args[1])->val; + t = do_constant_folding(op->opc, t, 0); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return false; +} + +static bool fold_const2(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { + uint64_t t1 = arg_info(op->args[1])->val; + uint64_t t2 = arg_info(op->args[2])->val; + + t1 = do_constant_folding(op->opc, t1, t2); + return tcg_opt_gen_movi(ctx, op, op->args[0], t1); + } + return false; +} + +/* + * These outermost fold_<op> functions are sorted alphabetically. + */ + +static bool fold_add(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_and(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_andc(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + static bool fold_call(OptContext *ctx, TCGOp *op) { TCGContext *s = ctx->tcg; @@ -XXX,XX +XXX,XX @@ static bool fold_call(OptContext *ctx, TCGOp *op) return true; } +static bool fold_ctpop(OptContext *ctx, TCGOp *op) +{ + return fold_const1(ctx, op); +} + +static bool fold_divide(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_eqv(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_exts(OptContext *ctx, TCGOp *op) +{ + return fold_const1(ctx, op); +} + +static bool fold_extu(OptContext *ctx, TCGOp *op) +{ + return fold_const1(ctx, op); +} + static bool fold_mb(OptContext *ctx, TCGOp *op) { /* Eliminate duplicate and redundant fence instructions. */ @@ -XXX,XX +XXX,XX @@ static bool fold_mb(OptContext *ctx, TCGOp *op) return true; } +static bool fold_mul(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_nand(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_neg(OptContext *ctx, TCGOp *op) +{ + return fold_const1(ctx, op); +} + +static bool fold_nor(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_not(OptContext *ctx, TCGOp *op) +{ + return fold_const1(ctx, op); +} + +static bool fold_or(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_orc(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) { /* Opcodes that touch guest memory stop the mb optimization. */ @@ -XXX,XX +XXX,XX @@ static bool fold_qemu_st(OptContext *ctx, TCGOp *op) return false; } +static bool fold_remainder(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_shift(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_sub(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_xor(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(not): - CASE_OP_32_64(neg): - CASE_OP_32_64(ext8s): - CASE_OP_32_64(ext8u): - CASE_OP_32_64(ext16s): - CASE_OP_32_64(ext16u): - CASE_OP_32_64(ctpop): - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_extrl_i64_i32: - case INDEX_op_extrh_i64_i32: - if (arg_is_const(op->args[1])) { - tmp = do_constant_folding(opc, arg_info(op->args[1])->val, 0); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - CASE_OP_32_64(bswap16): CASE_OP_32_64(bswap32): case INDEX_op_bswap64_i64: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(add): - CASE_OP_32_64(sub): - CASE_OP_32_64(mul): - CASE_OP_32_64(or): - CASE_OP_32_64(and): - CASE_OP_32_64(xor): - CASE_OP_32_64(shl): - CASE_OP_32_64(shr): - CASE_OP_32_64(sar): - CASE_OP_32_64(rotl): - CASE_OP_32_64(rotr): - CASE_OP_32_64(andc): - CASE_OP_32_64(orc): - CASE_OP_32_64(eqv): - CASE_OP_32_64(nand): - CASE_OP_32_64(nor): - CASE_OP_32_64(muluh): - CASE_OP_32_64(mulsh): - CASE_OP_32_64(div): - CASE_OP_32_64(divu): - CASE_OP_32_64(rem): - CASE_OP_32_64(remu): - if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - tmp = do_constant_folding(opc, arg_info(op->args[1])->val, - arg_info(op->args[2])->val); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - CASE_OP_32_64(clz): CASE_OP_32_64(ctz): if (arg_is_const(op->args[1])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; + default: + break; + + /* ---------------------------------------------------------- */ + /* Sorted alphabetically by opcode as much as possible. */ + + CASE_OP_32_64_VEC(add): + done = fold_add(&ctx, op); + break; + CASE_OP_32_64_VEC(and): + done = fold_and(&ctx, op); + break; + CASE_OP_32_64_VEC(andc): + done = fold_andc(&ctx, op); + break; + CASE_OP_32_64(ctpop): + done = fold_ctpop(&ctx, op); + break; + CASE_OP_32_64(div): + CASE_OP_32_64(divu): + done = fold_divide(&ctx, op); + break; + CASE_OP_32_64(eqv): + done = fold_eqv(&ctx, op); + break; + CASE_OP_32_64(ext8s): + CASE_OP_32_64(ext16s): + case INDEX_op_ext32s_i64: + case INDEX_op_ext_i32_i64: + done = fold_exts(&ctx, op); + break; + CASE_OP_32_64(ext8u): + CASE_OP_32_64(ext16u): + case INDEX_op_ext32u_i64: + case INDEX_op_extu_i32_i64: + case INDEX_op_extrl_i64_i32: + case INDEX_op_extrh_i64_i32: + done = fold_extu(&ctx, op); + break; case INDEX_op_mb: done = fold_mb(&ctx, op); break; + CASE_OP_32_64(mul): + done = fold_mul(&ctx, op); + break; + CASE_OP_32_64(mulsh): + CASE_OP_32_64(muluh): + done = fold_mul_highpart(&ctx, op); + break; + CASE_OP_32_64(nand): + done = fold_nand(&ctx, op); + break; + CASE_OP_32_64(neg): + done = fold_neg(&ctx, op); + break; + CASE_OP_32_64(nor): + done = fold_nor(&ctx, op); + break; + CASE_OP_32_64_VEC(not): + done = fold_not(&ctx, op); + break; + CASE_OP_32_64_VEC(or): + done = fold_or(&ctx, op); + break; + CASE_OP_32_64_VEC(orc): + done = fold_orc(&ctx, op); + break; case INDEX_op_qemu_ld_i32: case INDEX_op_qemu_ld_i64: done = fold_qemu_ld(&ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_qemu_st_i64: done = fold_qemu_st(&ctx, op); break; - - default: + CASE_OP_32_64(rem): + CASE_OP_32_64(remu): + done = fold_remainder(&ctx, op); + break; + CASE_OP_32_64(rotl): + CASE_OP_32_64(rotr): + CASE_OP_32_64(sar): + CASE_OP_32_64(shl): + CASE_OP_32_64(shr): + done = fold_shift(&ctx, op); + break; + CASE_OP_32_64_VEC(sub): + done = fold_sub(&ctx, op); + break; + CASE_OP_32_64_VEC(xor): + done = fold_xor(&ctx, op); break; } -- 2.25.1 Reduce some code duplication by folding the NE and EQ cases. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 145 ++++++++++++++++++++++++------------------------- 1 file changed, 72 insertions(+), 73 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_remainder(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_setcond2(OptContext *ctx, TCGOp *op) +{ + TCGCond cond = op->args[5]; + int i = do_constant_folding_cond2(&op->args[1], &op->args[3], cond); + int inv = 0; + + if (i >= 0) { + goto do_setcond_const; + } + + switch (cond) { + case TCG_COND_LT: + case TCG_COND_GE: + /* + * Simplify LT/GE comparisons vs zero to a single compare + * vs the high word of the input. + */ + if (arg_is_const(op->args[3]) && arg_info(op->args[3])->val == 0 && + arg_is_const(op->args[4]) && arg_info(op->args[4])->val == 0) { + goto do_setcond_high; + } + break; + + case TCG_COND_NE: + inv = 1; + QEMU_FALLTHROUGH; + case TCG_COND_EQ: + /* + * Simplify EQ/NE comparisons where one of the pairs + * can be simplified. + */ + i = do_constant_folding_cond(INDEX_op_setcond_i32, op->args[1], + op->args[3], cond); + switch (i ^ inv) { + case 0: + goto do_setcond_const; + case 1: + goto do_setcond_high; + } + + i = do_constant_folding_cond(INDEX_op_setcond_i32, op->args[2], + op->args[4], cond); + switch (i ^ inv) { + case 0: + goto do_setcond_const; + case 1: + op->args[2] = op->args[3]; + op->args[3] = cond; + op->opc = INDEX_op_setcond_i32; + break; + } + break; + + default: + break; + + do_setcond_high: + op->args[1] = op->args[2]; + op->args[2] = op->args[4]; + op->args[3] = cond; + op->opc = INDEX_op_setcond_i32; + break; + } + return false; + + do_setcond_const: + return tcg_opt_gen_movi(ctx, op, op->args[0], i); +} + static bool fold_shift(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - case INDEX_op_setcond2_i32: - i = do_constant_folding_cond2(&op->args[1], &op->args[3], - op->args[5]); - if (i >= 0) { - do_setcond_const: - tcg_opt_gen_movi(&ctx, op, op->args[0], i); - continue; - } - if ((op->args[5] == TCG_COND_LT || op->args[5] == TCG_COND_GE) - && arg_is_const(op->args[3]) - && arg_info(op->args[3])->val == 0 - && arg_is_const(op->args[4]) - && arg_info(op->args[4])->val == 0) { - /* Simplify LT/GE comparisons vs zero to a single compare - vs the high word of the input. */ - do_setcond_high: - reset_temp(op->args[0]); - arg_info(op->args[0])->z_mask = 1; - op->opc = INDEX_op_setcond_i32; - op->args[1] = op->args[2]; - op->args[2] = op->args[4]; - op->args[3] = op->args[5]; - break; - } - if (op->args[5] == TCG_COND_EQ) { - /* Simplify EQ comparisons where one of the pairs - can be simplified. */ - i = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[1], op->args[3], - TCG_COND_EQ); - if (i == 0) { - goto do_setcond_const; - } else if (i > 0) { - goto do_setcond_high; - } - i = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[2], op->args[4], - TCG_COND_EQ); - if (i == 0) { - goto do_setcond_high; - } else if (i < 0) { - break; - } - do_setcond_low: - reset_temp(op->args[0]); - arg_info(op->args[0])->z_mask = 1; - op->opc = INDEX_op_setcond_i32; - op->args[2] = op->args[3]; - op->args[3] = op->args[5]; - break; - } - if (op->args[5] == TCG_COND_NE) { - /* Simplify NE comparisons where one of the pairs - can be simplified. */ - i = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[1], op->args[3], - TCG_COND_NE); - if (i == 0) { - goto do_setcond_high; - } else if (i > 0) { - goto do_setcond_const; - } - i = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[2], op->args[4], - TCG_COND_NE); - if (i == 0) { - goto do_setcond_low; - } else if (i > 0) { - goto do_setcond_const; - } - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(shr): done = fold_shift(&ctx, op); break; + case INDEX_op_setcond2_i32: + done = fold_setcond2(&ctx, op); + break; CASE_OP_32_64_VEC(sub): done = fold_sub(&ctx, op); break; -- 2.25.1 Reduce some code duplication by folding the NE and EQ cases. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 159 +++++++++++++++++++++++++------------------------ 1 file changed, 81 insertions(+), 78 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_brcond2(OptContext *ctx, TCGOp *op) +{ + TCGCond cond = op->args[4]; + int i = do_constant_folding_cond2(&op->args[0], &op->args[2], cond); + TCGArg label = op->args[5]; + int inv = 0; + + if (i >= 0) { + goto do_brcond_const; + } + + switch (cond) { + case TCG_COND_LT: + case TCG_COND_GE: + /* + * Simplify LT/GE comparisons vs zero to a single compare + * vs the high word of the input. + */ + if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == 0 && + arg_is_const(op->args[3]) && arg_info(op->args[3])->val == 0) { + goto do_brcond_high; + } + break; + + case TCG_COND_NE: + inv = 1; + QEMU_FALLTHROUGH; + case TCG_COND_EQ: + /* + * Simplify EQ/NE comparisons where one of the pairs + * can be simplified. + */ + i = do_constant_folding_cond(INDEX_op_brcond_i32, op->args[0], + op->args[2], cond); + switch (i ^ inv) { + case 0: + goto do_brcond_const; + case 1: + goto do_brcond_high; + } + + i = do_constant_folding_cond(INDEX_op_brcond_i32, op->args[1], + op->args[3], cond); + switch (i ^ inv) { + case 0: + goto do_brcond_const; + case 1: + op->opc = INDEX_op_brcond_i32; + op->args[1] = op->args[2]; + op->args[2] = cond; + op->args[3] = label; + break; + } + break; + + default: + break; + + do_brcond_high: + op->opc = INDEX_op_brcond_i32; + op->args[0] = op->args[1]; + op->args[1] = op->args[3]; + op->args[2] = cond; + op->args[3] = label; + break; + + do_brcond_const: + if (i == 0) { + tcg_op_remove(ctx->tcg, op); + return true; + } + op->opc = INDEX_op_br; + op->args[0] = label; + break; + } + return false; +} + static bool fold_call(OptContext *ctx, TCGOp *op) { TCGContext *s = ctx->tcg; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - case INDEX_op_brcond2_i32: - i = do_constant_folding_cond2(&op->args[0], &op->args[2], - op->args[4]); - if (i == 0) { - do_brcond_false: - tcg_op_remove(s, op); - continue; - } - if (i > 0) { - do_brcond_true: - op->opc = opc = INDEX_op_br; - op->args[0] = op->args[5]; - break; - } - if ((op->args[4] == TCG_COND_LT || op->args[4] == TCG_COND_GE) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == 0 - && arg_is_const(op->args[3]) - && arg_info(op->args[3])->val == 0) { - /* Simplify LT/GE comparisons vs zero to a single compare - vs the high word of the input. */ - do_brcond_high: - op->opc = opc = INDEX_op_brcond_i32; - op->args[0] = op->args[1]; - op->args[1] = op->args[3]; - op->args[2] = op->args[4]; - op->args[3] = op->args[5]; - break; - } - if (op->args[4] == TCG_COND_EQ) { - /* Simplify EQ comparisons where one of the pairs - can be simplified. */ - i = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[0], op->args[2], - TCG_COND_EQ); - if (i == 0) { - goto do_brcond_false; - } else if (i > 0) { - goto do_brcond_high; - } - i = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[1], op->args[3], - TCG_COND_EQ); - if (i == 0) { - goto do_brcond_false; - } else if (i < 0) { - break; - } - do_brcond_low: - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - op->opc = INDEX_op_brcond_i32; - op->args[1] = op->args[2]; - op->args[2] = op->args[4]; - op->args[3] = op->args[5]; - break; - } - if (op->args[4] == TCG_COND_NE) { - /* Simplify NE comparisons where one of the pairs - can be simplified. */ - i = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[0], op->args[2], - TCG_COND_NE); - if (i == 0) { - goto do_brcond_high; - } else if (i > 0) { - goto do_brcond_true; - } - i = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[1], op->args[3], - TCG_COND_NE); - if (i == 0) { - goto do_brcond_low; - } else if (i > 0) { - goto do_brcond_true; - } - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(andc): done = fold_andc(&ctx, op); break; + case INDEX_op_brcond2_i32: + done = fold_brcond2(&ctx, op); + break; CASE_OP_32_64(ctpop): done = fold_ctpop(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_brcond(OptContext *ctx, TCGOp *op) +{ + TCGCond cond = op->args[2]; + int i = do_constant_folding_cond(op->opc, op->args[0], op->args[1], cond); + + if (i == 0) { + tcg_op_remove(ctx->tcg, op); + return true; + } + if (i > 0) { + op->opc = INDEX_op_br; + op->args[0] = op->args[3]; + } + return false; +} + static bool fold_brcond2(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[4]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(brcond): - i = do_constant_folding_cond(opc, op->args[0], - op->args[1], op->args[2]); - if (i == 0) { - tcg_op_remove(s, op); - continue; - } else if (i > 0) { - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - op->opc = opc = INDEX_op_br; - op->args[0] = op->args[3]; - break; - } - break; - CASE_OP_32_64(movcond): i = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[5]); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(andc): done = fold_andc(&ctx, op); break; + CASE_OP_32_64(brcond): + done = fold_brcond(&ctx, op); + break; case INDEX_op_brcond2_i32: done = fold_brcond2(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_remainder(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_setcond(OptContext *ctx, TCGOp *op) +{ + TCGCond cond = op->args[3]; + int i = do_constant_folding_cond(op->opc, op->args[1], op->args[2], cond); + + if (i >= 0) { + return tcg_opt_gen_movi(ctx, op, op->args[0], i); + } + return false; +} + static bool fold_setcond2(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[5]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(setcond): - i = do_constant_folding_cond(opc, op->args[1], - op->args[2], op->args[3]); - if (i >= 0) { - tcg_opt_gen_movi(&ctx, op, op->args[0], i); - continue; - } - break; - CASE_OP_32_64(movcond): i = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[5]); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(shr): done = fold_shift(&ctx, op); break; + CASE_OP_32_64(setcond): + done = fold_setcond(&ctx, op); + break; case INDEX_op_setcond2_i32: done = fold_setcond2(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_mulu2_i32(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { + uint32_t a = arg_info(op->args[2])->val; + uint32_t b = arg_info(op->args[3])->val; + uint64_t r = (uint64_t)a * b; + TCGArg rl, rh; + TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_mov_i32); + + rl = op->args[0]; + rh = op->args[1]; + tcg_opt_gen_movi(ctx, op, rl, (int32_t)r); + tcg_opt_gen_movi(ctx, op2, rh, (int32_t)(r >> 32)); + return true; + } + return false; +} + static bool fold_nand(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - case INDEX_op_mulu2_i32: - if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { - uint32_t a = arg_info(op->args[2])->val; - uint32_t b = arg_info(op->args[3])->val; - uint64_t r = (uint64_t)a * b; - TCGArg rl, rh; - TCGOp *op2 = tcg_op_insert_before(s, op, INDEX_op_mov_i32); - - rl = op->args[0]; - rh = op->args[1]; - tcg_opt_gen_movi(&ctx, op, rl, (int32_t)r); - tcg_opt_gen_movi(&ctx, op2, rh, (int32_t)(r >> 32)); - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(muluh): done = fold_mul_highpart(&ctx, op); break; + case INDEX_op_mulu2_i32: + done = fold_mulu2_i32(&ctx, op); + break; CASE_OP_32_64(nand): done = fold_nand(&ctx, op); break; -- 2.25.1 Add two additional helpers, fold_add2_i32 and fold_sub2_i32 which will not be simple wrappers forever. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 70 +++++++++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_add(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_addsub2_i32(OptContext *ctx, TCGOp *op, bool add) +{ + if (arg_is_const(op->args[2]) && arg_is_const(op->args[3]) && + arg_is_const(op->args[4]) && arg_is_const(op->args[5])) { + uint32_t al = arg_info(op->args[2])->val; + uint32_t ah = arg_info(op->args[3])->val; + uint32_t bl = arg_info(op->args[4])->val; + uint32_t bh = arg_info(op->args[5])->val; + uint64_t a = ((uint64_t)ah << 32) | al; + uint64_t b = ((uint64_t)bh << 32) | bl; + TCGArg rl, rh; + TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_mov_i32); + + if (add) { + a += b; + } else { + a -= b; + } + + rl = op->args[0]; + rh = op->args[1]; + tcg_opt_gen_movi(ctx, op, rl, (int32_t)a); + tcg_opt_gen_movi(ctx, op2, rh, (int32_t)(a >> 32)); + return true; + } + return false; +} + +static bool fold_add2_i32(OptContext *ctx, TCGOp *op) +{ + return fold_addsub2_i32(ctx, op, true); +} + static bool fold_and(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ static bool fold_sub(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) +{ + return fold_addsub2_i32(ctx, op, false); +} + static bool fold_xor(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - case INDEX_op_add2_i32: - case INDEX_op_sub2_i32: - if (arg_is_const(op->args[2]) && arg_is_const(op->args[3]) - && arg_is_const(op->args[4]) && arg_is_const(op->args[5])) { - uint32_t al = arg_info(op->args[2])->val; - uint32_t ah = arg_info(op->args[3])->val; - uint32_t bl = arg_info(op->args[4])->val; - uint32_t bh = arg_info(op->args[5])->val; - uint64_t a = ((uint64_t)ah << 32) | al; - uint64_t b = ((uint64_t)bh << 32) | bl; - TCGArg rl, rh; - TCGOp *op2 = tcg_op_insert_before(s, op, INDEX_op_mov_i32); - - if (opc == INDEX_op_add2_i32) { - a += b; - } else { - a -= b; - } - - rl = op->args[0]; - rh = op->args[1]; - tcg_opt_gen_movi(&ctx, op, rl, (int32_t)a); - tcg_opt_gen_movi(&ctx, op2, rh, (int32_t)(a >> 32)); - continue; - } - break; default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(add): done = fold_add(&ctx, op); break; + case INDEX_op_add2_i32: + done = fold_add2_i32(&ctx, op); + break; CASE_OP_32_64_VEC(and): done = fold_and(&ctx, op); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(sub): done = fold_sub(&ctx, op); break; + case INDEX_op_sub2_i32: + done = fold_sub2_i32(&ctx, op); + break; CASE_OP_32_64_VEC(xor): done = fold_xor(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 56 ++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_mb(OptContext *ctx, TCGOp *op) return true; } +static bool fold_movcond(OptContext *ctx, TCGOp *op) +{ + TCGOpcode opc = op->opc; + TCGCond cond = op->args[5]; + int i = do_constant_folding_cond(opc, op->args[1], op->args[2], cond); + + if (i >= 0) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]); + } + + if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { + uint64_t tv = arg_info(op->args[3])->val; + uint64_t fv = arg_info(op->args[4])->val; + + opc = (opc == INDEX_op_movcond_i32 + ? INDEX_op_setcond_i32 : INDEX_op_setcond_i64); + + if (tv == 1 && fv == 0) { + op->opc = opc; + op->args[3] = cond; + } else if (fv == 1 && tv == 0) { + op->opc = opc; + op->args[3] = tcg_invert_cond(cond); + } + } + return false; +} + static bool fold_mul(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(movcond): - i = do_constant_folding_cond(opc, op->args[1], - op->args[2], op->args[5]); - if (i >= 0) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[4 - i]); - continue; - } - if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { - uint64_t tv = arg_info(op->args[3])->val; - uint64_t fv = arg_info(op->args[4])->val; - TCGCond cond = op->args[5]; - - if (fv == 1 && tv == 0) { - cond = tcg_invert_cond(cond); - } else if (!(tv == 1 && fv == 0)) { - break; - } - op->args[3] = cond; - op->opc = opc = (opc == INDEX_op_movcond_i32 - ? INDEX_op_setcond_i32 - : INDEX_op_setcond_i64); - } - break; - - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_mb: done = fold_mb(&ctx, op); break; + CASE_OP_32_64(movcond): + done = fold_movcond(&ctx, op); + break; CASE_OP_32_64(mul): done = fold_mul(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_extract2(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { + uint64_t v1 = arg_info(op->args[1])->val; + uint64_t v2 = arg_info(op->args[2])->val; + int shr = op->args[3]; + + if (op->opc == INDEX_op_extract2_i64) { + v1 >>= shr; + v2 <<= 64 - shr; + } else { + v1 = (uint32_t)v1 >> shr; + v2 = (int32_t)v2 << (32 - shr); + } + return tcg_opt_gen_movi(ctx, op, op->args[0], v1 | v2); + } + return false; +} + static bool fold_exts(OptContext *ctx, TCGOp *op) { return fold_const1(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(extract2): - if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - uint64_t v1 = arg_info(op->args[1])->val; - uint64_t v2 = arg_info(op->args[2])->val; - int shr = op->args[3]; - - if (opc == INDEX_op_extract2_i64) { - tmp = (v1 >> shr) | (v2 << (64 - shr)); - } else { - tmp = (int32_t)(((uint32_t)v1 >> shr) | - ((uint32_t)v2 << (32 - shr))); - } - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(eqv): done = fold_eqv(&ctx, op); break; + CASE_OP_32_64(extract2): + done = fold_extract2(&ctx, op); + break; CASE_OP_32_64(ext8s): CASE_OP_32_64(ext16s): case INDEX_op_ext32s_i64: -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_extract(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t; + + t = arg_info(op->args[1])->val; + t = extract64(t, op->args[2], op->args[3]); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return false; +} + static bool fold_extract2(OptContext *ctx, TCGOp *op) { if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) return tcg_opt_gen_movi(ctx, op, op->args[0], i); } +static bool fold_sextract(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t; + + t = arg_info(op->args[1])->val; + t = sextract64(t, op->args[2], op->args[3]); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return false; +} + static bool fold_shift(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(extract): - if (arg_is_const(op->args[1])) { - tmp = extract64(arg_info(op->args[1])->val, - op->args[2], op->args[3]); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - - CASE_OP_32_64(sextract): - if (arg_is_const(op->args[1])) { - tmp = sextract64(arg_info(op->args[1])->val, - op->args[2], op->args[3]); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(eqv): done = fold_eqv(&ctx, op); break; + CASE_OP_32_64(extract): + done = fold_extract(&ctx, op); + break; CASE_OP_32_64(extract2): done = fold_extract2(&ctx, op); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_setcond2_i32: done = fold_setcond2(&ctx, op); break; + CASE_OP_32_64(sextract): + done = fold_sextract(&ctx, op); + break; CASE_OP_32_64_VEC(sub): done = fold_sub(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_ctpop(OptContext *ctx, TCGOp *op) return fold_const1(ctx, op); } +static bool fold_deposit(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { + uint64_t t1 = arg_info(op->args[1])->val; + uint64_t t2 = arg_info(op->args[2])->val; + + t1 = deposit64(t1, op->args[3], op->args[4], t2); + return tcg_opt_gen_movi(ctx, op, op->args[0], t1); + } + return false; +} + static bool fold_divide(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(deposit): - if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - tmp = deposit64(arg_info(op->args[1])->val, - op->args[3], op->args[4], - arg_info(op->args[2])->val); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(ctpop): done = fold_ctpop(&ctx, op); break; + CASE_OP_32_64(deposit): + done = fold_deposit(&ctx, op); + break; CASE_OP_32_64(div): CASE_OP_32_64(divu): done = fold_divide(&ctx, op); -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_call(OptContext *ctx, TCGOp *op) return true; } +static bool fold_count_zeros(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t = arg_info(op->args[1])->val; + + if (t != 0) { + t = do_constant_folding(op->opc, t, 0); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]); + } + return false; +} + static bool fold_ctpop(OptContext *ctx, TCGOp *op) { return fold_const1(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(clz): - CASE_OP_32_64(ctz): - if (arg_is_const(op->args[1])) { - TCGArg v = arg_info(op->args[1])->val; - if (v != 0) { - tmp = do_constant_folding(opc, v, 0); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - } else { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[2]); - } - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_brcond2_i32: done = fold_brcond2(&ctx, op); break; + CASE_OP_32_64(clz): + CASE_OP_32_64(ctz): + done = fold_count_zeros(&ctx, op); + break; CASE_OP_32_64(ctpop): done = fold_ctpop(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) return false; } +static bool fold_bswap(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t = arg_info(op->args[1])->val; + + t = do_constant_folding(op->opc, t, op->args[2]); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return false; +} + static bool fold_call(OptContext *ctx, TCGOp *op) { TCGContext *s = ctx->tcg; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(bswap16): - CASE_OP_32_64(bswap32): - case INDEX_op_bswap64_i64: - if (arg_is_const(op->args[1])) { - tmp = do_constant_folding(opc, arg_info(op->args[1])->val, - op->args[2]); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_brcond2_i32: done = fold_brcond2(&ctx, op); break; + CASE_OP_32_64(bswap16): + CASE_OP_32_64(bswap32): + case INDEX_op_bswap64_i64: + done = fold_bswap(&ctx, op); + break; CASE_OP_32_64(clz): CASE_OP_32_64(ctz): done = fold_count_zeros(&ctx, op); -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 53 +++++++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_divide(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_dup(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t = arg_info(op->args[1])->val; + t = dup_const(TCGOP_VECE(op), t); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return false; +} + +static bool fold_dup2(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { + uint64_t t = deposit64(arg_info(op->args[1])->val, 32, 32, + arg_info(op->args[2])->val); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + + if (args_are_copies(op->args[1], op->args[2])) { + op->opc = INDEX_op_dup_vec; + TCGOP_VECE(op) = MO_32; + } + return false; +} + static bool fold_eqv(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) done = tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); break; - case INDEX_op_dup_vec: - if (arg_is_const(op->args[1])) { - tmp = arg_info(op->args[1])->val; - tmp = dup_const(TCGOP_VECE(op), tmp); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - - case INDEX_op_dup2_vec: - assert(TCG_TARGET_REG_BITS == 32); - if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - tcg_opt_gen_movi(&ctx, op, op->args[0], - deposit64(arg_info(op->args[1])->val, 32, 32, - arg_info(op->args[2])->val)); - continue; - } else if (args_are_copies(op->args[1], op->args[2])) { - op->opc = INDEX_op_dup_vec; - TCGOP_VECE(op) = MO_32; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(divu): done = fold_divide(&ctx, op); break; + case INDEX_op_dup_vec: + done = fold_dup(&ctx, op); + break; + case INDEX_op_dup2_vec: + done = fold_dup2(&ctx, op); + break; CASE_OP_32_64(eqv): done = fold_eqv(&ctx, op); break; -- 2.25.1 This is the final entry in the main switch that was in a different form. After this, we have the option to convert the switch into a function dispatch table. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_mb(OptContext *ctx, TCGOp *op) return true; } +static bool fold_mov(OptContext *ctx, TCGOp *op) +{ + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); +} + static bool fold_movcond(OptContext *ctx, TCGOp *op) { TCGOpcode opc = op->opc; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Propagate constants through copy operations and do constant - folding. Constants will be substituted to arguments by register - allocator where needed and possible. Also detect copies. */ + /* + * Process each opcode. + * Sorted alphabetically by opcode as much as possible. + */ switch (opc) { - CASE_OP_32_64_VEC(mov): - done = tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - break; - - default: - break; - - /* ---------------------------------------------------------- */ - /* Sorted alphabetically by opcode as much as possible. */ - CASE_OP_32_64_VEC(add): done = fold_add(&ctx, op); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_mb: done = fold_mb(&ctx, op); break; + CASE_OP_32_64_VEC(mov): + done = fold_mov(&ctx, op); + break; CASE_OP_32_64(movcond): done = fold_movcond(&ctx, op); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(xor): done = fold_xor(&ctx, op); break; + default: + break; } if (!done) { -- 2.25.1 Pull the "op r, a, a => movi r, 0" optimization into a function, and use it in the outer opcode fold functions. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) return false; } +/* If the binary operation has both arguments equal, fold to @i. */ +static bool fold_xx_to_i(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (args_are_copies(op->args[1], op->args[2])) { + return tcg_opt_gen_movi(ctx, op, op->args[0], i); + } + return false; +} + /* * These outermost fold_<op> functions are sorted alphabetically. */ @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) static bool fold_andc(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_i(ctx, op, 0)) { + return true; + } + return false; } static bool fold_brcond(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_shift(OptContext *ctx, TCGOp *op) static bool fold_sub(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_i(ctx, op, 0)) { + return true; + } + return false; } static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) static bool fold_xor(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_i(ctx, op, 0)) { + return true; + } + return false; } /* Propagate constants and copies, fold constant expressions. */ @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Simplify expression for "op r, a, a => movi r, 0" cases */ - switch (opc) { - CASE_OP_32_64_VEC(andc): - CASE_OP_32_64_VEC(sub): - CASE_OP_32_64_VEC(xor): - if (args_are_copies(op->args[1], op->args[2])) { - tcg_opt_gen_movi(&ctx, op, op->args[0], 0); - continue; - } - break; - default: - break; - } - /* * Process each opcode. * Sorted alphabetically by opcode as much as possible. -- 2.25.1 Pull the "op r, a, a => mov r, a" optimization into a function, and use it in the outer opcode fold functions. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_xx_to_i(OptContext *ctx, TCGOp *op, uint64_t i) return false; } +/* If the binary operation has both arguments equal, fold to identity. */ +static bool fold_xx_to_x(OptContext *ctx, TCGOp *op) +{ + if (args_are_copies(op->args[1], op->args[2])) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); + } + return false; +} + /* * These outermost fold_<op> functions are sorted alphabetically. + * + * The ordering of the transformations should be: + * 1) those that produce a constant + * 2) those that produce a copy + * 3) those that produce information about the result value. */ static bool fold_add(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_add2_i32(OptContext *ctx, TCGOp *op) static bool fold_and(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_x(ctx, op)) { + return true; + } + return false; } static bool fold_andc(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_not(OptContext *ctx, TCGOp *op) static bool fold_or(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_x(ctx, op)) { + return true; + } + return false; } static bool fold_orc(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Simplify expression for "op r, a, a => mov r, a" cases */ - switch (opc) { - CASE_OP_32_64_VEC(or): - CASE_OP_32_64_VEC(and): - if (args_are_copies(op->args[1], op->args[2])) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - continue; - } - break; - default: - break; - } - /* * Process each opcode. * Sorted alphabetically by opcode as much as possible. -- 2.25.1 Pull the "op r, a, 0 => movi r, 0" optimization into a function, and use it in the outer opcode fold functions. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) return false; } +/* If the binary operation has second argument @i, fold to @i. */ +static bool fold_xi_to_i(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == i) { + return tcg_opt_gen_movi(ctx, op, op->args[0], i); + } + return false; +} + /* If the binary operation has both arguments equal, fold to @i. */ static bool fold_xx_to_i(OptContext *ctx, TCGOp *op, uint64_t i) { @@ -XXX,XX +XXX,XX @@ static bool fold_add2_i32(OptContext *ctx, TCGOp *op) static bool fold_and(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_xi_to_i(ctx, op, 0) || fold_xx_to_x(ctx, op)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) static bool fold_mul(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_i(ctx, op, 0)) { + return true; + } + return false; } static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_i(ctx, op, 0)) { + return true; + } + return false; } static bool fold_mulu2_i32(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) continue; } - /* Simplify expression for "op r, a, 0 => movi r, 0" cases */ - switch (opc) { - CASE_OP_32_64_VEC(and): - CASE_OP_32_64_VEC(mul): - CASE_OP_32_64(muluh): - CASE_OP_32_64(mulsh): - if (arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == 0) { - tcg_opt_gen_movi(&ctx, op, op->args[0], 0); - continue; - } - break; - default: - break; - } - /* * Process each opcode. * Sorted alphabetically by opcode as much as possible. -- 2.25.1 Compute the type of the operation early. There are at least 4 places that used a def->flags ladder to determine the type of the operation being optimized. There were two places that assumed !TCG_OPF_64BIT means TCG_TYPE_I32, and so could potentially compute incorrect results for vector operations. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 149 +++++++++++++++++++++++++++++-------------------- 1 file changed, 89 insertions(+), 60 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { /* In flight values from optimization. */ uint64_t z_mask; + TCGType type; } OptContext; static inline TempOptInfo *ts_info(TCGTemp *ts) @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) { TCGTemp *dst_ts = arg_temp(dst); TCGTemp *src_ts = arg_temp(src); - const TCGOpDef *def; TempOptInfo *di; TempOptInfo *si; uint64_t z_mask; @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) reset_ts(dst_ts); di = ts_info(dst_ts); si = ts_info(src_ts); - def = &tcg_op_defs[op->opc]; - if (def->flags & TCG_OPF_VECTOR) { - new_op = INDEX_op_mov_vec; - } else if (def->flags & TCG_OPF_64BIT) { - new_op = INDEX_op_mov_i64; - } else { + + switch (ctx->type) { + case TCG_TYPE_I32: new_op = INDEX_op_mov_i32; + break; + case TCG_TYPE_I64: + new_op = INDEX_op_mov_i64; + break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + /* TCGOP_VECL and TCGOP_VECE remain unchanged. */ + new_op = INDEX_op_mov_vec; + break; + default: + g_assert_not_reached(); } op->opc = new_op; - /* TCGOP_VECL and TCGOP_VECE remain unchanged. */ op->args[0] = dst; op->args[1] = src; @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) static bool tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, TCGArg dst, uint64_t val) { - const TCGOpDef *def = &tcg_op_defs[op->opc]; - TCGType type; - TCGTemp *tv; - - if (def->flags & TCG_OPF_VECTOR) { - type = TCGOP_VECL(op) + TCG_TYPE_V64; - } else if (def->flags & TCG_OPF_64BIT) { - type = TCG_TYPE_I64; - } else { - type = TCG_TYPE_I32; - } - /* Convert movi to mov with constant temp. */ - tv = tcg_constant_internal(type, val); + TCGTemp *tv = tcg_constant_internal(ctx->type, val); + init_ts_info(ctx, tv); return tcg_opt_gen_mov(ctx, op, dst, temp_arg(tv)); } @@ -XXX,XX +XXX,XX @@ static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y) } } -static uint64_t do_constant_folding(TCGOpcode op, uint64_t x, uint64_t y) +static uint64_t do_constant_folding(TCGOpcode op, TCGType type, + uint64_t x, uint64_t y) { - const TCGOpDef *def = &tcg_op_defs[op]; uint64_t res = do_constant_folding_2(op, x, y); - if (!(def->flags & TCG_OPF_64BIT)) { + if (type == TCG_TYPE_I32) { res = (int32_t)res; } return res; @@ -XXX,XX +XXX,XX @@ static bool do_constant_folding_cond_eq(TCGCond c) * Return -1 if the condition can't be simplified, * and the result of the condition (0 or 1) if it can. */ -static int do_constant_folding_cond(TCGOpcode op, TCGArg x, +static int do_constant_folding_cond(TCGType type, TCGArg x, TCGArg y, TCGCond c) { uint64_t xv = arg_info(x)->val; uint64_t yv = arg_info(y)->val; if (arg_is_const(x) && arg_is_const(y)) { - const TCGOpDef *def = &tcg_op_defs[op]; - tcg_debug_assert(!(def->flags & TCG_OPF_VECTOR)); - if (def->flags & TCG_OPF_64BIT) { - return do_constant_folding_cond_64(xv, yv, c); - } else { + switch (type) { + case TCG_TYPE_I32: return do_constant_folding_cond_32(xv, yv, c); + case TCG_TYPE_I64: + return do_constant_folding_cond_64(xv, yv, c); + default: + /* Only scalar comparisons are optimizable */ + return -1; } } else if (args_are_copies(x, y)) { return do_constant_folding_cond_eq(c); @@ -XXX,XX +XXX,XX @@ static bool fold_const1(OptContext *ctx, TCGOp *op) uint64_t t; t = arg_info(op->args[1])->val; - t = do_constant_folding(op->opc, t, 0); + t = do_constant_folding(op->opc, ctx->type, t, 0); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } return false; @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) uint64_t t1 = arg_info(op->args[1])->val; uint64_t t2 = arg_info(op->args[2])->val; - t1 = do_constant_folding(op->opc, t1, t2); + t1 = do_constant_folding(op->opc, ctx->type, t1, t2); return tcg_opt_gen_movi(ctx, op, op->args[0], t1); } return false; @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) static bool fold_brcond(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[2]; - int i = do_constant_folding_cond(op->opc, op->args[0], op->args[1], cond); + int i = do_constant_folding_cond(ctx->type, op->args[0], op->args[1], cond); if (i == 0) { tcg_op_remove(ctx->tcg, op); @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) * Simplify EQ/NE comparisons where one of the pairs * can be simplified. */ - i = do_constant_folding_cond(INDEX_op_brcond_i32, op->args[0], + i = do_constant_folding_cond(TCG_TYPE_I32, op->args[0], op->args[2], cond); switch (i ^ inv) { case 0: @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) goto do_brcond_high; } - i = do_constant_folding_cond(INDEX_op_brcond_i32, op->args[1], + i = do_constant_folding_cond(TCG_TYPE_I32, op->args[1], op->args[3], cond); switch (i ^ inv) { case 0: @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) if (arg_is_const(op->args[1])) { uint64_t t = arg_info(op->args[1])->val; - t = do_constant_folding(op->opc, t, op->args[2]); + t = do_constant_folding(op->opc, ctx->type, t, op->args[2]); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } return false; @@ -XXX,XX +XXX,XX @@ static bool fold_count_zeros(OptContext *ctx, TCGOp *op) uint64_t t = arg_info(op->args[1])->val; if (t != 0) { - t = do_constant_folding(op->opc, t, 0); + t = do_constant_folding(op->opc, ctx->type, t, 0); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]); @@ -XXX,XX +XXX,XX @@ static bool fold_mov(OptContext *ctx, TCGOp *op) static bool fold_movcond(OptContext *ctx, TCGOp *op) { - TCGOpcode opc = op->opc; TCGCond cond = op->args[5]; - int i = do_constant_folding_cond(opc, op->args[1], op->args[2], cond); + int i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); if (i >= 0) { return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]); @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { uint64_t tv = arg_info(op->args[3])->val; uint64_t fv = arg_info(op->args[4])->val; + TCGOpcode opc; - opc = (opc == INDEX_op_movcond_i32 - ? INDEX_op_setcond_i32 : INDEX_op_setcond_i64); + switch (ctx->type) { + case TCG_TYPE_I32: + opc = INDEX_op_setcond_i32; + break; + case TCG_TYPE_I64: + opc = INDEX_op_setcond_i64; + break; + default: + g_assert_not_reached(); + } if (tv == 1 && fv == 0) { op->opc = opc; @@ -XXX,XX +XXX,XX @@ static bool fold_remainder(OptContext *ctx, TCGOp *op) static bool fold_setcond(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[3]; - int i = do_constant_folding_cond(op->opc, op->args[1], op->args[2], cond); + int i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); if (i >= 0) { return tcg_opt_gen_movi(ctx, op, op->args[0], i); @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) * Simplify EQ/NE comparisons where one of the pairs * can be simplified. */ - i = do_constant_folding_cond(INDEX_op_setcond_i32, op->args[1], + i = do_constant_folding_cond(TCG_TYPE_I32, op->args[1], op->args[3], cond); switch (i ^ inv) { case 0: @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) goto do_setcond_high; } - i = do_constant_folding_cond(INDEX_op_setcond_i32, op->args[2], + i = do_constant_folding_cond(TCG_TYPE_I32, op->args[2], op->args[4], cond); switch (i ^ inv) { case 0: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) init_arguments(&ctx, op, def->nb_oargs + def->nb_iargs); copy_propagate(&ctx, op, def->nb_oargs, def->nb_iargs); + /* Pre-compute the type of the operation. */ + if (def->flags & TCG_OPF_VECTOR) { + ctx.type = TCG_TYPE_V64 + TCGOP_VECL(op); + } else if (def->flags & TCG_OPF_64BIT) { + ctx.type = TCG_TYPE_I64; + } else { + ctx.type = TCG_TYPE_I32; + } + /* For commutative operations make constant second argument */ switch (opc) { CASE_OP_32_64_VEC(add): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* Proceed with possible constant folding. */ break; } - if (opc == INDEX_op_sub_i32) { + switch (ctx.type) { + case TCG_TYPE_I32: neg_op = INDEX_op_neg_i32; have_neg = TCG_TARGET_HAS_neg_i32; - } else if (opc == INDEX_op_sub_i64) { + break; + case TCG_TYPE_I64: neg_op = INDEX_op_neg_i64; have_neg = TCG_TARGET_HAS_neg_i64; - } else if (TCG_TARGET_HAS_neg_vec) { - TCGType type = TCGOP_VECL(op) + TCG_TYPE_V64; - unsigned vece = TCGOP_VECE(op); - neg_op = INDEX_op_neg_vec; - have_neg = tcg_can_emit_vec_op(neg_op, type, vece) > 0; - } else { break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + neg_op = INDEX_op_neg_vec; + have_neg = tcg_can_emit_vec_op(neg_op, ctx.type, + TCGOP_VECE(op)) > 0; + break; + default: + g_assert_not_reached(); } if (!have_neg) { break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) TCGOpcode not_op; bool have_not; - if (def->flags & TCG_OPF_VECTOR) { - not_op = INDEX_op_not_vec; - have_not = TCG_TARGET_HAS_not_vec; - } else if (def->flags & TCG_OPF_64BIT) { - not_op = INDEX_op_not_i64; - have_not = TCG_TARGET_HAS_not_i64; - } else { + switch (ctx.type) { + case TCG_TYPE_I32: not_op = INDEX_op_not_i32; have_not = TCG_TARGET_HAS_not_i32; + break; + case TCG_TYPE_I64: + not_op = INDEX_op_not_i64; + have_not = TCG_TARGET_HAS_not_i64; + break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + not_op = INDEX_op_not_vec; + have_not = TCG_TARGET_HAS_not_vec; + break; + default: + g_assert_not_reached(); } if (!have_not) { break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) below, we can ignore high bits, but for further optimizations we need to record that the high bits contain garbage. */ partmask = z_mask; - if (!(def->flags & TCG_OPF_64BIT)) { + if (ctx.type == TCG_TYPE_I32) { z_mask |= ~(tcg_target_ulong)0xffffffffu; partmask &= 0xffffffffu; affected &= 0xffffffffu; -- 2.25.1 Split out the conditional conversion from a more complex logical operation to a simple NOT. Create a couple more helpers to make this easy for the outer-most logical operations. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 158 +++++++++++++++++++++++++++---------------------- 1 file changed, 86 insertions(+), 72 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) return false; } +/* + * Convert @op to NOT, if NOT is supported by the host. + * Return true f the conversion is successful, which will still + * indicate that the processing is complete. + */ +static bool fold_not(OptContext *ctx, TCGOp *op); +static bool fold_to_not(OptContext *ctx, TCGOp *op, int idx) +{ + TCGOpcode not_op; + bool have_not; + + switch (ctx->type) { + case TCG_TYPE_I32: + not_op = INDEX_op_not_i32; + have_not = TCG_TARGET_HAS_not_i32; + break; + case TCG_TYPE_I64: + not_op = INDEX_op_not_i64; + have_not = TCG_TARGET_HAS_not_i64; + break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + not_op = INDEX_op_not_vec; + have_not = TCG_TARGET_HAS_not_vec; + break; + default: + g_assert_not_reached(); + } + if (have_not) { + op->opc = not_op; + op->args[1] = op->args[idx]; + return fold_not(ctx, op); + } + return false; +} + +/* If the binary operation has first argument @i, fold to NOT. */ +static bool fold_ix_to_not(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (arg_is_const(op->args[1]) && arg_info(op->args[1])->val == i) { + return fold_to_not(ctx, op, 2); + } + return false; +} + /* If the binary operation has second argument @i, fold to @i. */ static bool fold_xi_to_i(OptContext *ctx, TCGOp *op, uint64_t i) { @@ -XXX,XX +XXX,XX @@ static bool fold_xi_to_i(OptContext *ctx, TCGOp *op, uint64_t i) return false; } +/* If the binary operation has second argument @i, fold to NOT. */ +static bool fold_xi_to_not(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == i) { + return fold_to_not(ctx, op, 1); + } + return false; +} + /* If the binary operation has both arguments equal, fold to @i. */ static bool fold_xx_to_i(OptContext *ctx, TCGOp *op, uint64_t i) { @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) static bool fold_andc(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || - fold_xx_to_i(ctx, op, 0)) { + fold_xx_to_i(ctx, op, 0) || + fold_ix_to_not(ctx, op, -1)) { return true; } return false; @@ -XXX,XX +XXX,XX @@ static bool fold_dup2(OptContext *ctx, TCGOp *op) static bool fold_eqv(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_not(ctx, op, 0)) { + return true; + } + return false; } static bool fold_extract(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_mulu2_i32(OptContext *ctx, TCGOp *op) static bool fold_nand(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_not(ctx, op, -1)) { + return true; + } + return false; } static bool fold_neg(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_neg(OptContext *ctx, TCGOp *op) static bool fold_nor(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_not(ctx, op, 0)) { + return true; + } + return false; } static bool fold_not(OptContext *ctx, TCGOp *op) { - return fold_const1(ctx, op); + if (fold_const1(ctx, op)) { + return true; + } + + /* Because of fold_to_not, we want to always return true, via finish. */ + finish_folding(ctx, op); + return true; } static bool fold_or(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_or(OptContext *ctx, TCGOp *op) static bool fold_orc(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_ix_to_not(ctx, op, 0)) { + return true; + } + return false; } static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) static bool fold_xor(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || - fold_xx_to_i(ctx, op, 0)) { + fold_xx_to_i(ctx, op, 0) || + fold_xi_to_not(ctx, op, -1)) { return true; } return false; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } } break; - CASE_OP_32_64_VEC(xor): - CASE_OP_32_64(nand): - if (!arg_is_const(op->args[1]) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == -1) { - i = 1; - goto try_not; - } - break; - CASE_OP_32_64(nor): - if (!arg_is_const(op->args[1]) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == 0) { - i = 1; - goto try_not; - } - break; - CASE_OP_32_64_VEC(andc): - if (!arg_is_const(op->args[2]) - && arg_is_const(op->args[1]) - && arg_info(op->args[1])->val == -1) { - i = 2; - goto try_not; - } - break; - CASE_OP_32_64_VEC(orc): - CASE_OP_32_64(eqv): - if (!arg_is_const(op->args[2]) - && arg_is_const(op->args[1]) - && arg_info(op->args[1])->val == 0) { - i = 2; - goto try_not; - } - break; - try_not: - { - TCGOpcode not_op; - bool have_not; - - switch (ctx.type) { - case TCG_TYPE_I32: - not_op = INDEX_op_not_i32; - have_not = TCG_TARGET_HAS_not_i32; - break; - case TCG_TYPE_I64: - not_op = INDEX_op_not_i64; - have_not = TCG_TARGET_HAS_not_i64; - break; - case TCG_TYPE_V64: - case TCG_TYPE_V128: - case TCG_TYPE_V256: - not_op = INDEX_op_not_vec; - have_not = TCG_TARGET_HAS_not_vec; - break; - default: - g_assert_not_reached(); - } - if (!have_not) { - break; - } - op->opc = not_op; - reset_temp(op->args[0]); - op->args[1] = op->args[i]; - continue; - } default: break; } -- 2.25.1 Even though there is only one user, place this more complex conversion into its own helper. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 89 ++++++++++++++++++++++++++------------------------ 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_nand(OptContext *ctx, TCGOp *op) static bool fold_neg(OptContext *ctx, TCGOp *op) { - return fold_const1(ctx, op); + if (fold_const1(ctx, op)) { + return true; + } + /* + * Because of fold_sub_to_neg, we want to always return true, + * via finish_folding. + */ + finish_folding(ctx, op); + return true; } static bool fold_nor(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_shift(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_sub_to_neg(OptContext *ctx, TCGOp *op) +{ + TCGOpcode neg_op; + bool have_neg; + + if (!arg_is_const(op->args[1]) || arg_info(op->args[1])->val != 0) { + return false; + } + + switch (ctx->type) { + case TCG_TYPE_I32: + neg_op = INDEX_op_neg_i32; + have_neg = TCG_TARGET_HAS_neg_i32; + break; + case TCG_TYPE_I64: + neg_op = INDEX_op_neg_i64; + have_neg = TCG_TARGET_HAS_neg_i64; + break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + neg_op = INDEX_op_neg_vec; + have_neg = (TCG_TARGET_HAS_neg_vec && + tcg_can_emit_vec_op(neg_op, ctx->type, TCGOP_VECE(op)) > 0); + break; + default: + g_assert_not_reached(); + } + if (have_neg) { + op->opc = neg_op; + op->args[1] = op->args[2]; + return fold_neg(ctx, op); + } + return false; +} + static bool fold_sub(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || - fold_xx_to_i(ctx, op, 0)) { + fold_xx_to_i(ctx, op, 0) || + fold_sub_to_neg(ctx, op)) { return true; } return false; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) continue; } break; - CASE_OP_32_64_VEC(sub): - { - TCGOpcode neg_op; - bool have_neg; - - if (arg_is_const(op->args[2])) { - /* Proceed with possible constant folding. */ - break; - } - switch (ctx.type) { - case TCG_TYPE_I32: - neg_op = INDEX_op_neg_i32; - have_neg = TCG_TARGET_HAS_neg_i32; - break; - case TCG_TYPE_I64: - neg_op = INDEX_op_neg_i64; - have_neg = TCG_TARGET_HAS_neg_i64; - break; - case TCG_TYPE_V64: - case TCG_TYPE_V128: - case TCG_TYPE_V256: - neg_op = INDEX_op_neg_vec; - have_neg = tcg_can_emit_vec_op(neg_op, ctx.type, - TCGOP_VECE(op)) > 0; - break; - default: - g_assert_not_reached(); - } - if (!have_neg) { - break; - } - if (arg_is_const(op->args[1]) - && arg_info(op->args[1])->val == 0) { - op->opc = neg_op; - reset_temp(op->args[0]); - op->args[1] = op->args[2]; - continue; - } - } - break; default: break; } -- 2.25.1 Pull the "op r, a, i => mov r, a" optimization into a function, and use them in the outer-most logical operations. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 61 +++++++++++++++++++++----------------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_xi_to_i(OptContext *ctx, TCGOp *op, uint64_t i) return false; } +/* If the binary operation has second argument @i, fold to identity. */ +static bool fold_xi_to_x(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == i) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); + } + return false; +} + /* If the binary operation has second argument @i, fold to NOT. */ static bool fold_xi_to_not(OptContext *ctx, TCGOp *op, uint64_t i) { @@ -XXX,XX +XXX,XX @@ static bool fold_xx_to_x(OptContext *ctx, TCGOp *op) static bool fold_add(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, 0)) { + return true; + } + return false; } static bool fold_addsub2_i32(OptContext *ctx, TCGOp *op, bool add) @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || fold_xi_to_i(ctx, op, 0) || + fold_xi_to_x(ctx, op, -1) || fold_xx_to_x(ctx, op)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || fold_xx_to_i(ctx, op, 0) || + fold_xi_to_x(ctx, op, 0) || fold_ix_to_not(ctx, op, -1)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_dup2(OptContext *ctx, TCGOp *op) static bool fold_eqv(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, -1) || fold_xi_to_not(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_not(OptContext *ctx, TCGOp *op) static bool fold_or(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, 0) || fold_xx_to_x(ctx, op)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_or(OptContext *ctx, TCGOp *op) static bool fold_orc(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, -1) || fold_ix_to_not(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) static bool fold_shift(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, 0)) { + return true; + } + return false; } static bool fold_sub_to_neg(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_sub(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || fold_xx_to_i(ctx, op, 0) || + fold_xi_to_x(ctx, op, 0) || fold_sub_to_neg(ctx, op)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_xor(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || fold_xx_to_i(ctx, op, 0) || + fold_xi_to_x(ctx, op, 0) || fold_xi_to_not(ctx, op, -1)) { return true; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Simplify expression for "op r, a, const => mov r, a" cases */ - switch (opc) { - CASE_OP_32_64_VEC(add): - CASE_OP_32_64_VEC(sub): - CASE_OP_32_64_VEC(or): - CASE_OP_32_64_VEC(xor): - CASE_OP_32_64_VEC(andc): - CASE_OP_32_64(shl): - CASE_OP_32_64(shr): - CASE_OP_32_64(sar): - CASE_OP_32_64(rotl): - CASE_OP_32_64(rotr): - if (!arg_is_const(op->args[1]) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == 0) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - continue; - } - break; - CASE_OP_32_64_VEC(and): - CASE_OP_32_64_VEC(orc): - CASE_OP_32_64(eqv): - if (!arg_is_const(op->args[1]) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == -1) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - continue; - } - break; - default: - break; - } - /* Simplify using known-zero bits. Currently only ops with a single output argument is supported. */ z_mask = -1; -- 2.25.1 Pull the "op r, 0, b => movi r, 0" optimization into a function, and use it in fold_shift. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_to_not(OptContext *ctx, TCGOp *op, int idx) return false; } +/* If the binary operation has first argument @i, fold to @i. */ +static bool fold_ix_to_i(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (arg_is_const(op->args[1]) && arg_info(op->args[1])->val == i) { + return tcg_opt_gen_movi(ctx, op, op->args[0], i); + } + return false; +} + /* If the binary operation has first argument @i, fold to NOT. */ static bool fold_ix_to_not(OptContext *ctx, TCGOp *op, uint64_t i) { @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) static bool fold_shift(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_ix_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Simplify expressions for "shift/rot r, 0, a => movi r, 0", - and "sub r, 0, a => neg r, a" case. */ - switch (opc) { - CASE_OP_32_64(shl): - CASE_OP_32_64(shr): - CASE_OP_32_64(sar): - CASE_OP_32_64(rotl): - CASE_OP_32_64(rotr): - if (arg_is_const(op->args[1]) - && arg_info(op->args[1])->val == 0) { - tcg_opt_gen_movi(&ctx, op, op->args[0], 0); - continue; - } - break; - default: - break; - } - /* Simplify using known-zero bits. Currently only ops with a single output argument is supported. */ z_mask = -1; -- 2.25.1 Move all of the known-zero optimizations into the per-opcode functions. Use fold_masks when there is a possibility of the result being determined, and simply set ctx->z_mask otherwise. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 545 ++++++++++++++++++++++++++----------------------- 1 file changed, 294 insertions(+), 251 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { TCGTempSet temps_used; /* In flight values from optimization. */ - uint64_t z_mask; + uint64_t a_mask; /* mask bit is 0 iff value identical to first input */ + uint64_t z_mask; /* mask bit is 0 iff value bit is 0 */ TCGType type; } OptContext; @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) return false; } +static bool fold_masks(OptContext *ctx, TCGOp *op) +{ + uint64_t a_mask = ctx->a_mask; + uint64_t z_mask = ctx->z_mask; + + /* + * 32-bit ops generate 32-bit results. For the result is zero test + * below, we can ignore high bits, but for further optimizations we + * need to record that the high bits contain garbage. + */ + if (ctx->type == TCG_TYPE_I32) { + ctx->z_mask |= MAKE_64BIT_MASK(32, 32); + a_mask &= MAKE_64BIT_MASK(0, 32); + z_mask &= MAKE_64BIT_MASK(0, 32); + } + + if (z_mask == 0) { + return tcg_opt_gen_movi(ctx, op, op->args[0], 0); + } + if (a_mask == 0) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); + } + return false; +} + /* * Convert @op to NOT, if NOT is supported by the host. * Return true f the conversion is successful, which will still @@ -XXX,XX +XXX,XX @@ static bool fold_add2_i32(OptContext *ctx, TCGOp *op) static bool fold_and(OptContext *ctx, TCGOp *op) { + uint64_t z1, z2; + if (fold_const2(ctx, op) || fold_xi_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, -1) || fold_xx_to_x(ctx, op)) { return true; } - return false; + + z1 = arg_info(op->args[1])->z_mask; + z2 = arg_info(op->args[2])->z_mask; + ctx->z_mask = z1 & z2; + + /* + * Known-zeros does not imply known-ones. Therefore unless + * arg2 is constant, we can't infer affected bits from it. + */ + if (arg_is_const(op->args[2])) { + ctx->a_mask = z1 & ~z2; + } + + return fold_masks(ctx, op); } static bool fold_andc(OptContext *ctx, TCGOp *op) { + uint64_t z1; + if (fold_const2(ctx, op) || fold_xx_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, 0) || fold_ix_to_not(ctx, op, -1)) { return true; } - return false; + + z1 = arg_info(op->args[1])->z_mask; + + /* + * Known-zeros does not imply known-ones. Therefore unless + * arg2 is constant, we can't infer anything from it. + */ + if (arg_is_const(op->args[2])) { + uint64_t z2 = ~arg_info(op->args[2])->z_mask; + ctx->a_mask = z1 & ~z2; + z1 &= z2; + } + ctx->z_mask = z1; + + return fold_masks(ctx, op); } static bool fold_brcond(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) static bool fold_bswap(OptContext *ctx, TCGOp *op) { + uint64_t z_mask, sign; + if (arg_is_const(op->args[1])) { uint64_t t = arg_info(op->args[1])->val; t = do_constant_folding(op->opc, ctx->type, t, op->args[2]); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } - return false; + + z_mask = arg_info(op->args[1])->z_mask; + switch (op->opc) { + case INDEX_op_bswap16_i32: + case INDEX_op_bswap16_i64: + z_mask = bswap16(z_mask); + sign = INT16_MIN; + break; + case INDEX_op_bswap32_i32: + case INDEX_op_bswap32_i64: + z_mask = bswap32(z_mask); + sign = INT32_MIN; + break; + case INDEX_op_bswap64_i64: + z_mask = bswap64(z_mask); + sign = INT64_MIN; + break; + default: + g_assert_not_reached(); + } + + switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { + case TCG_BSWAP_OZ: + break; + case TCG_BSWAP_OS: + /* If the sign bit may be 1, force all the bits above to 1. */ + if (z_mask & sign) { + z_mask |= sign; + } + break; + default: + /* The high bits are undefined: force all bits above the sign to 1. */ + z_mask |= sign << 1; + break; + } + ctx->z_mask = z_mask; + + return fold_masks(ctx, op); } static bool fold_call(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_call(OptContext *ctx, TCGOp *op) static bool fold_count_zeros(OptContext *ctx, TCGOp *op) { + uint64_t z_mask; + if (arg_is_const(op->args[1])) { uint64_t t = arg_info(op->args[1])->val; @@ -XXX,XX +XXX,XX @@ static bool fold_count_zeros(OptContext *ctx, TCGOp *op) } return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]); } + + switch (ctx->type) { + case TCG_TYPE_I32: + z_mask = 31; + break; + case TCG_TYPE_I64: + z_mask = 63; + break; + default: + g_assert_not_reached(); + } + ctx->z_mask = arg_info(op->args[2])->z_mask | z_mask; + return false; } static bool fold_ctpop(OptContext *ctx, TCGOp *op) { - return fold_const1(ctx, op); + if (fold_const1(ctx, op)) { + return true; + } + + switch (ctx->type) { + case TCG_TYPE_I32: + ctx->z_mask = 32 | 31; + break; + case TCG_TYPE_I64: + ctx->z_mask = 64 | 63; + break; + default: + g_assert_not_reached(); + } + return false; } static bool fold_deposit(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) t1 = deposit64(t1, op->args[3], op->args[4], t2); return tcg_opt_gen_movi(ctx, op, op->args[0], t1); } + + ctx->z_mask = deposit64(arg_info(op->args[1])->z_mask, + op->args[3], op->args[4], + arg_info(op->args[2])->z_mask); return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) static bool fold_extract(OptContext *ctx, TCGOp *op) { + uint64_t z_mask_old, z_mask; + if (arg_is_const(op->args[1])) { uint64_t t; @@ -XXX,XX +XXX,XX @@ static bool fold_extract(OptContext *ctx, TCGOp *op) t = extract64(t, op->args[2], op->args[3]); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } - return false; + + z_mask_old = arg_info(op->args[1])->z_mask; + z_mask = extract64(z_mask_old, op->args[2], op->args[3]); + if (op->args[2] == 0) { + ctx->a_mask = z_mask_old ^ z_mask; + } + ctx->z_mask = z_mask; + + return fold_masks(ctx, op); } static bool fold_extract2(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_extract2(OptContext *ctx, TCGOp *op) static bool fold_exts(OptContext *ctx, TCGOp *op) { - return fold_const1(ctx, op); + uint64_t z_mask_old, z_mask, sign; + bool type_change = false; + + if (fold_const1(ctx, op)) { + return true; + } + + z_mask_old = z_mask = arg_info(op->args[1])->z_mask; + + switch (op->opc) { + CASE_OP_32_64(ext8s): + sign = INT8_MIN; + z_mask = (uint8_t)z_mask; + break; + CASE_OP_32_64(ext16s): + sign = INT16_MIN; + z_mask = (uint16_t)z_mask; + break; + case INDEX_op_ext_i32_i64: + type_change = true; + QEMU_FALLTHROUGH; + case INDEX_op_ext32s_i64: + sign = INT32_MIN; + z_mask = (uint32_t)z_mask; + break; + default: + g_assert_not_reached(); + } + + if (z_mask & sign) { + z_mask |= sign; + } else if (!type_change) { + ctx->a_mask = z_mask_old ^ z_mask; + } + ctx->z_mask = z_mask; + + return fold_masks(ctx, op); } static bool fold_extu(OptContext *ctx, TCGOp *op) { - return fold_const1(ctx, op); + uint64_t z_mask_old, z_mask; + bool type_change = false; + + if (fold_const1(ctx, op)) { + return true; + } + + z_mask_old = z_mask = arg_info(op->args[1])->z_mask; + + switch (op->opc) { + CASE_OP_32_64(ext8u): + z_mask = (uint8_t)z_mask; + break; + CASE_OP_32_64(ext16u): + z_mask = (uint16_t)z_mask; + break; + case INDEX_op_extrl_i64_i32: + case INDEX_op_extu_i32_i64: + type_change = true; + QEMU_FALLTHROUGH; + case INDEX_op_ext32u_i64: + z_mask = (uint32_t)z_mask; + break; + case INDEX_op_extrh_i64_i32: + type_change = true; + z_mask >>= 32; + break; + default: + g_assert_not_reached(); + } + + ctx->z_mask = z_mask; + if (!type_change) { + ctx->a_mask = z_mask_old ^ z_mask; + } + return fold_masks(ctx, op); } static bool fold_mb(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]); } + ctx->z_mask = arg_info(op->args[3])->z_mask + | arg_info(op->args[4])->z_mask; + if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { uint64_t tv = arg_info(op->args[3])->val; uint64_t fv = arg_info(op->args[4])->val; @@ -XXX,XX +XXX,XX @@ static bool fold_nand(OptContext *ctx, TCGOp *op) static bool fold_neg(OptContext *ctx, TCGOp *op) { + uint64_t z_mask; + if (fold_const1(ctx, op)) { return true; } + + /* Set to 1 all bits to the left of the rightmost. */ + z_mask = arg_info(op->args[1])->z_mask; + ctx->z_mask = -(z_mask & -z_mask); + /* * Because of fold_sub_to_neg, we want to always return true, * via finish_folding. @@ -XXX,XX +XXX,XX @@ static bool fold_or(OptContext *ctx, TCGOp *op) fold_xx_to_x(ctx, op)) { return true; } - return false; + + ctx->z_mask = arg_info(op->args[1])->z_mask + | arg_info(op->args[2])->z_mask; + return fold_masks(ctx, op); } static bool fold_orc(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_orc(OptContext *ctx, TCGOp *op) static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) { + const TCGOpDef *def = &tcg_op_defs[op->opc]; + MemOpIdx oi = op->args[def->nb_oargs + def->nb_iargs]; + MemOp mop = get_memop(oi); + int width = 8 * memop_size(mop); + + if (!(mop & MO_SIGN) && width < 64) { + ctx->z_mask = MAKE_64BIT_MASK(0, width); + } + /* Opcodes that touch guest memory stop the mb optimization. */ ctx->prev_mb = NULL; return false; @@ -XXX,XX +XXX,XX @@ static bool fold_setcond(OptContext *ctx, TCGOp *op) if (i >= 0) { return tcg_opt_gen_movi(ctx, op, op->args[0], i); } + + ctx->z_mask = 1; return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) op->opc = INDEX_op_setcond_i32; break; } + + ctx->z_mask = 1; return false; do_setcond_const: @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) static bool fold_sextract(OptContext *ctx, TCGOp *op) { + int64_t z_mask_old, z_mask; + if (arg_is_const(op->args[1])) { uint64_t t; @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) t = sextract64(t, op->args[2], op->args[3]); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } - return false; + + z_mask_old = arg_info(op->args[1])->z_mask; + z_mask = sextract64(z_mask_old, op->args[2], op->args[3]); + if (op->args[2] == 0 && z_mask >= 0) { + ctx->a_mask = z_mask_old ^ z_mask; + } + ctx->z_mask = z_mask; + + return fold_masks(ctx, op); } static bool fold_shift(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_shift(OptContext *ctx, TCGOp *op) fold_xi_to_x(ctx, op, 0)) { return true; } + + if (arg_is_const(op->args[2])) { + ctx->z_mask = do_constant_folding(op->opc, ctx->type, + arg_info(op->args[1])->z_mask, + arg_info(op->args[2])->val); + return fold_masks(ctx, op); + } return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) return fold_addsub2_i32(ctx, op, false); } +static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) +{ + /* We can't do any folding with a load, but we can record bits. */ + switch (op->opc) { + CASE_OP_32_64(ld8u): + ctx->z_mask = MAKE_64BIT_MASK(0, 8); + break; + CASE_OP_32_64(ld16u): + ctx->z_mask = MAKE_64BIT_MASK(0, 16); + break; + case INDEX_op_ld32u_i64: + ctx->z_mask = MAKE_64BIT_MASK(0, 32); + break; + default: + g_assert_not_reached(); + } + return false; +} + static bool fold_xor(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || @@ -XXX,XX +XXX,XX @@ static bool fold_xor(OptContext *ctx, TCGOp *op) fold_xi_to_not(ctx, op, -1)) { return true; } - return false; + + ctx->z_mask = arg_info(op->args[1])->z_mask + | arg_info(op->args[2])->z_mask; + return fold_masks(ctx, op); } /* Propagate constants and copies, fold constant expressions. */ @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { - uint64_t z_mask, partmask, affected, tmp; TCGOpcode opc = op->opc; const TCGOpDef *def; bool done = false; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Simplify using known-zero bits. Currently only ops with a single - output argument is supported. */ - z_mask = -1; - affected = -1; - switch (opc) { - CASE_OP_32_64(ext8s): - if ((arg_info(op->args[1])->z_mask & 0x80) != 0) { - break; - } - QEMU_FALLTHROUGH; - CASE_OP_32_64(ext8u): - z_mask = 0xff; - goto and_const; - CASE_OP_32_64(ext16s): - if ((arg_info(op->args[1])->z_mask & 0x8000) != 0) { - break; - } - QEMU_FALLTHROUGH; - CASE_OP_32_64(ext16u): - z_mask = 0xffff; - goto and_const; - case INDEX_op_ext32s_i64: - if ((arg_info(op->args[1])->z_mask & 0x80000000) != 0) { - break; - } - QEMU_FALLTHROUGH; - case INDEX_op_ext32u_i64: - z_mask = 0xffffffffU; - goto and_const; - - CASE_OP_32_64(and): - z_mask = arg_info(op->args[2])->z_mask; - if (arg_is_const(op->args[2])) { - and_const: - affected = arg_info(op->args[1])->z_mask & ~z_mask; - } - z_mask = arg_info(op->args[1])->z_mask & z_mask; - break; - - case INDEX_op_ext_i32_i64: - if ((arg_info(op->args[1])->z_mask & 0x80000000) != 0) { - break; - } - QEMU_FALLTHROUGH; - case INDEX_op_extu_i32_i64: - /* We do not compute affected as it is a size changing op. */ - z_mask = (uint32_t)arg_info(op->args[1])->z_mask; - break; - - CASE_OP_32_64(andc): - /* Known-zeros does not imply known-ones. Therefore unless - op->args[2] is constant, we can't infer anything from it. */ - if (arg_is_const(op->args[2])) { - z_mask = ~arg_info(op->args[2])->z_mask; - goto and_const; - } - /* But we certainly know nothing outside args[1] may be set. */ - z_mask = arg_info(op->args[1])->z_mask; - break; - - case INDEX_op_sar_i32: - if (arg_is_const(op->args[2])) { - tmp = arg_info(op->args[2])->val & 31; - z_mask = (int32_t)arg_info(op->args[1])->z_mask >> tmp; - } - break; - case INDEX_op_sar_i64: - if (arg_is_const(op->args[2])) { - tmp = arg_info(op->args[2])->val & 63; - z_mask = (int64_t)arg_info(op->args[1])->z_mask >> tmp; - } - break; - - case INDEX_op_shr_i32: - if (arg_is_const(op->args[2])) { - tmp = arg_info(op->args[2])->val & 31; - z_mask = (uint32_t)arg_info(op->args[1])->z_mask >> tmp; - } - break; - case INDEX_op_shr_i64: - if (arg_is_const(op->args[2])) { - tmp = arg_info(op->args[2])->val & 63; - z_mask = (uint64_t)arg_info(op->args[1])->z_mask >> tmp; - } - break; - - case INDEX_op_extrl_i64_i32: - z_mask = (uint32_t)arg_info(op->args[1])->z_mask; - break; - case INDEX_op_extrh_i64_i32: - z_mask = (uint64_t)arg_info(op->args[1])->z_mask >> 32; - break; - - CASE_OP_32_64(shl): - if (arg_is_const(op->args[2])) { - tmp = arg_info(op->args[2])->val & (TCG_TARGET_REG_BITS - 1); - z_mask = arg_info(op->args[1])->z_mask << tmp; - } - break; - - CASE_OP_32_64(neg): - /* Set to 1 all bits to the left of the rightmost. */ - z_mask = -(arg_info(op->args[1])->z_mask - & -arg_info(op->args[1])->z_mask); - break; - - CASE_OP_32_64(deposit): - z_mask = deposit64(arg_info(op->args[1])->z_mask, - op->args[3], op->args[4], - arg_info(op->args[2])->z_mask); - break; - - CASE_OP_32_64(extract): - z_mask = extract64(arg_info(op->args[1])->z_mask, - op->args[2], op->args[3]); - if (op->args[2] == 0) { - affected = arg_info(op->args[1])->z_mask & ~z_mask; - } - break; - CASE_OP_32_64(sextract): - z_mask = sextract64(arg_info(op->args[1])->z_mask, - op->args[2], op->args[3]); - if (op->args[2] == 0 && (tcg_target_long)z_mask >= 0) { - affected = arg_info(op->args[1])->z_mask & ~z_mask; - } - break; - - CASE_OP_32_64(or): - CASE_OP_32_64(xor): - z_mask = arg_info(op->args[1])->z_mask - | arg_info(op->args[2])->z_mask; - break; - - case INDEX_op_clz_i32: - case INDEX_op_ctz_i32: - z_mask = arg_info(op->args[2])->z_mask | 31; - break; - - case INDEX_op_clz_i64: - case INDEX_op_ctz_i64: - z_mask = arg_info(op->args[2])->z_mask | 63; - break; - - case INDEX_op_ctpop_i32: - z_mask = 32 | 31; - break; - case INDEX_op_ctpop_i64: - z_mask = 64 | 63; - break; - - CASE_OP_32_64(setcond): - case INDEX_op_setcond2_i32: - z_mask = 1; - break; - - CASE_OP_32_64(movcond): - z_mask = arg_info(op->args[3])->z_mask - | arg_info(op->args[4])->z_mask; - break; - - CASE_OP_32_64(ld8u): - z_mask = 0xff; - break; - CASE_OP_32_64(ld16u): - z_mask = 0xffff; - break; - case INDEX_op_ld32u_i64: - z_mask = 0xffffffffu; - break; - - CASE_OP_32_64(qemu_ld): - { - MemOpIdx oi = op->args[def->nb_oargs + def->nb_iargs]; - MemOp mop = get_memop(oi); - if (!(mop & MO_SIGN)) { - z_mask = (2ULL << ((8 << (mop & MO_SIZE)) - 1)) - 1; - } - } - break; - - CASE_OP_32_64(bswap16): - z_mask = arg_info(op->args[1])->z_mask; - if (z_mask <= 0xffff) { - op->args[2] |= TCG_BSWAP_IZ; - } - z_mask = bswap16(z_mask); - switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { - case TCG_BSWAP_OZ: - break; - case TCG_BSWAP_OS: - z_mask = (int16_t)z_mask; - break; - default: /* undefined high bits */ - z_mask |= MAKE_64BIT_MASK(16, 48); - break; - } - break; - - case INDEX_op_bswap32_i64: - z_mask = arg_info(op->args[1])->z_mask; - if (z_mask <= 0xffffffffu) { - op->args[2] |= TCG_BSWAP_IZ; - } - z_mask = bswap32(z_mask); - switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { - case TCG_BSWAP_OZ: - break; - case TCG_BSWAP_OS: - z_mask = (int32_t)z_mask; - break; - default: /* undefined high bits */ - z_mask |= MAKE_64BIT_MASK(32, 32); - break; - } - break; - - default: - break; - } - - /* 32-bit ops generate 32-bit results. For the result is zero test - below, we can ignore high bits, but for further optimizations we - need to record that the high bits contain garbage. */ - partmask = z_mask; - if (ctx.type == TCG_TYPE_I32) { - z_mask |= ~(tcg_target_ulong)0xffffffffu; - partmask &= 0xffffffffu; - affected &= 0xffffffffu; - } - ctx.z_mask = z_mask; - - if (partmask == 0) { - tcg_opt_gen_movi(&ctx, op, op->args[0], 0); - continue; - } - if (affected == 0) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - continue; - } + /* Assume all bits affected, and no bits known zero. */ + ctx.a_mask = -1; + ctx.z_mask = -1; /* * Process each opcode. @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_extrh_i64_i32: done = fold_extu(&ctx, op); break; + CASE_OP_32_64(ld8u): + CASE_OP_32_64(ld16u): + case INDEX_op_ld32u_i64: + done = fold_tcg_ld(&ctx, op); + break; case INDEX_op_mb: done = fold_mb(&ctx, op); break; -- 2.25.1 Rename to fold_multiply2, and handle muls2_i32, mulu2_i64, and muls2_i64. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) return false; } -static bool fold_mulu2_i32(OptContext *ctx, TCGOp *op) +static bool fold_multiply2(OptContext *ctx, TCGOp *op) { if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { - uint32_t a = arg_info(op->args[2])->val; - uint32_t b = arg_info(op->args[3])->val; - uint64_t r = (uint64_t)a * b; + uint64_t a = arg_info(op->args[2])->val; + uint64_t b = arg_info(op->args[3])->val; + uint64_t h, l; TCGArg rl, rh; - TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_mov_i32); + TCGOp *op2; + + switch (op->opc) { + case INDEX_op_mulu2_i32: + l = (uint64_t)(uint32_t)a * (uint32_t)b; + h = (int32_t)(l >> 32); + l = (int32_t)l; + break; + case INDEX_op_muls2_i32: + l = (int64_t)(int32_t)a * (int32_t)b; + h = l >> 32; + l = (int32_t)l; + break; + case INDEX_op_mulu2_i64: + mulu64(&l, &h, a, b); + break; + case INDEX_op_muls2_i64: + muls64(&l, &h, a, b); + break; + default: + g_assert_not_reached(); + } rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(ctx, op, rl, (int32_t)r); - tcg_opt_gen_movi(ctx, op2, rh, (int32_t)(r >> 32)); + + /* The proper opcode is supplied by tcg_opt_gen_mov. */ + op2 = tcg_op_insert_before(ctx->tcg, op, 0); + + tcg_opt_gen_movi(ctx, op, rl, l); + tcg_opt_gen_movi(ctx, op2, rh, h); return true; } return false; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(muluh): done = fold_mul_highpart(&ctx, op); break; - case INDEX_op_mulu2_i32: - done = fold_mulu2_i32(&ctx, op); + CASE_OP_32_64(muls2): + CASE_OP_32_64(mulu2): + done = fold_multiply2(&ctx, op); break; CASE_OP_32_64(nand): done = fold_nand(&ctx, op); -- 2.25.1 Rename to fold_addsub2. Use Int128 to implement the wider operation. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 65 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ */ #include "qemu/osdep.h" +#include "qemu/int128.h" #include "tcg/tcg-op.h" #include "tcg-internal.h" @@ -XXX,XX +XXX,XX @@ static bool fold_add(OptContext *ctx, TCGOp *op) return false; } -static bool fold_addsub2_i32(OptContext *ctx, TCGOp *op, bool add) +static bool fold_addsub2(OptContext *ctx, TCGOp *op, bool add) { if (arg_is_const(op->args[2]) && arg_is_const(op->args[3]) && arg_is_const(op->args[4]) && arg_is_const(op->args[5])) { - uint32_t al = arg_info(op->args[2])->val; - uint32_t ah = arg_info(op->args[3])->val; - uint32_t bl = arg_info(op->args[4])->val; - uint32_t bh = arg_info(op->args[5])->val; - uint64_t a = ((uint64_t)ah << 32) | al; - uint64_t b = ((uint64_t)bh << 32) | bl; + uint64_t al = arg_info(op->args[2])->val; + uint64_t ah = arg_info(op->args[3])->val; + uint64_t bl = arg_info(op->args[4])->val; + uint64_t bh = arg_info(op->args[5])->val; TCGArg rl, rh; - TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_mov_i32); + TCGOp *op2; - if (add) { - a += b; + if (ctx->type == TCG_TYPE_I32) { + uint64_t a = deposit64(al, 32, 32, ah); + uint64_t b = deposit64(bl, 32, 32, bh); + + if (add) { + a += b; + } else { + a -= b; + } + + al = sextract64(a, 0, 32); + ah = sextract64(a, 32, 32); } else { - a -= b; + Int128 a = int128_make128(al, ah); + Int128 b = int128_make128(bl, bh); + + if (add) { + a = int128_add(a, b); + } else { + a = int128_sub(a, b); + } + + al = int128_getlo(a); + ah = int128_gethi(a); } rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(ctx, op, rl, (int32_t)a); - tcg_opt_gen_movi(ctx, op2, rh, (int32_t)(a >> 32)); + + /* The proper opcode is supplied by tcg_opt_gen_mov. */ + op2 = tcg_op_insert_before(ctx->tcg, op, 0); + + tcg_opt_gen_movi(ctx, op, rl, al); + tcg_opt_gen_movi(ctx, op2, rh, ah); return true; } return false; } -static bool fold_add2_i32(OptContext *ctx, TCGOp *op) +static bool fold_add2(OptContext *ctx, TCGOp *op) { - return fold_addsub2_i32(ctx, op, true); + return fold_addsub2(ctx, op, true); } static bool fold_and(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_sub(OptContext *ctx, TCGOp *op) return false; } -static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) +static bool fold_sub2(OptContext *ctx, TCGOp *op) { - return fold_addsub2_i32(ctx, op, false); + return fold_addsub2(ctx, op, false); } static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(add): done = fold_add(&ctx, op); break; - case INDEX_op_add2_i32: - done = fold_add2_i32(&ctx, op); + CASE_OP_32_64(add2): + done = fold_add2(&ctx, op); break; CASE_OP_32_64_VEC(and): done = fold_and(&ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(sub): done = fold_sub(&ctx, op); break; - case INDEX_op_sub2_i32: - done = fold_sub2_i32(&ctx, op); + CASE_OP_32_64(sub2): + done = fold_sub2(&ctx, op); break; CASE_OP_32_64_VEC(xor): done = fold_xor(&ctx, op); -- 2.25.1 Most of these are handled by creating a fold_const2_commutative to handle all of the binary operators. The rest were already handled on a case-by-case basis in the switch, and have their own fold function in which to place the call. We now have only one major switch on TCGOpcode. Introduce NO_DEST and a block comment for swap_commutative in order to make the handling of brcond and movcond opcodes cleaner. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 142 ++++++++++++++++++++++++------------------------- 1 file changed, 70 insertions(+), 72 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static int do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c) return -1; } +/** + * swap_commutative: + * @dest: TCGArg of the destination argument, or NO_DEST. + * @p1: first paired argument + * @p2: second paired argument + * + * If *@p1 is a constant and *@p2 is not, swap. + * If *@p2 matches @dest, swap. + * Return true if a swap was performed. + */ + +#define NO_DEST temp_arg(NULL) + static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2) { TCGArg a1 = *p1, a2 = *p2; @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) return false; } +static bool fold_const2_commutative(OptContext *ctx, TCGOp *op) +{ + swap_commutative(op->args[0], &op->args[1], &op->args[2]); + return fold_const2(ctx, op); +} + static bool fold_masks(OptContext *ctx, TCGOp *op) { uint64_t a_mask = ctx->a_mask; @@ -XXX,XX +XXX,XX @@ static bool fold_xx_to_x(OptContext *ctx, TCGOp *op) static bool fold_add(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_x(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_addsub2(OptContext *ctx, TCGOp *op, bool add) static bool fold_add2(OptContext *ctx, TCGOp *op) { + /* Note that the high and low parts may be independently swapped. */ + swap_commutative(op->args[0], &op->args[2], &op->args[4]); + swap_commutative(op->args[1], &op->args[3], &op->args[5]); + return fold_addsub2(ctx, op, true); } @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) { uint64_t z1, z2; - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, -1) || fold_xx_to_x(ctx, op)) { @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) static bool fold_brcond(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[2]; - int i = do_constant_folding_cond(ctx->type, op->args[0], op->args[1], cond); + int i; + if (swap_commutative(NO_DEST, &op->args[0], &op->args[1])) { + op->args[2] = cond = tcg_swap_cond(cond); + } + + i = do_constant_folding_cond(ctx->type, op->args[0], op->args[1], cond); if (i == 0) { tcg_op_remove(ctx->tcg, op); return true; @@ -XXX,XX +XXX,XX @@ static bool fold_brcond(OptContext *ctx, TCGOp *op) static bool fold_brcond2(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[4]; - int i = do_constant_folding_cond2(&op->args[0], &op->args[2], cond); TCGArg label = op->args[5]; - int inv = 0; + int i, inv = 0; + if (swap_commutative2(&op->args[0], &op->args[2])) { + op->args[4] = cond = tcg_swap_cond(cond); + } + + i = do_constant_folding_cond2(&op->args[0], &op->args[2], cond); if (i >= 0) { goto do_brcond_const; } @@ -XXX,XX +XXX,XX @@ static bool fold_dup2(OptContext *ctx, TCGOp *op) static bool fold_eqv(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_x(ctx, op, -1) || fold_xi_to_not(ctx, op, 0)) { return true; @@ -XXX,XX +XXX,XX @@ static bool fold_mov(OptContext *ctx, TCGOp *op) static bool fold_movcond(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[5]; - int i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); + int i; + if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) { + op->args[5] = cond = tcg_swap_cond(cond); + } + /* + * Canonicalize the "false" input reg to match the destination reg so + * that the tcg backend can implement a "move if true" operation. + */ + if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) { + op->args[5] = cond = tcg_invert_cond(cond); + } + + i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); if (i >= 0) { return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]); } @@ -XXX,XX +XXX,XX @@ static bool fold_mul(OptContext *ctx, TCGOp *op) static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_i(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) static bool fold_multiply2(OptContext *ctx, TCGOp *op) { + swap_commutative(op->args[0], &op->args[2], &op->args[3]); + if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { uint64_t a = arg_info(op->args[2])->val; uint64_t b = arg_info(op->args[3])->val; @@ -XXX,XX +XXX,XX @@ static bool fold_multiply2(OptContext *ctx, TCGOp *op) static bool fold_nand(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_not(ctx, op, -1)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_neg(OptContext *ctx, TCGOp *op) static bool fold_nor(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_not(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_not(OptContext *ctx, TCGOp *op) static bool fold_or(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_x(ctx, op, 0) || fold_xx_to_x(ctx, op)) { return true; @@ -XXX,XX +XXX,XX @@ static bool fold_remainder(OptContext *ctx, TCGOp *op) static bool fold_setcond(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[3]; - int i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); + int i; + if (swap_commutative(op->args[0], &op->args[1], &op->args[2])) { + op->args[3] = cond = tcg_swap_cond(cond); + } + + i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); if (i >= 0) { return tcg_opt_gen_movi(ctx, op, op->args[0], i); } @@ -XXX,XX +XXX,XX @@ static bool fold_setcond(OptContext *ctx, TCGOp *op) static bool fold_setcond2(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[5]; - int i = do_constant_folding_cond2(&op->args[1], &op->args[3], cond); - int inv = 0; + int i, inv = 0; + if (swap_commutative2(&op->args[1], &op->args[3])) { + op->args[5] = cond = tcg_swap_cond(cond); + } + + i = do_constant_folding_cond2(&op->args[1], &op->args[3], cond); if (i >= 0) { goto do_setcond_const; } @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) static bool fold_xor(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xx_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, 0) || fold_xi_to_not(ctx, op, -1)) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) ctx.type = TCG_TYPE_I32; } - /* For commutative operations make constant second argument */ - switch (opc) { - CASE_OP_32_64_VEC(add): - CASE_OP_32_64_VEC(mul): - CASE_OP_32_64_VEC(and): - CASE_OP_32_64_VEC(or): - CASE_OP_32_64_VEC(xor): - CASE_OP_32_64(eqv): - CASE_OP_32_64(nand): - CASE_OP_32_64(nor): - CASE_OP_32_64(muluh): - CASE_OP_32_64(mulsh): - swap_commutative(op->args[0], &op->args[1], &op->args[2]); - break; - CASE_OP_32_64(brcond): - if (swap_commutative(-1, &op->args[0], &op->args[1])) { - op->args[2] = tcg_swap_cond(op->args[2]); - } - break; - CASE_OP_32_64(setcond): - if (swap_commutative(op->args[0], &op->args[1], &op->args[2])) { - op->args[3] = tcg_swap_cond(op->args[3]); - } - break; - CASE_OP_32_64(movcond): - if (swap_commutative(-1, &op->args[1], &op->args[2])) { - op->args[5] = tcg_swap_cond(op->args[5]); - } - /* For movcond, we canonicalize the "false" input reg to match - the destination reg so that the tcg backend can implement - a "move if true" operation. */ - if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) { - op->args[5] = tcg_invert_cond(op->args[5]); - } - break; - CASE_OP_32_64(add2): - swap_commutative(op->args[0], &op->args[2], &op->args[4]); - swap_commutative(op->args[1], &op->args[3], &op->args[5]); - break; - CASE_OP_32_64(mulu2): - CASE_OP_32_64(muls2): - swap_commutative(op->args[0], &op->args[2], &op->args[3]); - break; - case INDEX_op_brcond2_i32: - if (swap_commutative2(&op->args[0], &op->args[2])) { - op->args[4] = tcg_swap_cond(op->args[4]); - } - break; - case INDEX_op_setcond2_i32: - if (swap_commutative2(&op->args[1], &op->args[3])) { - op->args[5] = tcg_swap_cond(op->args[5]); - } - break; - default: - break; - } - /* Assume all bits affected, and no bits known zero. */ ctx.a_mask = -1; ctx.z_mask = -1; -- 2.25.1 This "garbage" setting pre-dates the addition of the type changing opcodes INDEX_op_ext_i32_i64, INDEX_op_extu_i32_i64, and INDEX_op_extr{l,h}_i64_i32. So now we have a definitive points at which to adjust z_mask to eliminate such bits from the 32-bit operands. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void init_ts_info(OptContext *ctx, TCGTemp *ts) ti->is_const = true; ti->val = ts->val; ti->z_mask = ts->val; - if (TCG_TARGET_REG_BITS > 32 && ts->type == TCG_TYPE_I32) { - /* High bits of a 32-bit quantity are garbage. */ - ti->z_mask |= ~0xffffffffull; - } } else { ti->is_const = false; ti->z_mask = -1; @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) TCGTemp *src_ts = arg_temp(src); TempOptInfo *di; TempOptInfo *si; - uint64_t z_mask; TCGOpcode new_op; if (ts_are_copies(dst_ts, src_ts)) { @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) op->args[0] = dst; op->args[1] = src; - z_mask = si->z_mask; - if (TCG_TARGET_REG_BITS > 32 && new_op == INDEX_op_mov_i32) { - /* High bits of the destination are now garbage. */ - z_mask |= ~0xffffffffull; - } - di->z_mask = z_mask; + di->z_mask = si->z_mask; if (src_ts->type == dst_ts->type) { TempOptInfo *ni = ts_info(si->next_copy); @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) static bool tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, TCGArg dst, uint64_t val) { - /* Convert movi to mov with constant temp. */ - TCGTemp *tv = tcg_constant_internal(ctx->type, val); + TCGTemp *tv; + if (ctx->type == TCG_TYPE_I32) { + val = (int32_t)val; + } + + /* Convert movi to mov with constant temp. */ + tv = tcg_constant_internal(ctx->type, val); init_ts_info(ctx, tv); return tcg_opt_gen_mov(ctx, op, dst, temp_arg(tv)); } @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) uint64_t z_mask = ctx->z_mask; /* - * 32-bit ops generate 32-bit results. For the result is zero test - * below, we can ignore high bits, but for further optimizations we - * need to record that the high bits contain garbage. + * 32-bit ops generate 32-bit results, which for the purpose of + * simplifying tcg are sign-extended. Certainly that's how we + * represent our constants elsewhere. Note that the bits will + * be reset properly for a 64-bit value when encountering the + * type changing opcodes. */ if (ctx->type == TCG_TYPE_I32) { - ctx->z_mask |= MAKE_64BIT_MASK(32, 32); - a_mask &= MAKE_64BIT_MASK(0, 32); - z_mask &= MAKE_64BIT_MASK(0, 32); + a_mask = (int32_t)a_mask; + z_mask = (int32_t)z_mask; + ctx->z_mask = z_mask; } if (z_mask == 0) { -- 2.25.1 Recognize the constant function for or-complement. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_or(OptContext *ctx, TCGOp *op) static bool fold_orc(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_xx_to_i(ctx, op, -1) || fold_xi_to_x(ctx, op, -1) || fold_ix_to_not(ctx, op, 0)) { return true; -- 2.25.1 Recognize the identity function for low-part multiply. Suggested-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) static bool fold_mul(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || - fold_xi_to_i(ctx, op, 0)) { + fold_xi_to_i(ctx, op, 0) || + fold_xi_to_x(ctx, op, 1)) { return true; } return false; -- 2.25.1 Recognize the identity function for division. Suggested-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) static bool fold_divide(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, 1)) { + return true; + } + return false; } static bool fold_dup(OptContext *ctx, TCGOp *op) -- 2.25.1 Recognize the constant function for remainder. Suggested-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_qemu_st(OptContext *ctx, TCGOp *op) static bool fold_remainder(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_i(ctx, op, 0)) { + return true; + } + return false; } static bool fold_setcond(OptContext *ctx, TCGOp *op) -- 2.25.1 Certain targets, like riscv, produce signed 32-bit results. This can lead to lots of redundant extensions as values are manipulated. Begin by tracking only the obvious sign-extensions, and converting them to simple copies when possible. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 123 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 102 insertions(+), 21 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { TCGTemp *next_copy; uint64_t val; uint64_t z_mask; /* mask bit is 0 if and only if value bit is 0 */ + uint64_t s_mask; /* a left-aligned mask of clrsb(value) bits. */ } TempOptInfo; typedef struct OptContext { @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { /* In flight values from optimization. */ uint64_t a_mask; /* mask bit is 0 iff value identical to first input */ uint64_t z_mask; /* mask bit is 0 iff value bit is 0 */ + uint64_t s_mask; /* mask of clrsb(value) bits */ TCGType type; } OptContext; +/* Calculate the smask for a specific value. */ +static uint64_t smask_from_value(uint64_t value) +{ + int rep = clrsb64(value); + return ~(~0ull >> rep); +} + +/* + * Calculate the smask for a given set of known-zeros. + * If there are lots of zeros on the left, we can consider the remainder + * an unsigned field, and thus the corresponding signed field is one bit + * larger. + */ +static uint64_t smask_from_zmask(uint64_t zmask) +{ + /* + * Only the 0 bits are significant for zmask, thus the msb itself + * must be zero, else we have no sign information. + */ + int rep = clz64(zmask); + if (rep == 0) { + return 0; + } + rep -= 1; + return ~(~0ull >> rep); +} + static inline TempOptInfo *ts_info(TCGTemp *ts) { return ts->state_ptr; @@ -XXX,XX +XXX,XX @@ static void reset_ts(TCGTemp *ts) ti->prev_copy = ts; ti->is_const = false; ti->z_mask = -1; + ti->s_mask = 0; } static void reset_temp(TCGArg arg) @@ -XXX,XX +XXX,XX @@ static void init_ts_info(OptContext *ctx, TCGTemp *ts) ti->is_const = true; ti->val = ts->val; ti->z_mask = ts->val; + ti->s_mask = smask_from_value(ts->val); } else { ti->is_const = false; ti->z_mask = -1; + ti->s_mask = 0; } } @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) op->args[1] = src; di->z_mask = si->z_mask; + di->s_mask = si->s_mask; if (src_ts->type == dst_ts->type) { TempOptInfo *ni = ts_info(si->next_copy); @@ -XXX,XX +XXX,XX @@ static void finish_folding(OptContext *ctx, TCGOp *op) nb_oargs = def->nb_oargs; for (i = 0; i < nb_oargs; i++) { - reset_temp(op->args[i]); + TCGTemp *ts = arg_temp(op->args[i]); + reset_ts(ts); /* - * Save the corresponding known-zero bits mask for the + * Save the corresponding known-zero/sign bits mask for the * first output argument (only one supported so far). */ if (i == 0) { - arg_info(op->args[i])->z_mask = ctx->z_mask; + ts_info(ts)->z_mask = ctx->z_mask; + ts_info(ts)->s_mask = ctx->s_mask; } } } @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) { uint64_t a_mask = ctx->a_mask; uint64_t z_mask = ctx->z_mask; + uint64_t s_mask = ctx->s_mask; /* * 32-bit ops generate 32-bit results, which for the purpose of @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) if (ctx->type == TCG_TYPE_I32) { a_mask = (int32_t)a_mask; z_mask = (int32_t)z_mask; + s_mask |= MAKE_64BIT_MASK(32, 32); ctx->z_mask = z_mask; + ctx->s_mask = s_mask; } if (z_mask == 0) { @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) static bool fold_bswap(OptContext *ctx, TCGOp *op) { - uint64_t z_mask, sign; + uint64_t z_mask, s_mask, sign; if (arg_is_const(op->args[1])) { uint64_t t = arg_info(op->args[1])->val; @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) } z_mask = arg_info(op->args[1])->z_mask; + switch (op->opc) { case INDEX_op_bswap16_i32: case INDEX_op_bswap16_i64: @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) default: g_assert_not_reached(); } + s_mask = smask_from_zmask(z_mask); switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { case TCG_BSWAP_OZ: @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) /* If the sign bit may be 1, force all the bits above to 1. */ if (z_mask & sign) { z_mask |= sign; + s_mask = sign << 1; } break; default: /* The high bits are undefined: force all bits above the sign to 1. */ z_mask |= sign << 1; + s_mask = 0; break; } ctx->z_mask = z_mask; + ctx->s_mask = s_mask; return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) static bool fold_extract(OptContext *ctx, TCGOp *op) { uint64_t z_mask_old, z_mask; + int pos = op->args[2]; + int len = op->args[3]; if (arg_is_const(op->args[1])) { uint64_t t; t = arg_info(op->args[1])->val; - t = extract64(t, op->args[2], op->args[3]); + t = extract64(t, pos, len); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } z_mask_old = arg_info(op->args[1])->z_mask; - z_mask = extract64(z_mask_old, op->args[2], op->args[3]); - if (op->args[2] == 0) { + z_mask = extract64(z_mask_old, pos, len); + if (pos == 0) { ctx->a_mask = z_mask_old ^ z_mask; } ctx->z_mask = z_mask; + ctx->s_mask = smask_from_zmask(z_mask); return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_extract2(OptContext *ctx, TCGOp *op) static bool fold_exts(OptContext *ctx, TCGOp *op) { - uint64_t z_mask_old, z_mask, sign; + uint64_t s_mask_old, s_mask, z_mask, sign; bool type_change = false; if (fold_const1(ctx, op)) { return true; } - z_mask_old = z_mask = arg_info(op->args[1])->z_mask; + z_mask = arg_info(op->args[1])->z_mask; + s_mask = arg_info(op->args[1])->s_mask; + s_mask_old = s_mask; switch (op->opc) { CASE_OP_32_64(ext8s): @@ -XXX,XX +XXX,XX @@ static bool fold_exts(OptContext *ctx, TCGOp *op) if (z_mask & sign) { z_mask |= sign; - } else if (!type_change) { - ctx->a_mask = z_mask_old ^ z_mask; } + s_mask |= sign << 1; + ctx->z_mask = z_mask; + ctx->s_mask = s_mask; + if (!type_change) { + ctx->a_mask = s_mask & ~s_mask_old; + } return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_extu(OptContext *ctx, TCGOp *op) } ctx->z_mask = z_mask; + ctx->s_mask = smask_from_zmask(z_mask); if (!type_change) { ctx->a_mask = z_mask_old ^ z_mask; } @@ -XXX,XX +XXX,XX @@ static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) MemOp mop = get_memop(oi); int width = 8 * memop_size(mop); - if (!(mop & MO_SIGN) && width < 64) { - ctx->z_mask = MAKE_64BIT_MASK(0, width); + if (width < 64) { + ctx->s_mask = MAKE_64BIT_MASK(width, 64 - width); + if (!(mop & MO_SIGN)) { + ctx->z_mask = MAKE_64BIT_MASK(0, width); + ctx->s_mask <<= 1; + } } /* Opcodes that touch guest memory stop the mb optimization. */ @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) static bool fold_sextract(OptContext *ctx, TCGOp *op) { - int64_t z_mask_old, z_mask; + uint64_t z_mask, s_mask, s_mask_old; + int pos = op->args[2]; + int len = op->args[3]; if (arg_is_const(op->args[1])) { uint64_t t; t = arg_info(op->args[1])->val; - t = sextract64(t, op->args[2], op->args[3]); + t = sextract64(t, pos, len); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } - z_mask_old = arg_info(op->args[1])->z_mask; - z_mask = sextract64(z_mask_old, op->args[2], op->args[3]); - if (op->args[2] == 0 && z_mask >= 0) { - ctx->a_mask = z_mask_old ^ z_mask; - } + z_mask = arg_info(op->args[1])->z_mask; + z_mask = sextract64(z_mask, pos, len); ctx->z_mask = z_mask; + s_mask_old = arg_info(op->args[1])->s_mask; + s_mask = sextract64(s_mask_old, pos, len); + s_mask |= MAKE_64BIT_MASK(len, 64 - len); + ctx->s_mask = s_mask; + + if (pos == 0) { + ctx->a_mask = s_mask & ~s_mask_old; + } + return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) { /* We can't do any folding with a load, but we can record bits. */ switch (op->opc) { + CASE_OP_32_64(ld8s): + ctx->s_mask = MAKE_64BIT_MASK(8, 56); + break; CASE_OP_32_64(ld8u): ctx->z_mask = MAKE_64BIT_MASK(0, 8); + ctx->s_mask = MAKE_64BIT_MASK(9, 55); + break; + CASE_OP_32_64(ld16s): + ctx->s_mask = MAKE_64BIT_MASK(16, 48); break; CASE_OP_32_64(ld16u): ctx->z_mask = MAKE_64BIT_MASK(0, 16); + ctx->s_mask = MAKE_64BIT_MASK(17, 47); + break; + case INDEX_op_ld32s_i64: + ctx->s_mask = MAKE_64BIT_MASK(32, 32); break; case INDEX_op_ld32u_i64: ctx->z_mask = MAKE_64BIT_MASK(0, 32); + ctx->s_mask = MAKE_64BIT_MASK(33, 31); break; default: g_assert_not_reached(); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) ctx.type = TCG_TYPE_I32; } - /* Assume all bits affected, and no bits known zero. */ + /* Assume all bits affected, no bits known zero, no sign reps. */ ctx.a_mask = -1; ctx.z_mask = -1; + ctx.s_mask = 0; /* * Process each opcode. @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_extrh_i64_i32: done = fold_extu(&ctx, op); break; + CASE_OP_32_64(ld8s): CASE_OP_32_64(ld8u): + CASE_OP_32_64(ld16s): CASE_OP_32_64(ld16u): + case INDEX_op_ld32s_i64: case INDEX_op_ld32u_i64: done = fold_tcg_ld(&ctx, op); break; -- 2.25.1 Sign repetitions are perforce all identical, whether they are 1 or 0. Bitwise operations preserve the relative quantity of the repetitions. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) z2 = arg_info(op->args[2])->z_mask; ctx->z_mask = z1 & z2; + /* + * Sign repetitions are perforce all identical, whether they are 1 or 0. + * Bitwise operations preserve the relative quantity of the repetitions. + */ + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; + /* * Known-zeros does not imply known-ones. Therefore unless * arg2 is constant, we can't infer affected bits from it. @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) } ctx->z_mask = z1; + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) fold_xi_to_not(ctx, op, 0)) { return true; } + + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) ctx->z_mask = arg_info(op->args[3])->z_mask | arg_info(op->args[4])->z_mask; + ctx->s_mask = arg_info(op->args[3])->s_mask + & arg_info(op->args[4])->s_mask; if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { uint64_t tv = arg_info(op->args[3])->val; @@ -XXX,XX +XXX,XX @@ static bool fold_nand(OptContext *ctx, TCGOp *op) fold_xi_to_not(ctx, op, -1)) { return true; } + + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_nor(OptContext *ctx, TCGOp *op) fold_xi_to_not(ctx, op, 0)) { return true; } + + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_not(OptContext *ctx, TCGOp *op) return true; } + ctx->s_mask = arg_info(op->args[1])->s_mask; + /* Because of fold_to_not, we want to always return true, via finish. */ finish_folding(ctx, op); return true; @@ -XXX,XX +XXX,XX @@ static bool fold_or(OptContext *ctx, TCGOp *op) ctx->z_mask = arg_info(op->args[1])->z_mask | arg_info(op->args[2])->z_mask; + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_orc(OptContext *ctx, TCGOp *op) fold_ix_to_not(ctx, op, 0)) { return true; } + + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_xor(OptContext *ctx, TCGOp *op) ctx->z_mask = arg_info(op->args[1])->z_mask | arg_info(op->args[2])->z_mask; + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return fold_masks(ctx, op); } -- 2.25.1 The result is either 0 or 1, which means that we have a 2 bit signed result, and thus 62 bits of sign. For clarity, use the smask_from_zmask function. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_setcond(OptContext *ctx, TCGOp *op) } ctx->z_mask = 1; + ctx->s_mask = smask_from_zmask(1); return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) } ctx->z_mask = 1; + ctx->s_mask = smask_from_zmask(1); return false; do_setcond_const: -- 2.25.1 The results are generally 6 bit unsigned values, though the count leading and trailing bits may produce any value for a zero input. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_count_zeros(OptContext *ctx, TCGOp *op) g_assert_not_reached(); } ctx->z_mask = arg_info(op->args[2])->z_mask | z_mask; - + ctx->s_mask = smask_from_zmask(ctx->z_mask); return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_ctpop(OptContext *ctx, TCGOp *op) default: g_assert_not_reached(); } + ctx->s_mask = smask_from_zmask(ctx->z_mask); return false; } -- 2.25.1 For constant shifts, we can simply shift the s_mask. For variable shifts, we know that sar does not reduce the s_mask, which helps for sequences like ext32s_i64 t, in sar_i64 t, t, v ext32s_i64 out, t allowing the final extend to be eliminated. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static uint64_t smask_from_zmask(uint64_t zmask) return ~(~0ull >> rep); } +/* + * Recreate a properly left-aligned smask after manipulation. + * Some bit-shuffling, particularly shifts and rotates, may + * retain sign bits on the left, but may scatter disconnected + * sign bits on the right. Retain only what remains to the left. + */ +static uint64_t smask_from_smask(int64_t smask) +{ + /* Only the 1 bits are significant for smask */ + return smask_from_zmask(~smask); +} + static inline TempOptInfo *ts_info(TCGTemp *ts) { return ts->state_ptr; @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) static bool fold_shift(OptContext *ctx, TCGOp *op) { + uint64_t s_mask, z_mask, sign; + if (fold_const2(ctx, op) || fold_ix_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, 0)) { return true; } + s_mask = arg_info(op->args[1])->s_mask; + z_mask = arg_info(op->args[1])->z_mask; + if (arg_is_const(op->args[2])) { - ctx->z_mask = do_constant_folding(op->opc, ctx->type, - arg_info(op->args[1])->z_mask, - arg_info(op->args[2])->val); + int sh = arg_info(op->args[2])->val; + + ctx->z_mask = do_constant_folding(op->opc, ctx->type, z_mask, sh); + + s_mask = do_constant_folding(op->opc, ctx->type, s_mask, sh); + ctx->s_mask = smask_from_smask(s_mask); + return fold_masks(ctx, op); } + + switch (op->opc) { + CASE_OP_32_64(sar): + /* + * Arithmetic right shift will not reduce the number of + * input sign repetitions. + */ + ctx->s_mask = s_mask; + break; + CASE_OP_32_64(shr): + /* + * If the sign bit is known zero, then logical right shift + * will not reduced the number of input sign repetitions. + */ + sign = (s_mask & -s_mask) >> 1; + if (!(z_mask & sign)) { + ctx->s_mask = s_mask; + } + break; + default: + break; + } + return false; } -- 2.25.1 The following changes since commit 0a301624c2f4ced3331ffd5bce85b4274fe132af: Merge remote-tracking branch 'remotes/pmaydell/tags/pull-target-arm-20220208' into staging (2022-02-08 11:40:08 +0000) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20220211 for you to fetch changes up to 5c1a101ef6b85537a4ade93c39ea81cadd5c246e: tests/tcg/multiarch: Add sigbus.c (2022-02-09 09:00:01 +1100) ---------------------------------------------------------------- Fix safe_syscall_base for sparc64. Fix host signal handling for sparc64-linux. Speedups for jump cache and work list probing. Fix for exception replays. Raise guest SIGBUS for user-only misaligned accesses. ---------------------------------------------------------------- Idan Horowitz (2): accel/tcg: Optimize jump cache flush during tlb range flush softmmu/cpus: Check if the cpu work list is empty atomically Pavel Dovgalyuk (1): replay: use CF_NOIRQ for special exception-replaying TB Richard Henderson (29): common-user/host/sparc64: Fix safe_syscall_base linux-user: Introduce host_signal_mask linux-user: Introduce host_sigcontext linux-user: Move sparc/host-signal.h to sparc64/host-signal.h linux-user/include/host/sparc64: Fix host_sigcontext tcg/i386: Support raising sigbus for user-only tcg/aarch64: Support raising sigbus for user-only tcg/ppc: Support raising sigbus for user-only tcg/riscv: Support raising sigbus for user-only tcg/s390x: Support raising sigbus for user-only tcg/tci: Support raising sigbus for user-only tcg/arm: Drop support for armv4 and armv5 hosts tcg/arm: Remove use_armv5t_instructions tcg/arm: Remove use_armv6_instructions tcg/arm: Check alignment for ldrd and strd tcg/arm: Support unaligned access for softmmu tcg/arm: Reserve a register for guest_base tcg/arm: Support raising sigbus for user-only tcg/mips: Support unaligned access for user-only tcg/mips: Support unaligned access for softmmu tcg/sparc: Use tcg_out_movi_imm13 in tcg_out_addsub2_i64 tcg/sparc: Split out tcg_out_movi_imm32 tcg/sparc: Add scratch argument to tcg_out_movi_int tcg/sparc: Improve code gen for shifted 32-bit constants tcg/sparc: Convert patch_reloc to return bool tcg/sparc: Use the constant pool for 64-bit constants tcg/sparc: Add tcg_out_jmpl_const for better tail calls tcg/sparc: Support unaligned access for user-only tests/tcg/multiarch: Add sigbus.c WANG Xuerui (2): tcg/loongarch64: Fix fallout from recent MO_Q renaming tcg/loongarch64: Support raising sigbus for user-only linux-user/include/host/aarch64/host-signal.h | 16 +- linux-user/include/host/alpha/host-signal.h | 14 +- linux-user/include/host/arm/host-signal.h | 14 +- linux-user/include/host/i386/host-signal.h | 14 +- linux-user/include/host/loongarch64/host-signal.h | 14 +- linux-user/include/host/mips/host-signal.h | 14 +- linux-user/include/host/ppc/host-signal.h | 14 +- linux-user/include/host/riscv/host-signal.h | 14 +- linux-user/include/host/s390/host-signal.h | 14 +- linux-user/include/host/sparc/host-signal.h | 63 ---- linux-user/include/host/sparc64/host-signal.h | 65 +++- linux-user/include/host/x86_64/host-signal.h | 14 +- tcg/aarch64/tcg-target.h | 2 - tcg/arm/tcg-target.h | 6 +- tcg/i386/tcg-target.h | 2 - tcg/loongarch64/tcg-target.h | 2 - tcg/mips/tcg-target.h | 2 - tcg/ppc/tcg-target.h | 2 - tcg/riscv/tcg-target.h | 2 - tcg/s390x/tcg-target.h | 2 - accel/tcg/cpu-exec.c | 3 +- accel/tcg/cputlb.c | 9 + linux-user/signal.c | 22 +- softmmu/cpus.c | 7 +- tcg/tci.c | 20 +- tests/tcg/multiarch/sigbus.c | 68 ++++ tcg/aarch64/tcg-target.c.inc | 91 ++++- tcg/arm/tcg-target.c.inc | 410 +++++++++------------- tcg/i386/tcg-target.c.inc | 103 +++++- tcg/loongarch64/tcg-target.c.inc | 73 +++- tcg/mips/tcg-target.c.inc | 387 ++++++++++++++++++-- tcg/ppc/tcg-target.c.inc | 98 +++++- tcg/riscv/tcg-target.c.inc | 63 +++- tcg/s390x/tcg-target.c.inc | 59 +++- tcg/sparc/tcg-target.c.inc | 348 +++++++++++++++--- common-user/host/sparc64/safe-syscall.inc.S | 5 +- 36 files changed, 1561 insertions(+), 495 deletions(-) delete mode 100644 linux-user/include/host/sparc/host-signal.h create mode 100644 tests/tcg/multiarch/sigbus.c Use the "retl" instead of "ret" instruction alias, since we do not allocate a register window in this function. Fix the offset to the first stacked parameter, which lies beyond the register window save area. Fixes: 95c021dac835 ("linux-user/host/sparc64: Add safe-syscall.inc.S") Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- common-user/host/sparc64/safe-syscall.inc.S | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common-user/host/sparc64/safe-syscall.inc.S b/common-user/host/sparc64/safe-syscall.inc.S index XXXXXXX..XXXXXXX 100644 --- a/common-user/host/sparc64/safe-syscall.inc.S +++ b/common-user/host/sparc64/safe-syscall.inc.S @@ -XXX,XX +XXX,XX @@ .type safe_syscall_end, @function #define STACK_BIAS 2047 -#define PARAM(N) STACK_BIAS + N*8 +#define WINDOW_SIZE 16 * 8 +#define PARAM(N) STACK_BIAS + WINDOW_SIZE + N * 8 /* * This is the entry point for making a system call. The calling @@ -XXX,XX +XXX,XX @@ safe_syscall_end: /* code path for having successfully executed the syscall */ bcs,pn %xcc, 1f nop - ret + retl nop /* code path when we didn't execute the syscall */ -- 2.25.1 Do not directly access the uc_sigmask member. This is preparation for a sparc64 fix. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- linux-user/include/host/aarch64/host-signal.h | 5 +++++ linux-user/include/host/alpha/host-signal.h | 5 +++++ linux-user/include/host/arm/host-signal.h | 5 +++++ linux-user/include/host/i386/host-signal.h | 5 +++++ .../include/host/loongarch64/host-signal.h | 5 +++++ linux-user/include/host/mips/host-signal.h | 5 +++++ linux-user/include/host/ppc/host-signal.h | 5 +++++ linux-user/include/host/riscv/host-signal.h | 5 +++++ linux-user/include/host/s390/host-signal.h | 5 +++++ linux-user/include/host/sparc/host-signal.h | 5 +++++ linux-user/include/host/x86_64/host-signal.h | 5 +++++ linux-user/signal.c | 18 ++++++++---------- 12 files changed, 63 insertions(+), 10 deletions(-) diff --git a/linux-user/include/host/aarch64/host-signal.h b/linux-user/include/host/aarch64/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/aarch64/host-signal.h +++ b/linux-user/include/host/aarch64/host-signal.h @@ -XXX,XX +XXX,XX @@ static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) uc->uc_mcontext.pc = pc; } +static inline void *host_signal_mask(ucontext_t *uc) +{ + return &uc->uc_sigmask; +} + static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) { struct _aarch64_ctx *hdr; diff --git a/linux-user/include/host/alpha/host-signal.h b/linux-user/include/host/alpha/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/alpha/host-signal.h +++ b/linux-user/include/host/alpha/host-signal.h @@ -XXX,XX +XXX,XX @@ static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) uc->uc_mcontext.sc_pc = pc; } +static inline void *host_signal_mask(ucontext_t *uc) +{ + return &uc->uc_sigmask; +} + static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) { uint32_t *pc = (uint32_t *)host_signal_pc(uc); diff --git a/linux-user/include/host/arm/host-signal.h b/linux-user/include/host/arm/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/arm/host-signal.h +++ b/linux-user/include/host/arm/host-signal.h @@ -XXX,XX +XXX,XX @@ static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) uc->uc_mcontext.arm_pc = pc; } +static inline void *host_signal_mask(ucontext_t *uc) +{ + return &uc->uc_sigmask; +} + static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) { /* diff --git a/linux-user/include/host/i386/host-signal.h b/linux-user/include/host/i386/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/i386/host-signal.h +++ b/linux-user/include/host/i386/host-signal.h @@ -XXX,XX +XXX,XX @@ static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) uc->uc_mcontext.gregs[REG_EIP] = pc; } +static inline void *host_signal_mask(ucontext_t *uc) +{ + return &uc->uc_sigmask; +} + static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) { return uc->uc_mcontext.gregs[REG_TRAPNO] == 0xe diff --git a/linux-user/include/host/loongarch64/host-signal.h b/linux-user/include/host/loongarch64/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/loongarch64/host-signal.h +++ b/linux-user/include/host/loongarch64/host-signal.h @@ -XXX,XX +XXX,XX @@ static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) uc->uc_mcontext.__pc = pc; } +static inline void *host_signal_mask(ucontext_t *uc) +{ + return &uc->uc_sigmask; +} + static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) { const uint32_t *pinsn = (const uint32_t *)host_signal_pc(uc); diff --git a/linux-user/include/host/mips/host-signal.h b/linux-user/include/host/mips/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/mips/host-signal.h +++ b/linux-user/include/host/mips/host-signal.h @@ -XXX,XX +XXX,XX @@ static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) uc->uc_mcontext.pc = pc; } +static inline void *host_signal_mask(ucontext_t *uc) +{ + return &uc->uc_sigmask; +} + #if defined(__misp16) || defined(__mips_micromips) #error "Unsupported encoding" #endif diff --git a/linux-user/include/host/ppc/host-signal.h b/linux-user/include/host/ppc/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/ppc/host-signal.h +++ b/linux-user/include/host/ppc/host-signal.h @@ -XXX,XX +XXX,XX @@ static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) uc->uc_mcontext.regs->nip = pc; } +static inline void *host_signal_mask(ucontext_t *uc) +{ + return &uc->uc_sigmask; +} + static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) { return uc->uc_mcontext.regs->trap != 0x400 diff --git a/linux-user/include/host/riscv/host-signal.h b/linux-user/include/host/riscv/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/riscv/host-signal.h +++ b/linux-user/include/host/riscv/host-signal.h @@ -XXX,XX +XXX,XX @@ static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) uc->uc_mcontext.__gregs[REG_PC] = pc; } +static inline void *host_signal_mask(ucontext_t *uc) +{ + return &uc->uc_sigmask; +} + static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) { /* diff --git a/linux-user/include/host/s390/host-signal.h b/linux-user/include/host/s390/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/s390/host-signal.h +++ b/linux-user/include/host/s390/host-signal.h @@ -XXX,XX +XXX,XX @@ static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) uc->uc_mcontext.psw.addr = pc; } +static inline void *host_signal_mask(ucontext_t *uc) +{ + return &uc->uc_sigmask; +} + static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) { uint16_t *pinsn = (uint16_t *)host_signal_pc(uc); diff --git a/linux-user/include/host/sparc/host-signal.h b/linux-user/include/host/sparc/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/sparc/host-signal.h +++ b/linux-user/include/host/sparc/host-signal.h @@ -XXX,XX +XXX,XX @@ static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) #endif } +static inline void *host_signal_mask(ucontext_t *uc) +{ + return &uc->uc_sigmask; +} + static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) { uint32_t insn = *(uint32_t *)host_signal_pc(uc); diff --git a/linux-user/include/host/x86_64/host-signal.h b/linux-user/include/host/x86_64/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/x86_64/host-signal.h +++ b/linux-user/include/host/x86_64/host-signal.h @@ -XXX,XX +XXX,XX @@ static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) uc->uc_mcontext.gregs[REG_RIP] = pc; } +static inline void *host_signal_mask(ucontext_t *uc) +{ + return &uc->uc_sigmask; +} + static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) { return uc->uc_mcontext.gregs[REG_TRAPNO] == 0xe diff --git a/linux-user/signal.c b/linux-user/signal.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/signal.c +++ b/linux-user/signal.c @@ -XXX,XX +XXX,XX @@ static void host_signal_handler(int host_sig, siginfo_t *info, void *puc) int guest_sig; uintptr_t pc = 0; bool sync_sig = false; + void *sigmask = host_signal_mask(uc); /* * Non-spoofed SIGSEGV and SIGBUS are synchronous, and need special @@ -XXX,XX +XXX,XX @@ static void host_signal_handler(int host_sig, siginfo_t *info, void *puc) if (info->si_code == SEGV_ACCERR && h2g_valid(host_addr)) { /* If this was a write to a TB protected page, restart. */ if (is_write && - handle_sigsegv_accerr_write(cpu, &uc->uc_sigmask, - pc, guest_addr)) { + handle_sigsegv_accerr_write(cpu, sigmask, pc, guest_addr)) { return; } @@ -XXX,XX +XXX,XX @@ static void host_signal_handler(int host_sig, siginfo_t *info, void *puc) } } - sigprocmask(SIG_SETMASK, &uc->uc_sigmask, NULL); + sigprocmask(SIG_SETMASK, sigmask, NULL); cpu_loop_exit_sigsegv(cpu, guest_addr, access_type, maperr, pc); } else { - sigprocmask(SIG_SETMASK, &uc->uc_sigmask, NULL); + sigprocmask(SIG_SETMASK, sigmask, NULL); if (info->si_code == BUS_ADRALN) { cpu_loop_exit_sigbus(cpu, guest_addr, access_type, pc); } @@ -XXX,XX +XXX,XX @@ static void host_signal_handler(int host_sig, siginfo_t *info, void *puc) * now and it getting out to the main loop. Signals will be * unblocked again in process_pending_signals(). * - * WARNING: we cannot use sigfillset() here because the uc_sigmask + * WARNING: we cannot use sigfillset() here because the sigmask * field is a kernel sigset_t, which is much smaller than the * libc sigset_t which sigfillset() operates on. Using sigfillset() * would write 0xff bytes off the end of the structure and trash * data on the struct. - * We can't use sizeof(uc->uc_sigmask) either, because the libc - * headers define the struct field with the wrong (too large) type. */ - memset(&uc->uc_sigmask, 0xff, SIGSET_T_SIZE); - sigdelset(&uc->uc_sigmask, SIGSEGV); - sigdelset(&uc->uc_sigmask, SIGBUS); + memset(sigmask, 0xff, SIGSET_T_SIZE); + sigdelset(sigmask, SIGSEGV); + sigdelset(sigmask, SIGBUS); /* interrupt the virtual CPU as soon as possible */ cpu_exit(thread_cpu); -- 2.25.1 Do not directly access ucontext_t as the third signal parameter. This is preparation for a sparc64 fix. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- linux-user/include/host/aarch64/host-signal.h | 13 ++++++++----- linux-user/include/host/alpha/host-signal.h | 11 +++++++---- linux-user/include/host/arm/host-signal.h | 11 +++++++---- linux-user/include/host/i386/host-signal.h | 11 +++++++---- linux-user/include/host/loongarch64/host-signal.h | 11 +++++++---- linux-user/include/host/mips/host-signal.h | 11 +++++++---- linux-user/include/host/ppc/host-signal.h | 11 +++++++---- linux-user/include/host/riscv/host-signal.h | 11 +++++++---- linux-user/include/host/s390/host-signal.h | 11 +++++++---- linux-user/include/host/sparc/host-signal.h | 11 +++++++---- linux-user/include/host/x86_64/host-signal.h | 11 +++++++---- linux-user/signal.c | 4 ++-- 12 files changed, 80 insertions(+), 47 deletions(-) diff --git a/linux-user/include/host/aarch64/host-signal.h b/linux-user/include/host/aarch64/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/aarch64/host-signal.h +++ b/linux-user/include/host/aarch64/host-signal.h @@ -XXX,XX +XXX,XX @@ #ifndef AARCH64_HOST_SIGNAL_H #define AARCH64_HOST_SIGNAL_H +/* The third argument to a SA_SIGINFO handler is ucontext_t. */ +typedef ucontext_t host_sigcontext; + /* Pre-3.16 kernel headers don't have these, so provide fallback definitions */ #ifndef ESR_MAGIC #define ESR_MAGIC 0x45535201 @@ -XXX,XX +XXX,XX @@ struct esr_context { }; #endif -static inline struct _aarch64_ctx *first_ctx(ucontext_t *uc) +static inline struct _aarch64_ctx *first_ctx(host_sigcontext *uc) { return (struct _aarch64_ctx *)&uc->uc_mcontext.__reserved; } @@ -XXX,XX +XXX,XX @@ static inline struct _aarch64_ctx *next_ctx(struct _aarch64_ctx *hdr) return (struct _aarch64_ctx *)((char *)hdr + hdr->size); } -static inline uintptr_t host_signal_pc(ucontext_t *uc) +static inline uintptr_t host_signal_pc(host_sigcontext *uc) { return uc->uc_mcontext.pc; } -static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) +static inline void host_signal_set_pc(host_sigcontext *uc, uintptr_t pc) { uc->uc_mcontext.pc = pc; } -static inline void *host_signal_mask(ucontext_t *uc) +static inline void *host_signal_mask(host_sigcontext *uc) { return &uc->uc_sigmask; } -static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) +static inline bool host_signal_write(siginfo_t *info, host_sigcontext *uc) { struct _aarch64_ctx *hdr; uint32_t insn; diff --git a/linux-user/include/host/alpha/host-signal.h b/linux-user/include/host/alpha/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/alpha/host-signal.h +++ b/linux-user/include/host/alpha/host-signal.h @@ -XXX,XX +XXX,XX @@ #ifndef ALPHA_HOST_SIGNAL_H #define ALPHA_HOST_SIGNAL_H -static inline uintptr_t host_signal_pc(ucontext_t *uc) +/* The third argument to a SA_SIGINFO handler is ucontext_t. */ +typedef ucontext_t host_sigcontext; + +static inline uintptr_t host_signal_pc(host_sigcontext *uc) { return uc->uc_mcontext.sc_pc; } -static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) +static inline void host_signal_set_pc(host_sigcontext *uc, uintptr_t pc) { uc->uc_mcontext.sc_pc = pc; } -static inline void *host_signal_mask(ucontext_t *uc) +static inline void *host_signal_mask(host_sigcontext *uc) { return &uc->uc_sigmask; } -static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) +static inline bool host_signal_write(siginfo_t *info, host_sigcontext *uc) { uint32_t *pc = (uint32_t *)host_signal_pc(uc); uint32_t insn = *pc; diff --git a/linux-user/include/host/arm/host-signal.h b/linux-user/include/host/arm/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/arm/host-signal.h +++ b/linux-user/include/host/arm/host-signal.h @@ -XXX,XX +XXX,XX @@ #ifndef ARM_HOST_SIGNAL_H #define ARM_HOST_SIGNAL_H -static inline uintptr_t host_signal_pc(ucontext_t *uc) +/* The third argument to a SA_SIGINFO handler is ucontext_t. */ +typedef ucontext_t host_sigcontext; + +static inline uintptr_t host_signal_pc(host_sigcontext *uc) { return uc->uc_mcontext.arm_pc; } -static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) +static inline void host_signal_set_pc(host_sigcontext *uc, uintptr_t pc) { uc->uc_mcontext.arm_pc = pc; } -static inline void *host_signal_mask(ucontext_t *uc) +static inline void *host_signal_mask(host_sigcontext *uc) { return &uc->uc_sigmask; } -static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) +static inline bool host_signal_write(siginfo_t *info, host_sigcontext *uc) { /* * In the FSR, bit 11 is WnR, assuming a v6 or diff --git a/linux-user/include/host/i386/host-signal.h b/linux-user/include/host/i386/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/i386/host-signal.h +++ b/linux-user/include/host/i386/host-signal.h @@ -XXX,XX +XXX,XX @@ #ifndef I386_HOST_SIGNAL_H #define I386_HOST_SIGNAL_H -static inline uintptr_t host_signal_pc(ucontext_t *uc) +/* The third argument to a SA_SIGINFO handler is ucontext_t. */ +typedef ucontext_t host_sigcontext; + +static inline uintptr_t host_signal_pc(host_sigcontext *uc) { return uc->uc_mcontext.gregs[REG_EIP]; } -static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) +static inline void host_signal_set_pc(host_sigcontext *uc, uintptr_t pc) { uc->uc_mcontext.gregs[REG_EIP] = pc; } -static inline void *host_signal_mask(ucontext_t *uc) +static inline void *host_signal_mask(host_sigcontext *uc) { return &uc->uc_sigmask; } -static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) +static inline bool host_signal_write(siginfo_t *info, host_sigcontext *uc) { return uc->uc_mcontext.gregs[REG_TRAPNO] == 0xe && (uc->uc_mcontext.gregs[REG_ERR] & 0x2); diff --git a/linux-user/include/host/loongarch64/host-signal.h b/linux-user/include/host/loongarch64/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/loongarch64/host-signal.h +++ b/linux-user/include/host/loongarch64/host-signal.h @@ -XXX,XX +XXX,XX @@ #ifndef LOONGARCH64_HOST_SIGNAL_H #define LOONGARCH64_HOST_SIGNAL_H -static inline uintptr_t host_signal_pc(ucontext_t *uc) +/* The third argument to a SA_SIGINFO handler is ucontext_t. */ +typedef ucontext_t host_sigcontext; + +static inline uintptr_t host_signal_pc(host_sigcontext *uc) { return uc->uc_mcontext.__pc; } -static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) +static inline void host_signal_set_pc(host_sigcontext *uc, uintptr_t pc) { uc->uc_mcontext.__pc = pc; } -static inline void *host_signal_mask(ucontext_t *uc) +static inline void *host_signal_mask(host_sigcontext *uc) { return &uc->uc_sigmask; } -static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) +static inline bool host_signal_write(siginfo_t *info, host_sigcontext *uc) { const uint32_t *pinsn = (const uint32_t *)host_signal_pc(uc); uint32_t insn = pinsn[0]; diff --git a/linux-user/include/host/mips/host-signal.h b/linux-user/include/host/mips/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/mips/host-signal.h +++ b/linux-user/include/host/mips/host-signal.h @@ -XXX,XX +XXX,XX @@ #ifndef MIPS_HOST_SIGNAL_H #define MIPS_HOST_SIGNAL_H -static inline uintptr_t host_signal_pc(ucontext_t *uc) +/* The third argument to a SA_SIGINFO handler is ucontext_t. */ +typedef ucontext_t host_sigcontext; + +static inline uintptr_t host_signal_pc(host_sigcontext *uc) { return uc->uc_mcontext.pc; } -static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) +static inline void host_signal_set_pc(host_sigcontext *uc, uintptr_t pc) { uc->uc_mcontext.pc = pc; } -static inline void *host_signal_mask(ucontext_t *uc) +static inline void *host_signal_mask(host_sigcontext *uc) { return &uc->uc_sigmask; } @@ -XXX,XX +XXX,XX @@ static inline void *host_signal_mask(ucontext_t *uc) #error "Unsupported encoding" #endif -static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) +static inline bool host_signal_write(siginfo_t *info, host_sigcontext *uc) { uint32_t insn = *(uint32_t *)host_signal_pc(uc); diff --git a/linux-user/include/host/ppc/host-signal.h b/linux-user/include/host/ppc/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/ppc/host-signal.h +++ b/linux-user/include/host/ppc/host-signal.h @@ -XXX,XX +XXX,XX @@ #ifndef PPC_HOST_SIGNAL_H #define PPC_HOST_SIGNAL_H -static inline uintptr_t host_signal_pc(ucontext_t *uc) +/* The third argument to a SA_SIGINFO handler is ucontext_t. */ +typedef ucontext_t host_sigcontext; + +static inline uintptr_t host_signal_pc(host_sigcontext *uc) { return uc->uc_mcontext.regs->nip; } -static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) +static inline void host_signal_set_pc(host_sigcontext *uc, uintptr_t pc) { uc->uc_mcontext.regs->nip = pc; } -static inline void *host_signal_mask(ucontext_t *uc) +static inline void *host_signal_mask(host_sigcontext *uc) { return &uc->uc_sigmask; } -static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) +static inline bool host_signal_write(siginfo_t *info, host_sigcontext *uc) { return uc->uc_mcontext.regs->trap != 0x400 && (uc->uc_mcontext.regs->dsisr & 0x02000000); diff --git a/linux-user/include/host/riscv/host-signal.h b/linux-user/include/host/riscv/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/riscv/host-signal.h +++ b/linux-user/include/host/riscv/host-signal.h @@ -XXX,XX +XXX,XX @@ #ifndef RISCV_HOST_SIGNAL_H #define RISCV_HOST_SIGNAL_H -static inline uintptr_t host_signal_pc(ucontext_t *uc) +/* The third argument to a SA_SIGINFO handler is ucontext_t. */ +typedef ucontext_t host_sigcontext; + +static inline uintptr_t host_signal_pc(host_sigcontext *uc) { return uc->uc_mcontext.__gregs[REG_PC]; } -static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) +static inline void host_signal_set_pc(host_sigcontext *uc, uintptr_t pc) { uc->uc_mcontext.__gregs[REG_PC] = pc; } -static inline void *host_signal_mask(ucontext_t *uc) +static inline void *host_signal_mask(host_sigcontext *uc) { return &uc->uc_sigmask; } -static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) +static inline bool host_signal_write(siginfo_t *info, host_sigcontext *uc) { /* * Detect store by reading the instruction at the program counter. diff --git a/linux-user/include/host/s390/host-signal.h b/linux-user/include/host/s390/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/s390/host-signal.h +++ b/linux-user/include/host/s390/host-signal.h @@ -XXX,XX +XXX,XX @@ #ifndef S390_HOST_SIGNAL_H #define S390_HOST_SIGNAL_H -static inline uintptr_t host_signal_pc(ucontext_t *uc) +/* The third argument to a SA_SIGINFO handler is ucontext_t. */ +typedef ucontext_t host_sigcontext; + +static inline uintptr_t host_signal_pc(host_sigcontext *uc) { return uc->uc_mcontext.psw.addr; } -static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) +static inline void host_signal_set_pc(host_sigcontext *uc, uintptr_t pc) { uc->uc_mcontext.psw.addr = pc; } -static inline void *host_signal_mask(ucontext_t *uc) +static inline void *host_signal_mask(host_sigcontext *uc) { return &uc->uc_sigmask; } -static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) +static inline bool host_signal_write(siginfo_t *info, host_sigcontext *uc) { uint16_t *pinsn = (uint16_t *)host_signal_pc(uc); diff --git a/linux-user/include/host/sparc/host-signal.h b/linux-user/include/host/sparc/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/sparc/host-signal.h +++ b/linux-user/include/host/sparc/host-signal.h @@ -XXX,XX +XXX,XX @@ #ifndef SPARC_HOST_SIGNAL_H #define SPARC_HOST_SIGNAL_H -static inline uintptr_t host_signal_pc(ucontext_t *uc) +/* FIXME: the third argument to a SA_SIGINFO handler is *not* ucontext_t. */ +typedef ucontext_t host_sigcontext; + +static inline uintptr_t host_signal_pc(host_sigcontext *uc) { #ifdef __arch64__ return uc->uc_mcontext.mc_gregs[MC_PC]; @@ -XXX,XX +XXX,XX @@ static inline uintptr_t host_signal_pc(ucontext_t *uc) #endif } -static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) +static inline void host_signal_set_pc(host_sigcontext *uc, uintptr_t pc) { #ifdef __arch64__ uc->uc_mcontext.mc_gregs[MC_PC] = pc; @@ -XXX,XX +XXX,XX @@ static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) #endif } -static inline void *host_signal_mask(ucontext_t *uc) +static inline void *host_signal_mask(host_sigcontext *uc) { return &uc->uc_sigmask; } -static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) +static inline bool host_signal_write(siginfo_t *info, host_sigcontext *uc) { uint32_t insn = *(uint32_t *)host_signal_pc(uc); diff --git a/linux-user/include/host/x86_64/host-signal.h b/linux-user/include/host/x86_64/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/x86_64/host-signal.h +++ b/linux-user/include/host/x86_64/host-signal.h @@ -XXX,XX +XXX,XX @@ #ifndef X86_64_HOST_SIGNAL_H #define X86_64_HOST_SIGNAL_H -static inline uintptr_t host_signal_pc(ucontext_t *uc) +/* The third argument to a SA_SIGINFO handler is ucontext_t. */ +typedef ucontext_t host_sigcontext; + +static inline uintptr_t host_signal_pc(host_sigcontext *uc) { return uc->uc_mcontext.gregs[REG_RIP]; } -static inline void host_signal_set_pc(ucontext_t *uc, uintptr_t pc) +static inline void host_signal_set_pc(host_sigcontext *uc, uintptr_t pc) { uc->uc_mcontext.gregs[REG_RIP] = pc; } -static inline void *host_signal_mask(ucontext_t *uc) +static inline void *host_signal_mask(host_sigcontext *uc) { return &uc->uc_sigmask; } -static inline bool host_signal_write(siginfo_t *info, ucontext_t *uc) +static inline bool host_signal_write(siginfo_t *info, host_sigcontext *uc) { return uc->uc_mcontext.gregs[REG_TRAPNO] == 0xe && (uc->uc_mcontext.gregs[REG_ERR] & 0x2); diff --git a/linux-user/signal.c b/linux-user/signal.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/signal.c +++ b/linux-user/signal.c @@ -XXX,XX +XXX,XX @@ void queue_signal(CPUArchState *env, int sig, int si_type, /* Adjust the signal context to rewind out of safe-syscall if we're in it */ static inline void rewind_if_in_safe_syscall(void *puc) { - ucontext_t *uc = (ucontext_t *)puc; + host_sigcontext *uc = (host_sigcontext *)puc; uintptr_t pcreg = host_signal_pc(uc); if (pcreg > (uintptr_t)safe_syscall_start @@ -XXX,XX +XXX,XX @@ static void host_signal_handler(int host_sig, siginfo_t *info, void *puc) CPUState *cpu = env_cpu(env); TaskState *ts = cpu->opaque; target_siginfo_t tinfo; - ucontext_t *uc = puc; + host_sigcontext *uc = puc; struct emulated_sigtable *k; int guest_sig; uintptr_t pc = 0; -- 2.25.1 We do not support sparc32 as a host, so there's no point in sparc64 redirecting to sparc. Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- linux-user/include/host/sparc/host-signal.h | 71 ------------------- linux-user/include/host/sparc64/host-signal.h | 64 ++++++++++++++++- 2 files changed, 63 insertions(+), 72 deletions(-) delete mode 100644 linux-user/include/host/sparc/host-signal.h diff --git a/linux-user/include/host/sparc/host-signal.h b/linux-user/include/host/sparc/host-signal.h deleted file mode 100644 index XXXXXXX..XXXXXXX --- a/linux-user/include/host/sparc/host-signal.h +++ /dev/null @@ -XXX,XX +XXX,XX @@ -/* - * host-signal.h: signal info dependent on the host architecture - * - * Copyright (c) 2003-2005 Fabrice Bellard - * Copyright (c) 2021 Linaro Limited - * - * This work is licensed under the terms of the GNU LGPL, version 2.1 or later. - * See the COPYING file in the top-level directory. - */ - -#ifndef SPARC_HOST_SIGNAL_H -#define SPARC_HOST_SIGNAL_H - -/* FIXME: the third argument to a SA_SIGINFO handler is *not* ucontext_t. */ -typedef ucontext_t host_sigcontext; - -static inline uintptr_t host_signal_pc(host_sigcontext *uc) -{ -#ifdef __arch64__ - return uc->uc_mcontext.mc_gregs[MC_PC]; -#else - return uc->uc_mcontext.gregs[REG_PC]; -#endif -} - -static inline void host_signal_set_pc(host_sigcontext *uc, uintptr_t pc) -{ -#ifdef __arch64__ - uc->uc_mcontext.mc_gregs[MC_PC] = pc; -#else - uc->uc_mcontext.gregs[REG_PC] = pc; -#endif -} - -static inline void *host_signal_mask(host_sigcontext *uc) -{ - return &uc->uc_sigmask; -} - -static inline bool host_signal_write(siginfo_t *info, host_sigcontext *uc) -{ - uint32_t insn = *(uint32_t *)host_signal_pc(uc); - - if ((insn >> 30) == 3) { - switch ((insn >> 19) & 0x3f) { - case 0x05: /* stb */ - case 0x15: /* stba */ - case 0x06: /* sth */ - case 0x16: /* stha */ - case 0x04: /* st */ - case 0x14: /* sta */ - case 0x07: /* std */ - case 0x17: /* stda */ - case 0x0e: /* stx */ - case 0x1e: /* stxa */ - case 0x24: /* stf */ - case 0x34: /* stfa */ - case 0x27: /* stdf */ - case 0x37: /* stdfa */ - case 0x26: /* stqf */ - case 0x36: /* stqfa */ - case 0x25: /* stfsr */ - case 0x3c: /* casa */ - case 0x3e: /* casxa */ - return true; - } - } - return false; -} - -#endif diff --git a/linux-user/include/host/sparc64/host-signal.h b/linux-user/include/host/sparc64/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/sparc64/host-signal.h +++ b/linux-user/include/host/sparc64/host-signal.h @@ -1 +1,63 @@ -#include "../sparc/host-signal.h" +/* + * host-signal.h: signal info dependent on the host architecture + * + * Copyright (c) 2003-2005 Fabrice Bellard + * Copyright (c) 2021 Linaro Limited + * + * This work is licensed under the terms of the GNU LGPL, version 2.1 or later. + * See the COPYING file in the top-level directory. + */ + +#ifndef SPARC64_HOST_SIGNAL_H +#define SPARC64_HOST_SIGNAL_H + +/* FIXME: the third argument to a SA_SIGINFO handler is *not* ucontext_t. */ +typedef ucontext_t host_sigcontext; + +static inline uintptr_t host_signal_pc(host_sigcontext *uc) +{ + return uc->uc_mcontext.mc_gregs[MC_PC]; +} + +static inline void host_signal_set_pc(host_sigcontext *uc, uintptr_t pc) +{ + uc->uc_mcontext.mc_gregs[MC_PC] = pc; +} + +static inline void *host_signal_mask(host_sigcontext *uc) +{ + return &uc->uc_sigmask; +} + +static inline bool host_signal_write(siginfo_t *info, host_sigcontext *uc) +{ + uint32_t insn = *(uint32_t *)host_signal_pc(uc); + + if ((insn >> 30) == 3) { + switch ((insn >> 19) & 0x3f) { + case 0x05: /* stb */ + case 0x15: /* stba */ + case 0x06: /* sth */ + case 0x16: /* stha */ + case 0x04: /* st */ + case 0x14: /* sta */ + case 0x07: /* std */ + case 0x17: /* stda */ + case 0x0e: /* stx */ + case 0x1e: /* stxa */ + case 0x24: /* stf */ + case 0x34: /* stfa */ + case 0x27: /* stdf */ + case 0x37: /* stdfa */ + case 0x26: /* stqf */ + case 0x36: /* stqfa */ + case 0x25: /* stfsr */ + case 0x3c: /* casa */ + case 0x3e: /* casxa */ + return true; + } + } + return false; +} + +#endif -- 2.25.1 Sparc64 is unique on linux in *not* passing ucontext_t as the third argument to a SA_SIGINFO handler. It passes the old struct sigcontext instead. Set both pc and npc in host_signal_set_pc. Fixes: 8b5bd461935b ("linux-user/host/sparc: Populate host_signal.h") Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- linux-user/include/host/sparc64/host-signal.h | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/linux-user/include/host/sparc64/host-signal.h b/linux-user/include/host/sparc64/host-signal.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/include/host/sparc64/host-signal.h +++ b/linux-user/include/host/sparc64/host-signal.h @@ -XXX,XX +XXX,XX @@ #ifndef SPARC64_HOST_SIGNAL_H #define SPARC64_HOST_SIGNAL_H -/* FIXME: the third argument to a SA_SIGINFO handler is *not* ucontext_t. */ -typedef ucontext_t host_sigcontext; +/* The third argument to a SA_SIGINFO handler is struct sigcontext. */ +typedef struct sigcontext host_sigcontext; -static inline uintptr_t host_signal_pc(host_sigcontext *uc) +static inline uintptr_t host_signal_pc(host_sigcontext *sc) { - return uc->uc_mcontext.mc_gregs[MC_PC]; + return sc->sigc_regs.tpc; } -static inline void host_signal_set_pc(host_sigcontext *uc, uintptr_t pc) +static inline void host_signal_set_pc(host_sigcontext *sc, uintptr_t pc) { - uc->uc_mcontext.mc_gregs[MC_PC] = pc; + sc->sigc_regs.tpc = pc; + sc->sigc_regs.tnpc = pc + 4; } -static inline void *host_signal_mask(host_sigcontext *uc) +static inline void *host_signal_mask(host_sigcontext *sc) { - return &uc->uc_sigmask; + return &sc->sigc_mask; } static inline bool host_signal_write(siginfo_t *info, host_sigcontext *uc) -- 2.25.1 From: Idan Horowitz <idan.horowitz@gmail.com> When the length of the range is large enough, clearing the whole cache is faster than iterating over the (possibly extremely large) set of pages contained in the range. This mimics the pre-existing similar optimization done on the flush of the tlb itself. Signed-off-by: Idan Horowitz <idan.horowitz@gmail.com> Message-Id: <20220110164754.1066025-1-idan.horowitz@gmail.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cputlb.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ static void tlb_flush_range_by_mmuidx_async_0(CPUState *cpu, } qemu_spin_unlock(&env_tlb(env)->c.lock); + /* + * If the length is larger than the jump cache size, then it will take + * longer to clear each entry individually than it will to clear it all. + */ + if (d.len >= (TARGET_PAGE_SIZE * TB_JMP_CACHE_SIZE)) { + cpu_tb_jmp_cache_clear(cpu); + return; + } + for (target_ulong i = 0; i < d.len; i += TARGET_PAGE_SIZE) { tb_flush_jmp_cache(cpu, d.addr + i); } -- 2.25.1 From: Idan Horowitz <idan.horowitz@gmail.com> Instead of taking the lock of the cpu work list in order to check if it's empty, we can just read the head pointer atomically. This decreases cpu_work_list_empty's share from 5% to 1.3% in a profile of icount-enabled aarch64-softmmu. Signed-off-by: Idan Horowitz <idan.horowitz@gmail.com> Message-Id: <20220114004358.299534-1-idan.horowitz@gmail.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- softmmu/cpus.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/softmmu/cpus.c b/softmmu/cpus.c index XXXXXXX..XXXXXXX 100644 --- a/softmmu/cpus.c +++ b/softmmu/cpus.c @@ -XXX,XX +XXX,XX @@ bool cpu_is_stopped(CPUState *cpu) bool cpu_work_list_empty(CPUState *cpu) { - bool ret; - - qemu_mutex_lock(&cpu->work_mutex); - ret = QSIMPLEQ_EMPTY(&cpu->work_list); - qemu_mutex_unlock(&cpu->work_mutex); - return ret; + return QSIMPLEQ_EMPTY_ATOMIC(&cpu->work_list); } bool cpu_thread_is_idle(CPUState *cpu) -- 2.25.1 From: Pavel Dovgalyuk <pavel.dovgalyuk@ispras.ru> Commit aff0e204cb1f1c036a496c94c15f5dfafcd9b4b4 introduced CF_NOIRQ usage, but one case was forgotten. Record/replay uses one special TB which is not really executed, but used to cause a correct exception in replay mode. This patch adds CF_NOIRQ flag for such block. Signed-off-by: Pavel Dovgalyuk <Pavel.Dovgalyuk@ispras.ru> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <164362834054.1754532.7678416881159817273.stgit@pasha-ThinkPad-X280> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cpu-exec.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ static inline bool cpu_handle_exception(CPUState *cpu, int *ret) if (replay_has_exception() && cpu_neg(cpu)->icount_decr.u16.low + cpu->icount_extra == 0) { /* Execute just one insn to trigger exception pending in the log */ - cpu->cflags_next_tb = (curr_cflags(cpu) & ~CF_USE_ICOUNT) | 1; + cpu->cflags_next_tb = (curr_cflags(cpu) & ~CF_USE_ICOUNT) + | CF_NOIRQ | 1; } #endif return false; -- 2.25.1 From: WANG Xuerui <git@xen0n.name> Apparently we were left behind; just renaming MO_Q to MO_UQ is enough. Fixes: fc313c64345453c7 ("exec/memop: Adding signedness to quad definitions") Signed-off-by: WANG Xuerui <git@xen0n.name> Message-Id: <20220206162106.1092364-1-i.qemu@xen0n.name> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/loongarch64/tcg-target.c.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/loongarch64/tcg-target.c.inc +++ b/tcg/loongarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_indexed(TCGContext *s, TCGReg rd, TCGReg rj, case MO_SL: tcg_out_opc_ldx_w(s, rd, rj, rk); break; - case MO_Q: + case MO_UQ: tcg_out_opc_ldx_d(s, rd, rj, rk); break; default: -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/i386/tcg-target.h | 2 - tcg/i386/tcg-target.c.inc | 103 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 98 insertions(+), 7 deletions(-) diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, #define TCG_TARGET_HAS_MEMORY_BSWAP have_movbe -#ifdef CONFIG_SOFTMMU #define TCG_TARGET_NEED_LDST_LABELS -#endif #define TCG_TARGET_NEED_POOL_LABELS #endif diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ * THE SOFTWARE. */ +#include "../tcg-ldst.c.inc" #include "../tcg-pool.c.inc" #ifdef CONFIG_DEBUG_TCG @@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) #define OPC_VZEROUPPER (0x77 | P_EXT) #define OPC_XCHG_ax_r32 (0x90) -#define OPC_GRP3_Ev (0xf7) -#define OPC_GRP5 (0xff) +#define OPC_GRP3_Eb (0xf6) +#define OPC_GRP3_Ev (0xf7) +#define OPC_GRP5 (0xff) #define OPC_GRP14 (0x73 | P_EXT | P_DATA16) /* Group 1 opcode extensions for 0x80-0x83. @@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) #define SHIFT_SAR 7 /* Group 3 opcode extensions for 0xf6, 0xf7. To be used with OPC_GRP3. */ +#define EXT3_TESTi 0 #define EXT3_NOT 2 #define EXT3_NEG 3 #define EXT3_MUL 4 @@ -XXX,XX +XXX,XX @@ static void tcg_out_nopn(TCGContext *s, int n) } #if defined(CONFIG_SOFTMMU) -#include "../tcg-ldst.c.inc" - /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, * int mmu_idx, uintptr_t ra) */ @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) tcg_out_jmp(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]); return true; } -#elif TCG_TARGET_REG_BITS == 32 +#else + +static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addrlo, + TCGReg addrhi, unsigned a_bits) +{ + unsigned a_mask = (1 << a_bits) - 1; + TCGLabelQemuLdst *label; + + /* + * We are expecting a_bits to max out at 7, so we can usually use testb. + * For i686, we have to use testl for %esi/%edi. + */ + if (a_mask <= 0xff && (TCG_TARGET_REG_BITS == 64 || addrlo < 4)) { + tcg_out_modrm(s, OPC_GRP3_Eb | P_REXB_RM, EXT3_TESTi, addrlo); + tcg_out8(s, a_mask); + } else { + tcg_out_modrm(s, OPC_GRP3_Ev, EXT3_TESTi, addrlo); + tcg_out32(s, a_mask); + } + + /* jne slow_path */ + tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0); + + label = new_ldst_label(s); + label->is_ld = is_ld; + label->addrlo_reg = addrlo; + label->addrhi_reg = addrhi; + label->raddr = tcg_splitwx_to_rx(s->code_ptr + 4); + label->label_ptr[0] = s->code_ptr; + + s->code_ptr += 4; +} + +static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) +{ + /* resolve label address */ + tcg_patch32(l->label_ptr[0], s->code_ptr - l->label_ptr[0] - 4); + + if (TCG_TARGET_REG_BITS == 32) { + int ofs = 0; + + tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs); + ofs += 4; + + tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs); + ofs += 4; + if (TARGET_LONG_BITS == 64) { + tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs); + ofs += 4; + } + + tcg_out_pushi(s, (uintptr_t)l->raddr); + } else { + tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1], + l->addrlo_reg); + tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0); + + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RAX, (uintptr_t)l->raddr); + tcg_out_push(s, TCG_REG_RAX); + } + + /* "Tail call" to the helper, with the return address back inline. */ + tcg_out_jmp(s, (const void *)(l->is_ld ? helper_unaligned_ld + : helper_unaligned_st)); + return true; +} + +static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + return tcg_out_fail_alignment(s, l); +} + +static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + return tcg_out_fail_alignment(s, l); +} + +#if TCG_TARGET_REG_BITS == 32 # define x86_guest_base_seg 0 # define x86_guest_base_index -1 # define x86_guest_base_offset guest_base @@ -XXX,XX +XXX,XX @@ static inline int setup_guest_base_seg(void) return 0; } # endif +#endif #endif /* SOFTMMU */ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64) #if defined(CONFIG_SOFTMMU) int mem_index; tcg_insn_unit *label_ptr[2]; +#else + unsigned a_bits; #endif datalo = *args++; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64) add_qemu_ldst_label(s, true, is64, oi, datalo, datahi, addrlo, addrhi, s->code_ptr, label_ptr); #else + a_bits = get_alignment_bits(opc); + if (a_bits) { + tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); + } + tcg_out_qemu_ld_direct(s, datalo, datahi, addrlo, x86_guest_base_index, x86_guest_base_offset, x86_guest_base_seg, is64, opc); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) #if defined(CONFIG_SOFTMMU) int mem_index; tcg_insn_unit *label_ptr[2]; +#else + unsigned a_bits; #endif datalo = *args++; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) add_qemu_ldst_label(s, false, is64, oi, datalo, datahi, addrlo, addrhi, s->code_ptr, label_ptr); #else + a_bits = get_alignment_bits(opc); + if (a_bits) { + tcg_out_test_alignment(s, false, addrlo, addrhi, a_bits); + } + tcg_out_qemu_st_direct(s, datalo, datahi, addrlo, x86_guest_base_index, x86_guest_base_offset, x86_guest_base_seg, opc); #endif -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/aarch64/tcg-target.h | 2 - tcg/aarch64/tcg-target.c.inc | 91 +++++++++++++++++++++++++++++------- 2 files changed, 74 insertions(+), 19 deletions(-) diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ typedef enum { void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); -#ifdef CONFIG_SOFTMMU #define TCG_TARGET_NEED_LDST_LABELS -#endif #define TCG_TARGET_NEED_POOL_LABELS #endif /* AARCH64_TCG_TARGET_H */ diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ * See the COPYING file in the top-level directory for details. */ +#include "../tcg-ldst.c.inc" #include "../tcg-pool.c.inc" #include "qemu/bitops.h" @@ -XXX,XX +XXX,XX @@ typedef enum { I3404_ANDI = 0x12000000, I3404_ORRI = 0x32000000, I3404_EORI = 0x52000000, + I3404_ANDSI = 0x72000000, /* Move wide immediate instructions. */ I3405_MOVN = 0x12800000, @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_long(TCGContext *s, const tcg_insn_unit *target) if (offset == sextract64(offset, 0, 26)) { tcg_out_insn(s, 3206, B, offset); } else { - tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, (intptr_t)target); - tcg_out_insn(s, 3207, BR, TCG_REG_TMP); + /* Choose X9 as a call-clobbered non-LR temporary. */ + tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X9, (intptr_t)target); + tcg_out_insn(s, 3207, BR, TCG_REG_X9); } } @@ -XXX,XX +XXX,XX @@ static void tcg_out_cltz(TCGContext *s, TCGType ext, TCGReg d, } } -#ifdef CONFIG_SOFTMMU -#include "../tcg-ldst.c.inc" +static void tcg_out_adr(TCGContext *s, TCGReg rd, const void *target) +{ + ptrdiff_t offset = tcg_pcrel_diff(s, target); + tcg_debug_assert(offset == sextract64(offset, 0, 21)); + tcg_out_insn(s, 3406, ADR, rd, offset); +} +#ifdef CONFIG_SOFTMMU /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, * MemOpIdx oi, uintptr_t ra) */ @@ -XXX,XX +XXX,XX @@ static void * const qemu_st_helpers[MO_SIZE + 1] = { #endif }; -static inline void tcg_out_adr(TCGContext *s, TCGReg rd, const void *target) -{ - ptrdiff_t offset = tcg_pcrel_diff(s, target); - tcg_debug_assert(offset == sextract64(offset, 0, 21)); - tcg_out_insn(s, 3406, ADR, rd, offset); -} - static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) { MemOpIdx oi = lb->oi; @@ -XXX,XX +XXX,XX @@ static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, MemOp opc, tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0); } +#else +static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addr_reg, + unsigned a_bits) +{ + unsigned a_mask = (1 << a_bits) - 1; + TCGLabelQemuLdst *label = new_ldst_label(s); + + label->is_ld = is_ld; + label->addrlo_reg = addr_reg; + + /* tst addr, #mask */ + tcg_out_logicali(s, I3404_ANDSI, 0, TCG_REG_XZR, addr_reg, a_mask); + + label->label_ptr[0] = s->code_ptr; + + /* b.ne slow_path */ + tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0); + + label->raddr = tcg_splitwx_to_rx(s->code_ptr); +} + +static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) +{ + if (!reloc_pc19(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { + return false; + } + + tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_X1, l->addrlo_reg); + tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0); + + /* "Tail call" to the helper, with the return address back inline. */ + tcg_out_adr(s, TCG_REG_LR, l->raddr); + tcg_out_goto_long(s, (const void *)(l->is_ld ? helper_unaligned_ld + : helper_unaligned_st)); + return true; +} + +static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + return tcg_out_fail_alignment(s, l); +} + +static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + return tcg_out_fail_alignment(s, l); +} #endif /* CONFIG_SOFTMMU */ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp memop, TCGType ext, TCGReg data_r, TCGReg addr_r, TCGType otype, TCGReg off_r) { - /* Byte swapping is left to middle-end expansion. */ - tcg_debug_assert((memop & MO_BSWAP) == 0); - switch (memop & MO_SSIZE) { case MO_UB: tcg_out_ldst_r(s, I3312_LDRB, data_r, addr_r, otype, off_r); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp memop, TCGReg data_r, TCGReg addr_r, TCGType otype, TCGReg off_r) { - /* Byte swapping is left to middle-end expansion. */ - tcg_debug_assert((memop & MO_BSWAP) == 0); - switch (memop & MO_SIZE) { case MO_8: tcg_out_ldst_r(s, I3312_STRB, data_r, addr_r, otype, off_r); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, { MemOp memop = get_memop(oi); const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32; + + /* Byte swapping is left to middle-end expansion. */ + tcg_debug_assert((memop & MO_BSWAP) == 0); + #ifdef CONFIG_SOFTMMU unsigned mem_index = get_mmuidx(oi); tcg_insn_unit *label_ptr; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, add_qemu_ldst_label(s, true, oi, ext, data_reg, addr_reg, s->code_ptr, label_ptr); #else /* !CONFIG_SOFTMMU */ + unsigned a_bits = get_alignment_bits(memop); + if (a_bits) { + tcg_out_test_alignment(s, true, addr_reg, a_bits); + } if (USE_GUEST_BASE) { tcg_out_qemu_ld_direct(s, memop, ext, data_reg, TCG_REG_GUEST_BASE, otype, addr_reg); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, { MemOp memop = get_memop(oi); const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32; + + /* Byte swapping is left to middle-end expansion. */ + tcg_debug_assert((memop & MO_BSWAP) == 0); + #ifdef CONFIG_SOFTMMU unsigned mem_index = get_mmuidx(oi); tcg_insn_unit *label_ptr; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, add_qemu_ldst_label(s, false, oi, (memop & MO_SIZE)== MO_64, data_reg, addr_reg, s->code_ptr, label_ptr); #else /* !CONFIG_SOFTMMU */ + unsigned a_bits = get_alignment_bits(memop); + if (a_bits) { + tcg_out_test_alignment(s, false, addr_reg, a_bits); + } if (USE_GUEST_BASE) { tcg_out_qemu_st_direct(s, memop, data_reg, TCG_REG_GUEST_BASE, otype, addr_reg); -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/ppc/tcg-target.h | 2 - tcg/ppc/tcg-target.c.inc | 98 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 90 insertions(+), 10 deletions(-) diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_HAS_MEMORY_BSWAP 1 -#ifdef CONFIG_SOFTMMU #define TCG_TARGET_NEED_LDST_LABELS -#endif #define TCG_TARGET_NEED_POOL_LABELS #endif diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ #include "elf.h" #include "../tcg-pool.c.inc" +#include "../tcg-ldst.c.inc" /* * Standardize on the _CALL_FOO symbols used by GCC: @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, } } -static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target) +static void tcg_out_call_int(TCGContext *s, int lk, + const tcg_insn_unit *target) { #ifdef _CALL_AIX /* Look through the descriptor. If the branch is in range, and we @@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target) if (in_range_b(diff) && toc == (uint32_t)toc) { tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, toc); - tcg_out_b(s, LK, tgt); + tcg_out_b(s, lk, tgt); } else { /* Fold the low bits of the constant into the addresses below. */ intptr_t arg = (intptr_t)target; @@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target) tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_TMP1, ofs); tcg_out32(s, MTSPR | RA(TCG_REG_R0) | CTR); tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_REG_TMP1, ofs + SZP); - tcg_out32(s, BCCTR | BO_ALWAYS | LK); + tcg_out32(s, BCCTR | BO_ALWAYS | lk); } #elif defined(_CALL_ELF) && _CALL_ELF == 2 intptr_t diff; @@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target) diff = tcg_pcrel_diff(s, target); if (in_range_b(diff)) { - tcg_out_b(s, LK, target); + tcg_out_b(s, lk, target); } else { tcg_out32(s, MTSPR | RS(TCG_REG_R12) | CTR); - tcg_out32(s, BCCTR | BO_ALWAYS | LK); + tcg_out32(s, BCCTR | BO_ALWAYS | lk); } #else - tcg_out_b(s, LK, target); + tcg_out_b(s, lk, target); #endif } +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target) +{ + tcg_out_call_int(s, LK, target); +} + static const uint32_t qemu_ldx_opc[(MO_SSIZE + MO_BSWAP) + 1] = { [MO_UB] = LBZX, [MO_UW] = LHZX, @@ -XXX,XX +XXX,XX @@ static const uint32_t qemu_exts_opc[4] = { }; #if defined (CONFIG_SOFTMMU) -#include "../tcg-ldst.c.inc" - /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr, * int mmu_idx, uintptr_t ra) */ @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) tcg_out_b(s, 0, lb->raddr); return true; } +#else + +static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addrlo, + TCGReg addrhi, unsigned a_bits) +{ + unsigned a_mask = (1 << a_bits) - 1; + TCGLabelQemuLdst *label = new_ldst_label(s); + + label->is_ld = is_ld; + label->addrlo_reg = addrlo; + label->addrhi_reg = addrhi; + + /* We are expecting a_bits to max out at 7, much lower than ANDI. */ + tcg_debug_assert(a_bits < 16); + tcg_out32(s, ANDI | SAI(addrlo, TCG_REG_R0, a_mask)); + + label->label_ptr[0] = s->code_ptr; + tcg_out32(s, BC | BI(0, CR_EQ) | BO_COND_FALSE | LK); + + label->raddr = tcg_splitwx_to_rx(s->code_ptr); +} + +static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) +{ + if (!reloc_pc14(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { + return false; + } + + if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { + TCGReg arg = TCG_REG_R4; +#ifdef TCG_TARGET_CALL_ALIGN_ARGS + arg |= 1; +#endif + if (l->addrlo_reg != arg) { + tcg_out_mov(s, TCG_TYPE_I32, arg, l->addrhi_reg); + tcg_out_mov(s, TCG_TYPE_I32, arg + 1, l->addrlo_reg); + } else if (l->addrhi_reg != arg + 1) { + tcg_out_mov(s, TCG_TYPE_I32, arg + 1, l->addrlo_reg); + tcg_out_mov(s, TCG_TYPE_I32, arg, l->addrhi_reg); + } else { + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R0, arg); + tcg_out_mov(s, TCG_TYPE_I32, arg, arg + 1); + tcg_out_mov(s, TCG_TYPE_I32, arg + 1, TCG_REG_R0); + } + } else { + tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_R4, l->addrlo_reg); + } + tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_R3, TCG_AREG0); + + /* "Tail call" to the helper, with the return address back inline. */ + tcg_out_call_int(s, 0, (const void *)(l->is_ld ? helper_unaligned_ld + : helper_unaligned_st)); + return true; +} + +static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + return tcg_out_fail_alignment(s, l); +} + +static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + return tcg_out_fail_alignment(s, l); +} + #endif /* SOFTMMU */ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) #ifdef CONFIG_SOFTMMU int mem_index; tcg_insn_unit *label_ptr; +#else + unsigned a_bits; #endif datalo = *args++; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) rbase = TCG_REG_R3; #else /* !CONFIG_SOFTMMU */ + a_bits = get_alignment_bits(opc); + if (a_bits) { + tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); + } rbase = guest_base ? TCG_GUEST_BASE_REG : 0; if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { tcg_out_ext32u(s, TCG_REG_TMP1, addrlo); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) #ifdef CONFIG_SOFTMMU int mem_index; tcg_insn_unit *label_ptr; +#else + unsigned a_bits; #endif datalo = *args++; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) rbase = TCG_REG_R3; #else /* !CONFIG_SOFTMMU */ + a_bits = get_alignment_bits(opc); + if (a_bits) { + tcg_out_test_alignment(s, false, addrlo, addrhi, a_bits); + } rbase = guest_base ? TCG_GUEST_BASE_REG : 0; if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { tcg_out_ext32u(s, TCG_REG_TMP1, addrlo); -- 2.25.1 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/riscv/tcg-target.h | 2 -- tcg/riscv/tcg-target.c.inc | 63 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_DEFAULT_MO (0) -#ifdef CONFIG_SOFTMMU #define TCG_TARGET_NEED_LDST_LABELS -#endif #define TCG_TARGET_NEED_POOL_LABELS #define TCG_TARGET_HAS_MEMORY_BSWAP 0 diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ * THE SOFTWARE. */ +#include "../tcg-ldst.c.inc" #include "../tcg-pool.c.inc" #ifdef CONFIG_DEBUG_TCG @@ -XXX,XX +XXX,XX @@ static void tcg_out_mb(TCGContext *s, TCGArg a0) */ #if defined(CONFIG_SOFTMMU) -#include "../tcg-ldst.c.inc" - /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, * MemOpIdx oi, uintptr_t ra) */ @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) tcg_out_goto(s, l->raddr); return true; } +#else + +static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addr_reg, + unsigned a_bits) +{ + unsigned a_mask = (1 << a_bits) - 1; + TCGLabelQemuLdst *l = new_ldst_label(s); + + l->is_ld = is_ld; + l->addrlo_reg = addr_reg; + + /* We are expecting a_bits to max out at 7, so we can always use andi. */ + tcg_debug_assert(a_bits < 12); + tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addr_reg, a_mask); + + l->label_ptr[0] = s->code_ptr; + tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP1, TCG_REG_ZERO, 0); + + l->raddr = tcg_splitwx_to_rx(s->code_ptr); +} + +static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) +{ + /* resolve label address */ + if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { + return false; + } + + tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_A1, l->addrlo_reg); + tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_A0, TCG_AREG0); + + /* tail call, with the return address back inline. */ + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RA, (uintptr_t)l->raddr); + tcg_out_call_int(s, (const void *)(l->is_ld ? helper_unaligned_ld + : helper_unaligned_st), true); + return true; +} + +static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + return tcg_out_fail_alignment(s, l); +} + +static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + return tcg_out_fail_alignment(s, l); +} + #endif /* CONFIG_SOFTMMU */ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) MemOp opc; #if defined(CONFIG_SOFTMMU) tcg_insn_unit *label_ptr[1]; +#else + unsigned a_bits; #endif TCGReg base = TCG_REG_TMP0; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) tcg_out_ext32u(s, base, addr_regl); addr_regl = base; } + a_bits = get_alignment_bits(opc); + if (a_bits) { + tcg_out_test_alignment(s, true, addr_regl, a_bits); + } if (guest_base != 0) { tcg_out_opc_reg(s, OPC_ADD, base, TCG_GUEST_BASE_REG, addr_regl); } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) MemOp opc; #if defined(CONFIG_SOFTMMU) tcg_insn_unit *label_ptr[1]; +#else + unsigned a_bits; #endif TCGReg base = TCG_REG_TMP0; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) tcg_out_ext32u(s, base, addr_regl); addr_regl = base; } + a_bits = get_alignment_bits(opc); + if (a_bits) { + tcg_out_test_alignment(s, false, addr_regl, a_bits); + } if (guest_base != 0) { tcg_out_opc_reg(s, OPC_ADD, base, TCG_GUEST_BASE_REG, addr_regl); } -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/s390x/tcg-target.h | 2 -- tcg/s390x/tcg-target.c.inc | 59 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -XXX,XX +XXX,XX @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, /* no need to flush icache explicitly */ } -#ifdef CONFIG_SOFTMMU #define TCG_TARGET_NEED_LDST_LABELS -#endif #define TCG_TARGET_NEED_POOL_LABELS #endif diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ #error "unsupported code generation mode" #endif +#include "../tcg-ldst.c.inc" #include "../tcg-pool.c.inc" #include "elf.h" @@ -XXX,XX +XXX,XX @@ typedef enum S390Opcode { RI_OIHL = 0xa509, RI_OILH = 0xa50a, RI_OILL = 0xa50b, + RI_TMLL = 0xa701, RIE_CGIJ = 0xec7c, RIE_CGRJ = 0xec64, @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg data, } #if defined(CONFIG_SOFTMMU) -#include "../tcg-ldst.c.inc" - /* We're expecting to use a 20-bit negative offset on the tlb memory ops. */ QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 19)); @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) return true; } #else +static void tcg_out_test_alignment(TCGContext *s, bool is_ld, + TCGReg addrlo, unsigned a_bits) +{ + unsigned a_mask = (1 << a_bits) - 1; + TCGLabelQemuLdst *l = new_ldst_label(s); + + l->is_ld = is_ld; + l->addrlo_reg = addrlo; + + /* We are expecting a_bits to max out at 7, much lower than TMLL. */ + tcg_debug_assert(a_bits < 16); + tcg_out_insn(s, RI, TMLL, addrlo, a_mask); + + tcg_out16(s, RI_BRC | (7 << 4)); /* CC in {1,2,3} */ + l->label_ptr[0] = s->code_ptr; + s->code_ptr += 1; + + l->raddr = tcg_splitwx_to_rx(s->code_ptr); +} + +static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) +{ + if (!patch_reloc(l->label_ptr[0], R_390_PC16DBL, + (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) { + return false; + } + + tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_R3, l->addrlo_reg); + tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_AREG0); + + /* "Tail call" to the helper, with the return address back inline. */ + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R14, (uintptr_t)l->raddr); + tgen_gotoi(s, S390_CC_ALWAYS, (const void *)(l->is_ld ? helper_unaligned_ld + : helper_unaligned_st)); + return true; +} + +static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + return tcg_out_fail_alignment(s, l); +} + +static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + return tcg_out_fail_alignment(s, l); +} + static void tcg_prepare_user_ldst(TCGContext *s, TCGReg *addr_reg, TCGReg *index_reg, tcg_target_long *disp) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, #else TCGReg index_reg; tcg_target_long disp; + unsigned a_bits = get_alignment_bits(opc); + if (a_bits) { + tcg_out_test_alignment(s, true, addr_reg, a_bits); + } tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp); tcg_out_qemu_ld_direct(s, opc, data_reg, addr_reg, index_reg, disp); #endif @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, #else TCGReg index_reg; tcg_target_long disp; + unsigned a_bits = get_alignment_bits(opc); + if (a_bits) { + tcg_out_test_alignment(s, false, addr_reg, a_bits); + } tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp); tcg_out_qemu_st_direct(s, opc, data_reg, addr_reg, index_reg, disp); #endif -- 2.25.1 Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tci.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tcg/tci.c b/tcg/tci.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -XXX,XX +XXX,XX @@ static bool tci_compare64(uint64_t u0, uint64_t u1, TCGCond condition) static uint64_t tci_qemu_ld(CPUArchState *env, target_ulong taddr, MemOpIdx oi, const void *tb_ptr) { - MemOp mop = get_memop(oi) & (MO_BSWAP | MO_SSIZE); + MemOp mop = get_memop(oi); uintptr_t ra = (uintptr_t)tb_ptr; #ifdef CONFIG_SOFTMMU - switch (mop) { + switch (mop & (MO_BSWAP | MO_SSIZE)) { case MO_UB: return helper_ret_ldub_mmu(env, taddr, oi, ra); case MO_SB: @@ -XXX,XX +XXX,XX @@ static uint64_t tci_qemu_ld(CPUArchState *env, target_ulong taddr, } #else void *haddr = g2h(env_cpu(env), taddr); + unsigned a_mask = (1u << get_alignment_bits(mop)) - 1; uint64_t ret; set_helper_retaddr(ra); - switch (mop) { + if (taddr & a_mask) { + helper_unaligned_ld(env, taddr); + } + switch (mop & (MO_BSWAP | MO_SSIZE)) { case MO_UB: ret = ldub_p(haddr); break; @@ -XXX,XX +XXX,XX @@ static uint64_t tci_qemu_ld(CPUArchState *env, target_ulong taddr, static void tci_qemu_st(CPUArchState *env, target_ulong taddr, uint64_t val, MemOpIdx oi, const void *tb_ptr) { - MemOp mop = get_memop(oi) & (MO_BSWAP | MO_SSIZE); + MemOp mop = get_memop(oi); uintptr_t ra = (uintptr_t)tb_ptr; #ifdef CONFIG_SOFTMMU - switch (mop) { + switch (mop & (MO_BSWAP | MO_SIZE)) { case MO_UB: helper_ret_stb_mmu(env, taddr, val, oi, ra); break; @@ -XXX,XX +XXX,XX @@ static void tci_qemu_st(CPUArchState *env, target_ulong taddr, uint64_t val, } #else void *haddr = g2h(env_cpu(env), taddr); + unsigned a_mask = (1u << get_alignment_bits(mop)) - 1; set_helper_retaddr(ra); - switch (mop) { + if (taddr & a_mask) { + helper_unaligned_st(env, taddr); + } + switch (mop & (MO_BSWAP | MO_SIZE)) { case MO_UB: stb_p(haddr, val); break; -- 2.25.1 From: WANG Xuerui <git@xen0n.name> Signed-off-by: WANG Xuerui <git@xen0n.name> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20220106134238.3936163-1-git@xen0n.name> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/loongarch64/tcg-target.h | 2 - tcg/loongarch64/tcg-target.c.inc | 71 +++++++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/loongarch64/tcg-target.h +++ b/tcg/loongarch64/tcg-target.h @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); #define TCG_TARGET_DEFAULT_MO (0) -#ifdef CONFIG_SOFTMMU #define TCG_TARGET_NEED_LDST_LABELS -#endif #define TCG_TARGET_HAS_MEMORY_BSWAP 0 diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/loongarch64/tcg-target.c.inc +++ b/tcg/loongarch64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ * THE SOFTWARE. */ +#include "../tcg-ldst.c.inc" + #ifdef CONFIG_DEBUG_TCG static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { "zero", @@ -XXX,XX +XXX,XX @@ static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, */ #if defined(CONFIG_SOFTMMU) -#include "../tcg-ldst.c.inc" - /* * helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, * MemOpIdx oi, uintptr_t ra) @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) return tcg_out_goto(s, l->raddr); } +#else + +/* + * Alignment helpers for user-mode emulation + */ + +static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addr_reg, + unsigned a_bits) +{ + TCGLabelQemuLdst *l = new_ldst_label(s); + + l->is_ld = is_ld; + l->addrlo_reg = addr_reg; + + /* + * Without micro-architecture details, we don't know which of bstrpick or + * andi is faster, so use bstrpick as it's not constrained by imm field + * width. (Not to say alignments >= 2^12 are going to happen any time + * soon, though) + */ + tcg_out_opc_bstrpick_d(s, TCG_REG_TMP1, addr_reg, 0, a_bits - 1); + + l->label_ptr[0] = s->code_ptr; + tcg_out_opc_bne(s, TCG_REG_TMP1, TCG_REG_ZERO, 0); + + l->raddr = tcg_splitwx_to_rx(s->code_ptr); +} + +static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) +{ + /* resolve label address */ + if (!reloc_br_sk16(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { + return false; + } + + tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_A1, l->addrlo_reg); + tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_A0, TCG_AREG0); + + /* tail call, with the return address back inline. */ + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RA, (uintptr_t)l->raddr); + tcg_out_call_int(s, (const void *)(l->is_ld ? helper_unaligned_ld + : helper_unaligned_st), true); + return true; +} + +static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + return tcg_out_fail_alignment(s, l); +} + +static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + return tcg_out_fail_alignment(s, l); +} + #endif /* CONFIG_SOFTMMU */ /* @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, TCGType type) MemOp opc; #if defined(CONFIG_SOFTMMU) tcg_insn_unit *label_ptr[1]; +#else + unsigned a_bits; #endif TCGReg base; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, TCGType type) data_regl, addr_regl, s->code_ptr, label_ptr); #else + a_bits = get_alignment_bits(opc); + if (a_bits) { + tcg_out_test_alignment(s, true, addr_regl, a_bits); + } base = tcg_out_zext_addr_if_32_bit(s, addr_regl, TCG_REG_TMP0); TCGReg guest_base_reg = USE_GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_ZERO; tcg_out_qemu_ld_indexed(s, data_regl, base, guest_base_reg, opc, type); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args) MemOp opc; #if defined(CONFIG_SOFTMMU) tcg_insn_unit *label_ptr[1]; +#else + unsigned a_bits; #endif TCGReg base; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args) data_regl, addr_regl, s->code_ptr, label_ptr); #else + a_bits = get_alignment_bits(opc); + if (a_bits) { + tcg_out_test_alignment(s, false, addr_regl, a_bits); + } base = tcg_out_zext_addr_if_32_bit(s, addr_regl, TCG_REG_TMP0); TCGReg guest_base_reg = USE_GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_ZERO; tcg_out_qemu_st_indexed(s, data_regl, base, guest_base_reg, opc); -- 2.25.1 Support for unaligned accesses is difficult for pre-v6 hosts. While debian still builds for armv4, we cannot use a compile time test, so test the architecture at runtime and error out. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/arm/tcg-target.c.inc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_target_init(TCGContext *s) if (pl != NULL && pl[0] == 'v' && pl[1] >= '4' && pl[1] <= '9') { arm_arch = pl[1] - '0'; } + + if (arm_arch < 6) { + error_report("TCG: ARMv%d is unsupported; exiting", arm_arch); + exit(EXIT_FAILURE); + } } tcg_target_available_regs[TCG_TYPE_I32] = ALL_GENERAL_REGS; -- 2.25.1 This is now always true, since we require armv6. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/arm/tcg-target.h | 3 +-- tcg/arm/tcg-target.c.inc | 35 ++++++----------------------------- 2 files changed, 7 insertions(+), 31 deletions(-) diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern int arm_arch; -#define use_armv5t_instructions (__ARM_ARCH >= 5 || arm_arch >= 5) #define use_armv6_instructions (__ARM_ARCH >= 6 || arm_arch >= 6) #define use_armv7_instructions (__ARM_ARCH >= 7 || arm_arch >= 7) @@ -XXX,XX +XXX,XX @@ extern bool use_neon_instructions; #define TCG_TARGET_HAS_eqv_i32 0 #define TCG_TARGET_HAS_nand_i32 0 #define TCG_TARGET_HAS_nor_i32 0 -#define TCG_TARGET_HAS_clz_i32 use_armv5t_instructions +#define TCG_TARGET_HAS_clz_i32 1 #define TCG_TARGET_HAS_ctz_i32 use_armv7_instructions #define TCG_TARGET_HAS_ctpop_i32 0 #define TCG_TARGET_HAS_deposit_i32 use_armv7_instructions diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_b_reg(TCGContext *s, ARMCond cond, TCGReg rn) * Unless the C portion of QEMU is compiled as thumb, we don't need * true BX semantics; merely a branch to an address held in a register. */ - if (use_armv5t_instructions) { - tcg_out_bx_reg(s, cond, rn); - } else { - tcg_out_mov_reg(s, cond, TCG_REG_PC, rn); - } + tcg_out_bx_reg(s, cond, rn); } static void tcg_out_dat_imm(TCGContext *s, ARMCond cond, ARMInsn opc, @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto(TCGContext *s, ARMCond cond, const tcg_insn_unit *addr) } /* LDR is interworking from v5t. */ - if (arm_mode || use_armv5t_instructions) { - tcg_out_movi_pool(s, cond, TCG_REG_PC, addri); - return; - } - - /* else v4t */ - tcg_out_movi32(s, COND_AL, TCG_REG_TMP, addri); - tcg_out_bx_reg(s, COND_AL, TCG_REG_TMP); + tcg_out_movi_pool(s, cond, TCG_REG_PC, addri); } /* @@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *addr) if (disp - 8 < 0x02000000 && disp - 8 >= -0x02000000) { if (arm_mode) { tcg_out_bl_imm(s, COND_AL, disp); - return; - } - if (use_armv5t_instructions) { + } else { tcg_out_blx_imm(s, disp); - return; } + return; } - if (use_armv5t_instructions) { - tcg_out_movi32(s, COND_AL, TCG_REG_TMP, addri); - tcg_out_blx_reg(s, COND_AL, TCG_REG_TMP); - } else if (arm_mode) { - /* ??? Know that movi_pool emits exactly 1 insn. */ - tcg_out_mov_reg(s, COND_AL, TCG_REG_R14, TCG_REG_PC); - tcg_out_movi_pool(s, COND_AL, TCG_REG_PC, addri); - } else { - tcg_out_movi32(s, COND_AL, TCG_REG_TMP, addri); - tcg_out_mov_reg(s, COND_AL, TCG_REG_R14, TCG_REG_PC); - tcg_out_bx_reg(s, COND_AL, TCG_REG_TMP); - } + tcg_out_movi32(s, COND_AL, TCG_REG_TMP, addri); + tcg_out_blx_reg(s, COND_AL, TCG_REG_TMP); } static void tcg_out_goto_label(TCGContext *s, ARMCond cond, TCGLabel *l) -- 2.25.1 This is now always true, since we require armv6. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/arm/tcg-target.h | 1 - tcg/arm/tcg-target.c.inc | 192 ++++++--------------------------------- 2 files changed, 27 insertions(+), 166 deletions(-) diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern int arm_arch; -#define use_armv6_instructions (__ARM_ARCH >= 6 || arm_arch >= 6) #define use_armv7_instructions (__ARM_ARCH >= 7 || arm_arch >= 7) #undef TCG_TARGET_STACK_GROWSUP diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_dat_rIN(TCGContext *s, ARMCond cond, ARMInsn opc, static void tcg_out_mul32(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn, TCGReg rm) { - /* if ArchVersion() < 6 && d == n then UNPREDICTABLE; */ - if (!use_armv6_instructions && rd == rn) { - if (rd == rm) { - /* rd == rn == rm; copy an input to tmp first. */ - tcg_out_mov_reg(s, cond, TCG_REG_TMP, rn); - rm = rn = TCG_REG_TMP; - } else { - rn = rm; - rm = rd; - } - } /* mul */ tcg_out32(s, (cond << 28) | 0x90 | (rd << 16) | (rm << 8) | rn); } @@ -XXX,XX +XXX,XX @@ static void tcg_out_mul32(TCGContext *s, ARMCond cond, TCGReg rd, static void tcg_out_umull32(TCGContext *s, ARMCond cond, TCGReg rd0, TCGReg rd1, TCGReg rn, TCGReg rm) { - /* if ArchVersion() < 6 && (dHi == n || dLo == n) then UNPREDICTABLE; */ - if (!use_armv6_instructions && (rd0 == rn || rd1 == rn)) { - if (rd0 == rm || rd1 == rm) { - tcg_out_mov_reg(s, cond, TCG_REG_TMP, rn); - rn = TCG_REG_TMP; - } else { - TCGReg t = rn; - rn = rm; - rm = t; - } - } /* umull */ tcg_out32(s, (cond << 28) | 0x00800090 | (rd1 << 16) | (rd0 << 12) | (rm << 8) | rn); @@ -XXX,XX +XXX,XX @@ static void tcg_out_umull32(TCGContext *s, ARMCond cond, TCGReg rd0, static void tcg_out_smull32(TCGContext *s, ARMCond cond, TCGReg rd0, TCGReg rd1, TCGReg rn, TCGReg rm) { - /* if ArchVersion() < 6 && (dHi == n || dLo == n) then UNPREDICTABLE; */ - if (!use_armv6_instructions && (rd0 == rn || rd1 == rn)) { - if (rd0 == rm || rd1 == rm) { - tcg_out_mov_reg(s, cond, TCG_REG_TMP, rn); - rn = TCG_REG_TMP; - } else { - TCGReg t = rn; - rn = rm; - rm = t; - } - } /* smull */ tcg_out32(s, (cond << 28) | 0x00c00090 | (rd1 << 16) | (rd0 << 12) | (rm << 8) | rn); @@ -XXX,XX +XXX,XX @@ static void tcg_out_udiv(TCGContext *s, ARMCond cond, static void tcg_out_ext8s(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn) { - if (use_armv6_instructions) { - /* sxtb */ - tcg_out32(s, 0x06af0070 | (cond << 28) | (rd << 12) | rn); - } else { - tcg_out_dat_reg(s, cond, ARITH_MOV, - rd, 0, rn, SHIFT_IMM_LSL(24)); - tcg_out_dat_reg(s, cond, ARITH_MOV, - rd, 0, rd, SHIFT_IMM_ASR(24)); - } + /* sxtb */ + tcg_out32(s, 0x06af0070 | (cond << 28) | (rd << 12) | rn); } static void __attribute__((unused)) @@ -XXX,XX +XXX,XX @@ tcg_out_ext8u(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn) static void tcg_out_ext16s(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn) { - if (use_armv6_instructions) { - /* sxth */ - tcg_out32(s, 0x06bf0070 | (cond << 28) | (rd << 12) | rn); - } else { - tcg_out_dat_reg(s, cond, ARITH_MOV, - rd, 0, rn, SHIFT_IMM_LSL(16)); - tcg_out_dat_reg(s, cond, ARITH_MOV, - rd, 0, rd, SHIFT_IMM_ASR(16)); - } + /* sxth */ + tcg_out32(s, 0x06bf0070 | (cond << 28) | (rd << 12) | rn); } static void tcg_out_ext16u(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn) { - if (use_armv6_instructions) { - /* uxth */ - tcg_out32(s, 0x06ff0070 | (cond << 28) | (rd << 12) | rn); - } else { - tcg_out_dat_reg(s, cond, ARITH_MOV, - rd, 0, rn, SHIFT_IMM_LSL(16)); - tcg_out_dat_reg(s, cond, ARITH_MOV, - rd, 0, rd, SHIFT_IMM_LSR(16)); - } + /* uxth */ + tcg_out32(s, 0x06ff0070 | (cond << 28) | (rd << 12) | rn); } static void tcg_out_bswap16(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn, int flags) { - if (use_armv6_instructions) { - if (flags & TCG_BSWAP_OS) { - /* revsh */ - tcg_out32(s, 0x06ff0fb0 | (cond << 28) | (rd << 12) | rn); - return; - } - - /* rev16 */ - tcg_out32(s, 0x06bf0fb0 | (cond << 28) | (rd << 12) | rn); - if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { - /* uxth */ - tcg_out32(s, 0x06ff0070 | (cond << 28) | (rd << 12) | rd); - } + if (flags & TCG_BSWAP_OS) { + /* revsh */ + tcg_out32(s, 0x06ff0fb0 | (cond << 28) | (rd << 12) | rn); return; } - if (flags == 0) { - /* - * For stores, no input or output extension: - * rn = xxAB - * lsr tmp, rn, #8 tmp = 0xxA - * and tmp, tmp, #0xff tmp = 000A - * orr rd, tmp, rn, lsl #8 rd = xABA - */ - tcg_out_dat_reg(s, cond, ARITH_MOV, - TCG_REG_TMP, 0, rn, SHIFT_IMM_LSR(8)); - tcg_out_dat_imm(s, cond, ARITH_AND, TCG_REG_TMP, TCG_REG_TMP, 0xff); - tcg_out_dat_reg(s, cond, ARITH_ORR, - rd, TCG_REG_TMP, rn, SHIFT_IMM_LSL(8)); - return; + /* rev16 */ + tcg_out32(s, 0x06bf0fb0 | (cond << 28) | (rd << 12) | rn); + if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { + /* uxth */ + tcg_out32(s, 0x06ff0070 | (cond << 28) | (rd << 12) | rd); } - - /* - * Byte swap, leaving the result at the top of the register. - * We will then shift down, zero or sign-extending. - */ - if (flags & TCG_BSWAP_IZ) { - /* - * rn = 00AB - * ror tmp, rn, #8 tmp = B00A - * orr tmp, tmp, tmp, lsl #16 tmp = BA00 - */ - tcg_out_dat_reg(s, cond, ARITH_MOV, - TCG_REG_TMP, 0, rn, SHIFT_IMM_ROR(8)); - tcg_out_dat_reg(s, cond, ARITH_ORR, - TCG_REG_TMP, TCG_REG_TMP, TCG_REG_TMP, - SHIFT_IMM_LSL(16)); - } else { - /* - * rn = xxAB - * and tmp, rn, #0xff00 tmp = 00A0 - * lsl tmp, tmp, #8 tmp = 0A00 - * orr tmp, tmp, rn, lsl #24 tmp = BA00 - */ - tcg_out_dat_rI(s, cond, ARITH_AND, TCG_REG_TMP, rn, 0xff00, 1); - tcg_out_dat_reg(s, cond, ARITH_MOV, - TCG_REG_TMP, 0, TCG_REG_TMP, SHIFT_IMM_LSL(8)); - tcg_out_dat_reg(s, cond, ARITH_ORR, - TCG_REG_TMP, TCG_REG_TMP, rn, SHIFT_IMM_LSL(24)); - } - tcg_out_dat_reg(s, cond, ARITH_MOV, rd, 0, TCG_REG_TMP, - (flags & TCG_BSWAP_OS - ? SHIFT_IMM_ASR(8) : SHIFT_IMM_LSR(8))); } static void tcg_out_bswap32(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn) { - if (use_armv6_instructions) { - /* rev */ - tcg_out32(s, 0x06bf0f30 | (cond << 28) | (rd << 12) | rn); - } else { - tcg_out_dat_reg(s, cond, ARITH_EOR, - TCG_REG_TMP, rn, rn, SHIFT_IMM_ROR(16)); - tcg_out_dat_imm(s, cond, ARITH_BIC, - TCG_REG_TMP, TCG_REG_TMP, 0xff | 0x800); - tcg_out_dat_reg(s, cond, ARITH_MOV, - rd, 0, rn, SHIFT_IMM_ROR(8)); - tcg_out_dat_reg(s, cond, ARITH_EOR, - rd, rd, TCG_REG_TMP, SHIFT_IMM_LSR(8)); - } + /* rev */ + tcg_out32(s, 0x06bf0f30 | (cond << 28) | (rd << 12) | rn); } static void tcg_out_deposit(TCGContext *s, ARMCond cond, TCGReg rd, @@ -XXX,XX +XXX,XX @@ static void tcg_out_mb(TCGContext *s, TCGArg a0) { if (use_armv7_instructions) { tcg_out32(s, INSN_DMB_ISH); - } else if (use_armv6_instructions) { + } else { tcg_out32(s, INSN_DMB_MCR); } } @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_arg_reg64(TCGContext *s, TCGReg argreg, if (argreg & 1) { argreg++; } - if (use_armv6_instructions && argreg >= 4 - && (arglo & 1) == 0 && arghi == arglo + 1) { + if (argreg >= 4 && (arglo & 1) == 0 && arghi == arglo + 1) { tcg_out_strd_8(s, COND_AL, arglo, TCG_REG_CALL_STACK, (argreg - 4) * 4); return argreg + 2; @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi, int cmp_off = (is_load ? offsetof(CPUTLBEntry, addr_read) : offsetof(CPUTLBEntry, addr_write)); int fast_off = TLB_MASK_TABLE_OFS(mem_index); - int mask_off = fast_off + offsetof(CPUTLBDescFast, mask); - int table_off = fast_off + offsetof(CPUTLBDescFast, table); unsigned s_bits = opc & MO_SIZE; unsigned a_bits = get_alignment_bits(opc); @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi, } /* Load env_tlb(env)->f[mmu_idx].{mask,table} into {r0,r1}. */ - if (use_armv6_instructions) { - tcg_out_ldrd_8(s, COND_AL, TCG_REG_R0, TCG_AREG0, fast_off); - } else { - tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_R0, TCG_AREG0, mask_off); - tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_R1, TCG_AREG0, table_off); - } + tcg_out_ldrd_8(s, COND_AL, TCG_REG_R0, TCG_AREG0, fast_off); /* Extract the tlb index from the address into R0. */ tcg_out_dat_reg(s, COND_AL, ARITH_AND, TCG_REG_R0, TCG_REG_R0, addrlo, @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi, * Load the tlb comparator into R2/R3 and the fast path addend into R1. */ if (cmp_off == 0) { - if (use_armv6_instructions && TARGET_LONG_BITS == 64) { + if (TARGET_LONG_BITS == 64) { tcg_out_ldrd_rwb(s, COND_AL, TCG_REG_R2, TCG_REG_R1, TCG_REG_R0); } else { tcg_out_ld32_rwb(s, COND_AL, TCG_REG_R2, TCG_REG_R1, TCG_REG_R0); @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi, } else { tcg_out_dat_reg(s, COND_AL, ARITH_ADD, TCG_REG_R1, TCG_REG_R1, TCG_REG_R0, 0); - if (use_armv6_instructions && TARGET_LONG_BITS == 64) { + if (TARGET_LONG_BITS == 64) { tcg_out_ldrd_8(s, COND_AL, TCG_REG_R2, TCG_REG_R1, cmp_off); } else { tcg_out_ld32_12(s, COND_AL, TCG_REG_R2, TCG_REG_R1, cmp_off); } } - if (!use_armv6_instructions && TARGET_LONG_BITS == 64) { - tcg_out_ld32_12(s, COND_AL, TCG_REG_R3, TCG_REG_R1, cmp_off + 4); - } /* Load the tlb addend. */ tcg_out_ld32_12(s, COND_AL, TCG_REG_R1, TCG_REG_R1, @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) TCGReg argreg, datalo, datahi; MemOpIdx oi = lb->oi; MemOp opc = get_memop(oi); - void *func; if (!reloc_pc24(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { return false; @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) argreg = tcg_out_arg_imm32(s, argreg, oi); argreg = tcg_out_arg_reg32(s, argreg, TCG_REG_R14); - /* For armv6 we can use the canonical unsigned helpers and minimize - icache usage. For pre-armv6, use the signed helpers since we do - not have a single insn sign-extend. */ - if (use_armv6_instructions) { - func = qemu_ld_helpers[opc & MO_SIZE]; - } else { - func = qemu_ld_helpers[opc & MO_SSIZE]; - if (opc & MO_SIGN) { - opc = MO_UL; - } - } - tcg_out_call(s, func); + /* Use the canonical unsigned helpers and minimize icache usage. */ + tcg_out_call(s, qemu_ld_helpers[opc & MO_SIZE]); datalo = lb->datalo_reg; datahi = lb->datahi_reg; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_index(TCGContext *s, MemOp opc, break; case MO_UQ: /* Avoid ldrd for user-only emulation, to handle unaligned. */ - if (USING_SOFTMMU && use_armv6_instructions + if (USING_SOFTMMU && (datalo & 1) == 0 && datahi == datalo + 1) { tcg_out_ldrd_r(s, COND_AL, datalo, addrlo, addend); } else if (datalo != addend) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg datalo, break; case MO_UQ: /* Avoid ldrd for user-only emulation, to handle unaligned. */ - if (USING_SOFTMMU && use_armv6_instructions + if (USING_SOFTMMU && (datalo & 1) == 0 && datahi == datalo + 1) { tcg_out_ldrd_8(s, COND_AL, datalo, addrlo, 0); } else if (datalo == addrlo) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_index(TCGContext *s, ARMCond cond, MemOp opc, break; case MO_64: /* Avoid strd for user-only emulation, to handle unaligned. */ - if (USING_SOFTMMU && use_armv6_instructions + if (USING_SOFTMMU && (datalo & 1) == 0 && datahi == datalo + 1) { tcg_out_strd_r(s, cond, datalo, addrlo, addend); } else { @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg datalo, break; case MO_64: /* Avoid strd for user-only emulation, to handle unaligned. */ - if (USING_SOFTMMU && use_armv6_instructions + if (USING_SOFTMMU && (datalo & 1) == 0 && datahi == datalo + 1) { tcg_out_strd_8(s, COND_AL, datalo, addrlo, 0); } else { -- 2.25.1 We will shortly allow the use of unaligned memory accesses, and these require proper alignment. Use get_alignment_bits to verify and remove USING_SOFTMMU. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/arm/tcg-target.c.inc | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ bool use_idiv_instructions; bool use_neon_instructions; #endif -/* ??? Ought to think about changing CONFIG_SOFTMMU to always defined. */ -#ifdef CONFIG_SOFTMMU -# define USING_SOFTMMU 1 -#else -# define USING_SOFTMMU 0 -#endif - #ifdef CONFIG_DEBUG_TCG static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { "%r0", "%r1", "%r2", "%r3", "%r4", "%r5", "%r6", "%r7", @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_index(TCGContext *s, MemOp opc, tcg_out_ld32_r(s, COND_AL, datalo, addrlo, addend); break; case MO_UQ: - /* Avoid ldrd for user-only emulation, to handle unaligned. */ - if (USING_SOFTMMU + /* LDRD requires alignment; double-check that. */ + if (get_alignment_bits(opc) >= MO_64 && (datalo & 1) == 0 && datahi == datalo + 1) { tcg_out_ldrd_r(s, COND_AL, datalo, addrlo, addend); } else if (datalo != addend) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg datalo, tcg_out_ld32_12(s, COND_AL, datalo, addrlo, 0); break; case MO_UQ: - /* Avoid ldrd for user-only emulation, to handle unaligned. */ - if (USING_SOFTMMU + /* LDRD requires alignment; double-check that. */ + if (get_alignment_bits(opc) >= MO_64 && (datalo & 1) == 0 && datahi == datalo + 1) { tcg_out_ldrd_8(s, COND_AL, datalo, addrlo, 0); } else if (datalo == addrlo) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_index(TCGContext *s, ARMCond cond, MemOp opc, tcg_out_st32_r(s, cond, datalo, addrlo, addend); break; case MO_64: - /* Avoid strd for user-only emulation, to handle unaligned. */ - if (USING_SOFTMMU + /* STRD requires alignment; double-check that. */ + if (get_alignment_bits(opc) >= MO_64 && (datalo & 1) == 0 && datahi == datalo + 1) { tcg_out_strd_r(s, cond, datalo, addrlo, addend); } else { @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg datalo, tcg_out_st32_12(s, COND_AL, datalo, addrlo, 0); break; case MO_64: - /* Avoid strd for user-only emulation, to handle unaligned. */ - if (USING_SOFTMMU + /* STRD requires alignment; double-check that. */ + if (get_alignment_bits(opc) >= MO_64 && (datalo & 1) == 0 && datahi == datalo + 1) { tcg_out_strd_8(s, COND_AL, datalo, addrlo, 0); } else { -- 2.25.1 From armv6, the architecture supports unaligned accesses. All we need to do is perform the correct alignment check in tcg_out_tlb_read. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/arm/tcg-target.c.inc | 41 ++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi, int cmp_off = (is_load ? offsetof(CPUTLBEntry, addr_read) : offsetof(CPUTLBEntry, addr_write)); int fast_off = TLB_MASK_TABLE_OFS(mem_index); - unsigned s_bits = opc & MO_SIZE; - unsigned a_bits = get_alignment_bits(opc); - - /* - * We don't support inline unaligned acceses, but we can easily - * support overalignment checks. - */ - if (a_bits < s_bits) { - a_bits = s_bits; - } + unsigned s_mask = (1 << (opc & MO_SIZE)) - 1; + unsigned a_mask = (1 << get_alignment_bits(opc)) - 1; + TCGReg t_addr; /* Load env_tlb(env)->f[mmu_idx].{mask,table} into {r0,r1}. */ tcg_out_ldrd_8(s, COND_AL, TCG_REG_R0, TCG_AREG0, fast_off); @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi, /* * Check alignment, check comparators. - * Do this in no more than 3 insns. Use MOVW for v7, if possible, + * Do this in 2-4 insns. Use MOVW for v7, if possible, * to reduce the number of sequential conditional instructions. * Almost all guests have at least 4k pages, which means that we need * to clear at least 9 bits even for an 8-byte memory, which means it * isn't worth checking for an immediate operand for BIC. + * + * For unaligned accesses, test the page of the last unit of alignment. + * This leaves the least significant alignment bits unchanged, and of + * course must be zero. */ + t_addr = addrlo; + if (a_mask < s_mask) { + t_addr = TCG_REG_R0; + tcg_out_dat_imm(s, COND_AL, ARITH_ADD, t_addr, + addrlo, s_mask - a_mask); + } if (use_armv7_instructions && TARGET_PAGE_BITS <= 16) { - tcg_target_ulong mask = ~(TARGET_PAGE_MASK | ((1 << a_bits) - 1)); - - tcg_out_movi32(s, COND_AL, TCG_REG_TMP, mask); + tcg_out_movi32(s, COND_AL, TCG_REG_TMP, ~(TARGET_PAGE_MASK | a_mask)); tcg_out_dat_reg(s, COND_AL, ARITH_BIC, TCG_REG_TMP, - addrlo, TCG_REG_TMP, 0); + t_addr, TCG_REG_TMP, 0); tcg_out_dat_reg(s, COND_AL, ARITH_CMP, 0, TCG_REG_R2, TCG_REG_TMP, 0); } else { - if (a_bits) { - tcg_out_dat_imm(s, COND_AL, ARITH_TST, 0, addrlo, - (1 << a_bits) - 1); + if (a_mask) { + tcg_debug_assert(a_mask <= 0xff); + tcg_out_dat_imm(s, COND_AL, ARITH_TST, 0, addrlo, a_mask); } - tcg_out_dat_reg(s, COND_AL, ARITH_MOV, TCG_REG_TMP, 0, addrlo, + tcg_out_dat_reg(s, COND_AL, ARITH_MOV, TCG_REG_TMP, 0, t_addr, SHIFT_IMM_LSR(TARGET_PAGE_BITS)); - tcg_out_dat_reg(s, (a_bits ? COND_EQ : COND_AL), ARITH_CMP, + tcg_out_dat_reg(s, (a_mask ? COND_EQ : COND_AL), ARITH_CMP, 0, TCG_REG_R2, TCG_REG_TMP, SHIFT_IMM_LSL(TARGET_PAGE_BITS)); } -- 2.25.1 Reserve a register for the guest_base using aarch64 for reference. By doing so, we do not have to recompute it for every memory load. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/arm/tcg-target.c.inc | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const int tcg_target_call_oarg_regs[2] = { #define TCG_REG_TMP TCG_REG_R12 #define TCG_VEC_TMP TCG_REG_Q15 +#ifndef CONFIG_SOFTMMU +#define TCG_REG_GUEST_BASE TCG_REG_R11 +#endif typedef enum { COND_EQ = 0x0, @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) static void tcg_out_qemu_ld_index(TCGContext *s, MemOp opc, TCGReg datalo, TCGReg datahi, - TCGReg addrlo, TCGReg addend) + TCGReg addrlo, TCGReg addend, + bool scratch_addend) { /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((opc & MO_BSWAP) == 0); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_index(TCGContext *s, MemOp opc, if (get_alignment_bits(opc) >= MO_64 && (datalo & 1) == 0 && datahi == datalo + 1) { tcg_out_ldrd_r(s, COND_AL, datalo, addrlo, addend); - } else if (datalo != addend) { + } else if (scratch_addend) { tcg_out_ld32_rwb(s, COND_AL, datalo, addend, addrlo); tcg_out_ld32_12(s, COND_AL, datahi, addend, 4); } else { @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64) label_ptr = s->code_ptr; tcg_out_bl_imm(s, COND_NE, 0); - tcg_out_qemu_ld_index(s, opc, datalo, datahi, addrlo, addend); + tcg_out_qemu_ld_index(s, opc, datalo, datahi, addrlo, addend, true); add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi, s->code_ptr, label_ptr); #else /* !CONFIG_SOFTMMU */ if (guest_base) { - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, guest_base); - tcg_out_qemu_ld_index(s, opc, datalo, datahi, addrlo, TCG_REG_TMP); + tcg_out_qemu_ld_index(s, opc, datalo, datahi, + addrlo, TCG_REG_GUEST_BASE, false); } else { tcg_out_qemu_ld_direct(s, opc, datalo, datahi, addrlo); } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64) static void tcg_out_qemu_st_index(TCGContext *s, ARMCond cond, MemOp opc, TCGReg datalo, TCGReg datahi, - TCGReg addrlo, TCGReg addend) + TCGReg addrlo, TCGReg addend, + bool scratch_addend) { /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((opc & MO_BSWAP) == 0); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_index(TCGContext *s, ARMCond cond, MemOp opc, if (get_alignment_bits(opc) >= MO_64 && (datalo & 1) == 0 && datahi == datalo + 1) { tcg_out_strd_r(s, cond, datalo, addrlo, addend); - } else { + } else if (scratch_addend) { tcg_out_st32_rwb(s, cond, datalo, addend, addrlo); tcg_out_st32_12(s, cond, datahi, addend, 4); + } else { + tcg_out_dat_reg(s, cond, ARITH_ADD, TCG_REG_TMP, + addend, addrlo, SHIFT_IMM_LSL(0)); + tcg_out_st32_12(s, cond, datalo, TCG_REG_TMP, 0); + tcg_out_st32_12(s, cond, datahi, TCG_REG_TMP, 4); } break; default: @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) mem_index = get_mmuidx(oi); addend = tcg_out_tlb_read(s, addrlo, addrhi, opc, mem_index, 0); - tcg_out_qemu_st_index(s, COND_EQ, opc, datalo, datahi, addrlo, addend); + tcg_out_qemu_st_index(s, COND_EQ, opc, datalo, datahi, + addrlo, addend, true); /* The conditional call must come last, as we're going to return here. */ label_ptr = s->code_ptr; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) s->code_ptr, label_ptr); #else /* !CONFIG_SOFTMMU */ if (guest_base) { - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, guest_base); - tcg_out_qemu_st_index(s, COND_AL, opc, datalo, - datahi, addrlo, TCG_REG_TMP); + tcg_out_qemu_st_index(s, COND_AL, opc, datalo, datahi, + addrlo, TCG_REG_GUEST_BASE, false); } else { tcg_out_qemu_st_direct(s, opc, datalo, datahi, addrlo); } @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]); +#ifndef CONFIG_SOFTMMU + if (guest_base) { + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_GUEST_BASE, guest_base); + tcg_regset_set_reg(s->reserved_regs, TCG_REG_GUEST_BASE); + } +#endif + tcg_out_b_reg(s, COND_AL, tcg_target_call_iarg_regs[1]); /* -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/arm/tcg-target.h | 2 - tcg/arm/tcg-target.c.inc | 83 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_neon_instructions; /* not defined -- call should be eliminated at compile time */ void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); -#ifdef CONFIG_SOFTMMU #define TCG_TARGET_NEED_LDST_LABELS -#endif #define TCG_TARGET_NEED_POOL_LABELS #endif diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ */ #include "elf.h" +#include "../tcg-ldst.c.inc" #include "../tcg-pool.c.inc" int arm_arch = __ARM_ARCH; @@ -XXX,XX +XXX,XX @@ static void tcg_out_vldst(TCGContext *s, ARMInsn insn, } #ifdef CONFIG_SOFTMMU -#include "../tcg-ldst.c.inc" - /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, * int mmu_idx, uintptr_t ra) */ @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) tcg_out_goto(s, COND_AL, qemu_st_helpers[opc & MO_SIZE]); return true; } +#else + +static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addrlo, + TCGReg addrhi, unsigned a_bits) +{ + unsigned a_mask = (1 << a_bits) - 1; + TCGLabelQemuLdst *label = new_ldst_label(s); + + label->is_ld = is_ld; + label->addrlo_reg = addrlo; + label->addrhi_reg = addrhi; + + /* We are expecting a_bits to max out at 7, and can easily support 8. */ + tcg_debug_assert(a_mask <= 0xff); + /* tst addr, #mask */ + tcg_out_dat_imm(s, COND_AL, ARITH_TST, 0, addrlo, a_mask); + + /* blne slow_path */ + label->label_ptr[0] = s->code_ptr; + tcg_out_bl_imm(s, COND_NE, 0); + + label->raddr = tcg_splitwx_to_rx(s->code_ptr); +} + +static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) +{ + if (!reloc_pc24(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { + return false; + } + + if (TARGET_LONG_BITS == 64) { + /* 64-bit target address is aligned into R2:R3. */ + if (l->addrhi_reg != TCG_REG_R2) { + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R2, l->addrlo_reg); + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R3, l->addrhi_reg); + } else if (l->addrlo_reg != TCG_REG_R3) { + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R3, l->addrhi_reg); + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R2, l->addrlo_reg); + } else { + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R1, TCG_REG_R2); + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R2, TCG_REG_R3); + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R3, TCG_REG_R1); + } + } else { + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R1, l->addrlo_reg); + } + tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_AREG0); + + /* + * Tail call to the helper, with the return address back inline, + * just for the clarity of the debugging traceback -- the helper + * cannot return. We have used BLNE to arrive here, so LR is + * already set. + */ + tcg_out_goto(s, COND_AL, (const void *) + (l->is_ld ? helper_unaligned_ld : helper_unaligned_st)); + return true; +} + +static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + return tcg_out_fail_alignment(s, l); +} + +static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + return tcg_out_fail_alignment(s, l); +} #endif /* SOFTMMU */ static void tcg_out_qemu_ld_index(TCGContext *s, MemOp opc, @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64) int mem_index; TCGReg addend; tcg_insn_unit *label_ptr; +#else + unsigned a_bits; #endif datalo = *args++; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64) add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi, s->code_ptr, label_ptr); #else /* !CONFIG_SOFTMMU */ + a_bits = get_alignment_bits(opc); + if (a_bits) { + tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); + } if (guest_base) { tcg_out_qemu_ld_index(s, opc, datalo, datahi, addrlo, TCG_REG_GUEST_BASE, false); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) int mem_index; TCGReg addend; tcg_insn_unit *label_ptr; +#else + unsigned a_bits; #endif datalo = *args++; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi, s->code_ptr, label_ptr); #else /* !CONFIG_SOFTMMU */ + a_bits = get_alignment_bits(opc); + if (a_bits) { + tcg_out_test_alignment(s, false, addrlo, addrhi, a_bits); + } if (guest_base) { tcg_out_qemu_st_index(s, COND_AL, opc, datalo, datahi, addrlo, TCG_REG_GUEST_BASE, false); -- 2.25.1 This is kinda sorta the opposite of the other tcg hosts, where we get (normal) alignment checks for free with host SIGBUS and need to add code to support unaligned accesses. Fortunately, the ISA contains pairs of instructions that are used to implement unaligned memory accesses. Use them. Tested-by: Jiaxun Yang <jiaxun.yang@flygoat.com> Reviewed-by: Jiaxun Yang <jiaxun.yang@flygoat.com> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target.h | 2 - tcg/mips/tcg-target.c.inc | 334 +++++++++++++++++++++++++++++++++++++- 2 files changed, 328 insertions(+), 8 deletions(-) diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions; void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t) QEMU_ERROR("code path is reachable"); -#ifdef CONFIG_SOFTMMU #define TCG_TARGET_NEED_LDST_LABELS -#endif #endif diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ * THE SOFTWARE. */ +#include "../tcg-ldst.c.inc" + #ifdef HOST_WORDS_BIGENDIAN # define MIPS_BE 1 #else @@ -XXX,XX +XXX,XX @@ typedef enum { OPC_ORI = 015 << 26, OPC_XORI = 016 << 26, OPC_LUI = 017 << 26, + OPC_BNEL = 025 << 26, + OPC_BNEZALC_R6 = 030 << 26, OPC_DADDIU = 031 << 26, + OPC_LDL = 032 << 26, + OPC_LDR = 033 << 26, OPC_LB = 040 << 26, OPC_LH = 041 << 26, + OPC_LWL = 042 << 26, OPC_LW = 043 << 26, OPC_LBU = 044 << 26, OPC_LHU = 045 << 26, + OPC_LWR = 046 << 26, OPC_LWU = 047 << 26, OPC_SB = 050 << 26, OPC_SH = 051 << 26, + OPC_SWL = 052 << 26, OPC_SW = 053 << 26, + OPC_SDL = 054 << 26, + OPC_SDR = 055 << 26, + OPC_SWR = 056 << 26, OPC_LD = 067 << 26, OPC_SD = 077 << 26, @@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg) } #if defined(CONFIG_SOFTMMU) -#include "../tcg-ldst.c.inc" - static void * const qemu_ld_helpers[(MO_SSIZE | MO_BSWAP) + 1] = { [MO_UB] = helper_ret_ldub_mmu, [MO_SB] = helper_ret_ldsb_mmu, @@ -XXX,XX +XXX,XX @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0); return true; } -#endif + +#else + +static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addrlo, + TCGReg addrhi, unsigned a_bits) +{ + unsigned a_mask = (1 << a_bits) - 1; + TCGLabelQemuLdst *l = new_ldst_label(s); + + l->is_ld = is_ld; + l->addrlo_reg = addrlo; + l->addrhi_reg = addrhi; + + /* We are expecting a_bits to max out at 7, much lower than ANDI. */ + tcg_debug_assert(a_bits < 16); + tcg_out_opc_imm(s, OPC_ANDI, TCG_TMP0, addrlo, a_mask); + + l->label_ptr[0] = s->code_ptr; + if (use_mips32r6_instructions) { + tcg_out_opc_br(s, OPC_BNEZALC_R6, TCG_REG_ZERO, TCG_TMP0); + } else { + tcg_out_opc_br(s, OPC_BNEL, TCG_TMP0, TCG_REG_ZERO); + tcg_out_nop(s); + } + + l->raddr = tcg_splitwx_to_rx(s->code_ptr); +} + +static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) +{ + void *target; + + if (!reloc_pc16(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { + return false; + } + + if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { + /* A0 is env, A1 is skipped, A2:A3 is the uint64_t address. */ + TCGReg a2 = MIPS_BE ? l->addrhi_reg : l->addrlo_reg; + TCGReg a3 = MIPS_BE ? l->addrlo_reg : l->addrhi_reg; + + if (a3 != TCG_REG_A2) { + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_A2, a2); + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_A3, a3); + } else if (a2 != TCG_REG_A3) { + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_A3, a3); + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_A2, a2); + } else { + tcg_out_mov(s, TCG_TYPE_I32, TCG_TMP0, TCG_REG_A2); + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_A2, TCG_REG_A3); + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_A3, TCG_TMP0); + } + } else { + tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_A1, l->addrlo_reg); + } + tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_A0, TCG_AREG0); + + /* + * Tail call to the helper, with the return address back inline. + * We have arrived here via BNEL, so $31 is already set. + */ + target = (l->is_ld ? helper_unaligned_ld : helper_unaligned_st); + tcg_out_call_int(s, target, true); + return true; +} + +static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + return tcg_out_fail_alignment(s, l); +} + +static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + return tcg_out_fail_alignment(s, l); +} +#endif /* SOFTMMU */ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, TCGReg base, MemOp opc, bool is_64) @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, } } +static void __attribute__((unused)) +tcg_out_qemu_ld_unalign(TCGContext *s, TCGReg lo, TCGReg hi, + TCGReg base, MemOp opc, bool is_64) +{ + const MIPSInsn lw1 = MIPS_BE ? OPC_LWL : OPC_LWR; + const MIPSInsn lw2 = MIPS_BE ? OPC_LWR : OPC_LWL; + const MIPSInsn ld1 = MIPS_BE ? OPC_LDL : OPC_LDR; + const MIPSInsn ld2 = MIPS_BE ? OPC_LDR : OPC_LDL; + + bool sgn = (opc & MO_SIGN); + + switch (opc & (MO_SSIZE | MO_BSWAP)) { + case MO_SW | MO_BE: + case MO_UW | MO_BE: + tcg_out_opc_imm(s, sgn ? OPC_LB : OPC_LBU, TCG_TMP0, base, 0); + tcg_out_opc_imm(s, OPC_LBU, lo, base, 1); + if (use_mips32r2_instructions) { + tcg_out_opc_bf(s, OPC_INS, lo, TCG_TMP0, 31, 8); + } else { + tcg_out_opc_sa(s, OPC_SLL, TCG_TMP0, TCG_TMP0, 8); + tcg_out_opc_reg(s, OPC_OR, lo, TCG_TMP0, TCG_TMP1); + } + break; + + case MO_SW | MO_LE: + case MO_UW | MO_LE: + if (use_mips32r2_instructions && lo != base) { + tcg_out_opc_imm(s, OPC_LBU, lo, base, 0); + tcg_out_opc_imm(s, sgn ? OPC_LB : OPC_LBU, TCG_TMP0, base, 1); + tcg_out_opc_bf(s, OPC_INS, lo, TCG_TMP0, 31, 8); + } else { + tcg_out_opc_imm(s, OPC_LBU, TCG_TMP0, base, 0); + tcg_out_opc_imm(s, sgn ? OPC_LB : OPC_LBU, TCG_TMP1, base, 1); + tcg_out_opc_sa(s, OPC_SLL, TCG_TMP1, TCG_TMP1, 8); + tcg_out_opc_reg(s, OPC_OR, lo, TCG_TMP0, TCG_TMP1); + } + break; + + case MO_SL: + case MO_UL: + tcg_out_opc_imm(s, lw1, lo, base, 0); + tcg_out_opc_imm(s, lw2, lo, base, 3); + if (TCG_TARGET_REG_BITS == 64 && is_64 && !sgn) { + tcg_out_ext32u(s, lo, lo); + } + break; + + case MO_UL | MO_BSWAP: + case MO_SL | MO_BSWAP: + if (use_mips32r2_instructions) { + tcg_out_opc_imm(s, lw1, lo, base, 0); + tcg_out_opc_imm(s, lw2, lo, base, 3); + tcg_out_bswap32(s, lo, lo, + TCG_TARGET_REG_BITS == 64 && is_64 + ? (sgn ? TCG_BSWAP_OS : TCG_BSWAP_OZ) : 0); + } else { + const tcg_insn_unit *subr = + (TCG_TARGET_REG_BITS == 64 && is_64 && !sgn + ? bswap32u_addr : bswap32_addr); + + tcg_out_opc_imm(s, lw1, TCG_TMP0, base, 0); + tcg_out_bswap_subr(s, subr); + /* delay slot */ + tcg_out_opc_imm(s, lw2, TCG_TMP0, base, 3); + tcg_out_mov(s, is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32, lo, TCG_TMP3); + } + break; + + case MO_UQ: + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_opc_imm(s, ld1, lo, base, 0); + tcg_out_opc_imm(s, ld2, lo, base, 7); + } else { + tcg_out_opc_imm(s, lw1, MIPS_BE ? hi : lo, base, 0 + 0); + tcg_out_opc_imm(s, lw2, MIPS_BE ? hi : lo, base, 0 + 3); + tcg_out_opc_imm(s, lw1, MIPS_BE ? lo : hi, base, 4 + 0); + tcg_out_opc_imm(s, lw2, MIPS_BE ? lo : hi, base, 4 + 3); + } + break; + + case MO_UQ | MO_BSWAP: + if (TCG_TARGET_REG_BITS == 64) { + if (use_mips32r2_instructions) { + tcg_out_opc_imm(s, ld1, lo, base, 0); + tcg_out_opc_imm(s, ld2, lo, base, 7); + tcg_out_bswap64(s, lo, lo); + } else { + tcg_out_opc_imm(s, ld1, TCG_TMP0, base, 0); + tcg_out_bswap_subr(s, bswap64_addr); + /* delay slot */ + tcg_out_opc_imm(s, ld2, TCG_TMP0, base, 7); + tcg_out_mov(s, TCG_TYPE_I64, lo, TCG_TMP3); + } + } else if (use_mips32r2_instructions) { + tcg_out_opc_imm(s, lw1, TCG_TMP0, base, 0 + 0); + tcg_out_opc_imm(s, lw2, TCG_TMP0, base, 0 + 3); + tcg_out_opc_imm(s, lw1, TCG_TMP1, base, 4 + 0); + tcg_out_opc_imm(s, lw2, TCG_TMP1, base, 4 + 3); + tcg_out_opc_reg(s, OPC_WSBH, TCG_TMP0, 0, TCG_TMP0); + tcg_out_opc_reg(s, OPC_WSBH, TCG_TMP1, 0, TCG_TMP1); + tcg_out_opc_sa(s, OPC_ROTR, MIPS_BE ? lo : hi, TCG_TMP0, 16); + tcg_out_opc_sa(s, OPC_ROTR, MIPS_BE ? hi : lo, TCG_TMP1, 16); + } else { + tcg_out_opc_imm(s, lw1, TCG_TMP0, base, 0 + 0); + tcg_out_bswap_subr(s, bswap32_addr); + /* delay slot */ + tcg_out_opc_imm(s, lw2, TCG_TMP0, base, 0 + 3); + tcg_out_opc_imm(s, lw1, TCG_TMP0, base, 4 + 0); + tcg_out_mov(s, TCG_TYPE_I32, MIPS_BE ? lo : hi, TCG_TMP3); + tcg_out_bswap_subr(s, bswap32_addr); + /* delay slot */ + tcg_out_opc_imm(s, lw2, TCG_TMP0, base, 4 + 3); + tcg_out_mov(s, TCG_TYPE_I32, MIPS_BE ? hi : lo, TCG_TMP3); + } + break; + + default: + g_assert_not_reached(); + } +} + static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) { TCGReg addr_regl, addr_regh __attribute__((unused)); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) MemOp opc; #if defined(CONFIG_SOFTMMU) tcg_insn_unit *label_ptr[2]; +#else + unsigned a_bits, s_bits; #endif TCGReg base = TCG_REG_A0; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) } else { tcg_out_opc_reg(s, ALIAS_PADD, base, TCG_GUEST_BASE_REG, addr_regl); } - tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); + a_bits = get_alignment_bits(opc); + s_bits = opc & MO_SIZE; + /* + * R6 removes the left/right instructions but requires the + * system to support misaligned memory accesses. + */ + if (use_mips32r6_instructions) { + if (a_bits) { + tcg_out_test_alignment(s, true, addr_regl, addr_regh, a_bits); + } + tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); + } else { + if (a_bits && a_bits != s_bits) { + tcg_out_test_alignment(s, true, addr_regl, addr_regh, a_bits); + } + if (a_bits >= s_bits) { + tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); + } else { + tcg_out_qemu_ld_unalign(s, data_regl, data_regh, base, opc, is_64); + } + } #endif } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi, } } +static void __attribute__((unused)) +tcg_out_qemu_st_unalign(TCGContext *s, TCGReg lo, TCGReg hi, + TCGReg base, MemOp opc) +{ + const MIPSInsn sw1 = MIPS_BE ? OPC_SWL : OPC_SWR; + const MIPSInsn sw2 = MIPS_BE ? OPC_SWR : OPC_SWL; + const MIPSInsn sd1 = MIPS_BE ? OPC_SDL : OPC_SDR; + const MIPSInsn sd2 = MIPS_BE ? OPC_SDR : OPC_SDL; + + /* Don't clutter the code below with checks to avoid bswapping ZERO. */ + if ((lo | hi) == 0) { + opc &= ~MO_BSWAP; + } + + switch (opc & (MO_SIZE | MO_BSWAP)) { + case MO_16 | MO_BE: + tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, lo, 8); + tcg_out_opc_imm(s, OPC_SB, TCG_TMP0, base, 0); + tcg_out_opc_imm(s, OPC_SB, lo, base, 1); + break; + + case MO_16 | MO_LE: + tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, lo, 8); + tcg_out_opc_imm(s, OPC_SB, lo, base, 0); + tcg_out_opc_imm(s, OPC_SB, TCG_TMP0, base, 1); + break; + + case MO_32 | MO_BSWAP: + tcg_out_bswap32(s, TCG_TMP3, lo, 0); + lo = TCG_TMP3; + /* fall through */ + case MO_32: + tcg_out_opc_imm(s, sw1, lo, base, 0); + tcg_out_opc_imm(s, sw2, lo, base, 3); + break; + + case MO_64 | MO_BSWAP: + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_bswap64(s, TCG_TMP3, lo); + lo = TCG_TMP3; + } else if (use_mips32r2_instructions) { + tcg_out_opc_reg(s, OPC_WSBH, TCG_TMP0, 0, MIPS_BE ? hi : lo); + tcg_out_opc_reg(s, OPC_WSBH, TCG_TMP1, 0, MIPS_BE ? lo : hi); + tcg_out_opc_sa(s, OPC_ROTR, TCG_TMP0, TCG_TMP0, 16); + tcg_out_opc_sa(s, OPC_ROTR, TCG_TMP1, TCG_TMP1, 16); + hi = MIPS_BE ? TCG_TMP0 : TCG_TMP1; + lo = MIPS_BE ? TCG_TMP1 : TCG_TMP0; + } else { + tcg_out_bswap32(s, TCG_TMP3, MIPS_BE ? lo : hi, 0); + tcg_out_opc_imm(s, sw1, TCG_TMP3, base, 0 + 0); + tcg_out_opc_imm(s, sw2, TCG_TMP3, base, 0 + 3); + tcg_out_bswap32(s, TCG_TMP3, MIPS_BE ? hi : lo, 0); + tcg_out_opc_imm(s, sw1, TCG_TMP3, base, 4 + 0); + tcg_out_opc_imm(s, sw2, TCG_TMP3, base, 4 + 3); + break; + } + /* fall through */ + case MO_64: + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_opc_imm(s, sd1, lo, base, 0); + tcg_out_opc_imm(s, sd2, lo, base, 7); + } else { + tcg_out_opc_imm(s, sw1, MIPS_BE ? hi : lo, base, 0 + 0); + tcg_out_opc_imm(s, sw2, MIPS_BE ? hi : lo, base, 0 + 3); + tcg_out_opc_imm(s, sw1, MIPS_BE ? lo : hi, base, 4 + 0); + tcg_out_opc_imm(s, sw2, MIPS_BE ? lo : hi, base, 4 + 3); + } + break; + + default: + tcg_abort(); + } +} static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) { TCGReg addr_regl, addr_regh __attribute__((unused)); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) MemOp opc; #if defined(CONFIG_SOFTMMU) tcg_insn_unit *label_ptr[2]; +#else + unsigned a_bits, s_bits; #endif TCGReg base = TCG_REG_A0; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) data_regl, data_regh, addr_regl, addr_regh, s->code_ptr, label_ptr); #else - base = TCG_REG_A0; if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { tcg_out_ext32u(s, base, addr_regl); addr_regl = base; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) } else { tcg_out_opc_reg(s, ALIAS_PADD, base, TCG_GUEST_BASE_REG, addr_regl); } - tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); + a_bits = get_alignment_bits(opc); + s_bits = opc & MO_SIZE; + /* + * R6 removes the left/right instructions but requires the + * system to support misaligned memory accesses. + */ + if (use_mips32r6_instructions) { + if (a_bits) { + tcg_out_test_alignment(s, true, addr_regl, addr_regh, a_bits); + } + tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); + } else { + if (a_bits && a_bits != s_bits) { + tcg_out_test_alignment(s, true, addr_regl, addr_regh, a_bits); + } + if (a_bits >= s_bits) { + tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); + } else { + tcg_out_qemu_st_unalign(s, data_regl, data_regh, base, opc); + } + } #endif } -- 2.25.1 We can use the routines just added for user-only to emit unaligned accesses in softmmu mode too. Tested-by: Jiaxun Yang <jiaxun.yang@flygoat.com> Reviewed-by: Jiaxun Yang <jiaxun.yang@flygoat.com> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/mips/tcg-target.c.inc | 91 ++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 40 deletions(-) diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_tlb_load(TCGContext *s, TCGReg base, TCGReg addrl, tcg_insn_unit *label_ptr[2], bool is_load) { MemOp opc = get_memop(oi); - unsigned s_bits = opc & MO_SIZE; unsigned a_bits = get_alignment_bits(opc); + unsigned s_bits = opc & MO_SIZE; + unsigned a_mask = (1 << a_bits) - 1; + unsigned s_mask = (1 << s_bits) - 1; int mem_index = get_mmuidx(oi); int fast_off = TLB_MASK_TABLE_OFS(mem_index); int mask_off = fast_off + offsetof(CPUTLBDescFast, mask); @@ -XXX,XX +XXX,XX @@ static void tcg_out_tlb_load(TCGContext *s, TCGReg base, TCGReg addrl, int add_off = offsetof(CPUTLBEntry, addend); int cmp_off = (is_load ? offsetof(CPUTLBEntry, addr_read) : offsetof(CPUTLBEntry, addr_write)); - target_ulong mask; + target_ulong tlb_mask; /* Load tlb_mask[mmu_idx] and tlb_table[mmu_idx]. */ tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_AREG0, mask_off); @@ -XXX,XX +XXX,XX @@ static void tcg_out_tlb_load(TCGContext *s, TCGReg base, TCGReg addrl, /* Add the tlb_table pointer, creating the CPUTLBEntry address in TMP3. */ tcg_out_opc_reg(s, ALIAS_PADD, TCG_TMP3, TCG_TMP3, TCG_TMP1); - /* We don't currently support unaligned accesses. - We could do so with mips32r6. */ - if (a_bits < s_bits) { - a_bits = s_bits; - } - - /* Mask the page bits, keeping the alignment bits to compare against. */ - mask = (target_ulong)TARGET_PAGE_MASK | ((1 << a_bits) - 1); - /* Load the (low-half) tlb comparator. */ if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { - tcg_out_ld(s, TCG_TYPE_I32, TCG_TMP0, TCG_TMP3, cmp_off + LO_OFF); - tcg_out_movi(s, TCG_TYPE_I32, TCG_TMP1, mask); + tcg_out_ldst(s, OPC_LW, TCG_TMP0, TCG_TMP3, cmp_off + LO_OFF); } else { tcg_out_ldst(s, (TARGET_LONG_BITS == 64 ? OPC_LD : TCG_TARGET_REG_BITS == 64 ? OPC_LWU : OPC_LW), TCG_TMP0, TCG_TMP3, cmp_off); - tcg_out_movi(s, TCG_TYPE_TL, TCG_TMP1, mask); - /* No second compare is required here; - load the tlb addend for the fast path. */ - tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP2, TCG_TMP3, add_off); } /* Zero extend a 32-bit guest address for a 64-bit host. */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_tlb_load(TCGContext *s, TCGReg base, TCGReg addrl, tcg_out_ext32u(s, base, addrl); addrl = base; } - tcg_out_opc_reg(s, OPC_AND, TCG_TMP1, TCG_TMP1, addrl); + + /* + * Mask the page bits, keeping the alignment bits to compare against. + * For unaligned accesses, compare against the end of the access to + * verify that it does not cross a page boundary. + */ + tlb_mask = (target_ulong)TARGET_PAGE_MASK | a_mask; + tcg_out_movi(s, TCG_TYPE_I32, TCG_TMP1, tlb_mask); + if (a_mask >= s_mask) { + tcg_out_opc_reg(s, OPC_AND, TCG_TMP1, TCG_TMP1, addrl); + } else { + tcg_out_opc_imm(s, ALIAS_PADDI, TCG_TMP2, addrl, s_mask - a_mask); + tcg_out_opc_reg(s, OPC_AND, TCG_TMP1, TCG_TMP1, TCG_TMP2); + } + + if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { + /* Load the tlb addend for the fast path. */ + tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP2, TCG_TMP3, add_off); + } label_ptr[0] = s->code_ptr; tcg_out_opc_br(s, OPC_BNE, TCG_TMP1, TCG_TMP0); @@ -XXX,XX +XXX,XX @@ static void tcg_out_tlb_load(TCGContext *s, TCGReg base, TCGReg addrl, /* Load and test the high half tlb comparator. */ if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { /* delay slot */ - tcg_out_ld(s, TCG_TYPE_I32, TCG_TMP0, TCG_TMP3, cmp_off + HI_OFF); + tcg_out_ldst(s, OPC_LW, TCG_TMP0, TCG_TMP3, cmp_off + HI_OFF); /* Load the tlb addend for the fast path. */ tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP2, TCG_TMP3, add_off); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, } } -static void __attribute__((unused)) -tcg_out_qemu_ld_unalign(TCGContext *s, TCGReg lo, TCGReg hi, +static void tcg_out_qemu_ld_unalign(TCGContext *s, TCGReg lo, TCGReg hi, TCGReg base, MemOp opc, bool is_64) { const MIPSInsn lw1 = MIPS_BE ? OPC_LWL : OPC_LWR; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) #if defined(CONFIG_SOFTMMU) tcg_insn_unit *label_ptr[2]; #else - unsigned a_bits, s_bits; #endif + unsigned a_bits, s_bits; TCGReg base = TCG_REG_A0; data_regl = *args++; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); oi = *args++; opc = get_memop(oi); + a_bits = get_alignment_bits(opc); + s_bits = opc & MO_SIZE; + /* + * R6 removes the left/right instructions but requires the + * system to support misaligned memory accesses. + */ #if defined(CONFIG_SOFTMMU) tcg_out_tlb_load(s, base, addr_regl, addr_regh, oi, label_ptr, 1); - tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); + if (use_mips32r6_instructions || a_bits >= s_bits) { + tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); + } else { + tcg_out_qemu_ld_unalign(s, data_regl, data_regh, base, opc, is_64); + } add_qemu_ldst_label(s, 1, oi, (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), data_regl, data_regh, addr_regl, addr_regh, @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) } else { tcg_out_opc_reg(s, ALIAS_PADD, base, TCG_GUEST_BASE_REG, addr_regl); } - a_bits = get_alignment_bits(opc); - s_bits = opc & MO_SIZE; - /* - * R6 removes the left/right instructions but requires the - * system to support misaligned memory accesses. - */ if (use_mips32r6_instructions) { if (a_bits) { tcg_out_test_alignment(s, true, addr_regl, addr_regh, a_bits); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi, } } -static void __attribute__((unused)) -tcg_out_qemu_st_unalign(TCGContext *s, TCGReg lo, TCGReg hi, +static void tcg_out_qemu_st_unalign(TCGContext *s, TCGReg lo, TCGReg hi, TCGReg base, MemOp opc) { const MIPSInsn sw1 = MIPS_BE ? OPC_SWL : OPC_SWR; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) MemOp opc; #if defined(CONFIG_SOFTMMU) tcg_insn_unit *label_ptr[2]; -#else - unsigned a_bits, s_bits; #endif + unsigned a_bits, s_bits; TCGReg base = TCG_REG_A0; data_regl = *args++; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); oi = *args++; opc = get_memop(oi); + a_bits = get_alignment_bits(opc); + s_bits = opc & MO_SIZE; + /* + * R6 removes the left/right instructions but requires the + * system to support misaligned memory accesses. + */ #if defined(CONFIG_SOFTMMU) tcg_out_tlb_load(s, base, addr_regl, addr_regh, oi, label_ptr, 0); - tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); + if (use_mips32r6_instructions || a_bits >= s_bits) { + tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); + } else { + tcg_out_qemu_st_unalign(s, data_regl, data_regh, base, opc); + } add_qemu_ldst_label(s, 0, oi, (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), data_regl, data_regh, addr_regl, addr_regh, @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) } else { tcg_out_opc_reg(s, ALIAS_PADD, base, TCG_GUEST_BASE_REG, addr_regl); } - a_bits = get_alignment_bits(opc); - s_bits = opc & MO_SIZE; - /* - * R6 removes the left/right instructions but requires the - * system to support misaligned memory accesses. - */ if (use_mips32r6_instructions) { if (a_bits) { tcg_out_test_alignment(s, true, addr_regl, addr_regh, a_bits); -- 2.25.1 When BH is constant, it is constrained to 11 bits for use in MOVCC. For the cases in which we must load the constant BH into a register, we do not need the full logic of tcg_out_movi; we can use the simpler function for emitting a 13 bit constant. This eliminates the only case in which TCG_REG_T2 was passed to tcg_out_movi, which will shortly become invalid. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc/tcg-target.c.inc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_addsub2_i64(TCGContext *s, TCGReg rl, TCGReg rh, if (use_vis3_instructions && !is_sub) { /* Note that ADDXC doesn't accept immediates. */ if (bhconst && bh != 0) { - tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_T2, bh); + tcg_out_movi_imm13(s, TCG_REG_T2, bh); bh = TCG_REG_T2; } tcg_out_arith(s, rh, ah, bh, ARITH_ADDXC); @@ -XXX,XX +XXX,XX @@ static void tcg_out_addsub2_i64(TCGContext *s, TCGReg rl, TCGReg rh, tcg_out_movcc(s, TCG_COND_GEU, MOVCC_XCC, rh, ah, 0); } } else { - /* Otherwise adjust BH as if there is carry into T2 ... */ + /* + * Otherwise adjust BH as if there is carry into T2. + * Note that constant BH is constrained to 11 bits for the MOVCC, + * so the adjustment fits 12 bits. + */ if (bhconst) { - tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_T2, bh + (is_sub ? -1 : 1)); + tcg_out_movi_imm13(s, TCG_REG_T2, bh + (is_sub ? -1 : 1)); } else { tcg_out_arithi(s, TCG_REG_T2, bh, 1, is_sub ? ARITH_SUB : ARITH_ADD); -- 2.25.1 Handle 32-bit constants with a separate function, so that tcg_out_movi_int does not need to recurse. This slightly rearranges the order of tests for small constants, but produces the same output. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc/tcg-target.c.inc | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_imm13(TCGContext *s, TCGReg ret, int32_t arg) tcg_out_arithi(s, ret, TCG_REG_G0, arg, ARITH_OR); } +static void tcg_out_movi_imm32(TCGContext *s, TCGReg ret, int32_t arg) +{ + if (check_fit_i32(arg, 13)) { + /* A 13-bit constant sign-extended to 64-bits. */ + tcg_out_movi_imm13(s, ret, arg); + } else { + /* A 32-bit constant zero-extended to 64 bits. */ + tcg_out_sethi(s, ret, arg); + if (arg & 0x3ff) { + tcg_out_arithi(s, ret, ret, arg & 0x3ff, ARITH_OR); + } + } +} + static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, tcg_target_long arg, bool in_prologue) { tcg_target_long hi, lo = (int32_t)arg; tcg_target_long test, lsb; - /* Make sure we test 32-bit constants for imm13 properly. */ - if (type == TCG_TYPE_I32) { - arg = lo; + /* A 32-bit constant, or 32-bit zero-extended to 64-bits. */ + if (type == TCG_TYPE_I32 || arg == (uint32_t)arg) { + tcg_out_movi_imm32(s, ret, arg); + return; } /* A 13-bit constant sign-extended to 64-bits. */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, } } - /* A 32-bit constant, or 32-bit zero-extended to 64-bits. */ - if (type == TCG_TYPE_I32 || arg == (uint32_t)arg) { - tcg_out_sethi(s, ret, arg); - if (arg & 0x3ff) { - tcg_out_arithi(s, ret, ret, arg & 0x3ff, ARITH_OR); - } - return; - } - /* A 32-bit constant sign-extended to 64-bits. */ if (arg == lo) { tcg_out_sethi(s, ret, ~arg); @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, /* A 64-bit constant decomposed into 2 32-bit pieces. */ if (check_fit_i32(lo, 13)) { hi = (arg - lo) >> 32; - tcg_out_movi(s, TCG_TYPE_I32, ret, hi); + tcg_out_movi_imm32(s, ret, hi); tcg_out_arithi(s, ret, ret, 32, SHIFT_SLLX); tcg_out_arithi(s, ret, ret, lo, ARITH_ADD); } else { hi = arg >> 32; - tcg_out_movi(s, TCG_TYPE_I32, ret, hi); - tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_T2, lo); + tcg_out_movi_imm32(s, ret, hi); + tcg_out_movi_imm32(s, TCG_REG_T2, lo); tcg_out_arithi(s, ret, ret, 32, SHIFT_SLLX); tcg_out_arith(s, ret, ret, TCG_REG_T2, ARITH_OR); } -- 2.25.1 This will allow us to control exactly what scratch register is used for loading the constant. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc/tcg-target.c.inc | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_imm32(TCGContext *s, TCGReg ret, int32_t arg) } static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, - tcg_target_long arg, bool in_prologue) + tcg_target_long arg, bool in_prologue, + TCGReg scratch) { tcg_target_long hi, lo = (int32_t)arg; tcg_target_long test, lsb; @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, } else { hi = arg >> 32; tcg_out_movi_imm32(s, ret, hi); - tcg_out_movi_imm32(s, TCG_REG_T2, lo); + tcg_out_movi_imm32(s, scratch, lo); tcg_out_arithi(s, ret, ret, 32, SHIFT_SLLX); - tcg_out_arith(s, ret, ret, TCG_REG_T2, ARITH_OR); + tcg_out_arith(s, ret, ret, scratch, ARITH_OR); } } static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg ret, tcg_target_long arg) { - tcg_out_movi_int(s, type, ret, arg, false); + tcg_debug_assert(ret != TCG_REG_T2); + tcg_out_movi_int(s, type, ret, arg, false, TCG_REG_T2); } static void tcg_out_ldst_rr(TCGContext *s, TCGReg data, TCGReg a1, @@ -XXX,XX +XXX,XX @@ static void tcg_out_call_nodelay(TCGContext *s, const tcg_insn_unit *dest, } else { uintptr_t desti = (uintptr_t)dest; tcg_out_movi_int(s, TCG_TYPE_PTR, TCG_REG_T1, - desti & ~0xfff, in_prologue); + desti & ~0xfff, in_prologue, TCG_REG_O7); tcg_out_arithi(s, TCG_REG_O7, TCG_REG_T1, desti & 0xfff, JMPL); } } @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) #ifndef CONFIG_SOFTMMU if (guest_base != 0) { - tcg_out_movi_int(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base, true); + tcg_out_movi_int(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, + guest_base, true, TCG_REG_T1); tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG); } #endif -- 2.25.1 We had code for checking for 13 and 21-bit shifted constants, but we can do better and allow 32-bit shifted constants. This is still 2 insns shorter than the full 64-bit sequence. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc/tcg-target.c.inc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, return; } - /* A 21-bit constant, shifted. */ + /* A 32-bit constant, shifted. */ lsb = ctz64(arg); test = (tcg_target_long)arg >> lsb; - if (check_fit_tl(test, 13)) { - tcg_out_movi_imm13(s, ret, test); - tcg_out_arithi(s, ret, ret, lsb, SHIFT_SLLX); - return; - } else if (lsb > 10 && test == extract64(test, 0, 21)) { + if (lsb > 10 && test == extract64(test, 0, 21)) { tcg_out_sethi(s, ret, test << 10); tcg_out_arithi(s, ret, ret, lsb - 10, SHIFT_SLLX); return; + } else if (test == (uint32_t)test || test == (int32_t)test) { + tcg_out_movi_int(s, TCG_TYPE_I64, ret, test, in_prologue, scratch); + tcg_out_arithi(s, ret, ret, lsb, SHIFT_SLLX); + return; } /* A 64-bit constant decomposed into 2 32-bit pieces. */ -- 2.25.1 Since 7ecd02a06f8, if patch_reloc fails we restart translation with a smaller TB. SPARC had its function signature changed, but not the logic. Replace assert with return false. Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc/tcg-target.c.inc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool patch_reloc(tcg_insn_unit *src_rw, int type, switch (type) { case R_SPARC_WDISP16: - assert(check_fit_ptr(pcrel >> 2, 16)); + if (!check_fit_ptr(pcrel >> 2, 16)) { + return false; + } insn &= ~INSN_OFF16(-1); insn |= INSN_OFF16(pcrel); break; case R_SPARC_WDISP19: - assert(check_fit_ptr(pcrel >> 2, 19)); + if (!check_fit_ptr(pcrel >> 2, 19)) { + return false; + } insn &= ~INSN_OFF19(-1); insn |= INSN_OFF19(pcrel); break; -- 2.25.1 Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc/tcg-target.c.inc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static bool patch_reloc(tcg_insn_unit *src_rw, int type, insn &= ~INSN_OFF19(-1); insn |= INSN_OFF19(pcrel); break; + case R_SPARC_13: + if (!check_fit_ptr(value, 13)) { + return false; + } + insn &= ~INSN_IMM13(-1); + insn |= INSN_IMM13(value); + break; default: g_assert_not_reached(); } @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, return; } + /* Use the constant pool, if possible. */ + if (!in_prologue && USE_REG_TB) { + new_pool_label(s, arg, R_SPARC_13, s->code_ptr, + tcg_tbrel_diff(s, NULL)); + tcg_out32(s, LDX | INSN_RD(ret) | INSN_RS1(TCG_REG_TB)); + return; + } + /* A 64-bit constant decomposed into 2 32-bit pieces. */ if (check_fit_i32(lo, 13)) { hi = (arg - lo) >> 32; -- 2.25.1 Due to mapping changes, we now rarely place the code_gen_buffer near the main executable. Which means that direct calls will now rarely be in range. So, always use indirect calls for tail calls, which allows us to avoid clobbering %o7, and therefore we need not save and restore it. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc/tcg-target.c.inc | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static void tcg_out_addsub2_i64(TCGContext *s, TCGReg rl, TCGReg rh, tcg_out_mov(s, TCG_TYPE_I64, rl, tmp); } +static void tcg_out_jmpl_const(TCGContext *s, const tcg_insn_unit *dest, + bool in_prologue, bool tail_call) +{ + uintptr_t desti = (uintptr_t)dest; + + /* Be careful not to clobber %o7 for a tail call. */ + tcg_out_movi_int(s, TCG_TYPE_PTR, TCG_REG_T1, + desti & ~0xfff, in_prologue, + tail_call ? TCG_REG_G2 : TCG_REG_O7); + tcg_out_arithi(s, tail_call ? TCG_REG_G0 : TCG_REG_O7, + TCG_REG_T1, desti & 0xfff, JMPL); +} + static void tcg_out_call_nodelay(TCGContext *s, const tcg_insn_unit *dest, bool in_prologue) { @@ -XXX,XX +XXX,XX @@ static void tcg_out_call_nodelay(TCGContext *s, const tcg_insn_unit *dest, if (disp == (int32_t)disp) { tcg_out32(s, CALL | (uint32_t)disp >> 2); } else { - uintptr_t desti = (uintptr_t)dest; - tcg_out_movi_int(s, TCG_TYPE_PTR, TCG_REG_T1, - desti & ~0xfff, in_prologue, TCG_REG_O7); - tcg_out_arithi(s, TCG_REG_O7, TCG_REG_T1, desti & 0xfff, JMPL); + tcg_out_jmpl_const(s, dest, in_prologue, false); } } @@ -XXX,XX +XXX,XX @@ static void build_trampolines(TCGContext *s) /* Set the retaddr operand. */ tcg_out_mov(s, TCG_TYPE_PTR, ra, TCG_REG_O7); - /* Set the env operand. */ - tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O0, TCG_AREG0); /* Tail call. */ - tcg_out_call_nodelay(s, qemu_ld_helpers[i], true); - tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O7, ra); + tcg_out_jmpl_const(s, qemu_ld_helpers[i], true, true); + /* delay slot -- set the env argument */ + tcg_out_mov_delay(s, TCG_REG_O0, TCG_AREG0); } for (i = 0; i < ARRAY_SIZE(qemu_st_helpers); ++i) { @@ -XXX,XX +XXX,XX @@ static void build_trampolines(TCGContext *s) if (ra >= TCG_REG_O6) { tcg_out_st(s, TCG_TYPE_PTR, TCG_REG_O7, TCG_REG_CALL_STACK, TCG_TARGET_CALL_STACK_OFFSET); - ra = TCG_REG_G1; + } else { + tcg_out_mov(s, TCG_TYPE_PTR, ra, TCG_REG_O7); } - tcg_out_mov(s, TCG_TYPE_PTR, ra, TCG_REG_O7); - /* Set the env operand. */ - tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O0, TCG_AREG0); + /* Tail call. */ - tcg_out_call_nodelay(s, qemu_st_helpers[i], true); - tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O7, ra); + tcg_out_jmpl_const(s, qemu_st_helpers[i], true, true); + /* delay slot -- set the env argument */ + tcg_out_mov_delay(s, TCG_REG_O0, TCG_AREG0); } } #endif -- 2.25.1 This is kinda sorta the opposite of the other tcg hosts, where we get (normal) alignment checks for free with host SIGBUS and need to add code to support unaligned accesses. This inline code expansion is somewhat large, but it takes quite a few instructions to make a function call to a helper anyway. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc/tcg-target.c.inc | 219 +++++++++++++++++++++++++++++++++++-- 1 file changed, 211 insertions(+), 8 deletions(-) diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const int tcg_target_call_oarg_regs[] = { #define ARITH_ADD (INSN_OP(2) | INSN_OP3(0x00)) #define ARITH_ADDCC (INSN_OP(2) | INSN_OP3(0x10)) #define ARITH_AND (INSN_OP(2) | INSN_OP3(0x01)) +#define ARITH_ANDCC (INSN_OP(2) | INSN_OP3(0x11)) #define ARITH_ANDN (INSN_OP(2) | INSN_OP3(0x05)) #define ARITH_OR (INSN_OP(2) | INSN_OP3(0x02)) #define ARITH_ORCC (INSN_OP(2) | INSN_OP3(0x12)) @@ -XXX,XX +XXX,XX @@ static void build_trampolines(TCGContext *s) tcg_out_mov_delay(s, TCG_REG_O0, TCG_AREG0); } } +#else +static const tcg_insn_unit *qemu_unalign_ld_trampoline; +static const tcg_insn_unit *qemu_unalign_st_trampoline; + +static void build_trampolines(TCGContext *s) +{ + for (int ld = 0; ld < 2; ++ld) { + void *helper; + + while ((uintptr_t)s->code_ptr & 15) { + tcg_out_nop(s); + } + + if (ld) { + helper = helper_unaligned_ld; + qemu_unalign_ld_trampoline = tcg_splitwx_to_rx(s->code_ptr); + } else { + helper = helper_unaligned_st; + qemu_unalign_st_trampoline = tcg_splitwx_to_rx(s->code_ptr); + } + + if (!SPARC64 && TARGET_LONG_BITS == 64) { + /* Install the high part of the address. */ + tcg_out_arithi(s, TCG_REG_O1, TCG_REG_O2, 32, SHIFT_SRLX); + } + + /* Tail call. */ + tcg_out_jmpl_const(s, helper, true, true); + /* delay slot -- set the env argument */ + tcg_out_mov_delay(s, TCG_REG_O0, TCG_AREG0); + } +} #endif /* Generate global QEMU prologue and epilogue code */ @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) /* delay slot */ tcg_out_movi_imm13(s, TCG_REG_O0, 0); -#ifdef CONFIG_SOFTMMU build_trampolines(s); -#endif } static void tcg_out_nop_fill(tcg_insn_unit *p, int count) @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addr, int mem_index, static const int qemu_ld_opc[(MO_SSIZE | MO_BSWAP) + 1] = { [MO_UB] = LDUB, [MO_SB] = LDSB, + [MO_UB | MO_LE] = LDUB, + [MO_SB | MO_LE] = LDSB, [MO_BEUW] = LDUH, [MO_BESW] = LDSH, [MO_BEUL] = LDUW, [MO_BESL] = LDSW, [MO_BEUQ] = LDX, + [MO_BESQ] = LDX, [MO_LEUW] = LDUH_LE, [MO_LESW] = LDSH_LE, [MO_LEUL] = LDUW_LE, [MO_LESL] = LDSW_LE, [MO_LEUQ] = LDX_LE, + [MO_LESQ] = LDX_LE, }; static const int qemu_st_opc[(MO_SIZE | MO_BSWAP) + 1] = { @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, MemOpIdx oi, bool is_64) { MemOp memop = get_memop(oi); + tcg_insn_unit *label_ptr; + #ifdef CONFIG_SOFTMMU unsigned memi = get_mmuidx(oi); TCGReg addrz, param; const tcg_insn_unit *func; - tcg_insn_unit *label_ptr; addrz = tcg_out_tlb_load(s, addr, memi, memop, offsetof(CPUTLBEntry, addr_read)); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, *label_ptr |= INSN_OFF19(tcg_ptr_byte_diff(s->code_ptr, label_ptr)); #else + TCGReg index = (guest_base ? TCG_GUEST_BASE_REG : TCG_REG_G0); + unsigned a_bits = get_alignment_bits(memop); + unsigned s_bits = memop & MO_SIZE; + unsigned t_bits; + if (SPARC64 && TARGET_LONG_BITS == 32) { tcg_out_arithi(s, TCG_REG_T1, addr, 0, SHIFT_SRL); addr = TCG_REG_T1; } - tcg_out_ldst_rr(s, data, addr, - (guest_base ? TCG_GUEST_BASE_REG : TCG_REG_G0), + + /* + * Normal case: alignment equal to access size. + */ + if (a_bits == s_bits) { + tcg_out_ldst_rr(s, data, addr, index, + qemu_ld_opc[memop & (MO_BSWAP | MO_SSIZE)]); + return; + } + + /* + * Test for at least natural alignment, and assume most accesses + * will be aligned -- perform a straight load in the delay slot. + * This is required to preserve atomicity for aligned accesses. + */ + t_bits = MAX(a_bits, s_bits); + tcg_debug_assert(t_bits < 13); + tcg_out_arithi(s, TCG_REG_G0, addr, (1u << t_bits) - 1, ARITH_ANDCC); + + /* beq,a,pt %icc, label */ + label_ptr = s->code_ptr; + tcg_out_bpcc0(s, COND_E, BPCC_A | BPCC_PT | BPCC_ICC, 0); + /* delay slot */ + tcg_out_ldst_rr(s, data, addr, index, qemu_ld_opc[memop & (MO_BSWAP | MO_SSIZE)]); + + if (a_bits >= s_bits) { + /* + * Overalignment: A successful alignment test will perform the memory + * operation in the delay slot, and failure need only invoke the + * handler for SIGBUS. + */ + TCGReg arg_low = TCG_REG_O1 + (!SPARC64 && TARGET_LONG_BITS == 64); + tcg_out_call_nodelay(s, qemu_unalign_ld_trampoline, false); + /* delay slot -- move to low part of argument reg */ + tcg_out_mov_delay(s, arg_low, addr); + } else { + /* Underalignment: load by pieces of minimum alignment. */ + int ld_opc, a_size, s_size, i; + + /* + * Force full address into T1 early; avoids problems with + * overlap between @addr and @data. + */ + tcg_out_arith(s, TCG_REG_T1, addr, index, ARITH_ADD); + + a_size = 1 << a_bits; + s_size = 1 << s_bits; + if ((memop & MO_BSWAP) == MO_BE) { + ld_opc = qemu_ld_opc[a_bits | MO_BE | (memop & MO_SIGN)]; + tcg_out_ldst(s, data, TCG_REG_T1, 0, ld_opc); + ld_opc = qemu_ld_opc[a_bits | MO_BE]; + for (i = a_size; i < s_size; i += a_size) { + tcg_out_ldst(s, TCG_REG_T2, TCG_REG_T1, i, ld_opc); + tcg_out_arithi(s, data, data, a_size, SHIFT_SLLX); + tcg_out_arith(s, data, data, TCG_REG_T2, ARITH_OR); + } + } else if (a_bits == 0) { + ld_opc = LDUB; + tcg_out_ldst(s, data, TCG_REG_T1, 0, ld_opc); + for (i = a_size; i < s_size; i += a_size) { + if ((memop & MO_SIGN) && i == s_size - a_size) { + ld_opc = LDSB; + } + tcg_out_ldst(s, TCG_REG_T2, TCG_REG_T1, i, ld_opc); + tcg_out_arithi(s, TCG_REG_T2, TCG_REG_T2, i * 8, SHIFT_SLLX); + tcg_out_arith(s, data, data, TCG_REG_T2, ARITH_OR); + } + } else { + ld_opc = qemu_ld_opc[a_bits | MO_LE]; + tcg_out_ldst_rr(s, data, TCG_REG_T1, TCG_REG_G0, ld_opc); + for (i = a_size; i < s_size; i += a_size) { + tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, a_size, ARITH_ADD); + if ((memop & MO_SIGN) && i == s_size - a_size) { + ld_opc = qemu_ld_opc[a_bits | MO_LE | MO_SIGN]; + } + tcg_out_ldst_rr(s, TCG_REG_T2, TCG_REG_T1, TCG_REG_G0, ld_opc); + tcg_out_arithi(s, TCG_REG_T2, TCG_REG_T2, i * 8, SHIFT_SLLX); + tcg_out_arith(s, data, data, TCG_REG_T2, ARITH_OR); + } + } + } + + *label_ptr |= INSN_OFF19(tcg_ptr_byte_diff(s->code_ptr, label_ptr)); #endif /* CONFIG_SOFTMMU */ } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr, MemOpIdx oi) { MemOp memop = get_memop(oi); + tcg_insn_unit *label_ptr; + #ifdef CONFIG_SOFTMMU unsigned memi = get_mmuidx(oi); TCGReg addrz, param; const tcg_insn_unit *func; - tcg_insn_unit *label_ptr; addrz = tcg_out_tlb_load(s, addr, memi, memop, offsetof(CPUTLBEntry, addr_write)); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr, *label_ptr |= INSN_OFF19(tcg_ptr_byte_diff(s->code_ptr, label_ptr)); #else + TCGReg index = (guest_base ? TCG_GUEST_BASE_REG : TCG_REG_G0); + unsigned a_bits = get_alignment_bits(memop); + unsigned s_bits = memop & MO_SIZE; + unsigned t_bits; + if (SPARC64 && TARGET_LONG_BITS == 32) { tcg_out_arithi(s, TCG_REG_T1, addr, 0, SHIFT_SRL); addr = TCG_REG_T1; } - tcg_out_ldst_rr(s, data, addr, - (guest_base ? TCG_GUEST_BASE_REG : TCG_REG_G0), + + /* + * Normal case: alignment equal to access size. + */ + if (a_bits == s_bits) { + tcg_out_ldst_rr(s, data, addr, index, + qemu_st_opc[memop & (MO_BSWAP | MO_SIZE)]); + return; + } + + /* + * Test for at least natural alignment, and assume most accesses + * will be aligned -- perform a straight store in the delay slot. + * This is required to preserve atomicity for aligned accesses. + */ + t_bits = MAX(a_bits, s_bits); + tcg_debug_assert(t_bits < 13); + tcg_out_arithi(s, TCG_REG_G0, addr, (1u << t_bits) - 1, ARITH_ANDCC); + + /* beq,a,pt %icc, label */ + label_ptr = s->code_ptr; + tcg_out_bpcc0(s, COND_E, BPCC_A | BPCC_PT | BPCC_ICC, 0); + /* delay slot */ + tcg_out_ldst_rr(s, data, addr, index, qemu_st_opc[memop & (MO_BSWAP | MO_SIZE)]); + + if (a_bits >= s_bits) { + /* + * Overalignment: A successful alignment test will perform the memory + * operation in the delay slot, and failure need only invoke the + * handler for SIGBUS. + */ + TCGReg arg_low = TCG_REG_O1 + (!SPARC64 && TARGET_LONG_BITS == 64); + tcg_out_call_nodelay(s, qemu_unalign_st_trampoline, false); + /* delay slot -- move to low part of argument reg */ + tcg_out_mov_delay(s, arg_low, addr); + } else { + /* Underalignment: store by pieces of minimum alignment. */ + int st_opc, a_size, s_size, i; + + /* + * Force full address into T1 early; avoids problems with + * overlap between @addr and @data. + */ + tcg_out_arith(s, TCG_REG_T1, addr, index, ARITH_ADD); + + a_size = 1 << a_bits; + s_size = 1 << s_bits; + if ((memop & MO_BSWAP) == MO_BE) { + st_opc = qemu_st_opc[a_bits | MO_BE]; + for (i = 0; i < s_size; i += a_size) { + TCGReg d = data; + int shift = (s_size - a_size - i) * 8; + if (shift) { + d = TCG_REG_T2; + tcg_out_arithi(s, d, data, shift, SHIFT_SRLX); + } + tcg_out_ldst(s, d, TCG_REG_T1, i, st_opc); + } + } else if (a_bits == 0) { + tcg_out_ldst(s, data, TCG_REG_T1, 0, STB); + for (i = 1; i < s_size; i++) { + tcg_out_arithi(s, TCG_REG_T2, data, i * 8, SHIFT_SRLX); + tcg_out_ldst(s, TCG_REG_T2, TCG_REG_T1, i, STB); + } + } else { + /* Note that ST*A with immediate asi must use indexed address. */ + st_opc = qemu_st_opc[a_bits + MO_LE]; + tcg_out_ldst_rr(s, data, TCG_REG_T1, TCG_REG_G0, st_opc); + for (i = a_size; i < s_size; i += a_size) { + tcg_out_arithi(s, TCG_REG_T2, data, i * 8, SHIFT_SRLX); + tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, a_size, ARITH_ADD); + tcg_out_ldst_rr(s, TCG_REG_T2, TCG_REG_T1, TCG_REG_G0, st_opc); + } + } + } + + *label_ptr |= INSN_OFF19(tcg_ptr_byte_diff(s->code_ptr, label_ptr)); #endif /* CONFIG_SOFTMMU */ } -- 2.25.1 A mostly generic test for unaligned access raising SIGBUS. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tests/tcg/multiarch/sigbus.c | 68 ++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 tests/tcg/multiarch/sigbus.c diff --git a/tests/tcg/multiarch/sigbus.c b/tests/tcg/multiarch/sigbus.c new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/tcg/multiarch/sigbus.c @@ -XXX,XX +XXX,XX @@ +#define _GNU_SOURCE 1 + +#include <assert.h> +#include <stdlib.h> +#include <signal.h> +#include <endian.h> + + +unsigned long long x = 0x8877665544332211ull; +void * volatile p = (void *)&x + 1; + +void sigbus(int sig, siginfo_t *info, void *uc) +{ + assert(sig == SIGBUS); + assert(info->si_signo == SIGBUS); +#ifdef BUS_ADRALN + assert(info->si_code == BUS_ADRALN); +#endif + assert(info->si_addr == p); + exit(EXIT_SUCCESS); +} + +int main() +{ + struct sigaction sa = { + .sa_sigaction = sigbus, + .sa_flags = SA_SIGINFO + }; + int allow_fail = 0; + int tmp; + + tmp = sigaction(SIGBUS, &sa, NULL); + assert(tmp == 0); + + /* + * Select an operation that's likely to enforce alignment. + * On many guests that support unaligned accesses by default, + * this is often an atomic operation. + */ +#if defined(__aarch64__) + asm volatile("ldxr %w0,[%1]" : "=r"(tmp) : "r"(p) : "memory"); +#elif defined(__alpha__) + asm volatile("ldl_l %0,0(%1)" : "=r"(tmp) : "r"(p) : "memory"); +#elif defined(__arm__) + asm volatile("ldrex %0,[%1]" : "=r"(tmp) : "r"(p) : "memory"); +#elif defined(__powerpc__) + asm volatile("lwarx %0,0,%1" : "=r"(tmp) : "r"(p) : "memory"); +#elif defined(__riscv_atomic) + asm volatile("lr.w %0,(%1)" : "=r"(tmp) : "r"(p) : "memory"); +#else + /* No insn known to fault unaligned -- try for a straight load. */ + allow_fail = 1; + tmp = *(volatile int *)p; +#endif + + assert(allow_fail); + + /* + * We didn't see a signal. + * We might as well validate the unaligned load worked. + */ + if (BYTE_ORDER == LITTLE_ENDIAN) { + assert(tmp == 0x55443322); + } else { + assert(tmp == 0x77665544); + } + return EXIT_SUCCESS; +} -- 2.25.1