As promised, more Arm patches. The big thing in here is the MPS2-AN521 board model. thanks -- PMM The following changes since commit cfe6c547690b06fbce54a6d0f7b05dd7f18e36ea: Merge remote-tracking branch 'remotes/xanclic/tags/pull-block-2019-01-31' into staging (2019-01-31 19:26:09 +0000) are available in the Git repository at: https://git.linaro.org/people/pmaydell/qemu-arm.git tags/pull-target-arm-20190201 for you to fetch changes up to 7743b70ffe7a8ce168adce2cf50ad156b1fefb8c: tests/microbit-test: Add tests for nRF51 NVMC (2019-02-01 15:32:17 +0000) ---------------------------------------------------------------- target-arm queue: * New machine mps2-an521 -- this is a model of the AN521 FPGA image for the MPS2 devboard * Fix various places where we failed to UNDEF invalid A64 instructions * Don't UNDEF a valid FCMLA on 32-bit inputs * Fix some bugs in the newly-added PAuth implementation * microbit: Implement NVMC non-volatile memory controller ---------------------------------------------------------------- Aaron Lindsay OS (2): target/arm: Send interrupts on PMU counter overflow target/arm: Add a timer to predict PMU counter overflow Julia Suvorova (1): arm: Clarify the logic of set_pc() Peter Maydell (33): armv7m: Don't assume the NVIC's CPU is CPU 0 armv7m: Make cpu object a child of the armv7m container armv7m: Pass through start-powered-off CPU property hw/arm/iotkit: Rename IoTKit to ARMSSE hw/arm/iotkit: Refactor into abstract base class and subclass hw/arm/iotkit: Rename 'iotkit' local variables and functions hw/arm/iotkit: Rename files to hw/arm/armsse.[ch] hw/misc/iotkit-secctl: Support 4 internal MPCs hw/arm/armsse: Make number of SRAM banks parameterised hw/arm/armsse: Make SRAM bank size configurable hw/arm/armsse: Support dual-CPU configuration hw/arm/armsse: Give each CPU its own view of memory hw/arm/armsse: Put each CPU in its own cluster object iotkit-sysinfo: Make SYS_VERSION and SYS_CONFIG configurable hw/arm/armsse: Add unimplemented-device stubs for MHUs hw/arm/armsse: Add unimplemented-device stubs for PPUs hw/arm/armsse: Add unimplemented-device stub for cache control registers hw/arm/armsse: Add unimplemented-device stub for CPU local control registers hw/misc/armsse-cpuid: Implement SSE-200 CPU_IDENTITY register block hw/arm/armsse: Add CPU_IDENTITY block to SSE-200 hw/arm/armsse: Add SSE-200 model hw/arm/mps2-tz: Add IRQ infrastructure to support SSE-200 hw/arm/mps2-tz: Add mps2-an521 model target/arm/translate-a64: Don't underdecode system instructions target/arm/translate-a64: Don't underdecode PRFM target/arm/translate-a64: Don't underdecode SIMD ld/st multiple target/arm/translate-a64: Don't underdecode SIMD ld/st single target/arm/translate-a64: Don't underdecode add/sub extended register target/arm/translate-a64: Don't underdecode FP insns target/arm/translate-a64: Don't underdecode SDOT and UDOT exec.c: Don't reallocate IOMMUNotifiers that are in use target/arm/translate-a64: Fix FCMLA decoding error target/arm/translate-a64: Fix mishandling of size in FCMLA decode Remi Denis-Courmont (2): target/arm: fix AArch64 virtual address space size target/arm: fix decoding of B{,L}RA{A,B} Richard Henderson (5): target/arm: Enable API, APK bits in SCR, HCR target/arm: Always enable pac keys for user-only aarch64-linux-user: Update HWCAP bits from linux 5.0-rc1 aarch64-linux-user: Enable HWCAP bits for PAuth linux-user: Initialize aarch64 pac keys Steffen Görtz (3): hw/nvram/nrf51_nvm: Add nRF51 non-volatile memories arm: Instantiate NRF51 special NVM's and NVMC tests/microbit-test: Add tests for nRF51 NVMC kumar sourav (1): hw/arm/nrf51_soc: set object owner in memory_region_init_ram hw/arm/Makefile.objs | 2 +- hw/misc/Makefile.objs | 1 + hw/nvram/Makefile.objs | 1 + include/hw/arm/{iotkit.h => armsse.h} | 113 ++- include/hw/arm/armv7m.h | 1 + include/hw/arm/nrf51_soc.h | 2 + include/hw/misc/armsse-cpuid.h | 41 ++ include/hw/misc/iotkit-secctl.h | 6 +- include/hw/misc/iotkit-sysinfo.h | 6 + include/hw/nvram/nrf51_nvm.h | 64 ++ include/qom/cpu.h | 16 +- linux-user/aarch64/target_syscall.h | 2 + target/arm/cpu.h | 12 +- exec.c | 10 +- hw/arm/armsse.c | 1241 +++++++++++++++++++++++++++++++++ hw/arm/armv7m.c | 23 +- hw/arm/boot.c | 4 - hw/arm/iotkit.c | 759 -------------------- hw/arm/mps2-tz.c | 121 +++- hw/arm/nrf51_soc.c | 44 +- hw/intc/armv7m_nvic.c | 3 +- hw/misc/armsse-cpuid.c | 134 ++++ hw/misc/iotkit-secctl.c | 5 +- hw/misc/iotkit-sysinfo.c | 15 +- hw/nvram/nrf51_nvm.c | 388 +++++++++++ linux-user/aarch64/cpu_loop.c | 31 +- linux-user/elfload.c | 10 + target/arm/arm-powerctl.c | 3 - target/arm/cpu.c | 41 +- target/arm/cpu64.c | 75 -- target/arm/helper.c | 139 +++- target/arm/translate-a64.c | 59 +- tests/microbit-test.c | 108 +++ MAINTAINERS | 6 +- default-configs/arm-softmmu.mak | 3 +- hw/misc/trace-events | 4 + 36 files changed, 2552 insertions(+), 941 deletions(-) rename include/hw/arm/{iotkit.h => armsse.h} (53%) create mode 100644 include/hw/misc/armsse-cpuid.h create mode 100644 include/hw/nvram/nrf51_nvm.h create mode 100644 hw/arm/armsse.c delete mode 100644 hw/arm/iotkit.c create mode 100644 hw/misc/armsse-cpuid.c create mode 100644 hw/nvram/nrf51_nvm.c From: kumar sourav <sourav.jb1988@gmail.com> set object owner in memory_region_init_ram() instead of NULL. Signed-off-by: kumar sourav <sourav.jb1988@gmail.com> Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com> Message-id: 20190125155630.17430-1-sourav.jb1988@gmail.com Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- hw/arm/nrf51_soc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hw/arm/nrf51_soc.c b/hw/arm/nrf51_soc.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/nrf51_soc.c +++ b/hw/arm/nrf51_soc.c @@ -XXX,XX +XXX,XX @@ static void nrf51_soc_realize(DeviceState *dev_soc, Error **errp) } memory_region_add_subregion(&s->container, NRF51_FLASH_BASE, &s->flash); - memory_region_init_ram(&s->sram, NULL, "nrf51.sram", s->sram_size, &err); + memory_region_init_ram(&s->sram, OBJECT(s), "nrf51.sram", s->sram_size, + &err); if (err) { error_propagate(errp, err); return; -- 2.20.1 Currently the ARMv7M NVIC object's realize method assumes that the CPU the NVIC is attached to is CPU 0, because it thinks there can only ever be one CPU in the system. To allow a dual-Cortex-M33 setup we need to remove this assumption; instead the armv7m wrapper object tells the NVIC its CPU, in the same way that it already tells the CPU what the NVIC is. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-2-peter.maydell@linaro.org --- hw/arm/armv7m.c | 6 ++++-- hw/intc/armv7m_nvic.c | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/hw/arm/armv7m.c b/hw/arm/armv7m.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armv7m.c +++ b/hw/arm/armv7m.c @@ -XXX,XX +XXX,XX @@ static void armv7m_realize(DeviceState *dev, Error **errp) } } - /* Tell the CPU where the NVIC is; it will fail realize if it doesn't - * have one. + /* + * Tell the CPU where the NVIC is; it will fail realize if it doesn't + * have one. Similarly, tell the NVIC where its CPU is. */ s->cpu->env.nvic = &s->nvic; + s->nvic.cpu = s->cpu; object_property_set_bool(OBJECT(s->cpu), true, "realized", &err); if (err != NULL) { diff --git a/hw/intc/armv7m_nvic.c b/hw/intc/armv7m_nvic.c index XXXXXXX..XXXXXXX 100644 --- a/hw/intc/armv7m_nvic.c +++ b/hw/intc/armv7m_nvic.c @@ -XXX,XX +XXX,XX @@ static void armv7m_nvic_realize(DeviceState *dev, Error **errp) Error *err = NULL; int regionlen; - s->cpu = ARM_CPU(qemu_get_cpu(0)); - + /* The armv7m container object will have set our CPU pointer */ if (!s->cpu || !arm_feature(&s->cpu->env, ARM_FEATURE_M)) { error_setg(errp, "The NVIC can only be used with a Cortex-M CPU"); return; -- 2.20.1 Rather than just creating the CPUs with object_new, make them child objects of the armv7m container. This will allow the cluster code to find the CPUs if an armv7m object is made a child of a cluster object. object_new_with_props() will do the parenting for us. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-3-peter.maydell@linaro.org --- hw/arm/armv7m.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hw/arm/armv7m.c b/hw/arm/armv7m.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armv7m.c +++ b/hw/arm/armv7m.c @@ -XXX,XX +XXX,XX @@ static void armv7m_realize(DeviceState *dev, Error **errp) memory_region_add_subregion_overlap(&s->container, 0, s->board_memory, -1); - s->cpu = ARM_CPU(object_new(s->cpu_type)); + s->cpu = ARM_CPU(object_new_with_props(s->cpu_type, OBJECT(s), "cpu", + &err, NULL)); + if (err != NULL) { + error_propagate(errp, err); + return; + } object_property_set_link(OBJECT(s->cpu), OBJECT(&s->container), "memory", &error_abort); -- 2.20.1 Expose "start-powered-off" as a property of the ARMv7M container, which we just pass through to the CPU object in the same way that we do for "init-svtor" and "idau". (We want this for the SSE-200, which powers up only the first CPU at reset and leaves the second powered down.) As with the other CPU properties here, we can't just use alias properties, because the CPU QOM object is not created until armv7m realize time. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-4-peter.maydell@linaro.org --- include/hw/arm/armv7m.h | 1 + hw/arm/armv7m.c | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/include/hw/arm/armv7m.h b/include/hw/arm/armv7m.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/armv7m.h +++ b/include/hw/arm/armv7m.h @@ -XXX,XX +XXX,XX @@ typedef struct ARMv7MState { Object *idau; uint32_t init_svtor; bool enable_bitband; + bool start_powered_off; } ARMv7MState; #endif diff --git a/hw/arm/armv7m.c b/hw/arm/armv7m.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armv7m.c +++ b/hw/arm/armv7m.c @@ -XXX,XX +XXX,XX @@ static void armv7m_realize(DeviceState *dev, Error **errp) return; } } + if (object_property_find(OBJECT(s->cpu), "start-powered-off", NULL)) { + object_property_set_bool(OBJECT(s->cpu), s->start_powered_off, + "start-powered-off", &err); + if (err != NULL) { + error_propagate(errp, err); + return; + } + } /* * Tell the CPU where the NVIC is; it will fail realize if it doesn't @@ -XXX,XX +XXX,XX @@ static Property armv7m_properties[] = { DEFINE_PROP_LINK("idau", ARMv7MState, idau, TYPE_IDAU_INTERFACE, Object *), DEFINE_PROP_UINT32("init-svtor", ARMv7MState, init_svtor, 0), DEFINE_PROP_BOOL("enable-bitband", ARMv7MState, enable_bitband, false), + DEFINE_PROP_BOOL("start-powered-off", ARMv7MState, start_powered_off, + false), DEFINE_PROP_END_OF_LIST(), }; -- 2.20.1 The Arm IoTKit was effectively the forerunner of a series of subsystems for embedded SoCs, named the SSE-050, SSE-100 and SSE-200: https://developer.arm.com/products/system-design/subsystems These are generally quite similar, though later iterations have extra devices that earlier ones do not. We want to add a model of the SSE-200, which means refactoring the IoTKit code into an abstract base class and subclasses (using the same design that the bcm283x SoC and Aspeed SoC family implementations do). As a first step, rename the IoTKit struct and QOM macros to ARMSSE, which is what we're going to name the base class. We temporarily retain TYPE_IOTKIT to avoid changing the code that instantiates a TYPE_IOTKIT device here and then changing it back again when it is re-introduced as a subclass. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-5-peter.maydell@linaro.org --- include/hw/arm/iotkit.h | 22 ++++++++++----- hw/arm/iotkit.c | 59 +++++++++++++++++++++-------------------- hw/arm/mps2-tz.c | 2 +- 3 files changed, 47 insertions(+), 36 deletions(-) diff --git a/include/hw/arm/iotkit.h b/include/hw/arm/iotkit.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/iotkit.h +++ b/include/hw/arm/iotkit.h @@ -XXX,XX +XXX,XX @@ /* - * ARM IoT Kit + * ARM SSE (Subsystems for Embedded): IoTKit * * Copyright (c) 2018 Linaro Limited * Written by Peter Maydell @@ -XXX,XX +XXX,XX @@ * (at your option) any later version. */ -/* This is a model of the Arm IoT Kit which is documented in +/* + * This is a model of the Arm "Subsystems for Embedded" family of + * hardware, which include the IoT Kit and the SSE-050, SSE-100 and + * SSE-200. Currently we model only the Arm IoT Kit which is documented in * http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ecm0601256/index.html * It contains: * a Cortex-M33 @@ -XXX,XX +XXX,XX @@ #include "hw/or-irq.h" #include "hw/core/split-irq.h" -#define TYPE_IOTKIT "iotkit" -#define IOTKIT(obj) OBJECT_CHECK(IoTKit, (obj), TYPE_IOTKIT) +#define TYPE_ARMSSE "iotkit" +#define ARMSSE(obj) OBJECT_CHECK(ARMSSE, (obj), TYPE_ARMSSE) + +/* + * For the moment TYPE_IOTKIT is a synonym for TYPE_ARMSSE (and the + * latter's underlying name is left as "iotkit"); in a later + * commit it will become a subclass of TYPE_ARMSSE. + */ +#define TYPE_IOTKIT TYPE_ARMSSE /* We have an IRQ splitter and an OR gate input for each external PPC * and the 2 internal PPCs @@ -XXX,XX +XXX,XX @@ #define NUM_EXTERNAL_PPCS (IOTS_NUM_AHB_EXP_PPC + IOTS_NUM_APB_EXP_PPC) #define NUM_PPCS (NUM_EXTERNAL_PPCS + 2) -typedef struct IoTKit { +typedef struct ARMSSE { /*< private >*/ SysBusDevice parent_obj; @@ -XXX,XX +XXX,XX @@ typedef struct IoTKit { MemoryRegion *board_memory; uint32_t exp_numirq; uint32_t mainclk_frq; -} IoTKit; +} ARMSSE; #endif diff --git a/hw/arm/iotkit.c b/hw/arm/iotkit.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/iotkit.c +++ b/hw/arm/iotkit.c @@ -XXX,XX +XXX,XX @@ /* - * Arm IoT Kit + * Arm SSE (Subsystems for Embedded): IoTKit * * Copyright (c) 2018 Linaro Limited * Written by Peter Maydell @@ -XXX,XX +XXX,XX @@ /* Create an alias region of @size bytes starting at @base * which mirrors the memory starting at @orig. */ -static void make_alias(IoTKit *s, MemoryRegion *mr, const char *name, +static void make_alias(ARMSSE *s, MemoryRegion *mr, const char *name, hwaddr base, hwaddr size, hwaddr orig) { memory_region_init_alias(mr, NULL, name, &s->container, orig, size); @@ -XXX,XX +XXX,XX @@ static void irq_status_forwarder(void *opaque, int n, int level) static void nsccfg_handler(void *opaque, int n, int level) { - IoTKit *s = IOTKIT(opaque); + ARMSSE *s = ARMSSE(opaque); s->nsccfg = level; } -static void iotkit_forward_ppc(IoTKit *s, const char *ppcname, int ppcnum) +static void iotkit_forward_ppc(ARMSSE *s, const char *ppcname, int ppcnum) { /* Each of the 4 AHB and 4 APB PPCs that might be present in a - * system using the IoTKit has a collection of control lines which + * system using the ARMSSE has a collection of control lines which * are provided by the security controller and which we want to - * expose as control lines on the IoTKit device itself, so the - * code using the IoTKit can wire them up to the PPCs. + * expose as control lines on the ARMSSE device itself, so the + * code using the ARMSSE can wire them up to the PPCs. */ SplitIRQ *splitter = &s->ppc_irq_splitter[ppcnum]; DeviceState *iotkitdev = DEVICE(s); @@ -XXX,XX +XXX,XX @@ static void iotkit_forward_ppc(IoTKit *s, const char *ppcname, int ppcnum) g_free(name); } -static void iotkit_forward_sec_resp_cfg(IoTKit *s) +static void iotkit_forward_sec_resp_cfg(ARMSSE *s) { /* Forward the 3rd output from the splitter device as a * named GPIO output of the iotkit object. @@ -XXX,XX +XXX,XX @@ static void iotkit_forward_sec_resp_cfg(IoTKit *s) static void iotkit_init(Object *obj) { - IoTKit *s = IOTKIT(obj); + ARMSSE *s = ARMSSE(obj); int i; memory_region_init(&s->container, obj, "iotkit-container", UINT64_MAX); @@ -XXX,XX +XXX,XX @@ static void iotkit_init(Object *obj) static void iotkit_exp_irq(void *opaque, int n, int level) { - IoTKit *s = IOTKIT(opaque); + ARMSSE *s = ARMSSE(opaque); qemu_set_irq(s->exp_irqs[n], level); } static void iotkit_mpcexp_status(void *opaque, int n, int level) { - IoTKit *s = IOTKIT(opaque); + ARMSSE *s = ARMSSE(opaque); qemu_set_irq(s->mpcexp_status_in[n], level); } static void iotkit_realize(DeviceState *dev, Error **errp) { - IoTKit *s = IOTKIT(dev); + ARMSSE *s = ARMSSE(dev); int i; MemoryRegion *mr; Error *err = NULL; @@ -XXX,XX +XXX,XX @@ static void iotkit_realize(DeviceState *dev, Error **errp) * devices exist in both address spaces but with hard-wired security * permissions that will cause the CPU to fault for non-secure accesses. * - * The IoTKit has an IDAU (Implementation Defined Access Unit), + * The ARMSSE has an IDAU (Implementation Defined Access Unit), * which specifies hard-wired security permissions for different - * areas of the physical address space. For the IoTKit IDAU, the + * areas of the physical address space. For the ARMSSE IDAU, the * top 4 bits of the physical address are the IDAU region ID, and * if bit 28 (ie the lowest bit of the ID) is 0 then this is an NS * region, otherwise it is an S region. @@ -XXX,XX +XXX,XX @@ static void iotkit_realize(DeviceState *dev, Error **errp) * 0x20000000..0x2007ffff 32KB FPGA block RAM * 0x30000000..0x3fffffff alias of 0x20000000..0x2fffffff * 0x40000000..0x4000ffff base peripheral region 1 - * 0x40010000..0x4001ffff CPU peripherals (none for IoTKit) + * 0x40010000..0x4001ffff CPU peripherals (none for ARMSSE) * 0x40020000..0x4002ffff system control element peripherals * 0x40080000..0x400fffff base peripheral region 2 * 0x50000000..0x5fffffff alias of 0x40000000..0x4fffffff @@ -XXX,XX +XXX,XX @@ static void iotkit_realize(DeviceState *dev, Error **errp) qdev_connect_gpio_out_named(dev_secctl, "nsc_cfg", 0, s->nsc_cfg_in); /* The sec_resp_cfg output from the security controller must be split into - * multiple lines, one for each of the PPCs within the IoTKit and one - * that will be an output from the IoTKit to the system. + * multiple lines, one for each of the PPCs within the ARMSSE and one + * that will be an output from the ARMSSE to the system. */ object_property_set_int(OBJECT(&s->sec_resp_splitter), 3, "num-lines", &err); @@ -XXX,XX +XXX,XX @@ static void iotkit_realize(DeviceState *dev, Error **errp) /* 0x40010000 .. 0x4001ffff: private CPU region: unused in IoTKit */ - /* 0x40020000 .. 0x4002ffff : IoTKit system control peripheral region */ + /* 0x40020000 .. 0x4002ffff : ARMSSE system control peripheral region */ /* Devices behind APB PPC1: * 0x4002f000: S32K timer */ @@ -XXX,XX +XXX,XX @@ static void iotkit_realize(DeviceState *dev, Error **errp) qdev_get_gpio_in(DEVICE(&s->nmi_orgate), 0)); sysbus_mmio_map(SYS_BUS_DEVICE(&s->s32kwatchdog), 0, 0x5002e000); - /* 0x40080000 .. 0x4008ffff : IoTKit second Base peripheral region */ + /* 0x40080000 .. 0x4008ffff : ARMSSE second Base peripheral region */ qdev_prop_set_uint32(DEVICE(&s->nswatchdog), "wdogclk-frq", s->mainclk_frq); object_property_set_bool(OBJECT(&s->nswatchdog), true, "realized", &err); @@ -XXX,XX +XXX,XX @@ static void iotkit_realize(DeviceState *dev, Error **errp) * Expose our container region to the board model; this corresponds * to the AHB Slave Expansion ports which allow bus master devices * (eg DMA controllers) in the board model to make transactions into - * devices in the IoTKit. + * devices in the ARMSSE. */ sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->container); @@ -XXX,XX +XXX,XX @@ static void iotkit_realize(DeviceState *dev, Error **errp) static void iotkit_idau_check(IDAUInterface *ii, uint32_t address, int *iregion, bool *exempt, bool *ns, bool *nsc) { - /* For IoTKit systems the IDAU responses are simple logical functions + /* + * For ARMSSE systems the IDAU responses are simple logical functions * of the address bits. The NSC attribute is guest-adjustable via the * NSCCFG register in the security controller. */ - IoTKit *s = IOTKIT(ii); + ARMSSE *s = ARMSSE(ii); int region = extract32(address, 28, 4); *ns = !(region & 1); @@ -XXX,XX +XXX,XX @@ static const VMStateDescription iotkit_vmstate = { .version_id = 1, .minimum_version_id = 1, .fields = (VMStateField[]) { - VMSTATE_UINT32(nsccfg, IoTKit), + VMSTATE_UINT32(nsccfg, ARMSSE), VMSTATE_END_OF_LIST() } }; static Property iotkit_properties[] = { - DEFINE_PROP_LINK("memory", IoTKit, board_memory, TYPE_MEMORY_REGION, + DEFINE_PROP_LINK("memory", ARMSSE, board_memory, TYPE_MEMORY_REGION, MemoryRegion *), - DEFINE_PROP_UINT32("EXP_NUMIRQ", IoTKit, exp_numirq, 64), - DEFINE_PROP_UINT32("MAINCLK", IoTKit, mainclk_frq, 0), + DEFINE_PROP_UINT32("EXP_NUMIRQ", ARMSSE, exp_numirq, 64), + DEFINE_PROP_UINT32("MAINCLK", ARMSSE, mainclk_frq, 0), DEFINE_PROP_END_OF_LIST() }; static void iotkit_reset(DeviceState *dev) { - IoTKit *s = IOTKIT(dev); + ARMSSE *s = ARMSSE(dev); s->nsccfg = 0; } @@ -XXX,XX +XXX,XX @@ static void iotkit_class_init(ObjectClass *klass, void *data) } static const TypeInfo iotkit_info = { - .name = TYPE_IOTKIT, + .name = TYPE_ARMSSE, .parent = TYPE_SYS_BUS_DEVICE, - .instance_size = sizeof(IoTKit), + .instance_size = sizeof(ARMSSE), .instance_init = iotkit_init, .class_init = iotkit_class_init, .interfaces = (InterfaceInfo[]) { diff --git a/hw/arm/mps2-tz.c b/hw/arm/mps2-tz.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/mps2-tz.c +++ b/hw/arm/mps2-tz.c @@ -XXX,XX +XXX,XX @@ typedef struct { typedef struct { MachineState parent; - IoTKit iotkit; + ARMSSE iotkit; MemoryRegion psram; MemoryRegion ssram[3]; MemoryRegion ssram1_m; -- 2.20.1 The Arm SSE-200 Subsystem for Embedded is a revised and extended version of the older IoTKit SoC. Prepare for adding a model of it by refactoring the IoTKit code into an abstract base class which contains the functionality, driven by a class data block specific to each subclass. (This is the same approach used by the existing bcm283x SoC family implementation.) Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-6-peter.maydell@linaro.org --- include/hw/arm/iotkit.h | 22 +++++++++++++++++----- hw/arm/iotkit.c | 34 +++++++++++++++++++++++++++++----- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/include/hw/arm/iotkit.h b/include/hw/arm/iotkit.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/iotkit.h +++ b/include/hw/arm/iotkit.h @@ -XXX,XX +XXX,XX @@ #include "hw/or-irq.h" #include "hw/core/split-irq.h" -#define TYPE_ARMSSE "iotkit" +#define TYPE_ARMSSE "arm-sse" #define ARMSSE(obj) OBJECT_CHECK(ARMSSE, (obj), TYPE_ARMSSE) /* - * For the moment TYPE_IOTKIT is a synonym for TYPE_ARMSSE (and the - * latter's underlying name is left as "iotkit"); in a later - * commit it will become a subclass of TYPE_ARMSSE. + * These type names are for specific IoTKit subsystems; other than + * instantiating them, code using these devices should always handle + * them via the ARMSSE base class, so they have no IOTKIT() etc macros. */ -#define TYPE_IOTKIT TYPE_ARMSSE +#define TYPE_IOTKIT "iotkit" /* We have an IRQ splitter and an OR gate input for each external PPC * and the 2 internal PPCs @@ -XXX,XX +XXX,XX @@ typedef struct ARMSSE { uint32_t mainclk_frq; } ARMSSE; +typedef struct ARMSSEInfo ARMSSEInfo; + +typedef struct ARMSSEClass { + DeviceClass parent_class; + const ARMSSEInfo *info; +} ARMSSEClass; + +#define ARMSSE_CLASS(klass) \ + OBJECT_CLASS_CHECK(ARMSSEClass, (klass), TYPE_ARMSSE) +#define ARMSSE_GET_CLASS(obj) \ + OBJECT_GET_CLASS(ARMSSEClass, (obj), TYPE_ARMSSE) + #endif diff --git a/hw/arm/iotkit.c b/hw/arm/iotkit.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/iotkit.c +++ b/hw/arm/iotkit.c @@ -XXX,XX +XXX,XX @@ #include "hw/arm/iotkit.h" #include "hw/arm/arm.h" +struct ARMSSEInfo { + const char *name; +}; + +static const ARMSSEInfo armsse_variants[] = { + { + .name = TYPE_IOTKIT, + }, +}; + /* Clock frequency in HZ of the 32KHz "slow clock" */ #define S32KCLK (32 * 1000) @@ -XXX,XX +XXX,XX @@ static void iotkit_class_init(ObjectClass *klass, void *data) { DeviceClass *dc = DEVICE_CLASS(klass); IDAUInterfaceClass *iic = IDAU_INTERFACE_CLASS(klass); + ARMSSEClass *asc = ARMSSE_CLASS(klass); dc->realize = iotkit_realize; dc->vmsd = &iotkit_vmstate; dc->props = iotkit_properties; dc->reset = iotkit_reset; iic->check = iotkit_idau_check; + asc->info = data; } -static const TypeInfo iotkit_info = { +static const TypeInfo armsse_info = { .name = TYPE_ARMSSE, .parent = TYPE_SYS_BUS_DEVICE, .instance_size = sizeof(ARMSSE), .instance_init = iotkit_init, - .class_init = iotkit_class_init, + .abstract = true, .interfaces = (InterfaceInfo[]) { { TYPE_IDAU_INTERFACE }, { } } }; -static void iotkit_register_types(void) +static void armsse_register_types(void) { - type_register_static(&iotkit_info); + int i; + + type_register_static(&armsse_info); + + for (i = 0; i < ARRAY_SIZE(armsse_variants); i++) { + TypeInfo ti = { + .name = armsse_variants[i].name, + .parent = TYPE_ARMSSE, + .class_init = iotkit_class_init, + .class_data = (void *)&armsse_variants[i], + }; + type_register(&ti); + } } -type_init(iotkit_register_types); +type_init(armsse_register_types); -- 2.20.1 Rename various internal uses of 'iotkit' in hw/arm/iotkit.c to 'armsse', for consistency. The remaining occurences are: * related to the devices TYPE_IOTKIT_SYSCTL, TYPE_IOTKIT_SYSINFO, etc, which this refactor is not touching * references that apply specifically to the IoTKit (like the lack of a private CPU region) * the vmstate, which keeps its old "iotkit" name for migration compatibility reasons Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-7-peter.maydell@linaro.org --- hw/arm/iotkit.c | 68 ++++++++++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/hw/arm/iotkit.c b/hw/arm/iotkit.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/iotkit.c +++ b/hw/arm/iotkit.c @@ -XXX,XX +XXX,XX @@ static void nsccfg_handler(void *opaque, int n, int level) s->nsccfg = level; } -static void iotkit_forward_ppc(ARMSSE *s, const char *ppcname, int ppcnum) +static void armsse_forward_ppc(ARMSSE *s, const char *ppcname, int ppcnum) { /* Each of the 4 AHB and 4 APB PPCs that might be present in a * system using the ARMSSE has a collection of control lines which @@ -XXX,XX +XXX,XX @@ static void iotkit_forward_ppc(ARMSSE *s, const char *ppcname, int ppcnum) * code using the ARMSSE can wire them up to the PPCs. */ SplitIRQ *splitter = &s->ppc_irq_splitter[ppcnum]; - DeviceState *iotkitdev = DEVICE(s); + DeviceState *armssedev = DEVICE(s); DeviceState *dev_secctl = DEVICE(&s->secctl); DeviceState *dev_splitter = DEVICE(splitter); char *name; name = g_strdup_printf("%s_nonsec", ppcname); - qdev_pass_gpios(dev_secctl, iotkitdev, name); + qdev_pass_gpios(dev_secctl, armssedev, name); g_free(name); name = g_strdup_printf("%s_ap", ppcname); - qdev_pass_gpios(dev_secctl, iotkitdev, name); + qdev_pass_gpios(dev_secctl, armssedev, name); g_free(name); name = g_strdup_printf("%s_irq_enable", ppcname); - qdev_pass_gpios(dev_secctl, iotkitdev, name); + qdev_pass_gpios(dev_secctl, armssedev, name); g_free(name); name = g_strdup_printf("%s_irq_clear", ppcname); - qdev_pass_gpios(dev_secctl, iotkitdev, name); + qdev_pass_gpios(dev_secctl, armssedev, name); g_free(name); /* irq_status is a little more tricky, because we need to @@ -XXX,XX +XXX,XX @@ static void iotkit_forward_ppc(ARMSSE *s, const char *ppcname, int ppcnum) qdev_connect_gpio_out(dev_splitter, 1, qdev_get_gpio_in(DEVICE(&s->ppc_irq_orgate), ppcnum)); s->irq_status_in[ppcnum] = qdev_get_gpio_in(dev_splitter, 0); - qdev_init_gpio_in_named_with_opaque(iotkitdev, irq_status_forwarder, + qdev_init_gpio_in_named_with_opaque(armssedev, irq_status_forwarder, s->irq_status_in[ppcnum], name, 1); g_free(name); } -static void iotkit_forward_sec_resp_cfg(ARMSSE *s) +static void armsse_forward_sec_resp_cfg(ARMSSE *s) { /* Forward the 3rd output from the splitter device as a - * named GPIO output of the iotkit object. + * named GPIO output of the armsse object. */ DeviceState *dev = DEVICE(s); DeviceState *dev_splitter = DEVICE(&s->sec_resp_splitter); @@ -XXX,XX +XXX,XX @@ static void iotkit_forward_sec_resp_cfg(ARMSSE *s) qdev_connect_gpio_out(dev_splitter, 2, s->sec_resp_cfg_in); } -static void iotkit_init(Object *obj) +static void armsse_init(Object *obj) { ARMSSE *s = ARMSSE(obj); int i; - memory_region_init(&s->container, obj, "iotkit-container", UINT64_MAX); + memory_region_init(&s->container, obj, "armsse-container", UINT64_MAX); sysbus_init_child_obj(obj, "armv7m", &s->armv7m, sizeof(s->armv7m), TYPE_ARMV7M); @@ -XXX,XX +XXX,XX @@ static void iotkit_init(Object *obj) sizeof(s->nswatchdog), TYPE_CMSDK_APB_WATCHDOG); sysbus_init_child_obj(obj, "swatchdog", &s->swatchdog, sizeof(s->swatchdog), TYPE_CMSDK_APB_WATCHDOG); - sysbus_init_child_obj(obj, "iotkit-sysctl", &s->sysctl, + sysbus_init_child_obj(obj, "armsse-sysctl", &s->sysctl, sizeof(s->sysctl), TYPE_IOTKIT_SYSCTL); - sysbus_init_child_obj(obj, "iotkit-sysinfo", &s->sysinfo, + sysbus_init_child_obj(obj, "armsse-sysinfo", &s->sysinfo, sizeof(s->sysinfo), TYPE_IOTKIT_SYSINFO); object_initialize_child(obj, "nmi-orgate", &s->nmi_orgate, sizeof(s->nmi_orgate), TYPE_OR_IRQ, @@ -XXX,XX +XXX,XX @@ static void iotkit_init(Object *obj) } } -static void iotkit_exp_irq(void *opaque, int n, int level) +static void armsse_exp_irq(void *opaque, int n, int level) { ARMSSE *s = ARMSSE(opaque); qemu_set_irq(s->exp_irqs[n], level); } -static void iotkit_mpcexp_status(void *opaque, int n, int level) +static void armsse_mpcexp_status(void *opaque, int n, int level) { ARMSSE *s = ARMSSE(opaque); qemu_set_irq(s->mpcexp_status_in[n], level); } -static void iotkit_realize(DeviceState *dev, Error **errp) +static void armsse_realize(DeviceState *dev, Error **errp) { ARMSSE *s = ARMSSE(dev); int i; @@ -XXX,XX +XXX,XX @@ static void iotkit_realize(DeviceState *dev, Error **errp) for (i = 0; i < s->exp_numirq; i++) { s->exp_irqs[i] = qdev_get_gpio_in(DEVICE(&s->armv7m), i + 32); } - qdev_init_gpio_in_named(dev, iotkit_exp_irq, "EXP_IRQ", s->exp_numirq); + qdev_init_gpio_in_named(dev, armsse_exp_irq, "EXP_IRQ", s->exp_numirq); /* Set up the big aliases first */ make_alias(s, &s->alias1, "alias 1", 0x10000000, 0x10000000, 0x00000000); @@ -XXX,XX +XXX,XX @@ static void iotkit_realize(DeviceState *dev, Error **errp) qdev_get_gpio_in(dev_splitter, 0)); /* This RAM lives behind the Memory Protection Controller */ - memory_region_init_ram(&s->sram0, NULL, "iotkit.sram0", 0x00008000, &err); + memory_region_init_ram(&s->sram0, NULL, "armsse.sram0", 0x00008000, &err); if (err) { error_propagate(errp, err); return; @@ -XXX,XX +XXX,XX @@ static void iotkit_realize(DeviceState *dev, Error **errp) for (i = 0; i < IOTS_NUM_AHB_EXP_PPC; i++) { char *ppcname = g_strdup_printf("ahb_ppcexp%d", i); - iotkit_forward_ppc(s, ppcname, i); + armsse_forward_ppc(s, ppcname, i); g_free(ppcname); } for (i = 0; i < IOTS_NUM_APB_EXP_PPC; i++) { char *ppcname = g_strdup_printf("apb_ppcexp%d", i); - iotkit_forward_ppc(s, ppcname, i + IOTS_NUM_AHB_EXP_PPC); + armsse_forward_ppc(s, ppcname, i + IOTS_NUM_AHB_EXP_PPC); g_free(ppcname); } @@ -XXX,XX +XXX,XX @@ static void iotkit_realize(DeviceState *dev, Error **errp) /* Create GPIO inputs which will pass the line state for our * mpcexp_irq inputs to the correct splitter devices. */ - qdev_init_gpio_in_named(dev, iotkit_mpcexp_status, "mpcexp_status", + qdev_init_gpio_in_named(dev, armsse_mpcexp_status, "mpcexp_status", IOTS_NUM_EXP_MPC); - iotkit_forward_sec_resp_cfg(s); + armsse_forward_sec_resp_cfg(s); /* Forward the MSC related signals */ qdev_pass_gpios(dev_secctl, dev, "mscexp_status"); @@ -XXX,XX +XXX,XX @@ static void iotkit_realize(DeviceState *dev, Error **errp) system_clock_scale = NANOSECONDS_PER_SECOND / s->mainclk_frq; } -static void iotkit_idau_check(IDAUInterface *ii, uint32_t address, +static void armsse_idau_check(IDAUInterface *ii, uint32_t address, int *iregion, bool *exempt, bool *ns, bool *nsc) { /* @@ -XXX,XX +XXX,XX @@ static void iotkit_idau_check(IDAUInterface *ii, uint32_t address, *iregion = region; } -static const VMStateDescription iotkit_vmstate = { +static const VMStateDescription armsse_vmstate = { .name = "iotkit", .version_id = 1, .minimum_version_id = 1, @@ -XXX,XX +XXX,XX @@ static const VMStateDescription iotkit_vmstate = { } }; -static Property iotkit_properties[] = { +static Property armsse_properties[] = { DEFINE_PROP_LINK("memory", ARMSSE, board_memory, TYPE_MEMORY_REGION, MemoryRegion *), DEFINE_PROP_UINT32("EXP_NUMIRQ", ARMSSE, exp_numirq, 64), @@ -XXX,XX +XXX,XX @@ static Property iotkit_properties[] = { DEFINE_PROP_END_OF_LIST() }; -static void iotkit_reset(DeviceState *dev) +static void armsse_reset(DeviceState *dev) { ARMSSE *s = ARMSSE(dev); s->nsccfg = 0; } -static void iotkit_class_init(ObjectClass *klass, void *data) +static void armsse_class_init(ObjectClass *klass, void *data) { DeviceClass *dc = DEVICE_CLASS(klass); IDAUInterfaceClass *iic = IDAU_INTERFACE_CLASS(klass); ARMSSEClass *asc = ARMSSE_CLASS(klass); - dc->realize = iotkit_realize; - dc->vmsd = &iotkit_vmstate; - dc->props = iotkit_properties; - dc->reset = iotkit_reset; - iic->check = iotkit_idau_check; + dc->realize = armsse_realize; + dc->vmsd = &armsse_vmstate; + dc->props = armsse_properties; + dc->reset = armsse_reset; + iic->check = armsse_idau_check; asc->info = data; } @@ -XXX,XX +XXX,XX @@ static const TypeInfo armsse_info = { .name = TYPE_ARMSSE, .parent = TYPE_SYS_BUS_DEVICE, .instance_size = sizeof(ARMSSE), - .instance_init = iotkit_init, + .instance_init = armsse_init, .abstract = true, .interfaces = (InterfaceInfo[]) { { TYPE_IDAU_INTERFACE }, @@ -XXX,XX +XXX,XX @@ static void armsse_register_types(void) TypeInfo ti = { .name = armsse_variants[i].name, .parent = TYPE_ARMSSE, - .class_init = iotkit_class_init, + .class_init = armsse_class_init, .class_data = (void *)&armsse_variants[i], }; type_register(&ti); -- 2.20.1 Rename the files that used to be iotkit.[ch] to armsse.[ch] to reflect the fact they new cover multiple Arm subsystems for embedded. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-8-peter.maydell@linaro.org --- hw/arm/Makefile.objs | 2 +- include/hw/arm/{iotkit.h => armsse.h} | 4 ++-- hw/arm/{iotkit.c => armsse.c} | 2 +- hw/arm/mps2-tz.c | 2 +- MAINTAINERS | 4 ++-- default-configs/arm-softmmu.mak | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) rename include/hw/arm/{iotkit.h => armsse.h} (99%) rename hw/arm/{iotkit.c => armsse.c} (99%) diff --git a/hw/arm/Makefile.objs b/hw/arm/Makefile.objs index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/Makefile.objs +++ b/hw/arm/Makefile.objs @@ -XXX,XX +XXX,XX @@ obj-$(CONFIG_ASPEED_SOC) += aspeed_soc.o aspeed.o obj-$(CONFIG_MPS2) += mps2.o obj-$(CONFIG_MPS2) += mps2-tz.o obj-$(CONFIG_MSF2) += msf2-soc.o msf2-som.o -obj-$(CONFIG_IOTKIT) += iotkit.o +obj-$(CONFIG_ARMSSE) += armsse.o obj-$(CONFIG_FSL_IMX7) += fsl-imx7.o mcimx7d-sabre.o obj-$(CONFIG_ARM_SMMUV3) += smmu-common.o smmuv3.o obj-$(CONFIG_FSL_IMX6UL) += fsl-imx6ul.o mcimx6ul-evk.o diff --git a/include/hw/arm/iotkit.h b/include/hw/arm/armsse.h similarity index 99% rename from include/hw/arm/iotkit.h rename to include/hw/arm/armsse.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/iotkit.h +++ b/include/hw/arm/armsse.h @@ -XXX,XX +XXX,XX @@ * + named GPIO outputs mscexp_ns[0..15] */ -#ifndef IOTKIT_H -#define IOTKIT_H +#ifndef ARMSSE_H +#define ARMSSE_H #include "hw/sysbus.h" #include "hw/arm/armv7m.h" diff --git a/hw/arm/iotkit.c b/hw/arm/armsse.c similarity index 99% rename from hw/arm/iotkit.c rename to hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/iotkit.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ #include "trace.h" #include "hw/sysbus.h" #include "hw/registerfields.h" -#include "hw/arm/iotkit.h" +#include "hw/arm/armsse.h" #include "hw/arm/arm.h" struct ARMSSEInfo { diff --git a/hw/arm/mps2-tz.c b/hw/arm/mps2-tz.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/mps2-tz.c +++ b/hw/arm/mps2-tz.c @@ -XXX,XX +XXX,XX @@ #include "hw/misc/mps2-fpgaio.h" #include "hw/misc/tz-mpc.h" #include "hw/misc/tz-msc.h" -#include "hw/arm/iotkit.h" +#include "hw/arm/armsse.h" #include "hw/dma/pl080.h" #include "hw/ssi/pl022.h" #include "hw/devices.h" diff --git a/MAINTAINERS b/MAINTAINERS index XXXXXXX..XXXXXXX 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -XXX,XX +XXX,XX @@ F: hw/arm/mps2.c F: hw/arm/mps2-tz.c F: hw/misc/mps2-*.c F: include/hw/misc/mps2-*.h -F: hw/arm/iotkit.c -F: include/hw/arm/iotkit.h +F: hw/arm/armsse.c +F: include/hw/arm/armsse.h F: hw/misc/iotkit-secctl.c F: include/hw/misc/iotkit-secctl.h F: hw/misc/iotkit-sysctl.c diff --git a/default-configs/arm-softmmu.mak b/default-configs/arm-softmmu.mak index XXXXXXX..XXXXXXX 100644 --- a/default-configs/arm-softmmu.mak +++ b/default-configs/arm-softmmu.mak @@ -XXX,XX +XXX,XX @@ CONFIG_MPS2_SCC=y CONFIG_TZ_MPC=y CONFIG_TZ_MSC=y CONFIG_TZ_PPC=y -CONFIG_IOTKIT=y +CONFIG_ARMSSE=y CONFIG_IOTKIT_SECCTL=y CONFIG_IOTKIT_SYSCTL=y CONFIG_IOTKIT_SYSINFO=y -- 2.20.1 The SSE-200 has 4 banks of SRAM, each with its own internal Memory Protection Controller. The interrupt status for these extra MPCs appears in the same security controller SECMPCINTSTATUS register as the MPC for the IoTKit's single SRAM bank. Enhance the iotkit-secctl device to allow 4 MPCs. (If the particular IoTKit/SSE variant in use does not have all 4 MPCs then the unused inputs will simply result in the SECMPCINTSTATUS bits being zero as required.) The hardcoded constant "1"s in armsse.c indicate the actual number of SRAM MPCs the IoTKit has, and will be replaced in the following commit. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-9-peter.maydell@linaro.org --- include/hw/misc/iotkit-secctl.h | 6 +++--- hw/arm/armsse.c | 6 +++--- hw/misc/iotkit-secctl.c | 5 +++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/include/hw/misc/iotkit-secctl.h b/include/hw/misc/iotkit-secctl.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/misc/iotkit-secctl.h +++ b/include/hw/misc/iotkit-secctl.h @@ -XXX,XX +XXX,XX @@ * + named GPIO outputs ahb_ppcexp{0,1,2,3}_irq_enable * + named GPIO outputs ahb_ppcexp{0,1,2,3}_irq_clear * + named GPIO inputs ahb_ppcexp{0,1,2,3}_irq_status - * Controlling the MPC in the IoTKit: - * + named GPIO input mpc_status + * Controlling the (up to) 4 MPCs in the IoTKit/SSE: + * + named GPIO inputs mpc_status[0..3] * Controlling each of the 16 expansion MPCs which a system using the IoTKit * might provide: * + named GPIO inputs mpcexp_status[0..15] @@ -XXX,XX +XXX,XX @@ #define IOTS_NUM_APB_EXP_PPC 4 #define IOTS_NUM_AHB_EXP_PPC 4 #define IOTS_NUM_EXP_MPC 16 -#define IOTS_NUM_MPC 1 +#define IOTS_NUM_MPC 4 #define IOTS_NUM_EXP_MSC 16 typedef struct IoTKitSecCtl IoTKitSecCtl; diff --git a/hw/arm/armsse.c b/hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armsse.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ static void armsse_init(Object *obj) sizeof(s->mpc_irq_orgate), TYPE_OR_IRQ, &error_abort, NULL); - for (i = 0; i < ARRAY_SIZE(s->mpc_irq_splitter); i++) { + for (i = 0; i < IOTS_NUM_EXP_MPC + 1; i++) { char *name = g_strdup_printf("mpc-irq-splitter-%d", i); SplitIRQ *splitter = &s->mpc_irq_splitter[i]; @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) /* We must OR together lines from the MPC splitters to go to the NVIC */ object_property_set_int(OBJECT(&s->mpc_irq_orgate), - IOTS_NUM_EXP_MPC + IOTS_NUM_MPC, "num-lines", &err); + IOTS_NUM_EXP_MPC + 1, "num-lines", &err); if (err) { error_propagate(errp, err); return; @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) } /* Wire up the splitters for the MPC IRQs */ - for (i = 0; i < IOTS_NUM_EXP_MPC + IOTS_NUM_MPC; i++) { + for (i = 0; i < IOTS_NUM_EXP_MPC + 1; i++) { SplitIRQ *splitter = &s->mpc_irq_splitter[i]; DeviceState *dev_splitter = DEVICE(splitter); diff --git a/hw/misc/iotkit-secctl.c b/hw/misc/iotkit-secctl.c index XXXXXXX..XXXXXXX 100644 --- a/hw/misc/iotkit-secctl.c +++ b/hw/misc/iotkit-secctl.c @@ -XXX,XX +XXX,XX @@ static void iotkit_secctl_mpc_status(void *opaque, int n, int level) { IoTKitSecCtl *s = IOTKIT_SECCTL(opaque); - s->mpcintstatus = deposit32(s->mpcintstatus, 0, 1, !!level); + s->mpcintstatus = deposit32(s->mpcintstatus, n, 1, !!level); } static void iotkit_secctl_mpcexp_status(void *opaque, int n, int level) @@ -XXX,XX +XXX,XX @@ static void iotkit_secctl_init(Object *obj) qdev_init_gpio_out_named(dev, &s->sec_resp_cfg, "sec_resp_cfg", 1); qdev_init_gpio_out_named(dev, &s->nsc_cfg_irq, "nsc_cfg", 1); - qdev_init_gpio_in_named(dev, iotkit_secctl_mpc_status, "mpc_status", 1); + qdev_init_gpio_in_named(dev, iotkit_secctl_mpc_status, "mpc_status", + IOTS_NUM_MPC); qdev_init_gpio_in_named(dev, iotkit_secctl_mpcexp_status, "mpcexp_status", IOTS_NUM_EXP_MPC); -- 2.20.1 The SSE-200 has four banks of SRAM, each with its own Memory Protection Controller, where the IoTKit has only one. Make the number of SRAM banks a field in ARMSSEInfo. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-10-peter.maydell@linaro.org --- include/hw/arm/armsse.h | 9 +++-- hw/arm/armsse.c | 78 ++++++++++++++++++++++++++--------------- 2 files changed, 56 insertions(+), 31 deletions(-) diff --git a/include/hw/arm/armsse.h b/include/hw/arm/armsse.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/armsse.h +++ b/include/hw/arm/armsse.h @@ -XXX,XX +XXX,XX @@ #define NUM_EXTERNAL_PPCS (IOTS_NUM_AHB_EXP_PPC + IOTS_NUM_APB_EXP_PPC) #define NUM_PPCS (NUM_EXTERNAL_PPCS + 2) +#define MAX_SRAM_BANKS 4 +#if MAX_SRAM_BANKS > IOTS_NUM_MPC +#error Too many SRAM banks +#endif + typedef struct ARMSSE { /*< private >*/ SysBusDevice parent_obj; @@ -XXX,XX +XXX,XX @@ typedef struct ARMSSE { IoTKitSecCtl secctl; TZPPC apb_ppc0; TZPPC apb_ppc1; - TZMPC mpc; + TZMPC mpc[IOTS_NUM_MPC]; CMSDKAPBTIMER timer0; CMSDKAPBTIMER timer1; CMSDKAPBTIMER s32ktimer; @@ -XXX,XX +XXX,XX @@ typedef struct ARMSSE { MemoryRegion alias1; MemoryRegion alias2; MemoryRegion alias3; - MemoryRegion sram0; + MemoryRegion sram[MAX_SRAM_BANKS]; qemu_irq *exp_irqs; qemu_irq ppc0_irq; diff --git a/hw/arm/armsse.c b/hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armsse.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ struct ARMSSEInfo { const char *name; + int sram_banks; }; static const ARMSSEInfo armsse_variants[] = { { .name = TYPE_IOTKIT, + .sram_banks = 1, }, }; @@ -XXX,XX +XXX,XX @@ static void armsse_forward_sec_resp_cfg(ARMSSE *s) static void armsse_init(Object *obj) { ARMSSE *s = ARMSSE(obj); + ARMSSEClass *asc = ARMSSE_GET_CLASS(obj); + const ARMSSEInfo *info = asc->info; int i; + assert(info->sram_banks <= MAX_SRAM_BANKS); + memory_region_init(&s->container, obj, "armsse-container", UINT64_MAX); sysbus_init_child_obj(obj, "armv7m", &s->armv7m, sizeof(s->armv7m), @@ -XXX,XX +XXX,XX @@ static void armsse_init(Object *obj) TYPE_TZ_PPC); sysbus_init_child_obj(obj, "apb-ppc1", &s->apb_ppc1, sizeof(s->apb_ppc1), TYPE_TZ_PPC); - sysbus_init_child_obj(obj, "mpc", &s->mpc, sizeof(s->mpc), TYPE_TZ_MPC); + for (i = 0; i < info->sram_banks; i++) { + char *name = g_strdup_printf("mpc%d", i); + sysbus_init_child_obj(obj, name, &s->mpc[i], + sizeof(s->mpc[i]), TYPE_TZ_MPC); + g_free(name); + } object_initialize_child(obj, "mpc-irq-orgate", &s->mpc_irq_orgate, sizeof(s->mpc_irq_orgate), TYPE_OR_IRQ, &error_abort, NULL); - for (i = 0; i < IOTS_NUM_EXP_MPC + 1; i++) { + for (i = 0; i < IOTS_NUM_EXP_MPC + info->sram_banks; i++) { char *name = g_strdup_printf("mpc-irq-splitter-%d", i); SplitIRQ *splitter = &s->mpc_irq_splitter[i]; @@ -XXX,XX +XXX,XX @@ static void armsse_mpcexp_status(void *opaque, int n, int level) static void armsse_realize(DeviceState *dev, Error **errp) { ARMSSE *s = ARMSSE(dev); + ARMSSEClass *asc = ARMSSE_GET_CLASS(dev); + const ARMSSEInfo *info = asc->info; int i; MemoryRegion *mr; Error *err = NULL; @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) qdev_connect_gpio_out_named(dev_secctl, "sec_resp_cfg", 0, qdev_get_gpio_in(dev_splitter, 0)); - /* This RAM lives behind the Memory Protection Controller */ - memory_region_init_ram(&s->sram0, NULL, "armsse.sram0", 0x00008000, &err); - if (err) { - error_propagate(errp, err); - return; + /* Each SRAM bank lives behind its own Memory Protection Controller */ + for (i = 0; i < info->sram_banks; i++) { + char *ramname = g_strdup_printf("armsse.sram%d", i); + SysBusDevice *sbd_mpc; + + memory_region_init_ram(&s->sram[i], NULL, ramname, 0x00008000, &err); + g_free(ramname); + if (err) { + error_propagate(errp, err); + return; + } + object_property_set_link(OBJECT(&s->mpc[i]), OBJECT(&s->sram[i]), + "downstream", &err); + if (err) { + error_propagate(errp, err); + return; + } + object_property_set_bool(OBJECT(&s->mpc[i]), true, "realized", &err); + if (err) { + error_propagate(errp, err); + return; + } + /* Map the upstream end of the MPC into the right place... */ + sbd_mpc = SYS_BUS_DEVICE(&s->mpc[i]); + memory_region_add_subregion(&s->container, 0x20000000 + i * 0x8000, + sysbus_mmio_get_region(sbd_mpc, 1)); + /* ...and its register interface */ + memory_region_add_subregion(&s->container, 0x50083000 + i * 0x1000, + sysbus_mmio_get_region(sbd_mpc, 0)); } - object_property_set_link(OBJECT(&s->mpc), OBJECT(&s->sram0), - "downstream", &err); - if (err) { - error_propagate(errp, err); - return; - } - object_property_set_bool(OBJECT(&s->mpc), true, "realized", &err); - if (err) { - error_propagate(errp, err); - return; - } - /* Map the upstream end of the MPC into the right place... */ - memory_region_add_subregion(&s->container, 0x20000000, - sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->mpc), - 1)); - /* ...and its register interface */ - memory_region_add_subregion(&s->container, 0x50083000, - sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->mpc), - 0)); /* We must OR together lines from the MPC splitters to go to the NVIC */ object_property_set_int(OBJECT(&s->mpc_irq_orgate), - IOTS_NUM_EXP_MPC + 1, "num-lines", &err); + IOTS_NUM_EXP_MPC + info->sram_banks, + "num-lines", &err); if (err) { error_propagate(errp, err); return; @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) } /* Wire up the splitters for the MPC IRQs */ - for (i = 0; i < IOTS_NUM_EXP_MPC + 1; i++) { + for (i = 0; i < IOTS_NUM_EXP_MPC + info->sram_banks; i++) { SplitIRQ *splitter = &s->mpc_irq_splitter[i]; DeviceState *dev_splitter = DEVICE(splitter); @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) "mpcexp_status", i)); } else { /* Splitter input is from our own MPC */ - qdev_connect_gpio_out_named(DEVICE(&s->mpc), "irq", 0, + qdev_connect_gpio_out_named(DEVICE(&s->mpc[i - IOTS_NUM_EXP_MPC]), + "irq", 0, qdev_get_gpio_in(dev_splitter, 0)); qdev_connect_gpio_out(dev_splitter, 0, qdev_get_gpio_in_named(dev_secctl, -- 2.20.1 For the IoTKit the SRAM bank size is always 32K (15 bits); for the SSE-200 this is a configurable parameter, which defaults to 32K but can be changed when it is built into a particular SoC. For instance the Musca-B1 board sets it to 128K (17 bits). Make the bank size a QOM property. We follow the SSE-200 hardware in naming the parameter SRAM_ADDR_WIDTH, which specifies the number of address bits of a single SRAM bank. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-11-peter.maydell@linaro.org --- include/hw/arm/armsse.h | 1 + hw/arm/armsse.c | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/include/hw/arm/armsse.h b/include/hw/arm/armsse.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/armsse.h +++ b/include/hw/arm/armsse.h @@ -XXX,XX +XXX,XX @@ typedef struct ARMSSE { MemoryRegion *board_memory; uint32_t exp_numirq; uint32_t mainclk_frq; + uint32_t sram_addr_width; } ARMSSE; typedef struct ARMSSEInfo ARMSSEInfo; diff --git a/hw/arm/armsse.c b/hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armsse.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) DeviceState *dev_apb_ppc1; DeviceState *dev_secctl; DeviceState *dev_splitter; + uint32_t addr_width_max; if (!s->board_memory) { error_setg(errp, "memory property was not set"); @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) return; } + /* max SRAM_ADDR_WIDTH: 24 - log2(SRAM_NUM_BANK) */ + assert(is_power_of_2(info->sram_banks)); + addr_width_max = 24 - ctz32(info->sram_banks); + if (s->sram_addr_width < 1 || s->sram_addr_width > addr_width_max) { + error_setg(errp, "SRAM_ADDR_WIDTH must be between 1 and %d", + addr_width_max); + return; + } + /* Handling of which devices should be available only to secure * code is usually done differently for M profile than for A profile. * Instead of putting some devices only into the secure address space, @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) for (i = 0; i < info->sram_banks; i++) { char *ramname = g_strdup_printf("armsse.sram%d", i); SysBusDevice *sbd_mpc; + uint32_t sram_bank_size = 1 << s->sram_addr_width; - memory_region_init_ram(&s->sram[i], NULL, ramname, 0x00008000, &err); + memory_region_init_ram(&s->sram[i], NULL, ramname, + sram_bank_size, &err); g_free(ramname); if (err) { error_propagate(errp, err); @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) } /* Map the upstream end of the MPC into the right place... */ sbd_mpc = SYS_BUS_DEVICE(&s->mpc[i]); - memory_region_add_subregion(&s->container, 0x20000000 + i * 0x8000, + memory_region_add_subregion(&s->container, + 0x20000000 + i * sram_bank_size, sysbus_mmio_get_region(sbd_mpc, 1)); /* ...and its register interface */ memory_region_add_subregion(&s->container, 0x50083000 + i * 0x1000, @@ -XXX,XX +XXX,XX @@ static Property armsse_properties[] = { MemoryRegion *), DEFINE_PROP_UINT32("EXP_NUMIRQ", ARMSSE, exp_numirq, 64), DEFINE_PROP_UINT32("MAINCLK", ARMSSE, mainclk_frq, 0), + DEFINE_PROP_UINT32("SRAM_ADDR_WIDTH", ARMSSE, sram_addr_width, 15), DEFINE_PROP_END_OF_LIST() }; -- 2.20.1 The SSE-200 has two Cortex-M33 CPUs. These see the same view of memory, with the exception of the "private CPU region" which has per-CPU devices. Internal device interrupts for SSE-200 devices are mostly wired up to both CPUs, with the exception of a few per-CPU devices. External GPIO inputs on the SSE-200 device are provided for the second CPU's interrupts above 32, as is already the case for the first CPU. Refactor the code to support creation of multiple CPUs. For the moment we leave all CPUs with the same view of memory: this will not work in the multiple-CPU case, but we will fix this in the following commit. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-12-peter.maydell@linaro.org --- include/hw/arm/armsse.h | 21 +++- hw/arm/armsse.c | 206 ++++++++++++++++++++++++++++++++-------- 2 files changed, 180 insertions(+), 47 deletions(-) diff --git a/include/hw/arm/armsse.h b/include/hw/arm/armsse.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/armsse.h +++ b/include/hw/arm/armsse.h @@ -XXX,XX +XXX,XX @@ * + QOM property "memory" is a MemoryRegion containing the devices provided * by the board model. * + QOM property "MAINCLK" is the frequency of the main system clock - * + QOM property "EXP_NUMIRQ" sets the number of expansion interrupts - * + Named GPIO inputs "EXP_IRQ" 0..n are the expansion interrupts, which - * are wired to the NVIC lines 32 .. n+32 + * + QOM property "EXP_NUMIRQ" sets the number of expansion interrupts. + * (In hardware, the SSE-200 permits the number of expansion interrupts + * for the two CPUs to be configured separately, but we restrict it to + * being the same for both, to avoid having to have separate Property + * lists for different variants. This restriction can be relaxed later + * if necessary.) + * + Named GPIO inputs "EXP_IRQ" 0..n are the expansion interrupts for CPU 0, + * which are wired to its NVIC lines 32 .. n+32 + * + Named GPIO inputs "EXP_CPU1_IRQ" 0..n are the expansion interrupts for + * CPU 1, which are wired to its NVIC lines 32 .. n+32 * + sysbus MMIO region 0 is the "AHB Slave Expansion" which allows * bus master devices in the board model to make transactions into * all the devices and memory areas in the IoTKit @@ -XXX,XX +XXX,XX @@ #error Too many SRAM banks #endif +#define SSE_MAX_CPUS 2 + typedef struct ARMSSE { /*< private >*/ SysBusDevice parent_obj; /*< public >*/ - ARMv7MState armv7m; + ARMv7MState armv7m[SSE_MAX_CPUS]; IoTKitSecCtl secctl; TZPPC apb_ppc0; TZPPC apb_ppc1; @@ -XXX,XX +XXX,XX @@ typedef struct ARMSSE { qemu_or_irq mpc_irq_orgate; qemu_or_irq nmi_orgate; + SplitIRQ cpu_irq_splitter[32]; + CMSDKAPBDualTimer dualtimer; CMSDKAPBWatchdog s32kwatchdog; @@ -XXX,XX +XXX,XX @@ typedef struct ARMSSE { MemoryRegion alias3; MemoryRegion sram[MAX_SRAM_BANKS]; - qemu_irq *exp_irqs; + qemu_irq *exp_irqs[SSE_MAX_CPUS]; qemu_irq ppc0_irq; qemu_irq ppc1_irq; qemu_irq sec_resp_cfg; diff --git a/hw/arm/armsse.c b/hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armsse.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ struct ARMSSEInfo { const char *name; int sram_banks; + int num_cpus; }; static const ARMSSEInfo armsse_variants[] = { { .name = TYPE_IOTKIT, .sram_banks = 1, + .num_cpus = 1, }, }; /* Clock frequency in HZ of the 32KHz "slow clock" */ #define S32KCLK (32 * 1000) +/* Is internal IRQ n shared between CPUs in a multi-core SSE ? */ +static bool irq_is_common[32] = { + [0 ... 5] = true, + /* 6, 7: per-CPU MHU interrupts */ + [8 ... 12] = true, + /* 13: per-CPU icache interrupt */ + /* 14: reserved */ + [15 ... 20] = true, + /* 21: reserved */ + [22 ... 26] = true, + /* 27: reserved */ + /* 28, 29: per-CPU CTI interrupts */ + /* 30, 31: reserved */ +}; + /* Create an alias region of @size bytes starting at @base * which mirrors the memory starting at @orig. */ @@ -XXX,XX +XXX,XX @@ static void armsse_init(Object *obj) int i; assert(info->sram_banks <= MAX_SRAM_BANKS); + assert(info->num_cpus <= SSE_MAX_CPUS); memory_region_init(&s->container, obj, "armsse-container", UINT64_MAX); - sysbus_init_child_obj(obj, "armv7m", &s->armv7m, sizeof(s->armv7m), - TYPE_ARMV7M); - qdev_prop_set_string(DEVICE(&s->armv7m), "cpu-type", - ARM_CPU_TYPE_NAME("cortex-m33")); + for (i = 0; i < info->num_cpus; i++) { + char *name = g_strdup_printf("armv7m%d", i); + sysbus_init_child_obj(obj, name, &s->armv7m[i], sizeof(s->armv7m), + TYPE_ARMV7M); + qdev_prop_set_string(DEVICE(&s->armv7m[i]), "cpu-type", + ARM_CPU_TYPE_NAME("cortex-m33")); + g_free(name); + } sysbus_init_child_obj(obj, "secctl", &s->secctl, sizeof(s->secctl), TYPE_IOTKIT_SECCTL); @@ -XXX,XX +XXX,XX @@ static void armsse_init(Object *obj) TYPE_SPLIT_IRQ, &error_abort, NULL); g_free(name); } + if (info->num_cpus > 1) { + for (i = 0; i < ARRAY_SIZE(s->cpu_irq_splitter); i++) { + if (irq_is_common[i]) { + char *name = g_strdup_printf("cpu-irq-splitter%d", i); + SplitIRQ *splitter = &s->cpu_irq_splitter[i]; + + object_initialize_child(obj, name, splitter, sizeof(*splitter), + TYPE_SPLIT_IRQ, &error_abort, NULL); + g_free(name); + } + } + } } static void armsse_exp_irq(void *opaque, int n, int level) { - ARMSSE *s = ARMSSE(opaque); + qemu_irq *irqarray = opaque; - qemu_set_irq(s->exp_irqs[n], level); + qemu_set_irq(irqarray[n], level); } static void armsse_mpcexp_status(void *opaque, int n, int level) @@ -XXX,XX +XXX,XX @@ static void armsse_mpcexp_status(void *opaque, int n, int level) qemu_set_irq(s->mpcexp_status_in[n], level); } +static qemu_irq armsse_get_common_irq_in(ARMSSE *s, int irqno) +{ + /* + * Return a qemu_irq which can be used to signal IRQ n to + * all CPUs in the SSE. + */ + ARMSSEClass *asc = ARMSSE_GET_CLASS(s); + const ARMSSEInfo *info = asc->info; + + assert(irq_is_common[irqno]); + + if (info->num_cpus == 1) { + /* Only one CPU -- just connect directly to it */ + return qdev_get_gpio_in(DEVICE(&s->armv7m[0]), irqno); + } else { + /* Connect to the splitter which feeds all CPUs */ + return qdev_get_gpio_in(DEVICE(&s->cpu_irq_splitter[irqno]), 0); + } +} + static void armsse_realize(DeviceState *dev, Error **errp) { ARMSSE *s = ARMSSE(dev); @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) memory_region_add_subregion_overlap(&s->container, 0, s->board_memory, -1); - qdev_prop_set_uint32(DEVICE(&s->armv7m), "num-irq", s->exp_numirq + 32); - /* In real hardware the initial Secure VTOR is set from the INITSVTOR0 - * register in the IoT Kit System Control Register block, and the - * initial value of that is in turn specifiable by the FPGA that - * instantiates the IoT Kit. In QEMU we don't implement this wrinkle, - * and simply set the CPU's init-svtor to the IoT Kit default value. - */ - qdev_prop_set_uint32(DEVICE(&s->armv7m), "init-svtor", 0x10000000); - object_property_set_link(OBJECT(&s->armv7m), OBJECT(&s->container), - "memory", &err); - if (err) { - error_propagate(errp, err); - return; - } - object_property_set_link(OBJECT(&s->armv7m), OBJECT(s), "idau", &err); - if (err) { - error_propagate(errp, err); - return; - } - object_property_set_bool(OBJECT(&s->armv7m), true, "realized", &err); - if (err) { - error_propagate(errp, err); - return; + for (i = 0; i < info->num_cpus; i++) { + DeviceState *cpudev = DEVICE(&s->armv7m[i]); + Object *cpuobj = OBJECT(&s->armv7m[i]); + int j; + char *gpioname; + + qdev_prop_set_uint32(cpudev, "num-irq", s->exp_numirq + 32); + /* + * In real hardware the initial Secure VTOR is set from the INITSVTOR0 + * register in the IoT Kit System Control Register block, and the + * initial value of that is in turn specifiable by the FPGA that + * instantiates the IoT Kit. In QEMU we don't implement this wrinkle, + * and simply set the CPU's init-svtor to the IoT Kit default value. + * In SSE-200 the situation is similar, except that the default value + * is a reset-time signal input. Typically a board using the SSE-200 + * will have a system control processor whose boot firmware initializes + * the INITSVTOR* registers before powering up the CPUs in any case, + * so the hardware's default value doesn't matter. QEMU doesn't emulate + * the control processor, so instead we behave in the way that the + * firmware does. All boards currently known about have firmware that + * sets the INITSVTOR0 and INITSVTOR1 registers to 0x10000000, like the + * IoTKit default. We can make this more configurable if necessary. + */ + qdev_prop_set_uint32(cpudev, "init-svtor", 0x10000000); + /* + * Start all CPUs except CPU0 powered down. In real hardware it is + * a configurable property of the SSE-200 which CPUs start powered up + * (via the CPUWAIT0_RST and CPUWAIT1_RST parameters), but since all + * the boards we care about start CPU0 and leave CPU1 powered off, + * we hard-code that for now. We can add QOM properties for this + * later if necessary. + */ + if (i > 0) { + object_property_set_bool(cpuobj, true, "start-powered-off", &err); + if (err) { + error_propagate(errp, err); + return; + } + } + object_property_set_link(cpuobj, OBJECT(&s->container), "memory", &err); + if (err) { + error_propagate(errp, err); + return; + } + object_property_set_link(cpuobj, OBJECT(s), "idau", &err); + if (err) { + error_propagate(errp, err); + return; + } + object_property_set_bool(cpuobj, true, "realized", &err); + if (err) { + error_propagate(errp, err); + return; + } + + /* Connect EXP_IRQ/EXP_CPUn_IRQ GPIOs to the NVIC's lines 32 and up */ + s->exp_irqs[i] = g_new(qemu_irq, s->exp_numirq); + for (j = 0; j < s->exp_numirq; j++) { + s->exp_irqs[i][j] = qdev_get_gpio_in(cpudev, i + 32); + } + if (i == 0) { + gpioname = g_strdup("EXP_IRQ"); + } else { + gpioname = g_strdup_printf("EXP_CPU%d_IRQ", i); + } + qdev_init_gpio_in_named_with_opaque(dev, armsse_exp_irq, + s->exp_irqs[i], + gpioname, s->exp_numirq); + g_free(gpioname); } - /* Connect our EXP_IRQ GPIOs to the NVIC's lines 32 and up. */ - s->exp_irqs = g_new(qemu_irq, s->exp_numirq); - for (i = 0; i < s->exp_numirq; i++) { - s->exp_irqs[i] = qdev_get_gpio_in(DEVICE(&s->armv7m), i + 32); + /* Wire up the splitters that connect common IRQs to all CPUs */ + if (info->num_cpus > 1) { + for (i = 0; i < ARRAY_SIZE(s->cpu_irq_splitter); i++) { + if (irq_is_common[i]) { + Object *splitter = OBJECT(&s->cpu_irq_splitter[i]); + DeviceState *devs = DEVICE(splitter); + int cpunum; + + object_property_set_int(splitter, info->num_cpus, + "num-lines", &err); + if (err) { + error_propagate(errp, err); + return; + } + object_property_set_bool(splitter, true, "realized", &err); + if (err) { + error_propagate(errp, err); + return; + } + for (cpunum = 0; cpunum < info->num_cpus; cpunum++) { + DeviceState *cpudev = DEVICE(&s->armv7m[cpunum]); + + qdev_connect_gpio_out(devs, cpunum, + qdev_get_gpio_in(cpudev, i)); + } + } + } } - qdev_init_gpio_in_named(dev, armsse_exp_irq, "EXP_IRQ", s->exp_numirq); /* Set up the big aliases first */ make_alias(s, &s->alias1, "alias 1", 0x10000000, 0x10000000, 0x00000000); @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) return; } qdev_connect_gpio_out(DEVICE(&s->mpc_irq_orgate), 0, - qdev_get_gpio_in(DEVICE(&s->armv7m), 9)); + armsse_get_common_irq_in(s, 9)); /* Devices behind APB PPC0: * 0x40000000: timer0 @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) return; } sysbus_connect_irq(SYS_BUS_DEVICE(&s->timer0), 0, - qdev_get_gpio_in(DEVICE(&s->armv7m), 3)); + armsse_get_common_irq_in(s, 3)); mr = sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->timer0), 0); object_property_set_link(OBJECT(&s->apb_ppc0), OBJECT(mr), "port[0]", &err); if (err) { @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) return; } sysbus_connect_irq(SYS_BUS_DEVICE(&s->timer1), 0, - qdev_get_gpio_in(DEVICE(&s->armv7m), 4)); + armsse_get_common_irq_in(s, 4)); mr = sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->timer1), 0); object_property_set_link(OBJECT(&s->apb_ppc0), OBJECT(mr), "port[1]", &err); if (err) { @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) return; } sysbus_connect_irq(SYS_BUS_DEVICE(&s->dualtimer), 0, - qdev_get_gpio_in(DEVICE(&s->armv7m), 5)); + armsse_get_common_irq_in(s, 5)); mr = sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->dualtimer), 0); object_property_set_link(OBJECT(&s->apb_ppc0), OBJECT(mr), "port[2]", &err); if (err) { @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) return; } qdev_connect_gpio_out(DEVICE(&s->ppc_irq_orgate), 0, - qdev_get_gpio_in(DEVICE(&s->armv7m), 10)); + armsse_get_common_irq_in(s, 10)); /* 0x40010000 .. 0x4001ffff: private CPU region: unused in IoTKit */ @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) return; } sysbus_connect_irq(SYS_BUS_DEVICE(&s->s32ktimer), 0, - qdev_get_gpio_in(DEVICE(&s->armv7m), 2)); + armsse_get_common_irq_in(s, 2)); mr = sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->s32ktimer), 0); object_property_set_link(OBJECT(&s->apb_ppc1), OBJECT(mr), "port[0]", &err); if (err) { @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) return; } sysbus_connect_irq(SYS_BUS_DEVICE(&s->nswatchdog), 0, - qdev_get_gpio_in(DEVICE(&s->armv7m), 1)); + armsse_get_common_irq_in(s, 1)); sysbus_mmio_map(SYS_BUS_DEVICE(&s->nswatchdog), 0, 0x40081000); qdev_prop_set_uint32(DEVICE(&s->swatchdog), "wdogclk-frq", s->mainclk_frq); @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) qdev_pass_gpios(dev_secctl, dev, "mscexp_clear"); qdev_pass_gpios(dev_secctl, dev, "mscexp_ns"); qdev_connect_gpio_out_named(dev_secctl, "msc_irq", 0, - qdev_get_gpio_in(DEVICE(&s->armv7m), 11)); + armsse_get_common_irq_in(s, 11)); /* * Expose our container region to the board model; this corresponds -- 2.20.1 Give each CPU its own container memory region. This is necessary for two reasons: * some devices are instantiated one per CPU and the CPU sees only its own device * since a memory region can only be put into one container, we must give each armv7m object a different MemoryRegion as its 'memory' property, or a dual-CPU configuration will assert on realize when the second armv7m object tries to put the MR into a container when it is already in the first armv7m object's container Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-13-peter.maydell@linaro.org --- include/hw/arm/armsse.h | 10 ++++++++++ hw/arm/armsse.c | 22 ++++++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/include/hw/arm/armsse.h b/include/hw/arm/armsse.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/armsse.h +++ b/include/hw/arm/armsse.h @@ -XXX,XX +XXX,XX @@ typedef struct ARMSSE { IoTKitSysCtl sysctl; IoTKitSysCtl sysinfo; + /* + * 'container' holds all devices seen by all CPUs. + * 'cpu_container[i]' is the view that CPU i has: this has the + * per-CPU devices of that CPU, plus as the background 'container' + * (or an alias of it, since we can only use it directly once). + * container_alias[i] is the alias of 'container' used by CPU i+1; + * CPU 0 can use 'container' directly. + */ MemoryRegion container; + MemoryRegion container_alias[SSE_MAX_CPUS - 1]; + MemoryRegion cpu_container[SSE_MAX_CPUS]; MemoryRegion alias1; MemoryRegion alias2; MemoryRegion alias3; diff --git a/hw/arm/armsse.c b/hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armsse.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ static void armsse_init(Object *obj) qdev_prop_set_string(DEVICE(&s->armv7m[i]), "cpu-type", ARM_CPU_TYPE_NAME("cortex-m33")); g_free(name); + name = g_strdup_printf("arm-sse-cpu-container%d", i); + memory_region_init(&s->cpu_container[i], obj, name, UINT64_MAX); + g_free(name); + if (i > 0) { + name = g_strdup_printf("arm-sse-container-alias%d", i); + memory_region_init_alias(&s->container_alias[i - 1], obj, + name, &s->container, 0, UINT64_MAX); + g_free(name); + } } sysbus_init_child_obj(obj, "secctl", &s->secctl, sizeof(s->secctl), @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) * 0x50000000..0x5fffffff alias of 0x40000000..0x4fffffff */ - memory_region_add_subregion_overlap(&s->container, 0, s->board_memory, -1); + memory_region_add_subregion_overlap(&s->container, 0, s->board_memory, -2); for (i = 0; i < info->num_cpus; i++) { DeviceState *cpudev = DEVICE(&s->armv7m[i]); @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) return; } } - object_property_set_link(cpuobj, OBJECT(&s->container), "memory", &err); + + if (i > 0) { + memory_region_add_subregion_overlap(&s->cpu_container[i], 0, + &s->container_alias[i - 1], -1); + } else { + memory_region_add_subregion_overlap(&s->cpu_container[i], 0, + &s->container, -1); + } + object_property_set_link(cpuobj, OBJECT(&s->cpu_container[i]), + "memory", &err); if (err) { error_propagate(errp, err); return; -- 2.20.1 Create a cluster object to hold each CPU in the SSE. They are logically distinct and may be configured differently (for instance one may not have an FPU where the other does). Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-14-peter.maydell@linaro.org --- include/hw/arm/armsse.h | 2 ++ hw/arm/armsse.c | 31 ++++++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/include/hw/arm/armsse.h b/include/hw/arm/armsse.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/armsse.h +++ b/include/hw/arm/armsse.h @@ -XXX,XX +XXX,XX @@ #include "hw/misc/iotkit-sysinfo.h" #include "hw/or-irq.h" #include "hw/core/split-irq.h" +#include "hw/cpu/cluster.h" #define TYPE_ARMSSE "arm-sse" #define ARMSSE(obj) OBJECT_CHECK(ARMSSE, (obj), TYPE_ARMSSE) @@ -XXX,XX +XXX,XX @@ typedef struct ARMSSE { /*< public >*/ ARMv7MState armv7m[SSE_MAX_CPUS]; + CPUClusterState cluster[SSE_MAX_CPUS]; IoTKitSecCtl secctl; TZPPC apb_ppc0; TZPPC apb_ppc1; diff --git a/hw/arm/armsse.c b/hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armsse.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ static void armsse_init(Object *obj) memory_region_init(&s->container, obj, "armsse-container", UINT64_MAX); for (i = 0; i < info->num_cpus; i++) { - char *name = g_strdup_printf("armv7m%d", i); - sysbus_init_child_obj(obj, name, &s->armv7m[i], sizeof(s->armv7m), - TYPE_ARMV7M); + /* + * We put each CPU in its own cluster as they are logically + * distinct and may be configured differently. + */ + char *name; + + name = g_strdup_printf("cluster%d", i); + object_initialize_child(obj, name, &s->cluster[i], + sizeof(s->cluster[i]), TYPE_CPU_CLUSTER, + &error_abort, NULL); + qdev_prop_set_uint32(DEVICE(&s->cluster[i]), "cluster-id", i); + g_free(name); + + name = g_strdup_printf("armv7m%d", i); + sysbus_init_child_obj(OBJECT(&s->cluster[i]), name, + &s->armv7m[i], sizeof(s->armv7m), TYPE_ARMV7M); qdev_prop_set_string(DEVICE(&s->armv7m[i]), "cpu-type", ARM_CPU_TYPE_NAME("cortex-m33")); g_free(name); @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) error_propagate(errp, err); return; } + /* + * The cluster must be realized after the armv7m container, as + * the container's CPU object is only created on realize, and the + * CPU must exist and have been parented into the cluster before + * the cluster is realized. + */ + object_property_set_bool(OBJECT(&s->cluster[i]), + true, "realized", &err); + if (err) { + error_propagate(errp, err); + return; + } /* Connect EXP_IRQ/EXP_CPUn_IRQ GPIOs to the NVIC's lines 32 and up */ s->exp_irqs[i] = g_new(qemu_irq, s->exp_numirq); -- 2.20.1 The SYS_VERSION and SYS_CONFIG register values differ between the IoTKit and SSE-200. Make them configurable via QOM properties rather than hard-coded, and set them appropriately in the ARMSSE code that instantiates the IOTKIT_SYSINFO device. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-15-peter.maydell@linaro.org --- include/hw/misc/iotkit-sysinfo.h | 6 ++++ hw/arm/armsse.c | 51 ++++++++++++++++++++++++++++++++ hw/misc/iotkit-sysinfo.c | 15 ++++++++-- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/include/hw/misc/iotkit-sysinfo.h b/include/hw/misc/iotkit-sysinfo.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/misc/iotkit-sysinfo.h +++ b/include/hw/misc/iotkit-sysinfo.h @@ -XXX,XX +XXX,XX @@ * Arm IoTKit and documented in * http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ecm0601256/index.html * QEMU interface: + * + QOM property "SYS_VERSION": value to use for SYS_VERSION register + * + QOM property "SYS_CONFIG": value to use for SYS_CONFIG register * + sysbus MMIO region 0: the system information register bank */ @@ -XXX,XX +XXX,XX @@ typedef struct IoTKitSysInfo { /*< public >*/ MemoryRegion iomem; + + /* Properties */ + uint32_t sys_version; + uint32_t sys_config; } IoTKitSysInfo; #endif diff --git a/hw/arm/armsse.c b/hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armsse.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ #include "hw/arm/armsse.h" #include "hw/arm/arm.h" +/* Format of the System Information block SYS_CONFIG register */ +typedef enum SysConfigFormat { + IoTKitFormat, + SSE200Format, +} SysConfigFormat; + struct ARMSSEInfo { const char *name; int sram_banks; int num_cpus; + uint32_t sys_version; + SysConfigFormat sys_config_format; }; static const ARMSSEInfo armsse_variants[] = { @@ -XXX,XX +XXX,XX @@ static const ARMSSEInfo armsse_variants[] = { .name = TYPE_IOTKIT, .sram_banks = 1, .num_cpus = 1, + .sys_version = 0x41743, + .sys_config_format = IoTKitFormat, }, }; +static uint32_t armsse_sys_config_value(ARMSSE *s, const ARMSSEInfo *info) +{ + /* Return the SYS_CONFIG value for this SSE */ + uint32_t sys_config; + + switch (info->sys_config_format) { + case IoTKitFormat: + sys_config = 0; + sys_config = deposit32(sys_config, 0, 4, info->sram_banks); + sys_config = deposit32(sys_config, 4, 4, s->sram_addr_width - 12); + break; + case SSE200Format: + sys_config = 0; + sys_config = deposit32(sys_config, 0, 4, info->sram_banks); + sys_config = deposit32(sys_config, 4, 5, s->sram_addr_width); + sys_config = deposit32(sys_config, 24, 4, 2); + if (info->num_cpus > 1) { + sys_config = deposit32(sys_config, 10, 1, 1); + sys_config = deposit32(sys_config, 20, 4, info->sram_banks - 1); + sys_config = deposit32(sys_config, 28, 4, 2); + } + break; + default: + g_assert_not_reached(); + } + return sys_config; +} + /* Clock frequency in HZ of the 32KHz "slow clock" */ #define S32KCLK (32 * 1000) @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) qdev_get_gpio_in_named(dev_apb_ppc1, "cfg_sec_resp", 0)); + object_property_set_int(OBJECT(&s->sysinfo), info->sys_version, + "SYS_VERSION", &err); + if (err) { + error_propagate(errp, err); + return; + } + object_property_set_int(OBJECT(&s->sysinfo), + armsse_sys_config_value(s, info), + "SYS_CONFIG", &err); + if (err) { + error_propagate(errp, err); + return; + } object_property_set_bool(OBJECT(&s->sysinfo), true, "realized", &err); if (err) { error_propagate(errp, err); diff --git a/hw/misc/iotkit-sysinfo.c b/hw/misc/iotkit-sysinfo.c index XXXXXXX..XXXXXXX 100644 --- a/hw/misc/iotkit-sysinfo.c +++ b/hw/misc/iotkit-sysinfo.c @@ -XXX,XX +XXX,XX @@ static const int sysinfo_id[] = { static uint64_t iotkit_sysinfo_read(void *opaque, hwaddr offset, unsigned size) { + IoTKitSysInfo *s = IOTKIT_SYSINFO(opaque); uint64_t r; switch (offset) { case A_SYS_VERSION: - r = 0x41743; + r = s->sys_version; break; case A_SYS_CONFIG: - r = 0x31; + r = s->sys_config; break; case A_PID4 ... A_CID3: r = sysinfo_id[(offset - A_PID4) / 4]; @@ -XXX,XX +XXX,XX @@ static const MemoryRegionOps iotkit_sysinfo_ops = { .valid.max_access_size = 4, }; +static Property iotkit_sysinfo_props[] = { + DEFINE_PROP_UINT32("SYS_VERSION", IoTKitSysInfo, sys_version, 0), + DEFINE_PROP_UINT32("SYS_CONFIG", IoTKitSysInfo, sys_config, 0), + DEFINE_PROP_END_OF_LIST() +}; + static void iotkit_sysinfo_init(Object *obj) { SysBusDevice *sbd = SYS_BUS_DEVICE(obj); @@ -XXX,XX +XXX,XX @@ static void iotkit_sysinfo_init(Object *obj) static void iotkit_sysinfo_class_init(ObjectClass *klass, void *data) { + DeviceClass *dc = DEVICE_CLASS(klass); + /* * This device has no guest-modifiable state and so it * does not need a reset function or VMState. */ + + dc->props = iotkit_sysinfo_props; } static const TypeInfo iotkit_sysinfo_info = { -- 2.20.1 The SSE-200 has two Message Handling Units (MHUs), which sit behind the APB PPC0. Wire up some unimplemented-device stubs for these, since we don't yet implement a real model of this device. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-16-peter.maydell@linaro.org --- include/hw/arm/armsse.h | 3 +++ hw/arm/armsse.c | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/include/hw/arm/armsse.h b/include/hw/arm/armsse.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/armsse.h +++ b/include/hw/arm/armsse.h @@ -XXX,XX +XXX,XX @@ #include "hw/watchdog/cmsdk-apb-watchdog.h" #include "hw/misc/iotkit-sysctl.h" #include "hw/misc/iotkit-sysinfo.h" +#include "hw/misc/unimp.h" #include "hw/or-irq.h" #include "hw/core/split-irq.h" #include "hw/cpu/cluster.h" @@ -XXX,XX +XXX,XX @@ typedef struct ARMSSE { IoTKitSysCtl sysctl; IoTKitSysCtl sysinfo; + UnimplementedDeviceState mhu[2]; + /* * 'container' holds all devices seen by all CPUs. * 'cpu_container[i]' is the view that CPU i has: this has the diff --git a/hw/arm/armsse.c b/hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armsse.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ struct ARMSSEInfo { int num_cpus; uint32_t sys_version; SysConfigFormat sys_config_format; + bool has_mhus; }; static const ARMSSEInfo armsse_variants[] = { @@ -XXX,XX +XXX,XX @@ static const ARMSSEInfo armsse_variants[] = { .num_cpus = 1, .sys_version = 0x41743, .sys_config_format = IoTKitFormat, + .has_mhus = false, }, }; @@ -XXX,XX +XXX,XX @@ static void armsse_init(Object *obj) sizeof(s->sysctl), TYPE_IOTKIT_SYSCTL); sysbus_init_child_obj(obj, "armsse-sysinfo", &s->sysinfo, sizeof(s->sysinfo), TYPE_IOTKIT_SYSINFO); + if (info->has_mhus) { + sysbus_init_child_obj(obj, "mhu0", &s->mhu[0], sizeof(s->mhu[0]), + TYPE_UNIMPLEMENTED_DEVICE); + sysbus_init_child_obj(obj, "mhu1", &s->mhu[1], sizeof(s->mhu[1]), + TYPE_UNIMPLEMENTED_DEVICE); + } object_initialize_child(obj, "nmi-orgate", &s->nmi_orgate, sizeof(s->nmi_orgate), TYPE_OR_IRQ, &error_abort, NULL); @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) * 0x40000000: timer0 * 0x40001000: timer1 * 0x40002000: dual timer + * 0x40003000: MHU0 (SSE-200 only) + * 0x40004000: MHU1 (SSE-200 only) * We must configure and realize each downstream device and connect * it to the appropriate PPC port; then we can realize the PPC and * map its upstream ends to the right place in the container. @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) return; } + if (info->has_mhus) { + for (i = 0; i < ARRAY_SIZE(s->mhu); i++) { + char *name = g_strdup_printf("MHU%d", i); + char *port = g_strdup_printf("port[%d]", i + 3); + + qdev_prop_set_string(DEVICE(&s->mhu[i]), "name", name); + qdev_prop_set_uint64(DEVICE(&s->mhu[i]), "size", 0x1000); + object_property_set_bool(OBJECT(&s->mhu[i]), true, + "realized", &err); + if (err) { + error_propagate(errp, err); + return; + } + mr = sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->mhu[i]), 0); + object_property_set_link(OBJECT(&s->apb_ppc0), OBJECT(mr), + port, &err); + if (err) { + error_propagate(errp, err); + return; + } + g_free(name); + g_free(port); + } + } + object_property_set_bool(OBJECT(&s->apb_ppc0), true, "realized", &err); if (err) { error_propagate(errp, err); @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) memory_region_add_subregion(&s->container, 0x40001000, mr); mr = sysbus_mmio_get_region(sbd_apb_ppc0, 2); memory_region_add_subregion(&s->container, 0x40002000, mr); + if (info->has_mhus) { + mr = sysbus_mmio_get_region(sbd_apb_ppc0, 3); + memory_region_add_subregion(&s->container, 0x40003000, mr); + mr = sysbus_mmio_get_region(sbd_apb_ppc0, 4); + memory_region_add_subregion(&s->container, 0x40004000, mr); + } for (i = 0; i < IOTS_APB_PPC0_NUM_PORTS; i++) { qdev_connect_gpio_out_named(dev_secctl, "apb_ppc0_nonsec", i, qdev_get_gpio_in_named(dev_apb_ppc0, -- 2.20.1 Add unimplemented-device stubs for the various Power Policy Unit devices that the SSE-200 has. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-17-peter.maydell@linaro.org --- include/hw/arm/armsse.h | 11 ++++++++ hw/arm/armsse.c | 58 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/include/hw/arm/armsse.h b/include/hw/arm/armsse.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/armsse.h +++ b/include/hw/arm/armsse.h @@ -XXX,XX +XXX,XX @@ #define SSE_MAX_CPUS 2 +/* These define what each PPU in the ppu[] index is for */ +#define CPU0CORE_PPU 0 +#define CPU1CORE_PPU 1 +#define DBG_PPU 2 +#define RAM0_PPU 3 +#define RAM1_PPU 4 +#define RAM2_PPU 5 +#define RAM3_PPU 6 +#define NUM_PPUS 7 + typedef struct ARMSSE { /*< private >*/ SysBusDevice parent_obj; @@ -XXX,XX +XXX,XX @@ typedef struct ARMSSE { IoTKitSysCtl sysinfo; UnimplementedDeviceState mhu[2]; + UnimplementedDeviceState ppu[NUM_PPUS]; /* * 'container' holds all devices seen by all CPUs. diff --git a/hw/arm/armsse.c b/hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armsse.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ struct ARMSSEInfo { uint32_t sys_version; SysConfigFormat sys_config_format; bool has_mhus; + bool has_ppus; }; static const ARMSSEInfo armsse_variants[] = { @@ -XXX,XX +XXX,XX @@ static const ARMSSEInfo armsse_variants[] = { .sys_version = 0x41743, .sys_config_format = IoTKitFormat, .has_mhus = false, + .has_ppus = false, }, }; @@ -XXX,XX +XXX,XX @@ static void armsse_init(Object *obj) sysbus_init_child_obj(obj, "mhu1", &s->mhu[1], sizeof(s->mhu[1]), TYPE_UNIMPLEMENTED_DEVICE); } + if (info->has_ppus) { + for (i = 0; i < info->num_cpus; i++) { + char *name = g_strdup_printf("CPU%dCORE_PPU", i); + int ppuidx = CPU0CORE_PPU + i; + + sysbus_init_child_obj(obj, name, &s->ppu[ppuidx], + sizeof(s->ppu[ppuidx]), + TYPE_UNIMPLEMENTED_DEVICE); + g_free(name); + } + sysbus_init_child_obj(obj, "DBG_PPU", &s->ppu[DBG_PPU], + sizeof(s->ppu[DBG_PPU]), + TYPE_UNIMPLEMENTED_DEVICE); + for (i = 0; i < info->sram_banks; i++) { + char *name = g_strdup_printf("RAM%d_PPU", i); + int ppuidx = RAM0_PPU + i; + + sysbus_init_child_obj(obj, name, &s->ppu[ppuidx], + sizeof(s->ppu[ppuidx]), + TYPE_UNIMPLEMENTED_DEVICE); + g_free(name); + } + } object_initialize_child(obj, "nmi-orgate", &s->nmi_orgate, sizeof(s->nmi_orgate), TYPE_OR_IRQ, &error_abort, NULL); @@ -XXX,XX +XXX,XX @@ static qemu_irq armsse_get_common_irq_in(ARMSSE *s, int irqno) } } +static void map_ppu(ARMSSE *s, int ppuidx, const char *name, hwaddr addr) +{ + /* Map a PPU unimplemented device stub */ + DeviceState *dev = DEVICE(&s->ppu[ppuidx]); + + qdev_prop_set_string(dev, "name", name); + qdev_prop_set_uint64(dev, "size", 0x1000); + qdev_init_nofail(dev); + sysbus_mmio_map(SYS_BUS_DEVICE(&s->ppu[ppuidx]), 0, addr); +} + static void armsse_realize(DeviceState *dev, Error **errp) { ARMSSE *s = ARMSSE(dev); @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) } sysbus_mmio_map(SYS_BUS_DEVICE(&s->sysctl), 0, 0x50021000); + if (info->has_ppus) { + /* CPUnCORE_PPU for each CPU */ + for (i = 0; i < info->num_cpus; i++) { + char *name = g_strdup_printf("CPU%dCORE_PPU", i); + + map_ppu(s, CPU0CORE_PPU + i, name, 0x50023000 + i * 0x2000); + /* + * We don't support CPU debug so don't create the + * CPU0DEBUG_PPU at 0x50024000 and 0x50026000. + */ + g_free(name); + } + map_ppu(s, DBG_PPU, "DBG_PPU", 0x50029000); + + for (i = 0; i < info->sram_banks; i++) { + char *name = g_strdup_printf("RAM%d_PPU", i); + + map_ppu(s, RAM0_PPU + i, name, 0x5002a000 + i * 0x1000); + g_free(name); + } + } + /* This OR gate wires together outputs from the secure watchdogs to NMI */ object_property_set_int(OBJECT(&s->nmi_orgate), 2, "num-lines", &err); if (err) { -- 2.20.1 The SSE-200 gives each CPU a register bank to use to control its L1 instruction cache. Put in an unimplemented-device stub for this. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-18-peter.maydell@linaro.org --- include/hw/arm/armsse.h | 1 + hw/arm/armsse.c | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/include/hw/arm/armsse.h b/include/hw/arm/armsse.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/armsse.h +++ b/include/hw/arm/armsse.h @@ -XXX,XX +XXX,XX @@ typedef struct ARMSSE { UnimplementedDeviceState mhu[2]; UnimplementedDeviceState ppu[NUM_PPUS]; + UnimplementedDeviceState cachectrl[SSE_MAX_CPUS]; /* * 'container' holds all devices seen by all CPUs. diff --git a/hw/arm/armsse.c b/hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armsse.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ struct ARMSSEInfo { SysConfigFormat sys_config_format; bool has_mhus; bool has_ppus; + bool has_cachectrl; }; static const ARMSSEInfo armsse_variants[] = { @@ -XXX,XX +XXX,XX @@ static const ARMSSEInfo armsse_variants[] = { .sys_config_format = IoTKitFormat, .has_mhus = false, .has_ppus = false, + .has_cachectrl = false, }, }; @@ -XXX,XX +XXX,XX @@ static void armsse_init(Object *obj) g_free(name); } } + if (info->has_cachectrl) { + for (i = 0; i < info->num_cpus; i++) { + char *name = g_strdup_printf("cachectrl%d", i); + + sysbus_init_child_obj(obj, name, &s->cachectrl[i], + sizeof(s->cachectrl[i]), + TYPE_UNIMPLEMENTED_DEVICE); + g_free(name); + } + } object_initialize_child(obj, "nmi-orgate", &s->nmi_orgate, sizeof(s->nmi_orgate), TYPE_OR_IRQ, &error_abort, NULL); @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) qdev_connect_gpio_out(DEVICE(&s->ppc_irq_orgate), 0, armsse_get_common_irq_in(s, 10)); - /* 0x40010000 .. 0x4001ffff: private CPU region: unused in IoTKit */ + /* + * 0x40010000 .. 0x4001ffff (and the 0x5001000... secure-only alias): + * private per-CPU region (all these devices are SSE-200 only): + * 0x50010000: L1 icache control registers + * 0x50011000: CPUSECCTRL (CPU local security control registers) + * 0x4001f000 and 0x5001f000: CPU_IDENTITY register block + */ + if (info->has_cachectrl) { + for (i = 0; i < info->num_cpus; i++) { + char *name = g_strdup_printf("cachectrl%d", i); + MemoryRegion *mr; + + qdev_prop_set_string(DEVICE(&s->cachectrl[i]), "name", name); + g_free(name); + qdev_prop_set_uint64(DEVICE(&s->cachectrl[i]), "size", 0x1000); + object_property_set_bool(OBJECT(&s->cachectrl[i]), true, + "realized", &err); + if (err) { + error_propagate(errp, err); + return; + } + + mr = sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->cachectrl[i]), 0); + memory_region_add_subregion(&s->cpu_container[i], 0x50010000, mr); + } + } /* 0x40020000 .. 0x4002ffff : ARMSSE system control peripheral region */ /* Devices behind APB PPC1: -- 2.20.1 The SSE-200 has a "CPU local security control" register bank; add an unimplemented-device stub for it. (The register bank has only one interesting register, which allows the guest to lock down changes to various CPU registers so they cannot be modified further. We don't support that in our Cortex-M33 model anyway.) Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-19-peter.maydell@linaro.org --- include/hw/arm/armsse.h | 1 + hw/arm/armsse.c | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/include/hw/arm/armsse.h b/include/hw/arm/armsse.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/armsse.h +++ b/include/hw/arm/armsse.h @@ -XXX,XX +XXX,XX @@ typedef struct ARMSSE { UnimplementedDeviceState mhu[2]; UnimplementedDeviceState ppu[NUM_PPUS]; UnimplementedDeviceState cachectrl[SSE_MAX_CPUS]; + UnimplementedDeviceState cpusecctrl[SSE_MAX_CPUS]; /* * 'container' holds all devices seen by all CPUs. diff --git a/hw/arm/armsse.c b/hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armsse.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ struct ARMSSEInfo { bool has_mhus; bool has_ppus; bool has_cachectrl; + bool has_cpusecctrl; }; static const ARMSSEInfo armsse_variants[] = { @@ -XXX,XX +XXX,XX @@ static const ARMSSEInfo armsse_variants[] = { .has_mhus = false, .has_ppus = false, .has_cachectrl = false, + .has_cpusecctrl = false, }, }; @@ -XXX,XX +XXX,XX @@ static void armsse_init(Object *obj) g_free(name); } } + if (info->has_cpusecctrl) { + for (i = 0; i < info->num_cpus; i++) { + char *name = g_strdup_printf("cpusecctrl%d", i); + + sysbus_init_child_obj(obj, name, &s->cpusecctrl[i], + sizeof(s->cpusecctrl[i]), + TYPE_UNIMPLEMENTED_DEVICE); + g_free(name); + } + } object_initialize_child(obj, "nmi-orgate", &s->nmi_orgate, sizeof(s->nmi_orgate), TYPE_OR_IRQ, &error_abort, NULL); @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) memory_region_add_subregion(&s->cpu_container[i], 0x50010000, mr); } } + if (info->has_cpusecctrl) { + for (i = 0; i < info->num_cpus; i++) { + char *name = g_strdup_printf("CPUSECCTRL%d", i); + MemoryRegion *mr; + + qdev_prop_set_string(DEVICE(&s->cpusecctrl[i]), "name", name); + g_free(name); + qdev_prop_set_uint64(DEVICE(&s->cpusecctrl[i]), "size", 0x1000); + object_property_set_bool(OBJECT(&s->cpusecctrl[i]), true, + "realized", &err); + if (err) { + error_propagate(errp, err); + return; + } + + mr = sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->cpusecctrl[i]), 0); + memory_region_add_subregion(&s->cpu_container[i], 0x50011000, mr); + } + } /* 0x40020000 .. 0x4002ffff : ARMSSE system control peripheral region */ /* Devices behind APB PPC1: -- 2.20.1 The SSE-200 has a CPU_IDENTITY register block, which is a set of read-only registers. As well as the usual PID/CID registers, there is a single CPUID register which indicates whether the CPU is CPU 0 or CPU 1. Implement a model of this register block. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-20-peter.maydell@linaro.org --- hw/misc/Makefile.objs | 1 + include/hw/misc/armsse-cpuid.h | 41 ++++++++++ hw/misc/armsse-cpuid.c | 134 ++++++++++++++++++++++++++++++++ MAINTAINERS | 2 + default-configs/arm-softmmu.mak | 1 + hw/misc/trace-events | 4 + 6 files changed, 183 insertions(+) create mode 100644 include/hw/misc/armsse-cpuid.h create mode 100644 hw/misc/armsse-cpuid.c diff --git a/hw/misc/Makefile.objs b/hw/misc/Makefile.objs index XXXXXXX..XXXXXXX 100644 --- a/hw/misc/Makefile.objs +++ b/hw/misc/Makefile.objs @@ -XXX,XX +XXX,XX @@ obj-$(CONFIG_TZ_PPC) += tz-ppc.o obj-$(CONFIG_IOTKIT_SECCTL) += iotkit-secctl.o obj-$(CONFIG_IOTKIT_SYSCTL) += iotkit-sysctl.o obj-$(CONFIG_IOTKIT_SYSINFO) += iotkit-sysinfo.o +obj-$(CONFIG_ARMSSE_CPUID) += armsse-cpuid.o obj-$(CONFIG_PVPANIC) += pvpanic.o obj-$(CONFIG_AUX) += auxbus.o diff --git a/include/hw/misc/armsse-cpuid.h b/include/hw/misc/armsse-cpuid.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/include/hw/misc/armsse-cpuid.h @@ -XXX,XX +XXX,XX @@ +/* + * ARM SSE-200 CPU_IDENTITY register block + * + * Copyright (c) 2019 Linaro Limited + * Written by Peter Maydell + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 or + * (at your option) any later version. + */ + +/* + * This is a model of the "CPU_IDENTITY" register block which is part of the + * Arm SSE-200 and documented in + * http://infocenter.arm.com/help/topic/com.arm.doc.101104_0100_00_en/corelink_sse200_subsystem_for_embedded_technical_reference_manual_101104_0100_00_en.pdf + * + * QEMU interface: + * + QOM property "CPUID": the value to use for the CPUID register + * + sysbus MMIO region 0: the system information register bank + */ + +#ifndef HW_MISC_ARMSSE_CPUID_H +#define HW_MISC_ARMSSE_CPUID_H + +#include "hw/sysbus.h" + +#define TYPE_ARMSSE_CPUID "armsse-cpuid" +#define ARMSSE_CPUID(obj) OBJECT_CHECK(ARMSSECPUID, (obj), TYPE_ARMSSE_CPUID) + +typedef struct ARMSSECPUID { + /*< private >*/ + SysBusDevice parent_obj; + + /*< public >*/ + MemoryRegion iomem; + + /* Properties */ + uint32_t cpuid; +} ARMSSECPUID; + +#endif diff --git a/hw/misc/armsse-cpuid.c b/hw/misc/armsse-cpuid.c new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/hw/misc/armsse-cpuid.c @@ -XXX,XX +XXX,XX @@ +/* + * ARM SSE-200 CPU_IDENTITY register block + * + * Copyright (c) 2019 Linaro Limited + * Written by Peter Maydell + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 or + * (at your option) any later version. + */ + +/* + * This is a model of the "CPU_IDENTITY" register block which is part of the + * Arm SSE-200 and documented in + * http://infocenter.arm.com/help/topic/com.arm.doc.101104_0100_00_en/corelink_sse200_subsystem_for_embedded_technical_reference_manual_101104_0100_00_en.pdf + * + * It consists of one read-only CPUID register (set by QOM property), plus the + * usual ID registers. + */ + +#include "qemu/osdep.h" +#include "qemu/log.h" +#include "trace.h" +#include "qapi/error.h" +#include "sysemu/sysemu.h" +#include "hw/sysbus.h" +#include "hw/registerfields.h" +#include "hw/misc/armsse-cpuid.h" + +REG32(CPUID, 0x0) +REG32(PID4, 0xfd0) +REG32(PID5, 0xfd4) +REG32(PID6, 0xfd8) +REG32(PID7, 0xfdc) +REG32(PID0, 0xfe0) +REG32(PID1, 0xfe4) +REG32(PID2, 0xfe8) +REG32(PID3, 0xfec) +REG32(CID0, 0xff0) +REG32(CID1, 0xff4) +REG32(CID2, 0xff8) +REG32(CID3, 0xffc) + +/* PID/CID values */ +static const int sysinfo_id[] = { + 0x04, 0x00, 0x00, 0x00, /* PID4..PID7 */ + 0x58, 0xb8, 0x0b, 0x00, /* PID0..PID3 */ + 0x0d, 0xf0, 0x05, 0xb1, /* CID0..CID3 */ +}; + +static uint64_t armsse_cpuid_read(void *opaque, hwaddr offset, + unsigned size) +{ + ARMSSECPUID *s = ARMSSE_CPUID(opaque); + uint64_t r; + + switch (offset) { + case A_CPUID: + r = s->cpuid; + break; + case A_PID4 ... A_CID3: + r = sysinfo_id[(offset - A_PID4) / 4]; + break; + default: + qemu_log_mask(LOG_GUEST_ERROR, + "SSE CPU_IDENTITY read: bad offset 0x%x\n", (int)offset); + r = 0; + break; + } + trace_armsse_cpuid_read(offset, r, size); + return r; +} + +static void armsse_cpuid_write(void *opaque, hwaddr offset, + uint64_t value, unsigned size) +{ + trace_armsse_cpuid_write(offset, value, size); + + qemu_log_mask(LOG_GUEST_ERROR, + "SSE CPU_IDENTITY: write to RO offset 0x%x\n", (int)offset); +} + +static const MemoryRegionOps armsse_cpuid_ops = { + .read = armsse_cpuid_read, + .write = armsse_cpuid_write, + .endianness = DEVICE_LITTLE_ENDIAN, + /* byte/halfword accesses are just zero-padded on reads and writes */ + .impl.min_access_size = 4, + .impl.max_access_size = 4, + .valid.min_access_size = 1, + .valid.max_access_size = 4, +}; + +static Property armsse_cpuid_props[] = { + DEFINE_PROP_UINT32("CPUID", ARMSSECPUID, cpuid, 0), + DEFINE_PROP_END_OF_LIST() +}; + +static void armsse_cpuid_init(Object *obj) +{ + SysBusDevice *sbd = SYS_BUS_DEVICE(obj); + ARMSSECPUID *s = ARMSSE_CPUID(obj); + + memory_region_init_io(&s->iomem, obj, &armsse_cpuid_ops, + s, "armsse-cpuid", 0x1000); + sysbus_init_mmio(sbd, &s->iomem); +} + +static void armsse_cpuid_class_init(ObjectClass *klass, void *data) +{ + DeviceClass *dc = DEVICE_CLASS(klass); + + /* + * This device has no guest-modifiable state and so it + * does not need a reset function or VMState. + */ + + dc->props = armsse_cpuid_props; +} + +static const TypeInfo armsse_cpuid_info = { + .name = TYPE_ARMSSE_CPUID, + .parent = TYPE_SYS_BUS_DEVICE, + .instance_size = sizeof(ARMSSECPUID), + .instance_init = armsse_cpuid_init, + .class_init = armsse_cpuid_class_init, +}; + +static void armsse_cpuid_register_types(void) +{ + type_register_static(&armsse_cpuid_info); +} + +type_init(armsse_cpuid_register_types); diff --git a/MAINTAINERS b/MAINTAINERS index XXXXXXX..XXXXXXX 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -XXX,XX +XXX,XX @@ F: hw/misc/iotkit-sysctl.c F: include/hw/misc/iotkit-sysctl.h F: hw/misc/iotkit-sysinfo.c F: include/hw/misc/iotkit-sysinfo.h +F: hw/misc/armsse-cpuid.c +F: include/hw/misc/armsse-cpuid.h Musicpal M: Jan Kiszka <jan.kiszka@web.de> diff --git a/default-configs/arm-softmmu.mak b/default-configs/arm-softmmu.mak index XXXXXXX..XXXXXXX 100644 --- a/default-configs/arm-softmmu.mak +++ b/default-configs/arm-softmmu.mak @@ -XXX,XX +XXX,XX @@ CONFIG_ARMSSE=y CONFIG_IOTKIT_SECCTL=y CONFIG_IOTKIT_SYSCTL=y CONFIG_IOTKIT_SYSINFO=y +CONFIG_ARMSSE_CPUID=y CONFIG_VERSATILE=y CONFIG_VERSATILE_PCI=y diff --git a/hw/misc/trace-events b/hw/misc/trace-events index XXXXXXX..XXXXXXX 100644 --- a/hw/misc/trace-events +++ b/hw/misc/trace-events @@ -XXX,XX +XXX,XX @@ iotkit_sysinfo_write(uint64_t offset, uint64_t data, unsigned size) "IoTKit SysI iotkit_sysctl_read(uint64_t offset, uint64_t data, unsigned size) "IoTKit SysCtl read: offset 0x%" PRIx64 " data 0x%" PRIx64 " size %u" iotkit_sysctl_write(uint64_t offset, uint64_t data, unsigned size) "IoTKit SysCtl write: offset 0x%" PRIx64 " data 0x%" PRIx64 " size %u" iotkit_sysctl_reset(void) "IoTKit SysCtl: reset" + +# hw/misc/armsse-cpuid.c +armsse_cpuid_read(uint64_t offset, uint64_t data, unsigned size) "SSE-200 CPU_IDENTITY read: offset 0x%" PRIx64 " data 0x%" PRIx64 " size %u" +armsse_cpuid_write(uint64_t offset, uint64_t data, unsigned size) "SSE-200 CPU_IDENTITY write: offset 0x%" PRIx64 " data 0x%" PRIx64 " size %u" -- 2.20.1 Instantiate a copy of the CPU_IDENTITY register block for each CPU in an SSE-200. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-21-peter.maydell@linaro.org --- include/hw/arm/armsse.h | 3 +++ hw/arm/armsse.c | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/include/hw/arm/armsse.h b/include/hw/arm/armsse.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/armsse.h +++ b/include/hw/arm/armsse.h @@ -XXX,XX +XXX,XX @@ #include "hw/watchdog/cmsdk-apb-watchdog.h" #include "hw/misc/iotkit-sysctl.h" #include "hw/misc/iotkit-sysinfo.h" +#include "hw/misc/armsse-cpuid.h" #include "hw/misc/unimp.h" #include "hw/or-irq.h" #include "hw/core/split-irq.h" @@ -XXX,XX +XXX,XX @@ typedef struct ARMSSE { UnimplementedDeviceState cachectrl[SSE_MAX_CPUS]; UnimplementedDeviceState cpusecctrl[SSE_MAX_CPUS]; + ARMSSECPUID cpuid[SSE_MAX_CPUS]; + /* * 'container' holds all devices seen by all CPUs. * 'cpu_container[i]' is the view that CPU i has: this has the diff --git a/hw/arm/armsse.c b/hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armsse.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ struct ARMSSEInfo { bool has_ppus; bool has_cachectrl; bool has_cpusecctrl; + bool has_cpuid; }; static const ARMSSEInfo armsse_variants[] = { @@ -XXX,XX +XXX,XX @@ static const ARMSSEInfo armsse_variants[] = { .has_ppus = false, .has_cachectrl = false, .has_cpusecctrl = false, + .has_cpuid = false, }, }; @@ -XXX,XX +XXX,XX @@ static void armsse_init(Object *obj) g_free(name); } } + if (info->has_cpuid) { + for (i = 0; i < info->num_cpus; i++) { + char *name = g_strdup_printf("cpuid%d", i); + + sysbus_init_child_obj(obj, name, &s->cpuid[i], + sizeof(s->cpuid[i]), + TYPE_ARMSSE_CPUID); + g_free(name); + } + } object_initialize_child(obj, "nmi-orgate", &s->nmi_orgate, sizeof(s->nmi_orgate), TYPE_OR_IRQ, &error_abort, NULL); @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) memory_region_add_subregion(&s->cpu_container[i], 0x50011000, mr); } } + if (info->has_cpuid) { + for (i = 0; i < info->num_cpus; i++) { + MemoryRegion *mr; + + qdev_prop_set_uint32(DEVICE(&s->cpuid[i]), "CPUID", i); + object_property_set_bool(OBJECT(&s->cpuid[i]), true, + "realized", &err); + if (err) { + error_propagate(errp, err); + return; + } + + mr = sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->cpuid[i]), 0); + memory_region_add_subregion(&s->cpu_container[i], 0x4001F000, mr); + } + } /* 0x40020000 .. 0x4002ffff : ARMSSE system control peripheral region */ /* Devices behind APB PPC1: -- 2.20.1 Add a model of the SSE-200, now we have put in all the code that lets us make it different from the IoTKit. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-22-peter.maydell@linaro.org --- include/hw/arm/armsse.h | 19 ++++++++++++++++--- hw/arm/armsse.c | 12 ++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/include/hw/arm/armsse.h b/include/hw/arm/armsse.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/armsse.h +++ b/include/hw/arm/armsse.h @@ -XXX,XX +XXX,XX @@ /* - * ARM SSE (Subsystems for Embedded): IoTKit + * ARM SSE (Subsystems for Embedded): IoTKit, SSE-200 * * Copyright (c) 2018 Linaro Limited * Written by Peter Maydell @@ -XXX,XX +XXX,XX @@ /* * This is a model of the Arm "Subsystems for Embedded" family of * hardware, which include the IoT Kit and the SSE-050, SSE-100 and - * SSE-200. Currently we model only the Arm IoT Kit which is documented in + * SSE-200. Currently we model: + * - the Arm IoT Kit which is documented in * http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ecm0601256/index.html - * It contains: + * - the SSE-200 which is documented in + * http://infocenter.arm.com/help/topic/com.arm.doc.101104_0100_00_en/corelink_sse200_subsystem_for_embedded_technical_reference_manual_101104_0100_00_en.pdf + * + * The IoTKit contains: * a Cortex-M33 * the IDAU * some timers and watchdogs @@ -XXX,XX +XXX,XX @@ * a security controller * a bus fabric which arranges that some parts of the address * space are secure and non-secure aliases of each other + * The SSE-200 additionally contains: + * a second Cortex-M33 + * two Message Handling Units (MHUs) + * an optional CryptoCell (which we do not model) + * more SRAM banks with associated MPCs + * multiple Power Policy Units (PPUs) + * a control interface for an icache for each CPU + * per-CPU identity and control register blocks * * QEMU interface: * + QOM property "memory" is a MemoryRegion containing the devices provided @@ -XXX,XX +XXX,XX @@ * them via the ARMSSE base class, so they have no IOTKIT() etc macros. */ #define TYPE_IOTKIT "iotkit" +#define TYPE_SSE200 "sse-200" /* We have an IRQ splitter and an OR gate input for each external PPC * and the 2 internal PPCs diff --git a/hw/arm/armsse.c b/hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armsse.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ static const ARMSSEInfo armsse_variants[] = { .has_cpusecctrl = false, .has_cpuid = false, }, + { + .name = TYPE_SSE200, + .sram_banks = 4, + .num_cpus = 2, + .sys_version = 0x22041743, + .sys_config_format = SSE200Format, + .has_mhus = true, + .has_ppus = true, + .has_cachectrl = true, + .has_cpusecctrl = true, + .has_cpuid = true, + }, }; static uint32_t armsse_sys_config_value(ARMSSE *s, const ARMSSEInfo *info) -- 2.20.1 In preparation for adding support for the AN521 MPS2 image, we need to handle wiring up the MPS2 device interrupt lines to both CPUs in the SSE-200, rather than just the one that the IoTKit has. Abstract out a "connect to the IoTKit interrupt line" function and make it connect to a splitter which feeds both sets of inputs for the SSE-200 case. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-23-peter.maydell@linaro.org --- hw/arm/mps2-tz.c | 79 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 20 deletions(-) diff --git a/hw/arm/mps2-tz.c b/hw/arm/mps2-tz.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/mps2-tz.c +++ b/hw/arm/mps2-tz.c @@ -XXX,XX +XXX,XX @@ #include "net/net.h" #include "hw/core/split-irq.h" +#define MPS2TZ_NUMIRQ 92 + typedef enum MPS2TZFPGAType { FPGA_AN505, + FPGA_AN521, } MPS2TZFPGAType; typedef struct { @@ -XXX,XX +XXX,XX @@ typedef struct { SplitIRQ sec_resp_splitter; qemu_or_irq uart_irq_orgate; DeviceState *lan9118; + SplitIRQ cpu_irq_splitter[MPS2TZ_NUMIRQ]; } MPS2TZMachineState; #define TYPE_MPS2TZ_MACHINE "mps2tz" @@ -XXX,XX +XXX,XX @@ static void make_ram_alias(MemoryRegion *mr, const char *name, memory_region_add_subregion(get_system_memory(), base, mr); } +static qemu_irq get_sse_irq_in(MPS2TZMachineState *mms, int irqno) +{ + /* Return a qemu_irq which will signal IRQ n to all CPUs in the SSE. */ + MPS2TZMachineClass *mmc = MPS2TZ_MACHINE_GET_CLASS(mms); + + assert(irqno < MPS2TZ_NUMIRQ); + + switch (mmc->fpga_type) { + case FPGA_AN505: + return qdev_get_gpio_in_named(DEVICE(&mms->iotkit), "EXP_IRQ", irqno); + case FPGA_AN521: + return qdev_get_gpio_in(DEVICE(&mms->cpu_irq_splitter[irqno]), 0); + default: + g_assert_not_reached(); + } +} + /* Most of the devices in the AN505 FPGA image sit behind * Peripheral Protection Controllers. These data structures * define the layout of which devices sit behind which PPCs. @@ -XXX,XX +XXX,XX @@ static MemoryRegion *make_uart(MPS2TZMachineState *mms, void *opaque, int txirqno = i * 2 + 1; int combirqno = i + 10; SysBusDevice *s; - DeviceState *iotkitdev = DEVICE(&mms->iotkit); DeviceState *orgate_dev = DEVICE(&mms->uart_irq_orgate); sysbus_init_child_obj(OBJECT(mms), name, uart, sizeof(mms->uart[0]), @@ -XXX,XX +XXX,XX @@ static MemoryRegion *make_uart(MPS2TZMachineState *mms, void *opaque, qdev_prop_set_uint32(DEVICE(uart), "pclk-frq", SYSCLK_FRQ); object_property_set_bool(OBJECT(uart), true, "realized", &error_fatal); s = SYS_BUS_DEVICE(uart); - sysbus_connect_irq(s, 0, qdev_get_gpio_in_named(iotkitdev, - "EXP_IRQ", txirqno)); - sysbus_connect_irq(s, 1, qdev_get_gpio_in_named(iotkitdev, - "EXP_IRQ", rxirqno)); + sysbus_connect_irq(s, 0, get_sse_irq_in(mms, txirqno)); + sysbus_connect_irq(s, 1, get_sse_irq_in(mms, rxirqno)); sysbus_connect_irq(s, 2, qdev_get_gpio_in(orgate_dev, i * 2)); sysbus_connect_irq(s, 3, qdev_get_gpio_in(orgate_dev, i * 2 + 1)); - sysbus_connect_irq(s, 4, qdev_get_gpio_in_named(iotkitdev, - "EXP_IRQ", combirqno)); + sysbus_connect_irq(s, 4, get_sse_irq_in(mms, combirqno)); return sysbus_mmio_get_region(SYS_BUS_DEVICE(uart), 0); } @@ -XXX,XX +XXX,XX @@ static MemoryRegion *make_eth_dev(MPS2TZMachineState *mms, void *opaque, const char *name, hwaddr size) { SysBusDevice *s; - DeviceState *iotkitdev = DEVICE(&mms->iotkit); NICInfo *nd = &nd_table[0]; /* In hardware this is a LAN9220; the LAN9118 is software compatible @@ -XXX,XX +XXX,XX @@ static MemoryRegion *make_eth_dev(MPS2TZMachineState *mms, void *opaque, qdev_init_nofail(mms->lan9118); s = SYS_BUS_DEVICE(mms->lan9118); - sysbus_connect_irq(s, 0, qdev_get_gpio_in_named(iotkitdev, "EXP_IRQ", 16)); + sysbus_connect_irq(s, 0, get_sse_irq_in(mms, 16)); return sysbus_mmio_get_region(s, 0); } @@ -XXX,XX +XXX,XX @@ static MemoryRegion *make_dma(MPS2TZMachineState *mms, void *opaque, s = SYS_BUS_DEVICE(dma); /* Wire up DMACINTR, DMACINTERR, DMACINTTC */ - sysbus_connect_irq(s, 0, qdev_get_gpio_in_named(iotkitdev, - "EXP_IRQ", 58 + i * 3)); - sysbus_connect_irq(s, 1, qdev_get_gpio_in_named(iotkitdev, - "EXP_IRQ", 56 + i * 3)); - sysbus_connect_irq(s, 2, qdev_get_gpio_in_named(iotkitdev, - "EXP_IRQ", 57 + i * 3)); + sysbus_connect_irq(s, 0, get_sse_irq_in(mms, 58 + i * 3)); + sysbus_connect_irq(s, 1, get_sse_irq_in(mms, 56 + i * 3)); + sysbus_connect_irq(s, 2, get_sse_irq_in(mms, 57 + i * 3)); g_free(mscname); return sysbus_mmio_get_region(s, 0); @@ -XXX,XX +XXX,XX @@ static MemoryRegion *make_spi(MPS2TZMachineState *mms, void *opaque, */ PL022State *spi = opaque; int i = spi - &mms->spi[0]; - DeviceState *iotkitdev = DEVICE(&mms->iotkit); SysBusDevice *s; sysbus_init_child_obj(OBJECT(mms), name, spi, sizeof(mms->spi[0]), TYPE_PL022); object_property_set_bool(OBJECT(spi), true, "realized", &error_fatal); s = SYS_BUS_DEVICE(spi); - sysbus_connect_irq(s, 0, - qdev_get_gpio_in_named(iotkitdev, "EXP_IRQ", 51 + i)); + sysbus_connect_irq(s, 0, get_sse_irq_in(mms, 51 + i)); return sysbus_mmio_get_region(s, 0); } static void mps2tz_common_init(MachineState *machine) { MPS2TZMachineState *mms = MPS2TZ_MACHINE(machine); + MPS2TZMachineClass *mmc = MPS2TZ_MACHINE_GET_CLASS(mms); MachineClass *mc = MACHINE_GET_CLASS(machine); MemoryRegion *system_memory = get_system_memory(); DeviceState *iotkitdev; @@ -XXX,XX +XXX,XX @@ static void mps2tz_common_init(MachineState *machine) iotkitdev = DEVICE(&mms->iotkit); object_property_set_link(OBJECT(&mms->iotkit), OBJECT(system_memory), "memory", &error_abort); - qdev_prop_set_uint32(iotkitdev, "EXP_NUMIRQ", 92); + qdev_prop_set_uint32(iotkitdev, "EXP_NUMIRQ", MPS2TZ_NUMIRQ); qdev_prop_set_uint32(iotkitdev, "MAINCLK", SYSCLK_FRQ); object_property_set_bool(OBJECT(&mms->iotkit), true, "realized", &error_fatal); + /* + * The AN521 needs us to create splitters to feed the IRQ inputs + * for each CPU in the SSE-200 from each device in the board. + */ + if (mmc->fpga_type == FPGA_AN521) { + for (i = 0; i < MPS2TZ_NUMIRQ; i++) { + char *name = g_strdup_printf("mps2-irq-splitter%d", i); + SplitIRQ *splitter = &mms->cpu_irq_splitter[i]; + + object_initialize_child(OBJECT(machine), name, + splitter, sizeof(*splitter), + TYPE_SPLIT_IRQ, &error_fatal, NULL); + g_free(name); + + object_property_set_int(OBJECT(splitter), 2, "num-lines", + &error_fatal); + object_property_set_bool(OBJECT(splitter), true, "realized", + &error_fatal); + qdev_connect_gpio_out(DEVICE(splitter), 0, + qdev_get_gpio_in_named(DEVICE(&mms->iotkit), + "EXP_IRQ", i)); + qdev_connect_gpio_out(DEVICE(splitter), 1, + qdev_get_gpio_in_named(DEVICE(&mms->iotkit), + "EXP_CPU1_IRQ", i)); + } + } + /* The sec_resp_cfg output from the IoTKit must be split into multiple * lines, one for each of the PPCs we create here, plus one per MSC. */ @@ -XXX,XX +XXX,XX @@ static void mps2tz_common_init(MachineState *machine) object_property_set_bool(OBJECT(&mms->uart_irq_orgate), true, "realized", &error_fatal); qdev_connect_gpio_out(DEVICE(&mms->uart_irq_orgate), 0, - qdev_get_gpio_in_named(iotkitdev, "EXP_IRQ", 15)); + get_sse_irq_in(mms, 15)); /* Most of the devices in the FPGA are behind Peripheral Protection * Controllers. The required order for initializing things is: -- 2.20.1 Add a model of the MPS2 FPGA image described in Application Note AN521. This is identical to the AN505 image, except that it uses the SSE-200 rather than the IoTKit and so has two Cortex-M33 CPUs. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190121185118.18550-24-peter.maydell@linaro.org --- hw/arm/mps2-tz.c | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/hw/arm/mps2-tz.c b/hw/arm/mps2-tz.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/mps2-tz.c +++ b/hw/arm/mps2-tz.c @@ -XXX,XX +XXX,XX @@ * as seen by the guest depend significantly on the FPGA image. * This source file covers the following FPGA images, for TrustZone cores: * "mps2-an505" -- Cortex-M33 as documented in ARM Application Note AN505 + * "mps2-an521" -- Dual Cortex-M33 as documented in Application Note AN521 * * Links to the TRM for the board itself and to the various Application * Notes which document the FPGA images can be found here: @@ -XXX,XX +XXX,XX @@ * http://infocenter.arm.com/help/topic/com.arm.doc.100112_0200_06_en/versatile_express_cortex_m_prototyping_systems_v2m_mps2_and_v2m_mps2plus_technical_reference_100112_0200_06_en.pdf * Application Note AN505: * http://infocenter.arm.com/help/topic/com.arm.doc.dai0505b/index.html + * Application Note AN521: + * http://infocenter.arm.com/help/topic/com.arm.doc.dai0521c/index.html * * The AN505 defers to the Cortex-M33 processor ARMv8M IoT Kit FVP User Guide * (ARM ECM0601256) for the details of some of the device layout: * http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ecm0601256/index.html + * Similarly, the AN521 uses the SSE-200, and the SSE-200 TRM defines + * most of the device layout: + * http://infocenter.arm.com/help/topic/com.arm.doc.101104_0100_00_en/corelink_sse200_subsystem_for_embedded_technical_reference_manual_101104_0100_00_en.pdf + * */ #include "qemu/osdep.h" @@ -XXX,XX +XXX,XX @@ typedef struct { MachineClass parent; MPS2TZFPGAType fpga_type; uint32_t scc_id; + const char *armsse_type; } MPS2TZMachineClass; typedef struct { @@ -XXX,XX +XXX,XX @@ typedef struct { #define TYPE_MPS2TZ_MACHINE "mps2tz" #define TYPE_MPS2TZ_AN505_MACHINE MACHINE_TYPE_NAME("mps2-an505") +#define TYPE_MPS2TZ_AN521_MACHINE MACHINE_TYPE_NAME("mps2-an521") #define MPS2TZ_MACHINE(obj) \ OBJECT_CHECK(MPS2TZMachineState, obj, TYPE_MPS2TZ_MACHINE) @@ -XXX,XX +XXX,XX @@ static void mps2tz_common_init(MachineState *machine) } sysbus_init_child_obj(OBJECT(machine), "iotkit", &mms->iotkit, - sizeof(mms->iotkit), TYPE_IOTKIT); + sizeof(mms->iotkit), mmc->armsse_type); iotkitdev = DEVICE(&mms->iotkit); object_property_set_link(OBJECT(&mms->iotkit), OBJECT(system_memory), "memory", &error_abort); @@ -XXX,XX +XXX,XX @@ static void mps2tz_class_init(ObjectClass *oc, void *data) IDAUInterfaceClass *iic = IDAU_INTERFACE_CLASS(oc); mc->init = mps2tz_common_init; - mc->max_cpus = 1; iic->check = mps2_tz_idau_check; } @@ -XXX,XX +XXX,XX @@ static void mps2tz_an505_class_init(ObjectClass *oc, void *data) MPS2TZMachineClass *mmc = MPS2TZ_MACHINE_CLASS(oc); mc->desc = "ARM MPS2 with AN505 FPGA image for Cortex-M33"; + mc->default_cpus = 1; + mc->min_cpus = mc->default_cpus; + mc->max_cpus = mc->default_cpus; mmc->fpga_type = FPGA_AN505; mc->default_cpu_type = ARM_CPU_TYPE_NAME("cortex-m33"); mmc->scc_id = 0x41045050; + mmc->armsse_type = TYPE_IOTKIT; +} + +static void mps2tz_an521_class_init(ObjectClass *oc, void *data) +{ + MachineClass *mc = MACHINE_CLASS(oc); + MPS2TZMachineClass *mmc = MPS2TZ_MACHINE_CLASS(oc); + + mc->desc = "ARM MPS2 with AN521 FPGA image for dual Cortex-M33"; + mc->default_cpus = 2; + mc->min_cpus = mc->default_cpus; + mc->max_cpus = mc->default_cpus; + mmc->fpga_type = FPGA_AN521; + mc->default_cpu_type = ARM_CPU_TYPE_NAME("cortex-m33"); + mmc->scc_id = 0x41045210; + mmc->armsse_type = TYPE_SSE200; } static const TypeInfo mps2tz_info = { @@ -XXX,XX +XXX,XX @@ static const TypeInfo mps2tz_an505_info = { .class_init = mps2tz_an505_class_init, }; +static const TypeInfo mps2tz_an521_info = { + .name = TYPE_MPS2TZ_AN521_MACHINE, + .parent = TYPE_MPS2TZ_MACHINE, + .class_init = mps2tz_an521_class_init, +}; + static void mps2tz_machine_init(void) { type_register_static(&mps2tz_info); type_register_static(&mps2tz_an505_info); + type_register_static(&mps2tz_an521_info); } type_init(mps2tz_machine_init); -- 2.20.1 The "system instructions" and "system register move" subcategories of "branches, exception generating and system instructions" for A64 only apply if bits [23:22] are zero; other values are currently unallocated. Correctly UNDEF these unallocated encodings. Reported-by: Laurent Desnogues <laurent.desnogues@gmail.com> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Laurent Desnogues <laurent.desnogues@gmail.com> Message-id: 20190125182626.9221-2-peter.maydell@linaro.org --- target/arm/translate-a64.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void disas_b_exc_sys(DisasContext *s, uint32_t insn) break; case 0x6a: /* Exception generation / System */ if (insn & (1 << 24)) { - disas_system(s, insn); + if (extract32(insn, 22, 2) == 0) { + disas_system(s, insn); + } else { + unallocated_encoding(s); + } } else { disas_exc(s, insn); } -- 2.20.1 The PRFM prefetch insn in the load/store with imm9 encodings requires idx field 0b00; we were underdecoding this by only checking !is_unpriv (which is equivalent to idx != 2). Correctly UNDEF the unallocated encodings where idx == 0b01 and 0b11 as well as 0b10. Reported-by: Laurent Desnogues <laurent.desnogues@gmail.com> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Laurent Desnogues <laurent.desnogues@gmail.com> Message-id: 20190125182626.9221-3-peter.maydell@linaro.org --- target/arm/translate-a64.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void disas_ldst_reg_imm9(DisasContext *s, uint32_t insn, } else { if (size == 3 && opc == 2) { /* PRFM - prefetch */ - if (is_unpriv) { + if (idx != 0) { unallocated_encoding(s); return; } -- 2.20.1 In the AdvSIMD load/store multiple structures encodings, the non-post-indexed case should have zeroes in [20:16] (which is the Rm field for the post-indexed case). Correctly UNDEF the currently unallocated encodings which have non-zeroes in those bits. Reported-by: Laurent Desnogues <laurent.desnogues@gmail.com> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Laurent Desnogues <laurent.desnogues@gmail.com> Message-id: 20190125182626.9221-4-peter.maydell@linaro.org --- target/arm/translate-a64.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void disas_ldst_multiple_struct(DisasContext *s, uint32_t insn) { int rt = extract32(insn, 0, 5); int rn = extract32(insn, 5, 5); + int rm = extract32(insn, 16, 5); int size = extract32(insn, 10, 2); int opcode = extract32(insn, 12, 4); bool is_store = !extract32(insn, 22, 1); @@ -XXX,XX +XXX,XX @@ static void disas_ldst_multiple_struct(DisasContext *s, uint32_t insn) return; } + if (!is_postidx && rm != 0) { + unallocated_encoding(s); + return; + } + /* From the shared decode logic */ switch (opcode) { case 0x0: @@ -XXX,XX +XXX,XX @@ static void disas_ldst_multiple_struct(DisasContext *s, uint32_t insn) } if (is_postidx) { - int rm = extract32(insn, 16, 5); if (rm == 31) { tcg_gen_mov_i64(tcg_rn, tcg_addr); } else { -- 2.20.1 In the AdvSIMD load/store single structure encodings, the non-post-indexed case should have zeroes in [20:16] (which is the Rm field for the post-indexed case). Bit 31 must also be zero (a check we got right in ldst_multiple but not here). Correctly UNDEF these unallocated encodings. Reported-by: Laurent Desnogues <laurent.desnogues@gmail.com> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Laurent Desnogues <laurent.desnogues@gmail.com> Message-id: 20190125182626.9221-5-peter.maydell@linaro.org --- target/arm/translate-a64.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void disas_ldst_single_struct(DisasContext *s, uint32_t insn) { int rt = extract32(insn, 0, 5); int rn = extract32(insn, 5, 5); + int rm = extract32(insn, 16, 5); int size = extract32(insn, 10, 2); int S = extract32(insn, 12, 1); int opc = extract32(insn, 13, 3); @@ -XXX,XX +XXX,XX @@ static void disas_ldst_single_struct(DisasContext *s, uint32_t insn) int ebytes, xs; TCGv_i64 tcg_addr, tcg_rn, tcg_ebytes; + if (extract32(insn, 31, 1)) { + unallocated_encoding(s); + return; + } + if (!is_postidx && rm != 0) { + unallocated_encoding(s); + return; + } + switch (scale) { case 3: if (!is_load || S) { @@ -XXX,XX +XXX,XX @@ static void disas_ldst_single_struct(DisasContext *s, uint32_t insn) } if (is_postidx) { - int rm = extract32(insn, 16, 5); if (rm == 31) { tcg_gen_mov_i64(tcg_rn, tcg_addr); } else { -- 2.20.1 In the "add/subtract (extended register)" encoding group, the "opt" field in bits [23:22] must be zero. Correctly UNDEF the unallocated encodings where this field is not zero. Reported-by: Laurent Desnogues <laurent.desnogues@gmail.com> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Laurent Desnogues <laurent.desnogues@gmail.com> Message-id: 20190125182626.9221-6-peter.maydell@linaro.org --- target/arm/translate-a64.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void disas_add_sub_ext_reg(DisasContext *s, uint32_t insn) int imm3 = extract32(insn, 10, 3); int option = extract32(insn, 13, 3); int rm = extract32(insn, 16, 5); + int opt = extract32(insn, 22, 2); bool setflags = extract32(insn, 29, 1); bool sub_op = extract32(insn, 30, 1); bool sf = extract32(insn, 31, 1); @@ -XXX,XX +XXX,XX @@ static void disas_add_sub_ext_reg(DisasContext *s, uint32_t insn) TCGv_i64 tcg_rd; TCGv_i64 tcg_result; - if (imm3 > 4) { + if (imm3 > 4 || opt != 0) { unallocated_encoding(s); return; } -- 2.20.1 In the encoding groups * floating-point data-processing (1 source) * floating-point data-processing (2 source) * floating-point data-processing (3 source) * floating-point immediate * floating-point compare * floating-ponit conditional compare * floating-point conditional select bit 31 is M and bit 29 is S (and bit 30 is 0, already checked at this point in the decode). None of these groups allocate any encoding for M=1 or S=1. We checked this in disas_fp_compare(), disas_fp_ccomp() and disas_fp_csel(), but missed it in disas_fp_1src(), disas_fp_2src(), disas_fp_3src() and disas_fp_imm(). We also missed that in the fp immediate encoding the imm5 field must be all zeroes. Correctly UNDEF the unallocated encodings here. Reported-by: Laurent Desnogues <laurent.desnogues@gmail.com> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Laurent Desnogues <laurent.desnogues@gmail.com> Message-id: 20190125182626.9221-7-peter.maydell@linaro.org --- target/arm/translate-a64.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void handle_fp_fcvt(DisasContext *s, int opcode, */ static void disas_fp_1src(DisasContext *s, uint32_t insn) { + int mos = extract32(insn, 29, 3); int type = extract32(insn, 22, 2); int opcode = extract32(insn, 15, 6); int rn = extract32(insn, 5, 5); int rd = extract32(insn, 0, 5); + if (mos) { + unallocated_encoding(s); + return; + } + switch (opcode) { case 0x4: case 0x5: case 0x7: { @@ -XXX,XX +XXX,XX @@ static void handle_fp_2src_half(DisasContext *s, int opcode, */ static void disas_fp_2src(DisasContext *s, uint32_t insn) { + int mos = extract32(insn, 29, 3); int type = extract32(insn, 22, 2); int rd = extract32(insn, 0, 5); int rn = extract32(insn, 5, 5); int rm = extract32(insn, 16, 5); int opcode = extract32(insn, 12, 4); - if (opcode > 8) { + if (opcode > 8 || mos) { unallocated_encoding(s); return; } @@ -XXX,XX +XXX,XX @@ static void handle_fp_3src_half(DisasContext *s, bool o0, bool o1, */ static void disas_fp_3src(DisasContext *s, uint32_t insn) { + int mos = extract32(insn, 29, 3); int type = extract32(insn, 22, 2); int rd = extract32(insn, 0, 5); int rn = extract32(insn, 5, 5); @@ -XXX,XX +XXX,XX @@ static void disas_fp_3src(DisasContext *s, uint32_t insn) bool o0 = extract32(insn, 15, 1); bool o1 = extract32(insn, 21, 1); + if (mos) { + unallocated_encoding(s); + return; + } + switch (type) { case 0: if (!fp_access_check(s)) { @@ -XXX,XX +XXX,XX @@ uint64_t vfp_expand_imm(int size, uint8_t imm8) static void disas_fp_imm(DisasContext *s, uint32_t insn) { int rd = extract32(insn, 0, 5); + int imm5 = extract32(insn, 5, 5); int imm8 = extract32(insn, 13, 8); int type = extract32(insn, 22, 2); + int mos = extract32(insn, 29, 3); uint64_t imm; TCGv_i64 tcg_res; TCGMemOp sz; + if (mos || imm5) { + unallocated_encoding(s); + return; + } + switch (type) { case 0: sz = MO_32; -- 2.20.1 In the AdvSIMD scalar x indexed element and vector x indexed element encoding group, the SDOT and UDOT instructions are vector only, and their opcode is unallocated in the scalar group. Correctly UNDEF this unallocated encoding. Reported-by: Laurent Desnogues <laurent.desnogues@gmail.com> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Laurent Desnogues <laurent.desnogues@gmail.com> Message-id: 20190125182626.9221-8-peter.maydell@linaro.org --- target/arm/translate-a64.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn) break; case 0x0e: /* SDOT */ case 0x1e: /* UDOT */ - if (size != MO_32 || !dc_isar_feature(aa64_dp, s)) { + if (is_scalar || size != MO_32 || !dc_isar_feature(aa64_dp, s)) { unallocated_encoding(s); return; } -- 2.20.1 The tcg_register_iommu_notifier() code has a GArray of TCGIOMMUNotifier structs which it has registered by passing memory_region_register_iommu_notifier() a pointer to the embedded IOMMUNotifier field. Unfortunately, if we need to enlarge the array via g_array_set_size() this can cause a realloc(), which invalidates the pointer that memory_region_register_iommu_notifier() put into the MemoryRegion's iommu_notify list. This can result in segfaults. Switch the GArray to holding pointers to the TCGIOMMUNotifier structs, so that we can individually allocate and free them. Cc: qemu-stable@nongnu.org Fixes: 1f871c5e6b0f30644a60a ("exec.c: Handle IOMMUs in address_space_translate_for_iotlb()") Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190128174241.5860-1-peter.maydell@linaro.org --- exec.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/exec.c b/exec.c index XXXXXXX..XXXXXXX 100644 --- a/exec.c +++ b/exec.c @@ -XXX,XX +XXX,XX @@ static void tcg_register_iommu_notifier(CPUState *cpu, int i; for (i = 0; i < cpu->iommu_notifiers->len; i++) { - notifier = &g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier, i); + notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i); if (notifier->mr == mr && notifier->iommu_idx == iommu_idx) { break; } @@ -XXX,XX +XXX,XX @@ static void tcg_register_iommu_notifier(CPUState *cpu, if (i == cpu->iommu_notifiers->len) { /* Not found, add a new entry at the end of the array */ cpu->iommu_notifiers = g_array_set_size(cpu->iommu_notifiers, i + 1); - notifier = &g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier, i); + notifier = g_new0(TCGIOMMUNotifier, 1); + g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i) = notifier; notifier->mr = mr; notifier->iommu_idx = iommu_idx; @@ -XXX,XX +XXX,XX @@ static void tcg_iommu_free_notifier_list(CPUState *cpu) TCGIOMMUNotifier *notifier; for (i = 0; i < cpu->iommu_notifiers->len; i++) { - notifier = &g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier, i); + notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i); memory_region_unregister_iommu_notifier(notifier->mr, &notifier->n); + g_free(notifier); } g_array_free(cpu->iommu_notifiers, true); } @@ -XXX,XX +XXX,XX @@ void cpu_exec_realizefn(CPUState *cpu, Error **errp) vmstate_register(NULL, cpu->cpu_index, cc->vmsd, cpu); } - cpu->iommu_notifiers = g_array_new(false, true, sizeof(TCGIOMMUNotifier)); + cpu->iommu_notifiers = g_array_new(false, true, sizeof(TCGIOMMUNotifier *)); #endif } -- 2.20.1 The FCMLA (by element) instruction exists in the "vector x indexed element" encoding group, but not in the "scalar x indexed element" group. Correctly UNDEF the unallocated encodings. Reported-by: Laurent Desnogues <laurent.desnogues@gmail.com> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Laurent Desnogues <laurent.desnogues@gmail.com> Message-id: 20190129140411.682-2-peter.maydell@linaro.org --- target/arm/translate-a64.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn) case 0x13: /* FCMLA #90 */ case 0x15: /* FCMLA #180 */ case 0x17: /* FCMLA #270 */ - if (!dc_isar_feature(aa64_fcma, s)) { + if (is_scalar || !dc_isar_feature(aa64_fcma, s)) { unallocated_encoding(s); return; } -- 2.20.1 In disas_simd_indexed(), for the case of "complex fp", each indexable element is a complex pair, so the total size is twice that indicated in the 'size' field in the encoding. We were trying to do this "double the size" operation with a left shift by 1, but this is incorrect because the 'size' field is a MO_8/MO_16/MO_32/MO_64 value, and doubling the size should be done by a simple increment. This meant we were mishandling FCMLA (by element) of values where the real and imaginary parts are 32-bit floats, and would incorrectly UNDEF this encoding. (No other insns take this code path, and for 16-bit floats it happens that 1 << 1 and 1 + 1 are both the same). Reported-by: Laurent Desnogues <laurent.desnogues@gmail.com> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Laurent Desnogues <laurent.desnogues@gmail.com> Message-id: 20190129140411.682-3-peter.maydell@linaro.org --- target/arm/translate-a64.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn) case 2: /* complex fp */ /* Each indexable element is a complex pair. */ - size <<= 1; + size += 1; switch (size) { case MO_32: if (h && !is_q) { -- 2.20.1 From: Aaron Lindsay OS <aaron@os.amperecomputing.com> Whenever we notice that a counter overflow has occurred, send an interrupt. This is made more reliable with the addition of a timer in a follow-on commit. Signed-off-by: Aaron Lindsay <aaron@os.amperecomputing.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190124162401.5111-2-aaron@os.amperecomputing.com Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper.c | 61 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -XXX,XX +XXX,XX @@ static const ARMCPRegInfo v6_cp_reginfo[] = { /* Definitions for the PMU registers */ #define PMCRN_MASK 0xf800 #define PMCRN_SHIFT 11 +#define PMCRLC 0x40 #define PMCRDP 0x10 #define PMCRD 0x8 #define PMCRC 0x4 @@ -XXX,XX +XXX,XX @@ static bool pmu_counter_enabled(CPUARMState *env, uint8_t counter) return enabled && !prohibited && !filtered; } +static void pmu_update_irq(CPUARMState *env) +{ + ARMCPU *cpu = arm_env_get_cpu(env); + qemu_set_irq(cpu->pmu_interrupt, (env->cp15.c9_pmcr & PMCRE) && + (env->cp15.c9_pminten & env->cp15.c9_pmovsr)); +} + /* * Ensure c15_ccnt is the guest-visible count so that operations such as * enabling/disabling the counter or filtering, modifying the count itself, @@ -XXX,XX +XXX,XX @@ void pmccntr_op_start(CPUARMState *env) eff_cycles /= 64; } - env->cp15.c15_ccnt = eff_cycles - env->cp15.c15_ccnt_delta; + uint64_t new_pmccntr = eff_cycles - env->cp15.c15_ccnt_delta; + + uint64_t overflow_mask = env->cp15.c9_pmcr & PMCRLC ? \ + 1ull << 63 : 1ull << 31; + if (env->cp15.c15_ccnt & ~new_pmccntr & overflow_mask) { + env->cp15.c9_pmovsr |= (1 << 31); + pmu_update_irq(env); + } + + env->cp15.c15_ccnt = new_pmccntr; } env->cp15.c15_ccnt_delta = cycles; } @@ -XXX,XX +XXX,XX @@ static void pmevcntr_op_start(CPUARMState *env, uint8_t counter) } if (pmu_counter_enabled(env, counter)) { - env->cp15.c14_pmevcntr[counter] = - count - env->cp15.c14_pmevcntr_delta[counter]; + uint32_t new_pmevcntr = count - env->cp15.c14_pmevcntr_delta[counter]; + + if (env->cp15.c14_pmevcntr[counter] & ~new_pmevcntr & INT32_MIN) { + env->cp15.c9_pmovsr |= (1 << counter); + pmu_update_irq(env); + } + env->cp15.c14_pmevcntr[counter] = new_pmevcntr; } env->cp15.c14_pmevcntr_delta[counter] = count; } @@ -XXX,XX +XXX,XX @@ static void pmswinc_write(CPUARMState *env, const ARMCPRegInfo *ri, /* counter is SW_INCR */ (env->cp15.c14_pmevtyper[i] & PMXEVTYPER_EVTCOUNT) == 0x0) { pmevcntr_op_start(env, i); - env->cp15.c14_pmevcntr[i]++; + + /* + * Detect if this write causes an overflow since we can't predict + * PMSWINC overflows like we can for other events + */ + uint32_t new_pmswinc = env->cp15.c14_pmevcntr[i] + 1; + + if (env->cp15.c14_pmevcntr[i] & ~new_pmswinc & INT32_MIN) { + env->cp15.c9_pmovsr |= (1 << i); + pmu_update_irq(env); + } + + env->cp15.c14_pmevcntr[i] = new_pmswinc; + pmevcntr_op_finish(env, i); } } @@ -XXX,XX +XXX,XX @@ static void pmovsr_write(CPUARMState *env, const ARMCPRegInfo *ri, { value &= pmu_counter_mask(env); env->cp15.c9_pmovsr &= ~value; + pmu_update_irq(env); } static void pmovsset_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -XXX,XX +XXX,XX @@ static void pmovsset_write(CPUARMState *env, const ARMCPRegInfo *ri, { value &= pmu_counter_mask(env); env->cp15.c9_pmovsr |= value; + pmu_update_irq(env); } static void pmevtyper_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -XXX,XX +XXX,XX @@ static void pmintenset_write(CPUARMState *env, const ARMCPRegInfo *ri, /* We have no event counters so only the C bit can be changed */ value &= pmu_counter_mask(env); env->cp15.c9_pminten |= value; + pmu_update_irq(env); } static void pmintenclr_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -XXX,XX +XXX,XX @@ static void pmintenclr_write(CPUARMState *env, const ARMCPRegInfo *ri, { value &= pmu_counter_mask(env); env->cp15.c9_pminten &= ~value; + pmu_update_irq(env); } static void vbar_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -XXX,XX +XXX,XX @@ static const ARMCPRegInfo v7_cp_reginfo[] = { .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten), .writefn = pmcntenclr_write }, { .name = "PMOVSR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 3, - .access = PL0_RW, + .access = PL0_RW, .type = ARM_CP_IO, .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmovsr), .accessfn = pmreg_access, .writefn = pmovsr_write, @@ -XXX,XX +XXX,XX @@ static const ARMCPRegInfo v7_cp_reginfo[] = { { .name = "PMOVSCLR_EL0", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 3, .access = PL0_RW, .accessfn = pmreg_access, - .type = ARM_CP_ALIAS, + .type = ARM_CP_ALIAS | ARM_CP_IO, .fieldoffset = offsetof(CPUARMState, cp15.c9_pmovsr), .writefn = pmovsr_write, .raw_writefn = raw_write }, { .name = "PMSWINC", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 4, - .access = PL0_W, .accessfn = pmreg_access_swinc, .type = ARM_CP_NO_RAW, + .access = PL0_W, .accessfn = pmreg_access_swinc, + .type = ARM_CP_NO_RAW | ARM_CP_IO, .writefn = pmswinc_write }, { .name = "PMSWINC_EL0", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 4, - .access = PL0_W, .accessfn = pmreg_access_swinc, .type = ARM_CP_NO_RAW, + .access = PL0_W, .accessfn = pmreg_access_swinc, + .type = ARM_CP_NO_RAW | ARM_CP_IO, .writefn = pmswinc_write }, { .name = "PMSELR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 5, .access = PL0_RW, .type = ARM_CP_ALIAS, @@ -XXX,XX +XXX,XX @@ static const ARMCPRegInfo pmovsset_cp_reginfo[] = { /* PMOVSSET is not implemented in v7 before v7ve */ { .name = "PMOVSSET", .cp = 15, .opc1 = 0, .crn = 9, .crm = 14, .opc2 = 3, .access = PL0_RW, .accessfn = pmreg_access, - .type = ARM_CP_ALIAS, + .type = ARM_CP_ALIAS | ARM_CP_IO, .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmovsr), .writefn = pmovsset_write, .raw_writefn = raw_write }, { .name = "PMOVSSET_EL0", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 14, .opc2 = 3, .access = PL0_RW, .accessfn = pmreg_access, - .type = ARM_CP_ALIAS, + .type = ARM_CP_ALIAS | ARM_CP_IO, .fieldoffset = offsetof(CPUARMState, cp15.c9_pmovsr), .writefn = pmovsset_write, .raw_writefn = raw_write }, -- 2.20.1 From: Aaron Lindsay OS <aaron@os.amperecomputing.com> Make PMU overflow interrupts more accurate by using a timer to predict when they will overflow rather than waiting for an event to occur which allows us to otherwise check them. Signed-off-by: Aaron Lindsay <aaron@os.amperecomputing.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190124162401.5111-3-aaron@os.amperecomputing.com Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/cpu.h | 10 +++++++ target/arm/cpu.c | 12 ++++++++ target/arm/helper.c | 72 +++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 92 insertions(+), 2 deletions(-) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -XXX,XX +XXX,XX @@ struct ARMCPU { /* Timers used by the generic (architected) timer */ QEMUTimer *gt_timer[NUM_GTIMERS]; + /* + * Timer used by the PMU. Its state is restored after migration by + * pmu_op_finish() - it does not need other handling during migration + */ + QEMUTimer *pmu_timer; /* GPIO outputs for generic timer */ qemu_irq gt_timer_outputs[NUM_GTIMERS]; /* GPIO output for GICv3 maintenance interrupt signal */ @@ -XXX,XX +XXX,XX @@ void pmccntr_op_finish(CPUARMState *env); void pmu_op_start(CPUARMState *env); void pmu_op_finish(CPUARMState *env); +/* + * Called when a PMU counter is due to overflow + */ +void arm_pmu_timer_cb(void *opaque); + /** * Functions to register as EL change hooks for PMU mode filtering */ diff --git a/target/arm/cpu.c b/target/arm/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -XXX,XX +XXX,XX @@ static void arm_cpu_finalizefn(Object *obj) QLIST_REMOVE(hook, node); g_free(hook); } +#ifndef CONFIG_USER_ONLY + if (cpu->pmu_timer) { + timer_del(cpu->pmu_timer); + timer_deinit(cpu->pmu_timer); + timer_free(cpu->pmu_timer); + } +#endif } static void arm_cpu_realizefn(DeviceState *dev, Error **errp) @@ -XXX,XX +XXX,XX @@ static void arm_cpu_realizefn(DeviceState *dev, Error **errp) arm_register_pre_el_change_hook(cpu, &pmu_pre_el_change, 0); arm_register_el_change_hook(cpu, &pmu_post_el_change, 0); } + +#ifndef CONFIG_USER_ONLY + cpu->pmu_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, arm_pmu_timer_cb, + cpu); +#endif } else { cpu->id_aa64dfr0 &= ~0xf00; cpu->pmceid0 = 0; diff --git a/target/arm/helper.c b/target/arm/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -XXX,XX +XXX,XX @@ typedef struct pm_event { * counters hold a difference from the return value from this function */ uint64_t (*get_count)(CPUARMState *); + /* + * Return how many nanoseconds it will take (at a minimum) for count events + * to occur. A negative value indicates the counter will never overflow, or + * that the counter has otherwise arranged for the overflow bit to be set + * and the PMU interrupt to be raised on overflow. + */ + int64_t (*ns_per_count)(uint64_t); } pm_event; static bool event_always_supported(CPUARMState *env) @@ -XXX,XX +XXX,XX @@ static uint64_t swinc_get_count(CPUARMState *env) return 0; } +static int64_t swinc_ns_per(uint64_t ignored) +{ + return -1; +} + /* * Return the underlying cycle count for the PMU cycle counters. If we're in * usermode, simply return 0. @@ -XXX,XX +XXX,XX @@ static uint64_t cycles_get_count(CPUARMState *env) } #ifndef CONFIG_USER_ONLY +static int64_t cycles_ns_per(uint64_t cycles) +{ + return (ARM_CPU_FREQ / NANOSECONDS_PER_SECOND) * cycles; +} + static bool instructions_supported(CPUARMState *env) { return use_icount == 1 /* Precise instruction counting */; @@ -XXX,XX +XXX,XX @@ static uint64_t instructions_get_count(CPUARMState *env) { return (uint64_t)cpu_get_icount_raw(); } + +static int64_t instructions_ns_per(uint64_t icount) +{ + return cpu_icount_to_ns((int64_t)icount); +} #endif static const pm_event pm_events[] = { { .number = 0x000, /* SW_INCR */ .supported = event_always_supported, .get_count = swinc_get_count, + .ns_per_count = swinc_ns_per, }, #ifndef CONFIG_USER_ONLY { .number = 0x008, /* INST_RETIRED, Instruction architecturally executed */ .supported = instructions_supported, .get_count = instructions_get_count, + .ns_per_count = instructions_ns_per, }, { .number = 0x011, /* CPU_CYCLES, Cycle */ .supported = event_always_supported, .get_count = cycles_get_count, + .ns_per_count = cycles_ns_per, } #endif }; @@ -XXX,XX +XXX,XX @@ void pmccntr_op_start(CPUARMState *env) void pmccntr_op_finish(CPUARMState *env) { if (pmu_counter_enabled(env, 31)) { - uint64_t prev_cycles = env->cp15.c15_ccnt_delta; +#ifndef CONFIG_USER_ONLY + /* Calculate when the counter will next overflow */ + uint64_t remaining_cycles = -env->cp15.c15_ccnt; + if (!(env->cp15.c9_pmcr & PMCRLC)) { + remaining_cycles = (uint32_t)remaining_cycles; + } + int64_t overflow_in = cycles_ns_per(remaining_cycles); + if (overflow_in > 0) { + int64_t overflow_at = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + + overflow_in; + ARMCPU *cpu = arm_env_get_cpu(env); + timer_mod_anticipate_ns(cpu->pmu_timer, overflow_at); + } +#endif + + uint64_t prev_cycles = env->cp15.c15_ccnt_delta; if (env->cp15.c9_pmcr & PMCRD) { /* Increment once every 64 processor clock cycles */ prev_cycles /= 64; } - env->cp15.c15_ccnt_delta = prev_cycles - env->cp15.c15_ccnt; } } @@ -XXX,XX +XXX,XX @@ static void pmevcntr_op_start(CPUARMState *env, uint8_t counter) static void pmevcntr_op_finish(CPUARMState *env, uint8_t counter) { if (pmu_counter_enabled(env, counter)) { +#ifndef CONFIG_USER_ONLY + uint16_t event = env->cp15.c14_pmevtyper[counter] & PMXEVTYPER_EVTCOUNT; + uint16_t event_idx = supported_event_map[event]; + uint64_t delta = UINT32_MAX - + (uint32_t)env->cp15.c14_pmevcntr[counter] + 1; + int64_t overflow_in = pm_events[event_idx].ns_per_count(delta); + + if (overflow_in > 0) { + int64_t overflow_at = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + + overflow_in; + ARMCPU *cpu = arm_env_get_cpu(env); + timer_mod_anticipate_ns(cpu->pmu_timer, overflow_at); + } +#endif + env->cp15.c14_pmevcntr_delta[counter] -= env->cp15.c14_pmevcntr[counter]; } @@ -XXX,XX +XXX,XX @@ void pmu_post_el_change(ARMCPU *cpu, void *ignored) pmu_op_finish(&cpu->env); } +void arm_pmu_timer_cb(void *opaque) +{ + ARMCPU *cpu = opaque; + + /* + * Update all the counter values based on the current underlying counts, + * triggering interrupts to be raised, if necessary. pmu_op_finish() also + * has the effect of setting the cpu->pmu_timer to the next earliest time a + * counter may expire. + */ + pmu_op_start(&cpu->env); + pmu_op_finish(&cpu->env); +} + static void pmcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> These bits become writable with the ARMv8.3-PAuth extension. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20190129143511.12311-1-richard.henderson@linaro.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/target/arm/helper.c b/target/arm/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -XXX,XX +XXX,XX @@ static void scr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) if (cpu_isar_feature(aa64_lor, cpu)) { valid_mask |= SCR_TLOR; } + if (cpu_isar_feature(aa64_pauth, cpu)) { + valid_mask |= SCR_API | SCR_APK; + } /* Clear all-context RES0 bits. */ value &= valid_mask; @@ -XXX,XX +XXX,XX @@ static void hcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) if (cpu_isar_feature(aa64_lor, cpu)) { valid_mask |= HCR_TLOR; } + if (cpu_isar_feature(aa64_pauth, cpu)) { + valid_mask |= HCR_API | HCR_APK; + } /* Clear RES0 bits. */ value &= valid_mask; -- 2.20.1 From: Julia Suvorova <jusual@mail.ru> Until now, the set_pc logic was unclear, which raised questions about whether it should be used directly, applying a value to PC or adding additional checks, for example, set the Thumb bit in Arm cpu. Let's set the set_pc logic for “Configure the PC, as was done in the ELF file” and implement synchronize_with_tb hook for preserving PC to cpu_tb_exec. Signed-off-by: Julia Suvorova <jusual@mail.ru> Acked-by: Stefan Hajnoczi <stefanha@redhat.com> Message-id: 20190129121817.7109-1-jusual@mail.ru Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- include/qom/cpu.h | 16 ++++++++++++++-- hw/arm/boot.c | 4 ---- target/arm/arm-powerctl.c | 3 --- target/arm/cpu.c | 26 +++++++++++++++++++++++++- target/arm/cpu64.c | 15 --------------- 5 files changed, 39 insertions(+), 25 deletions(-) diff --git a/include/qom/cpu.h b/include/qom/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/include/qom/cpu.h +++ b/include/qom/cpu.h @@ -XXX,XX +XXX,XX @@ struct TranslationBlock; * @get_arch_id: Callback for getting architecture-dependent CPU ID. * @get_paging_enabled: Callback for inquiring whether paging is enabled. * @get_memory_mapping: Callback for obtaining the memory mappings. - * @set_pc: Callback for setting the Program Counter register. + * @set_pc: Callback for setting the Program Counter register. This + * should have the semantics used by the target architecture when + * setting the PC from a source such as an ELF file entry point; + * for example on Arm it will also set the Thumb mode bit based + * on the least significant bit of the new PC value. + * If the target behaviour here is anything other than "set + * the PC register to the value passed in" then the target must + * also implement the synchronize_from_tb hook. * @synchronize_from_tb: Callback for synchronizing state from a TCG - * #TranslationBlock. + * #TranslationBlock. This is called when we abandon execution + * of a TB before starting it, and must set all parts of the CPU + * state which the previous TB in the chain may not have updated. + * This always includes at least the program counter; some targets + * will need to do more. If this hook is not implemented then the + * default is to call @set_pc(tb->pc). * @handle_mmu_fault: Callback for handling an MMU fault. * @get_phys_page_debug: Callback for obtaining a physical address. * @get_phys_page_attrs_debug: Callback for obtaining a physical address and the diff --git a/hw/arm/boot.c b/hw/arm/boot.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/boot.c +++ b/hw/arm/boot.c @@ -XXX,XX +XXX,XX @@ static void do_cpu_reset(void *opaque) g_assert_not_reached(); } - if (!env->aarch64) { - env->thumb = info->entry & 1; - entry &= 0xfffffffe; - } cpu_set_pc(cs, entry); } else { /* If we are booting Linux then we need to check whether we are diff --git a/target/arm/arm-powerctl.c b/target/arm/arm-powerctl.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/arm-powerctl.c +++ b/target/arm/arm-powerctl.c @@ -XXX,XX +XXX,XX @@ static void arm_set_cpu_on_async_work(CPUState *target_cpu_state, if (info->target_aa64) { target_cpu->env.xregs[0] = info->context_id; - target_cpu->env.thumb = false; } else { target_cpu->env.regs[0] = info->context_id; - target_cpu->env.thumb = info->entry & 1; - info->entry &= 0xfffffffe; } /* Start the new CPU at the requested address */ diff --git a/target/arm/cpu.c b/target/arm/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -XXX,XX +XXX,XX @@ static void arm_cpu_set_pc(CPUState *cs, vaddr value) { ARMCPU *cpu = ARM_CPU(cs); + CPUARMState *env = &cpu->env; - cpu->env.regs[15] = value; + if (is_a64(env)) { + env->pc = value; + env->thumb = 0; + } else { + env->regs[15] = value & ~1; + env->thumb = value & 1; + } +} + +static void arm_cpu_synchronize_from_tb(CPUState *cs, TranslationBlock *tb) +{ + ARMCPU *cpu = ARM_CPU(cs); + CPUARMState *env = &cpu->env; + + /* + * It's OK to look at env for the current mode here, because it's + * never possible for an AArch64 TB to chain to an AArch32 TB. + */ + if (is_a64(env)) { + env->pc = tb->pc; + } else { + env->regs[15] = tb->pc; + } } static bool arm_cpu_has_work(CPUState *cs) @@ -XXX,XX +XXX,XX @@ static void arm_cpu_class_init(ObjectClass *oc, void *data) cc->cpu_exec_interrupt = arm_cpu_exec_interrupt; cc->dump_state = arm_cpu_dump_state; cc->set_pc = arm_cpu_set_pc; + cc->synchronize_from_tb = arm_cpu_synchronize_from_tb; cc->gdb_read_register = arm_cpu_gdb_read_register; cc->gdb_write_register = arm_cpu_gdb_write_register; #ifdef CONFIG_USER_ONLY diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu64.c +++ b/target/arm/cpu64.c @@ -XXX,XX +XXX,XX @@ static void aarch64_cpu_finalizefn(Object *obj) { } -static void aarch64_cpu_set_pc(CPUState *cs, vaddr value) -{ - ARMCPU *cpu = ARM_CPU(cs); - /* It's OK to look at env for the current mode here, because it's - * never possible for an AArch64 TB to chain to an AArch32 TB. - * (Otherwise we would need to use synchronize_from_tb instead.) - */ - if (is_a64(&cpu->env)) { - cpu->env.pc = value; - } else { - cpu->env.regs[15] = value; - } -} - static gchar *aarch64_gdb_arch_name(CPUState *cs) { return g_strdup("aarch64"); @@ -XXX,XX +XXX,XX @@ static void aarch64_cpu_class_init(ObjectClass *oc, void *data) CPUClass *cc = CPU_CLASS(oc); cc->cpu_exec_interrupt = arm_cpu_exec_interrupt; - cc->set_pc = aarch64_cpu_set_pc; cc->gdb_read_register = aarch64_cpu_gdb_read_register; cc->gdb_write_register = aarch64_cpu_gdb_write_register; cc->gdb_num_core_regs = 34; -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Drop the pac properties. This approach cannot work as written because the properties are applied before arm_cpu_reset, which zeros SCTLR_EL1 (amongst everything else). We can re-introduce the properties if they turn out to be useful. But since linux 5.0 enables all of the keys, they may not be. Fixes: 1ae9cfbd470 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/cpu.c | 3 +++ target/arm/cpu64.c | 60 ---------------------------------------------- 2 files changed, 3 insertions(+), 60 deletions(-) diff --git a/target/arm/cpu.c b/target/arm/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -XXX,XX +XXX,XX @@ static void arm_cpu_reset(CPUState *s) env->pstate = PSTATE_MODE_EL0t; /* Userspace expects access to DC ZVA, CTL_EL0 and the cache ops */ env->cp15.sctlr_el[1] |= SCTLR_UCT | SCTLR_UCI | SCTLR_DZE; + /* Enable all PAC keys. */ + env->cp15.sctlr_el[1] |= (SCTLR_EnIA | SCTLR_EnIB | + SCTLR_EnDA | SCTLR_EnDB); /* Enable all PAC instructions */ env->cp15.hcr_el2 |= HCR_API; env->cp15.scr_el3 |= SCR_API; diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu64.c +++ b/target/arm/cpu64.c @@ -XXX,XX +XXX,XX @@ static void cpu_max_set_sve_vq(Object *obj, Visitor *v, const char *name, error_propagate(errp, err); } -#ifdef CONFIG_USER_ONLY -static void cpu_max_get_packey(Object *obj, Visitor *v, const char *name, - void *opaque, Error **errp) -{ - ARMCPU *cpu = ARM_CPU(obj); - const uint64_t *bit = opaque; - bool enabled = (cpu->env.cp15.sctlr_el[1] & *bit) != 0; - - visit_type_bool(v, name, &enabled, errp); -} - -static void cpu_max_set_packey(Object *obj, Visitor *v, const char *name, - void *opaque, Error **errp) -{ - ARMCPU *cpu = ARM_CPU(obj); - Error *err = NULL; - const uint64_t *bit = opaque; - bool enabled; - - visit_type_bool(v, name, &enabled, errp); - - if (!err) { - if (enabled) { - cpu->env.cp15.sctlr_el[1] |= *bit; - } else { - cpu->env.cp15.sctlr_el[1] &= ~*bit; - } - } - error_propagate(errp, err); -} -#endif - /* -cpu max: if KVM is enabled, like -cpu host (best possible with this host); * otherwise, a CPU with as many features enabled as our emulation supports. * The version of '-cpu max' for qemu-system-arm is defined in cpu.c; @@ -XXX,XX +XXX,XX @@ static void aarch64_max_initfn(Object *obj) */ cpu->ctr = 0x80038003; /* 32 byte I and D cacheline size, VIPT icache */ cpu->dcz_blocksize = 7; /* 512 bytes */ - - /* - * Note that Linux will enable enable all of the keys at once. - * But doing it this way will allow experimentation beyond that. - */ - { - static const uint64_t apia_bit = SCTLR_EnIA; - static const uint64_t apib_bit = SCTLR_EnIB; - static const uint64_t apda_bit = SCTLR_EnDA; - static const uint64_t apdb_bit = SCTLR_EnDB; - - object_property_add(obj, "apia", "bool", cpu_max_get_packey, - cpu_max_set_packey, NULL, - (void *)&apia_bit, &error_fatal); - object_property_add(obj, "apib", "bool", cpu_max_get_packey, - cpu_max_set_packey, NULL, - (void *)&apib_bit, &error_fatal); - object_property_add(obj, "apda", "bool", cpu_max_get_packey, - cpu_max_set_packey, NULL, - (void *)&apda_bit, &error_fatal); - object_property_add(obj, "apdb", "bool", cpu_max_get_packey, - cpu_max_set_packey, NULL, - (void *)&apdb_bit, &error_fatal); - - /* Enable all PAC keys by default. */ - cpu->env.cp15.sctlr_el[1] |= SCTLR_EnIA | SCTLR_EnIB; - cpu->env.cp15.sctlr_el[1] |= SCTLR_EnDA | SCTLR_EnDB; - } #endif cpu->sve_max_vq = ARM_MAX_VQ; -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Laurent Vivier <laurent@vivier.eu> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- linux-user/elfload.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/linux-user/elfload.c b/linux-user/elfload.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/elfload.c +++ b/linux-user/elfload.c @@ -XXX,XX +XXX,XX @@ enum { ARM_HWCAP_A64_ASIMDDP = 1 << 20, ARM_HWCAP_A64_SHA512 = 1 << 21, ARM_HWCAP_A64_SVE = 1 << 22, + ARM_HWCAP_A64_ASIMDFHM = 1 << 23, + ARM_HWCAP_A64_DIT = 1 << 24, + ARM_HWCAP_A64_USCAT = 1 << 25, + ARM_HWCAP_A64_ILRCPC = 1 << 26, + ARM_HWCAP_A64_FLAGM = 1 << 27, + ARM_HWCAP_A64_SSBS = 1 << 28, + ARM_HWCAP_A64_SB = 1 << 29, + ARM_HWCAP_A64_PACA = 1 << 30, + ARM_HWCAP_A64_PACG = 1UL << 31, }; #define ELF_HWCAP get_elf_hwcap() -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- linux-user/elfload.c | 1 + 1 file changed, 1 insertion(+) diff --git a/linux-user/elfload.c b/linux-user/elfload.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/elfload.c +++ b/linux-user/elfload.c @@ -XXX,XX +XXX,XX @@ static uint32_t get_elf_hwcap(void) GET_FEATURE_ID(aa64_dp, ARM_HWCAP_A64_ASIMDDP); GET_FEATURE_ID(aa64_fcma, ARM_HWCAP_A64_FCMA); GET_FEATURE_ID(aa64_sve, ARM_HWCAP_A64_SVE); + GET_FEATURE_ID(aa64_pauth, ARM_HWCAP_A64_PACA | ARM_HWCAP_A64_PACG); #undef GET_FEATURE_ID -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Initialize the keys to a non-zero value on process start. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- linux-user/aarch64/target_syscall.h | 2 ++ linux-user/aarch64/cpu_loop.c | 31 +++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/linux-user/aarch64/target_syscall.h b/linux-user/aarch64/target_syscall.h index XXXXXXX..XXXXXXX 100644 --- a/linux-user/aarch64/target_syscall.h +++ b/linux-user/aarch64/target_syscall.h @@ -XXX,XX +XXX,XX @@ struct target_pt_regs { #define TARGET_PR_SVE_SET_VL 50 #define TARGET_PR_SVE_GET_VL 51 +void arm_init_pauth_key(ARMPACKey *key); + #endif /* AARCH64_TARGET_SYSCALL_H */ diff --git a/linux-user/aarch64/cpu_loop.c b/linux-user/aarch64/cpu_loop.c index XXXXXXX..XXXXXXX 100644 --- a/linux-user/aarch64/cpu_loop.c +++ b/linux-user/aarch64/cpu_loop.c @@ -XXX,XX +XXX,XX @@ void cpu_loop(CPUARMState *env) } } +static uint64_t arm_rand64(void) +{ + int shift = 64 - clz64(RAND_MAX); + int i, n = 64 / shift + (64 % shift != 0); + uint64_t ret = 0; + + for (i = 0; i < n; i++) { + ret = (ret << shift) | rand(); + } + return ret; +} + +void arm_init_pauth_key(ARMPACKey *key) +{ + key->lo = arm_rand64(); + key->hi = arm_rand64(); +} + void target_cpu_copy_regs(CPUArchState *env, struct target_pt_regs *regs) { - CPUState *cpu = ENV_GET_CPU(env); - TaskState *ts = cpu->opaque; + ARMCPU *cpu = arm_env_get_cpu(env); + CPUState *cs = CPU(cpu); + TaskState *ts = cs->opaque; struct image_info *info = ts->info; int i; @@ -XXX,XX +XXX,XX @@ void target_cpu_copy_regs(CPUArchState *env, struct target_pt_regs *regs) } #endif + if (cpu_isar_feature(aa64_pauth, cpu)) { + arm_init_pauth_key(&env->apia_key); + arm_init_pauth_key(&env->apib_key); + arm_init_pauth_key(&env->apda_key); + arm_init_pauth_key(&env->apdb_key); + arm_init_pauth_key(&env->apga_key); + } + ts->stack_base = info->start_stack; ts->heap_base = info->brk; /* This will be filled in on the first SYS_HEAPINFO call. */ -- 2.20.1 From: Remi Denis-Courmont <remi.denis.courmont@huawei.com> Since QEMU does not support the ARMv8.2-LVA, Large Virtual Address, extension (yet), the VA address space is 48-bits plus a sign bit. User mode can only handle the positive half of the address space, so that makes a limit of 48 bits. (With LVA, it would be 53 and 52 bits respectively.) The incorrectly large address space conflicts with PAuth instructions, which use bits 48-54 and 56-63 for the pointer authentication code. This also conflicts with (as yet unsupported by QEMU) data tagging and with the ARMv8.5-MTE extension. Signed-off-by: Remi Denis-Courmont <remi.denis.courmont@huawei.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/cpu.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -XXX,XX +XXX,XX @@ bool write_cpustate_to_list(ARMCPU *cpu); #if defined(TARGET_AARCH64) # define TARGET_PHYS_ADDR_SPACE_BITS 48 -# define TARGET_VIRT_ADDR_SPACE_BITS 64 +# define TARGET_VIRT_ADDR_SPACE_BITS 48 #else # define TARGET_PHYS_ADDR_SPACE_BITS 40 # define TARGET_VIRT_ADDR_SPACE_BITS 32 -- 2.20.1 From: Remi Denis-Courmont <remi.denis.courmont@huawei.com> A flawed test lead to the instructions always being treated as unallocated encodings. Fixes: https://bugs.launchpad.net/bugs/1813460 Signed-off-by: Remi Denis-Courmont <remi.denis.courmont@huawei.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-a64.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void disas_uncond_b_reg(DisasContext *s, uint32_t insn) if (!dc_isar_feature(aa64_pauth, s)) { goto do_unallocated; } - if (op3 != 2 || op3 != 3) { + if ((op3 & ~1) != 2) { goto do_unallocated; } if (s->pauth_active) { -- 2.20.1 From: Steffen Görtz <contrib@steffen-goertz.de> The nRF51 contains three regions of non-volatile memory (NVM): - CODE (R/W): contains code - FICR (R): Factory information like code size, chip id etc. - UICR (R/W): Changeable configuration data. Lock bits, Code protection configuration, Bootloader address, Nordic SoftRadio configuration, Firmware configuration. Read and write access to the memories is managed by the Non-volatile memory controller. Memory schema: [ CPU ] -+- [ NVM, either FICR, UICR or CODE ] | | \- [ NVMC ] Signed-off-by: Steffen Görtz <contrib@steffen-goertz.de> Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Tested-by: Joel Stanley <joel@jms.id.au> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20190201023357.22596-2-stefanha@redhat.com Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- hw/nvram/Makefile.objs | 1 + include/hw/nvram/nrf51_nvm.h | 64 ++++++ hw/nvram/nrf51_nvm.c | 388 +++++++++++++++++++++++++++++++++++ 3 files changed, 453 insertions(+) create mode 100644 include/hw/nvram/nrf51_nvm.h create mode 100644 hw/nvram/nrf51_nvm.c diff --git a/hw/nvram/Makefile.objs b/hw/nvram/Makefile.objs index XXXXXXX..XXXXXXX 100644 --- a/hw/nvram/Makefile.objs +++ b/hw/nvram/Makefile.objs @@ -XXX,XX +XXX,XX @@ common-obj-y += fw_cfg.o common-obj-y += chrp_nvram.o common-obj-$(CONFIG_MAC_NVRAM) += mac_nvram.o obj-$(CONFIG_PSERIES) += spapr_nvram.o +obj-$(CONFIG_NRF51_SOC) += nrf51_nvm.o diff --git a/include/hw/nvram/nrf51_nvm.h b/include/hw/nvram/nrf51_nvm.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/include/hw/nvram/nrf51_nvm.h @@ -XXX,XX +XXX,XX @@ +/* + * Nordic Semiconductor nRF51 non-volatile memory + * + * It provides an interface to erase regions in flash memory. + * Furthermore it provides the user and factory information registers. + * + * QEMU interface: + * + sysbus MMIO regions 0: NVMC peripheral registers + * + sysbus MMIO regions 1: FICR peripheral registers + * + sysbus MMIO regions 2: UICR peripheral registers + * + flash-size property: flash size in bytes. + * + * Accuracy of the peripheral model: + * + Code regions (MPU configuration) are disregarded. + * + * Copyright 2018 Steffen Görtz <contrib@steffen-goertz.de> + * + * This code is licensed under the GPL version 2 or later. See + * the COPYING file in the top-level directory. + * + */ +#ifndef NRF51_NVM_H +#define NRF51_NVM_H + +#include "hw/sysbus.h" +#define TYPE_NRF51_NVM "nrf51_soc.nvm" +#define NRF51_NVM(obj) OBJECT_CHECK(NRF51NVMState, (obj), TYPE_NRF51_NVM) + +#define NRF51_UICR_FIXTURE_SIZE 64 + +#define NRF51_NVMC_SIZE 0x1000 + +#define NRF51_NVMC_READY 0x400 +#define NRF51_NVMC_READY_READY 0x01 +#define NRF51_NVMC_CONFIG 0x504 +#define NRF51_NVMC_CONFIG_MASK 0x03 +#define NRF51_NVMC_CONFIG_WEN 0x01 +#define NRF51_NVMC_CONFIG_EEN 0x02 +#define NRF51_NVMC_ERASEPCR1 0x508 +#define NRF51_NVMC_ERASEPCR0 0x510 +#define NRF51_NVMC_ERASEALL 0x50C +#define NRF51_NVMC_ERASEUICR 0x514 +#define NRF51_NVMC_ERASE 0x01 + +#define NRF51_UICR_SIZE 0x100 + +typedef struct NRF51NVMState { + SysBusDevice parent_obj; + + MemoryRegion mmio; + MemoryRegion ficr; + MemoryRegion uicr; + MemoryRegion flash; + + uint32_t uicr_content[NRF51_UICR_FIXTURE_SIZE]; + uint32_t flash_size; + uint8_t *storage; + + uint32_t config; + +} NRF51NVMState; + + +#endif diff --git a/hw/nvram/nrf51_nvm.c b/hw/nvram/nrf51_nvm.c new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/hw/nvram/nrf51_nvm.c @@ -XXX,XX +XXX,XX @@ +/* + * Nordic Semiconductor nRF51 non-volatile memory + * + * It provides an interface to erase regions in flash memory. + * Furthermore it provides the user and factory information registers. + * + * Reference Manual: http://infocenter.nordicsemi.com/pdf/nRF51_RM_v3.0.pdf + * + * See nRF51 reference manual and product sheet sections: + * + Non-Volatile Memory Controller (NVMC) + * + Factory Information Configuration Registers (FICR) + * + User Information Configuration Registers (UICR) + * + * Copyright 2018 Steffen Görtz <contrib@steffen-goertz.de> + * + * This code is licensed under the GPL version 2 or later. See + * the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "qapi/error.h" +#include "qemu/log.h" +#include "exec/address-spaces.h" +#include "hw/arm/nrf51.h" +#include "hw/nvram/nrf51_nvm.h" + +/* + * FICR Registers Assignments + * CODEPAGESIZE 0x010 + * CODESIZE 0x014 + * CLENR0 0x028 + * PPFC 0x02C + * NUMRAMBLOCK 0x034 + * SIZERAMBLOCKS 0x038 + * SIZERAMBLOCK[0] 0x038 + * SIZERAMBLOCK[1] 0x03C + * SIZERAMBLOCK[2] 0x040 + * SIZERAMBLOCK[3] 0x044 + * CONFIGID 0x05C + * DEVICEID[0] 0x060 + * DEVICEID[1] 0x064 + * ER[0] 0x080 + * ER[1] 0x084 + * ER[2] 0x088 + * ER[3] 0x08C + * IR[0] 0x090 + * IR[1] 0x094 + * IR[2] 0x098 + * IR[3] 0x09C + * DEVICEADDRTYPE 0x0A0 + * DEVICEADDR[0] 0x0A4 + * DEVICEADDR[1] 0x0A8 + * OVERRIDEEN 0x0AC + * NRF_1MBIT[0] 0x0B0 + * NRF_1MBIT[1] 0x0B4 + * NRF_1MBIT[2] 0x0B8 + * NRF_1MBIT[3] 0x0BC + * NRF_1MBIT[4] 0x0C0 + * BLE_1MBIT[0] 0x0EC + * BLE_1MBIT[1] 0x0F0 + * BLE_1MBIT[2] 0x0F4 + * BLE_1MBIT[3] 0x0F8 + * BLE_1MBIT[4] 0x0FC + */ +static const uint32_t ficr_content[64] = { + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000400, + 0x00000100, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000002, 0x00002000, + 0x00002000, 0x00002000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000003, + 0x12345678, 0x9ABCDEF1, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF +}; + +static uint64_t ficr_read(void *opaque, hwaddr offset, unsigned int size) +{ + assert(offset < sizeof(ficr_content)); + return ficr_content[offset / 4]; +} + +static void ficr_write(void *opaque, hwaddr offset, uint64_t value, + unsigned int size) +{ + /* Intentionally do nothing */ +} + +static const MemoryRegionOps ficr_ops = { + .read = ficr_read, + .write = ficr_write, + .impl.min_access_size = 4, + .impl.max_access_size = 4, + .endianness = DEVICE_LITTLE_ENDIAN +}; + +/* + * UICR Registers Assignments + * CLENR0 0x000 + * RBPCONF 0x004 + * XTALFREQ 0x008 + * FWID 0x010 + * BOOTLOADERADDR 0x014 + * NRFFW[0] 0x014 + * NRFFW[1] 0x018 + * NRFFW[2] 0x01C + * NRFFW[3] 0x020 + * NRFFW[4] 0x024 + * NRFFW[5] 0x028 + * NRFFW[6] 0x02C + * NRFFW[7] 0x030 + * NRFFW[8] 0x034 + * NRFFW[9] 0x038 + * NRFFW[10] 0x03C + * NRFFW[11] 0x040 + * NRFFW[12] 0x044 + * NRFFW[13] 0x048 + * NRFFW[14] 0x04C + * NRFHW[0] 0x050 + * NRFHW[1] 0x054 + * NRFHW[2] 0x058 + * NRFHW[3] 0x05C + * NRFHW[4] 0x060 + * NRFHW[5] 0x064 + * NRFHW[6] 0x068 + * NRFHW[7] 0x06C + * NRFHW[8] 0x070 + * NRFHW[9] 0x074 + * NRFHW[10] 0x078 + * NRFHW[11] 0x07C + * CUSTOMER[0] 0x080 + * CUSTOMER[1] 0x084 + * CUSTOMER[2] 0x088 + * CUSTOMER[3] 0x08C + * CUSTOMER[4] 0x090 + * CUSTOMER[5] 0x094 + * CUSTOMER[6] 0x098 + * CUSTOMER[7] 0x09C + * CUSTOMER[8] 0x0A0 + * CUSTOMER[9] 0x0A4 + * CUSTOMER[10] 0x0A8 + * CUSTOMER[11] 0x0AC + * CUSTOMER[12] 0x0B0 + * CUSTOMER[13] 0x0B4 + * CUSTOMER[14] 0x0B8 + * CUSTOMER[15] 0x0BC + * CUSTOMER[16] 0x0C0 + * CUSTOMER[17] 0x0C4 + * CUSTOMER[18] 0x0C8 + * CUSTOMER[19] 0x0CC + * CUSTOMER[20] 0x0D0 + * CUSTOMER[21] 0x0D4 + * CUSTOMER[22] 0x0D8 + * CUSTOMER[23] 0x0DC + * CUSTOMER[24] 0x0E0 + * CUSTOMER[25] 0x0E4 + * CUSTOMER[26] 0x0E8 + * CUSTOMER[27] 0x0EC + * CUSTOMER[28] 0x0F0 + * CUSTOMER[29] 0x0F4 + * CUSTOMER[30] 0x0F8 + * CUSTOMER[31] 0x0FC + */ + +static uint64_t uicr_read(void *opaque, hwaddr offset, unsigned int size) +{ + NRF51NVMState *s = NRF51_NVM(opaque); + + assert(offset < sizeof(s->uicr_content)); + return s->uicr_content[offset / 4]; +} + +static void uicr_write(void *opaque, hwaddr offset, uint64_t value, + unsigned int size) +{ + NRF51NVMState *s = NRF51_NVM(opaque); + + assert(offset < sizeof(s->uicr_content)); + s->uicr_content[offset / 4] = value; +} + +static const MemoryRegionOps uicr_ops = { + .read = uicr_read, + .write = uicr_write, + .impl.min_access_size = 4, + .impl.max_access_size = 4, + .endianness = DEVICE_LITTLE_ENDIAN +}; + + +static uint64_t io_read(void *opaque, hwaddr offset, unsigned int size) +{ + NRF51NVMState *s = NRF51_NVM(opaque); + uint64_t r = 0; + + switch (offset) { + case NRF51_NVMC_READY: + r = NRF51_NVMC_READY_READY; + break; + case NRF51_NVMC_CONFIG: + r = s->config; + break; + default: + qemu_log_mask(LOG_GUEST_ERROR, + "%s: bad read offset 0x%" HWADDR_PRIx "\n", __func__, offset); + break; + } + + return r; +} + +static void io_write(void *opaque, hwaddr offset, uint64_t value, + unsigned int size) +{ + NRF51NVMState *s = NRF51_NVM(opaque); + + switch (offset) { + case NRF51_NVMC_CONFIG: + s->config = value & NRF51_NVMC_CONFIG_MASK; + break; + case NRF51_NVMC_ERASEPCR0: + case NRF51_NVMC_ERASEPCR1: + if (s->config & NRF51_NVMC_CONFIG_EEN) { + /* Mask in-page sub address */ + value &= ~(NRF51_PAGE_SIZE - 1); + if (value <= (s->flash_size - NRF51_PAGE_SIZE)) { + memset(s->storage + value, 0xFF, NRF51_PAGE_SIZE); + memory_region_flush_rom_device(&s->flash, value, + NRF51_PAGE_SIZE); + } + } else { + qemu_log_mask(LOG_GUEST_ERROR, + "%s: Flash erase at 0x%" HWADDR_PRIx" while flash not erasable.\n", + __func__, offset); + } + break; + case NRF51_NVMC_ERASEALL: + if (value == NRF51_NVMC_ERASE) { + if (s->config & NRF51_NVMC_CONFIG_EEN) { + memset(s->storage, 0xFF, s->flash_size); + memory_region_flush_rom_device(&s->flash, 0, s->flash_size); + memset(s->uicr_content, 0xFF, sizeof(s->uicr_content)); + } else { + qemu_log_mask(LOG_GUEST_ERROR, "%s: Flash not erasable.\n", + __func__); + } + } + break; + case NRF51_NVMC_ERASEUICR: + if (value == NRF51_NVMC_ERASE) { + memset(s->uicr_content, 0xFF, sizeof(s->uicr_content)); + } + break; + + default: + qemu_log_mask(LOG_GUEST_ERROR, + "%s: bad write offset 0x%" HWADDR_PRIx "\n", __func__, offset); + } +} + +static const MemoryRegionOps io_ops = { + .read = io_read, + .write = io_write, + .impl.min_access_size = 4, + .impl.max_access_size = 4, + .endianness = DEVICE_LITTLE_ENDIAN, +}; + + +static void flash_write(void *opaque, hwaddr offset, uint64_t value, + unsigned int size) +{ + NRF51NVMState *s = NRF51_NVM(opaque); + + if (s->config & NRF51_NVMC_CONFIG_WEN) { + uint32_t oldval; + + assert(offset + size <= s->flash_size); + + /* NOR Flash only allows bits to be flipped from 1's to 0's on write */ + oldval = ldl_le_p(s->storage + offset); + oldval &= value; + stl_le_p(s->storage + offset, oldval); + + memory_region_flush_rom_device(&s->flash, offset, size); + } else { + qemu_log_mask(LOG_GUEST_ERROR, + "%s: Flash write 0x%" HWADDR_PRIx" while flash not writable.\n", + __func__, offset); + } +} + + + +static const MemoryRegionOps flash_ops = { + .write = flash_write, + .valid.min_access_size = 4, + .valid.max_access_size = 4, + .endianness = DEVICE_LITTLE_ENDIAN, +}; + +static void nrf51_nvm_init(Object *obj) +{ + NRF51NVMState *s = NRF51_NVM(obj); + SysBusDevice *sbd = SYS_BUS_DEVICE(obj); + + memory_region_init_io(&s->mmio, obj, &io_ops, s, "nrf51_soc.nvmc", + NRF51_NVMC_SIZE); + sysbus_init_mmio(sbd, &s->mmio); + + memory_region_init_io(&s->ficr, obj, &ficr_ops, s, "nrf51_soc.ficr", + sizeof(ficr_content)); + sysbus_init_mmio(sbd, &s->ficr); + + memory_region_init_io(&s->uicr, obj, &uicr_ops, s, "nrf51_soc.uicr", + sizeof(s->uicr_content)); + sysbus_init_mmio(sbd, &s->uicr); +} + +static void nrf51_nvm_realize(DeviceState *dev, Error **errp) +{ + NRF51NVMState *s = NRF51_NVM(dev); + Error *err = NULL; + + memory_region_init_rom_device(&s->flash, OBJECT(dev), &flash_ops, s, + "nrf51_soc.flash", s->flash_size, &err); + if (err) { + error_propagate(errp, err); + return; + } + + s->storage = memory_region_get_ram_ptr(&s->flash); + sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->flash); +} + +static void nrf51_nvm_reset(DeviceState *dev) +{ + NRF51NVMState *s = NRF51_NVM(dev); + + s->config = 0x00; + memset(s->uicr_content, 0xFF, sizeof(s->uicr_content)); +} + +static Property nrf51_nvm_properties[] = { + DEFINE_PROP_UINT32("flash-size", NRF51NVMState, flash_size, 0x40000), + DEFINE_PROP_END_OF_LIST(), +}; + +static const VMStateDescription vmstate_nvm = { + .name = "nrf51_soc.nvm", + .version_id = 1, + .minimum_version_id = 1, + .fields = (VMStateField[]) { + VMSTATE_UINT32_ARRAY(uicr_content, NRF51NVMState, + NRF51_UICR_FIXTURE_SIZE), + VMSTATE_UINT32(config, NRF51NVMState), + VMSTATE_END_OF_LIST() + } +}; + +static void nrf51_nvm_class_init(ObjectClass *klass, void *data) +{ + DeviceClass *dc = DEVICE_CLASS(klass); + + dc->props = nrf51_nvm_properties; + dc->vmsd = &vmstate_nvm; + dc->realize = nrf51_nvm_realize; + dc->reset = nrf51_nvm_reset; +} + +static const TypeInfo nrf51_nvm_info = { + .name = TYPE_NRF51_NVM, + .parent = TYPE_SYS_BUS_DEVICE, + .instance_size = sizeof(NRF51NVMState), + .instance_init = nrf51_nvm_init, + .class_init = nrf51_nvm_class_init +}; + +static void nrf51_nvm_register_types(void) +{ + type_register_static(&nrf51_nvm_info); +} + +type_init(nrf51_nvm_register_types) -- 2.20.1 From: Steffen Görtz <contrib@steffen-goertz.de> Instantiates UICR, FICR, FLASH and NVMC in nRF51 SOC. Signed-off-by: Steffen Görtz <contrib@steffen-goertz.de> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com> Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Message-id: 20190201023357.22596-3-stefanha@redhat.com Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- include/hw/arm/nrf51_soc.h | 2 ++ hw/arm/nrf51_soc.c | 41 +++++++++++++++++++++++++++----------- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/include/hw/arm/nrf51_soc.h b/include/hw/arm/nrf51_soc.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/nrf51_soc.h +++ b/include/hw/arm/nrf51_soc.h @@ -XXX,XX +XXX,XX @@ #include "hw/char/nrf51_uart.h" #include "hw/misc/nrf51_rng.h" #include "hw/gpio/nrf51_gpio.h" +#include "hw/nvram/nrf51_nvm.h" #include "hw/timer/nrf51_timer.h" #define TYPE_NRF51_SOC "nrf51-soc" @@ -XXX,XX +XXX,XX @@ typedef struct NRF51State { NRF51UARTState uart; NRF51RNGState rng; + NRF51NVMState nvm; NRF51GPIOState gpio; NRF51TimerState timer[NRF51_NUM_TIMERS]; diff --git a/hw/arm/nrf51_soc.c b/hw/arm/nrf51_soc.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/nrf51_soc.c +++ b/hw/arm/nrf51_soc.c @@ -XXX,XX +XXX,XX @@ * are supported in the future, add a sub-class of NRF51SoC for * the specific variants */ -#define NRF51822_FLASH_SIZE (256 * NRF51_PAGE_SIZE) -#define NRF51822_SRAM_SIZE (16 * NRF51_PAGE_SIZE) +#define NRF51822_FLASH_PAGES 256 +#define NRF51822_SRAM_PAGES 16 +#define NRF51822_FLASH_SIZE (NRF51822_FLASH_PAGES * NRF51_PAGE_SIZE) +#define NRF51822_SRAM_SIZE (NRF51822_SRAM_PAGES * NRF51_PAGE_SIZE) #define BASE_TO_IRQ(base) ((base >> 12) & 0x1F) @@ -XXX,XX +XXX,XX @@ static void nrf51_soc_realize(DeviceState *dev_soc, Error **errp) memory_region_add_subregion_overlap(&s->container, 0, s->board_memory, -1); - memory_region_init_rom(&s->flash, OBJECT(s), "nrf51.flash", s->flash_size, - &err); - if (err) { - error_propagate(errp, err); - return; - } - memory_region_add_subregion(&s->container, NRF51_FLASH_BASE, &s->flash); - memory_region_init_ram(&s->sram, OBJECT(s), "nrf51.sram", s->sram_size, &err); if (err) { @@ -XXX,XX +XXX,XX @@ static void nrf51_soc_realize(DeviceState *dev_soc, Error **errp) qdev_get_gpio_in(DEVICE(&s->cpu), BASE_TO_IRQ(NRF51_RNG_BASE))); + /* UICR, FICR, NVMC, FLASH */ + object_property_set_uint(OBJECT(&s->nvm), s->flash_size, "flash-size", + &err); + if (err) { + error_propagate(errp, err); + return; + } + + object_property_set_bool(OBJECT(&s->nvm), true, "realized", &err); + if (err) { + error_propagate(errp, err); + return; + } + + mr = sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->nvm), 0); + memory_region_add_subregion_overlap(&s->container, NRF51_NVMC_BASE, mr, 0); + mr = sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->nvm), 1); + memory_region_add_subregion_overlap(&s->container, NRF51_FICR_BASE, mr, 0); + mr = sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->nvm), 2); + memory_region_add_subregion_overlap(&s->container, NRF51_UICR_BASE, mr, 0); + mr = sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->nvm), 3); + memory_region_add_subregion_overlap(&s->container, NRF51_FLASH_BASE, mr, 0); + /* GPIO */ object_property_set_bool(OBJECT(&s->gpio), true, "realized", &err); if (err) { @@ -XXX,XX +XXX,XX @@ static void nrf51_soc_realize(DeviceState *dev_soc, Error **errp) create_unimplemented_device("nrf51_soc.io", NRF51_IOMEM_BASE, NRF51_IOMEM_SIZE); - create_unimplemented_device("nrf51_soc.ficr", NRF51_FICR_BASE, - NRF51_FICR_SIZE); create_unimplemented_device("nrf51_soc.private", NRF51_PRIVATE_BASE, NRF51_PRIVATE_SIZE); } @@ -XXX,XX +XXX,XX @@ static void nrf51_soc_init(Object *obj) sysbus_init_child_obj(obj, "rng", &s->rng, sizeof(s->rng), TYPE_NRF51_RNG); + sysbus_init_child_obj(obj, "nvm", &s->nvm, sizeof(s->nvm), TYPE_NRF51_NVM); + sysbus_init_child_obj(obj, "gpio", &s->gpio, sizeof(s->gpio), TYPE_NRF51_GPIO); -- 2.20.1 From: Steffen Görtz <contrib@steffen-goertz.de> Signed-off-by: Steffen Görtz <contrib@steffen-goertz.de> Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Acked-by: Thomas Huth <thuth@redhat.com> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20190201023357.22596-4-stefanha@redhat.com Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- tests/microbit-test.c | 108 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/tests/microbit-test.c b/tests/microbit-test.c index XXXXXXX..XXXXXXX 100644 --- a/tests/microbit-test.c +++ b/tests/microbit-test.c @@ -XXX,XX +XXX,XX @@ #include "hw/arm/nrf51.h" #include "hw/char/nrf51_uart.h" #include "hw/gpio/nrf51_gpio.h" +#include "hw/nvram/nrf51_nvm.h" #include "hw/timer/nrf51_timer.h" #include "hw/i2c/microbit_i2c.h" @@ -XXX,XX +XXX,XX @@ static void test_microbit_i2c(void) qtest_quit(qts); } +#define FLASH_SIZE (256 * NRF51_PAGE_SIZE) + +static void fill_and_erase(QTestState *qts, hwaddr base, hwaddr size, + uint32_t address_reg) +{ + hwaddr i; + + /* Erase Page */ + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x02); + qtest_writel(qts, NRF51_NVMC_BASE + address_reg, base); + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x00); + + /* Check memory */ + for (i = 0; i < size / 4; i++) { + g_assert_cmpuint(qtest_readl(qts, base + i * 4), ==, 0xFFFFFFFF); + } + + /* Fill memory */ + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x01); + for (i = 0; i < size / 4; i++) { + qtest_writel(qts, base + i * 4, i); + g_assert_cmpuint(qtest_readl(qts, base + i * 4), ==, i); + } + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x00); +} + +static void test_nrf51_nvmc(void) +{ + uint32_t value; + hwaddr i; + QTestState *qts = qtest_init("-M microbit"); + + /* Test always ready */ + value = qtest_readl(qts, NRF51_NVMC_BASE + NRF51_NVMC_READY); + g_assert_cmpuint(value & 0x01, ==, 0x01); + + /* Test write-read config register */ + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x03); + g_assert_cmpuint(qtest_readl(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG), + ==, 0x03); + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x00); + g_assert_cmpuint(qtest_readl(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG), + ==, 0x00); + + /* Test PCR0 */ + fill_and_erase(qts, NRF51_FLASH_BASE, NRF51_PAGE_SIZE, + NRF51_NVMC_ERASEPCR0); + fill_and_erase(qts, NRF51_FLASH_BASE + NRF51_PAGE_SIZE, + NRF51_PAGE_SIZE, NRF51_NVMC_ERASEPCR0); + + /* Test PCR1 */ + fill_and_erase(qts, NRF51_FLASH_BASE, NRF51_PAGE_SIZE, + NRF51_NVMC_ERASEPCR1); + fill_and_erase(qts, NRF51_FLASH_BASE + NRF51_PAGE_SIZE, + NRF51_PAGE_SIZE, NRF51_NVMC_ERASEPCR1); + + /* Erase all */ + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x02); + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_ERASEALL, 0x01); + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x00); + + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x01); + for (i = 0; i < FLASH_SIZE / 4; i++) { + qtest_writel(qts, NRF51_FLASH_BASE + i * 4, i); + g_assert_cmpuint(qtest_readl(qts, NRF51_FLASH_BASE + i * 4), ==, i); + } + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x00); + + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x02); + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_ERASEALL, 0x01); + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x00); + + for (i = 0; i < FLASH_SIZE / 4; i++) { + g_assert_cmpuint(qtest_readl(qts, NRF51_FLASH_BASE + i * 4), + ==, 0xFFFFFFFF); + } + + /* Erase UICR */ + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x02); + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_ERASEUICR, 0x01); + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x00); + + for (i = 0; i < NRF51_UICR_SIZE / 4; i++) { + g_assert_cmpuint(qtest_readl(qts, NRF51_UICR_BASE + i * 4), + ==, 0xFFFFFFFF); + } + + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x01); + for (i = 0; i < NRF51_UICR_SIZE / 4; i++) { + qtest_writel(qts, NRF51_UICR_BASE + i * 4, i); + g_assert_cmpuint(qtest_readl(qts, NRF51_UICR_BASE + i * 4), ==, i); + } + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x00); + + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x02); + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_ERASEUICR, 0x01); + qtest_writel(qts, NRF51_NVMC_BASE + NRF51_NVMC_CONFIG, 0x00); + + for (i = 0; i < NRF51_UICR_SIZE / 4; i++) { + g_assert_cmpuint(qtest_readl(qts, NRF51_UICR_BASE + i * 4), + ==, 0xFFFFFFFF); + } + + qtest_quit(qts); +} + static void test_nrf51_gpio(void) { size_t i; @@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv) qtest_add_func("/microbit/nrf51/uart", test_nrf51_uart); qtest_add_func("/microbit/nrf51/gpio", test_nrf51_gpio); + qtest_add_func("/microbit/nrf51/nvmc", test_nrf51_nvmc); qtest_add_func("/microbit/nrf51/timer", test_nrf51_timer); qtest_add_func("/microbit/microbit/i2c", test_microbit_i2c); -- 2.20.1 Big fat pullreq this time around, because it has all of RTH's SVE2 emulation patchset in it. -- PMM The following changes since commit 0dab1d36f55c3ed649bb8e4c74b9269ef3a63049: Merge remote-tracking branch 'remotes/stefanha-gitlab/tags/block-pull-request' into staging (2021-05-24 15:48:08 +0100) are available in the Git repository at: https://git.linaro.org/people/pmaydell/qemu-arm.git tags/pull-target-arm-20210525 for you to fetch changes up to f8680aaa6e5bfc6022b75157c23db7d2ea98ab11: target/arm: Enable SVE2 and related extensions (2021-05-25 16:01:44 +0100) ---------------------------------------------------------------- target-arm queue: * Implement SVE2 emulation * Implement integer matrix multiply accumulate * Implement FEAT_TLBIOS * Implement FEAT_TLBRANGE * disas/libvixl: Protect C system header for C++ compiler * Use correct SP in M-profile exception return * AN524, AN547: Correct modelling of internal SRAMs * hw/intc/arm_gicv3_cpuif: Fix EOIR write access check logic * hw/arm/smmuv3: Another range invalidation fix ---------------------------------------------------------------- Eric Auger (1): hw/arm/smmuv3: Another range invalidation fix Peter Maydell (8): hw/intc/arm_gicv3_cpuif: Fix EOIR write access check logic hw/arm/mps2-tz: Don't duplicate modelling of SRAM in AN524 hw/arm/mps2-tz: Make SRAM_ADDR_WIDTH board-specific hw/arm/armsse.c: Correct modelling of SSE-300 internal SRAMs hw/arm/armsse: Convert armsse_realize() to use ERRP_GUARD hw/arm/mps2-tz: Allow board to specify a boot RAM size hw/arm: Model TCMs in the SSE-300, not the AN547 target/arm: Use correct SP in M-profile exception return Philippe Mathieu-Daudé (1): disas/libvixl: Protect C system header for C++ compiler Rebecca Cran (3): target/arm: Add support for FEAT_TLBIRANGE target/arm: Add support for FEAT_TLBIOS target/arm: set ID_AA64ISAR0.TLB to 2 for max AARCH64 CPU type Richard Henderson (84): accel/tcg: Replace g_new() + memcpy() by g_memdup() accel/tcg: Pass length argument to tlb_flush_range_locked() accel/tlb: Rename TLBFlushPageBitsByMMUIdxData -> TLBFlushRangeData accel/tcg: Remove {encode,decode}_pbm_to_runon accel/tcg: Add tlb_flush_range_by_mmuidx() accel/tcg: Add tlb_flush_range_by_mmuidx_all_cpus() accel/tlb: Add tlb_flush_range_by_mmuidx_all_cpus_synced() accel/tcg: Rename tlb_flush_page_bits -> range]_by_mmuidx_async_0 accel/tlb: Rename tlb_flush_[page_bits > range]_by_mmuidx_async_[2 > 1] target/arm: Add ID_AA64ZFR0 fields and isar_feature_aa64_sve2 target/arm: Implement SVE2 Integer Multiply - Unpredicated target/arm: Implement SVE2 integer pairwise add and accumulate long target/arm: Implement SVE2 integer unary operations (predicated) target/arm: Split out saturating/rounding shifts from neon target/arm: Implement SVE2 saturating/rounding bitwise shift left (predicated) target/arm: Implement SVE2 integer halving add/subtract (predicated) target/arm: Implement SVE2 integer pairwise arithmetic target/arm: Implement SVE2 saturating add/subtract (predicated) target/arm: Implement SVE2 integer add/subtract long target/arm: Implement SVE2 integer add/subtract interleaved long target/arm: Implement SVE2 integer add/subtract wide target/arm: Implement SVE2 integer multiply long target/arm: Implement SVE2 PMULLB, PMULLT target/arm: Implement SVE2 bitwise shift left long target/arm: Implement SVE2 bitwise exclusive-or interleaved target/arm: Implement SVE2 bitwise permute target/arm: Implement SVE2 complex integer add target/arm: Implement SVE2 integer absolute difference and accumulate long target/arm: Implement SVE2 integer add/subtract long with carry target/arm: Implement SVE2 bitwise shift right and accumulate target/arm: Implement SVE2 bitwise shift and insert target/arm: Implement SVE2 integer absolute difference and accumulate target/arm: Implement SVE2 saturating extract narrow target/arm: Implement SVE2 SHRN, RSHRN target/arm: Implement SVE2 SQSHRUN, SQRSHRUN target/arm: Implement SVE2 UQSHRN, UQRSHRN target/arm: Implement SVE2 SQSHRN, SQRSHRN target/arm: Implement SVE2 WHILEGT, WHILEGE, WHILEHI, WHILEHS target/arm: Implement SVE2 WHILERW, WHILEWR target/arm: Implement SVE2 bitwise ternary operations target/arm: Implement SVE2 saturating multiply-add long target/arm: Implement SVE2 saturating multiply-add high target/arm: Implement SVE2 integer multiply-add long target/arm: Implement SVE2 complex integer multiply-add target/arm: Implement SVE2 XAR target/arm: Use correct output type for gvec_sdot_*_b target/arm: Pass separate addend to {U, S}DOT helpers target/arm: Pass separate addend to FCMLA helpers target/arm: Split out formats for 2 vectors + 1 index target/arm: Split out formats for 3 vectors + 1 index target/arm: Implement SVE2 integer multiply (indexed) target/arm: Implement SVE2 integer multiply-add (indexed) target/arm: Implement SVE2 saturating multiply-add high (indexed) target/arm: Implement SVE2 saturating multiply-add (indexed) target/arm: Implement SVE2 saturating multiply (indexed) target/arm: Implement SVE2 signed saturating doubling multiply high target/arm: Implement SVE2 saturating multiply high (indexed) target/arm: Implement SVE2 multiply-add long (indexed) target/arm: Implement SVE2 integer multiply long (indexed) target/arm: Implement SVE2 complex integer multiply-add (indexed) target/arm: Implement SVE2 complex integer dot product target/arm: Macroize helper_gvec_{s,u}dot_{b,h} target/arm: Macroize helper_gvec_{s,u}dot_idx_{b,h} target/arm: Implement SVE mixed sign dot product (indexed) target/arm: Implement SVE mixed sign dot product target/arm: Implement SVE2 crypto unary operations target/arm: Implement SVE2 crypto destructive binary operations target/arm: Implement SVE2 crypto constructive binary operations target/arm: Implement SVE2 FCVTNT target/arm: Share table of sve load functions target/arm: Tidy do_ldrq target/arm: Implement SVE2 LD1RO target/arm: Implement 128-bit ZIP, UZP, TRN target/arm: Move endian adjustment macros to vec_internal.h target/arm: Implement aarch64 SUDOT, USDOT target/arm: Split out do_neon_ddda_fpst target/arm: Remove unused fpst from VDOT_scalar target/arm: Fix decode for VDOT (indexed) target/arm: Split out do_neon_ddda target/arm: Split decode of VSDOT and VUDOT target/arm: Implement aarch32 VSUDOT, VUSDOT target/arm: Implement integer matrix multiply accumulate linux-user/aarch64: Enable hwcap bits for sve2 and related extensions target/arm: Enable SVE2 and related extensions Stephen Long (17): target/arm: Implement SVE2 floating-point pairwise target/arm: Implement SVE2 MATCH, NMATCH target/arm: Implement SVE2 ADDHNB, ADDHNT target/arm: Implement SVE2 RADDHNB, RADDHNT target/arm: Implement SVE2 SUBHNB, SUBHNT target/arm: Implement SVE2 RSUBHNB, RSUBHNT target/arm: Implement SVE2 HISTCNT, HISTSEG target/arm: Implement SVE2 scatter store insns target/arm: Implement SVE2 gather load insns target/arm: Implement SVE2 FMMLA target/arm: Implement SVE2 SPLICE, EXT target/arm: Implement SVE2 TBL, TBX target/arm: Implement SVE2 FCVTLT target/arm: Implement SVE2 FCVTXNT, FCVTX target/arm: Implement SVE2 FLOGB target/arm: Implement SVE2 bitwise shift immediate target/arm: Implement SVE2 fp multiply-add long disas/libvixl/vixl/code-buffer.h | 2 +- disas/libvixl/vixl/globals.h | 16 +- disas/libvixl/vixl/invalset.h | 2 +- disas/libvixl/vixl/platform.h | 2 + disas/libvixl/vixl/utils.h | 2 +- include/exec/exec-all.h | 44 + include/hw/arm/armsse.h | 2 + target/arm/cpu.h | 76 + target/arm/helper-sve.h | 722 ++++++++- target/arm/helper.h | 110 +- target/arm/translate-a64.h | 3 + target/arm/vec_internal.h | 167 ++ target/arm/neon-shared.decode | 24 +- target/arm/sve.decode | 574 ++++++- accel/tcg/cputlb.c | 231 ++- hw/arm/armsse.c | 35 +- hw/arm/mps2-tz.c | 39 +- hw/arm/smmuv3.c | 50 +- hw/intc/arm_gicv3_cpuif.c | 48 +- linux-user/elfload.c | 10 + target/arm/cpu.c | 2 + target/arm/cpu64.c | 14 + target/arm/cpu_tcg.c | 1 + target/arm/helper.c | 327 +++- target/arm/kvm64.c | 21 +- target/arm/m_helper.c | 3 +- target/arm/neon_helper.c | 507 +----- target/arm/sve_helper.c | 2110 +++++++++++++++++++++++-- target/arm/translate-a64.c | 111 +- target/arm/translate-neon.c | 231 +-- target/arm/translate-sve.c | 3200 +++++++++++++++++++++++++++++++++++--- target/arm/vec_helper.c | 887 ++++++++--- disas/libvixl/vixl/utils.cc | 2 +- 33 files changed, 8275 insertions(+), 1300 deletions(-) From: Eric Auger <eric.auger@redhat.com> 6d9cd115b9 ("hw/arm/smmuv3: Enforce invalidation on a power of two range") failed to completely fix misalignment issues with range invalidation. For instance invalidations patterns like "invalidate 32 4kB pages starting from 0xff395000 are not correctly handled" due to the fact the previous fix only made sure the number of invalidated pages were a power of 2 but did not properly handle the start address was not aligned with the range. This can be noticed when boothing a fedora 33 with protected virtio-blk-pci. Signed-off-by: Eric Auger <eric.auger@redhat.com> Fixes: 6d9cd115b9 ("hw/arm/smmuv3: Enforce invalidation on a power of two range") Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- hw/arm/smmuv3.c | 50 +++++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/hw/arm/smmuv3.c b/hw/arm/smmuv3.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/smmuv3.c +++ b/hw/arm/smmuv3.c @@ -XXX,XX +XXX,XX @@ static void smmuv3_inv_notifiers_iova(SMMUState *s, int asid, dma_addr_t iova, static void smmuv3_s1_range_inval(SMMUState *s, Cmd *cmd) { - uint8_t scale = 0, num = 0, ttl = 0; - dma_addr_t addr = CMD_ADDR(cmd); + dma_addr_t end, addr = CMD_ADDR(cmd); uint8_t type = CMD_TYPE(cmd); uint16_t vmid = CMD_VMID(cmd); + uint8_t scale = CMD_SCALE(cmd); + uint8_t num = CMD_NUM(cmd); + uint8_t ttl = CMD_TTL(cmd); bool leaf = CMD_LEAF(cmd); uint8_t tg = CMD_TG(cmd); - uint64_t first_page = 0, last_page; - uint64_t num_pages = 1; + uint64_t num_pages; + uint8_t granule; int asid = -1; - if (tg) { - scale = CMD_SCALE(cmd); - num = CMD_NUM(cmd); - ttl = CMD_TTL(cmd); - num_pages = (num + 1) * BIT_ULL(scale); - } - if (type == SMMU_CMD_TLBI_NH_VA) { asid = CMD_ASID(cmd); } + if (!tg) { + trace_smmuv3_s1_range_inval(vmid, asid, addr, tg, 1, ttl, leaf); + smmuv3_inv_notifiers_iova(s, asid, addr, tg, 1); + smmu_iotlb_inv_iova(s, asid, addr, tg, 1, ttl); + return; + } + + /* RIL in use */ + + num_pages = (num + 1) * BIT_ULL(scale); + granule = tg * 2 + 10; + /* Split invalidations into ^2 range invalidations */ - last_page = num_pages - 1; - while (num_pages) { - uint8_t granule = tg * 2 + 10; - uint64_t mask, count; + end = addr + (num_pages << granule) - 1; - mask = dma_aligned_pow2_mask(first_page, last_page, 64 - granule); - count = mask + 1; + while (addr != end + 1) { + uint64_t mask = dma_aligned_pow2_mask(addr, end, 64); - trace_smmuv3_s1_range_inval(vmid, asid, addr, tg, count, ttl, leaf); - smmuv3_inv_notifiers_iova(s, asid, addr, tg, count); - smmu_iotlb_inv_iova(s, asid, addr, tg, count, ttl); - - num_pages -= count; - first_page += count; - addr += count * BIT_ULL(granule); + num_pages = (mask + 1) >> granule; + trace_smmuv3_s1_range_inval(vmid, asid, addr, tg, num_pages, ttl, leaf); + smmuv3_inv_notifiers_iova(s, asid, addr, tg, num_pages); + smmu_iotlb_inv_iova(s, asid, addr, tg, num_pages, ttl); + addr += mask + 1; } } -- 2.20.1 In icc_eoir_write() we assume that we can identify the group of the IRQ being completed based purely on which register is being written to and the current CPU state, and that "CPU state matches group indicated by register" is the only necessary access check. This isn't correct: if the CPU is not in Secure state then EOIR1 will only complete Group 1 NS IRQs, but if the CPU is in EL3 it can complete both Group 1 S and Group 1 NS IRQs. (The pseudocode ICC_EOIR1_EL1 makes this clear.) We were also missing the logic to prevent EOIR0 writes completing G0 IRQs when they should not. Rearrange the logic to first identify the group of the current highest priority interrupt and then look at whether we should complete it or ignore the access based on which register was accessed and the state of the CPU. The resulting behavioural change is: * EL3 can now complete G1NS interrupts * G0 interrupt completion is now ignored if the GIC and the CPU have the security extension enabled and the CPU is not secure Reported-by: Chan Kim <ckim@etri.re.kr> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210510150016.24910-1-peter.maydell@linaro.org --- hw/intc/arm_gicv3_cpuif.c | 48 ++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/hw/intc/arm_gicv3_cpuif.c b/hw/intc/arm_gicv3_cpuif.c index XXXXXXX..XXXXXXX 100644 --- a/hw/intc/arm_gicv3_cpuif.c +++ b/hw/intc/arm_gicv3_cpuif.c @@ -XXX,XX +XXX,XX @@ static void icc_eoir_write(CPUARMState *env, const ARMCPRegInfo *ri, GICv3CPUState *cs = icc_cs_from_env(env); int irq = value & 0xffffff; int grp; + bool is_eoir0 = ri->crm == 8; - if (icv_access(env, ri->crm == 8 ? HCR_FMO : HCR_IMO)) { + if (icv_access(env, is_eoir0 ? HCR_FMO : HCR_IMO)) { icv_eoir_write(env, ri, value); return; } - trace_gicv3_icc_eoir_write(ri->crm == 8 ? 0 : 1, + trace_gicv3_icc_eoir_write(is_eoir0 ? 0 : 1, gicv3_redist_affid(cs), value); - if (ri->crm == 8) { - /* EOIR0 */ - grp = GICV3_G0; - } else { - /* EOIR1 */ - if (arm_is_secure(env)) { - grp = GICV3_G1; - } else { - grp = GICV3_G1NS; - } - } - if (irq >= cs->gic->num_irq) { /* This handles two cases: * 1. If software writes the ID of a spurious interrupt [ie 1020-1023] @@ -XXX,XX +XXX,XX @@ static void icc_eoir_write(CPUARMState *env, const ARMCPRegInfo *ri, return; } - if (icc_highest_active_group(cs) != grp) { - return; + grp = icc_highest_active_group(cs); + switch (grp) { + case GICV3_G0: + if (!is_eoir0) { + return; + } + if (!(cs->gic->gicd_ctlr & GICD_CTLR_DS) + && arm_feature(env, ARM_FEATURE_EL3) && !arm_is_secure(env)) { + return; + } + break; + case GICV3_G1: + if (is_eoir0) { + return; + } + if (!arm_is_secure(env)) { + return; + } + break; + case GICV3_G1NS: + if (is_eoir0) { + return; + } + if (!arm_is_el3_or_mon(env) && arm_is_secure(env)) { + return; + } + break; + default: + g_assert_not_reached(); } icc_drop_prio(cs, grp); -- 2.20.1 The SRAM at 0x2000_0000 is part of the SSE-200 itself, and we model it that way in hw/arm/armsse.c (along with the associated MPCs). We incorrectly also added an entry to the RAMInfo array for the AN524 in hw/arm/mps2-tz.c, which was pointless because the CPU would never see it. Delete it. The bug had no guest-visible effect because devices in the SSE-200 take priority over those in the board model (armsse.c maps s->board_memory at priority -2). Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210510190844.17799-2-peter.maydell@linaro.org --- hw/arm/mps2-tz.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/hw/arm/mps2-tz.c b/hw/arm/mps2-tz.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/mps2-tz.c +++ b/hw/arm/mps2-tz.c @@ -XXX,XX +XXX,XX @@ static const RAMInfo an524_raminfo[] = { { .size = 512 * KiB, .mpc = 0, .mrindex = 0, - }, { - .name = "sram", - .base = 0x20000000, - .size = 32 * 4 * KiB, - .mpc = -1, - .mrindex = 1, }, { /* We don't model QSPI flash yet; for now expose it as simple ROM */ .name = "QSPI", .base = 0x28000000, .size = 8 * MiB, .mpc = 1, - .mrindex = 2, + .mrindex = 1, .flags = IS_ROM, }, { .name = "DDR", -- 2.20.1 The AN547 sets the SRAM_ADDR_WIDTH for the SSE-300 to 21; since this is not the default value for the SSE-300, model this in mps2-tz.c as a per-board value. Reported-by: Devaraj Ranganna <devaraj.ranganna@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210510190844.17799-3-peter.maydell@linaro.org --- hw/arm/mps2-tz.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hw/arm/mps2-tz.c b/hw/arm/mps2-tz.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/mps2-tz.c +++ b/hw/arm/mps2-tz.c @@ -XXX,XX +XXX,XX @@ struct MPS2TZMachineClass { int numirq; /* Number of external interrupts */ int uart_overflow_irq; /* number of the combined UART overflow IRQ */ uint32_t init_svtor; /* init-svtor setting for SSE */ + uint32_t sram_addr_width; /* SRAM_ADDR_WIDTH setting for SSE */ const RAMInfo *raminfo; const char *armsse_type; }; @@ -XXX,XX +XXX,XX @@ static void mps2tz_common_init(MachineState *machine) OBJECT(system_memory), &error_abort); qdev_prop_set_uint32(iotkitdev, "EXP_NUMIRQ", mmc->numirq); qdev_prop_set_uint32(iotkitdev, "init-svtor", mmc->init_svtor); + qdev_prop_set_uint32(iotkitdev, "SRAM_ADDR_WIDTH", mmc->sram_addr_width); qdev_connect_clock_in(iotkitdev, "MAINCLK", mms->sysclk); qdev_connect_clock_in(iotkitdev, "S32KCLK", mms->s32kclk); sysbus_realize(SYS_BUS_DEVICE(&mms->iotkit), &error_fatal); @@ -XXX,XX +XXX,XX @@ static void mps2tz_an505_class_init(ObjectClass *oc, void *data) mmc->numirq = 92; mmc->uart_overflow_irq = 47; mmc->init_svtor = 0x10000000; + mmc->sram_addr_width = 15; mmc->raminfo = an505_raminfo; mmc->armsse_type = TYPE_IOTKIT; mps2tz_set_default_ram_info(mmc); @@ -XXX,XX +XXX,XX @@ static void mps2tz_an521_class_init(ObjectClass *oc, void *data) mmc->numirq = 92; mmc->uart_overflow_irq = 47; mmc->init_svtor = 0x10000000; + mmc->sram_addr_width = 15; mmc->raminfo = an505_raminfo; /* AN521 is the same as AN505 here */ mmc->armsse_type = TYPE_SSE200; mps2tz_set_default_ram_info(mmc); @@ -XXX,XX +XXX,XX @@ static void mps3tz_an524_class_init(ObjectClass *oc, void *data) mmc->numirq = 95; mmc->uart_overflow_irq = 47; mmc->init_svtor = 0x10000000; + mmc->sram_addr_width = 15; mmc->raminfo = an524_raminfo; mmc->armsse_type = TYPE_SSE200; mps2tz_set_default_ram_info(mmc); @@ -XXX,XX +XXX,XX @@ static void mps3tz_an547_class_init(ObjectClass *oc, void *data) mmc->numirq = 96; mmc->uart_overflow_irq = 48; mmc->init_svtor = 0x00000000; + mmc->sram_addr_width = 21; mmc->raminfo = an547_raminfo; mmc->armsse_type = TYPE_SSE300; mps2tz_set_default_ram_info(mmc); -- 2.20.1 The SSE-300 was not correctly modelling its internal SRAMs: * the SRAM address width default is 18 * the SRAM is mapped at 0x2100_0000, not 0x2000_0000 like the SSE-200 and IoTKit The default address width is no longer guest-visible since our only SSE-300 board sets it explicitly to a non-default value, but following the hardware's default will help for any future boards we need to model. Reported-by: Devaraj Ranganna <devaraj.ranganna@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210510190844.17799-4-peter.maydell@linaro.org --- hw/arm/armsse.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hw/arm/armsse.c b/hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armsse.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ struct ARMSSEInfo { const char *cpu_type; uint32_t sse_version; int sram_banks; + uint32_t sram_bank_base; int num_cpus; uint32_t sys_version; uint32_t iidr; @@ -XXX,XX +XXX,XX @@ static Property sse300_properties[] = { DEFINE_PROP_LINK("memory", ARMSSE, board_memory, TYPE_MEMORY_REGION, MemoryRegion *), DEFINE_PROP_UINT32("EXP_NUMIRQ", ARMSSE, exp_numirq, 64), - DEFINE_PROP_UINT32("SRAM_ADDR_WIDTH", ARMSSE, sram_addr_width, 15), + DEFINE_PROP_UINT32("SRAM_ADDR_WIDTH", ARMSSE, sram_addr_width, 18), DEFINE_PROP_UINT32("init-svtor", ARMSSE, init_svtor, 0x10000000), DEFINE_PROP_BOOL("CPU0_FPU", ARMSSE, cpu_fpu[0], true), DEFINE_PROP_BOOL("CPU0_DSP", ARMSSE, cpu_dsp[0], true), @@ -XXX,XX +XXX,XX @@ static const ARMSSEInfo armsse_variants[] = { .sse_version = ARMSSE_IOTKIT, .cpu_type = ARM_CPU_TYPE_NAME("cortex-m33"), .sram_banks = 1, + .sram_bank_base = 0x20000000, .num_cpus = 1, .sys_version = 0x41743, .iidr = 0, @@ -XXX,XX +XXX,XX @@ static const ARMSSEInfo armsse_variants[] = { .sse_version = ARMSSE_SSE200, .cpu_type = ARM_CPU_TYPE_NAME("cortex-m33"), .sram_banks = 4, + .sram_bank_base = 0x20000000, .num_cpus = 2, .sys_version = 0x22041743, .iidr = 0, @@ -XXX,XX +XXX,XX @@ static const ARMSSEInfo armsse_variants[] = { .sse_version = ARMSSE_SSE300, .cpu_type = ARM_CPU_TYPE_NAME("cortex-m55"), .sram_banks = 2, + .sram_bank_base = 0x21000000, .num_cpus = 1, .sys_version = 0x7e00043b, .iidr = 0x74a0043b, @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) /* Map the upstream end of the MPC into the right place... */ sbd_mpc = SYS_BUS_DEVICE(&s->mpc[i]); memory_region_add_subregion(&s->container, - 0x20000000 + i * sram_bank_size, + info->sram_bank_base + i * sram_bank_size, sysbus_mmio_get_region(sbd_mpc, 1)); /* ...and its register interface */ memory_region_add_subregion(&s->container, 0x50083000 + i * 0x1000, -- 2.20.1 Convert armsse_realize() to use ERRP_GUARD(), following the rules in include/qapi/error.h. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210510190844.17799-5-peter.maydell@linaro.org --- hw/arm/armsse.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hw/arm/armsse.c b/hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armsse.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) const ARMSSEDeviceInfo *devinfo; int i; MemoryRegion *mr; - Error *err = NULL; SysBusDevice *sbd_apb_ppc0; SysBusDevice *sbd_secctl; DeviceState *dev_apb_ppc0; @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) DeviceState *dev_splitter; uint32_t addr_width_max; + ERRP_GUARD(); + if (!s->board_memory) { error_setg(errp, "memory property was not set"); return; @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) uint32_t sram_bank_size = 1 << s->sram_addr_width; memory_region_init_ram(&s->sram[i], NULL, ramname, - sram_bank_size, &err); + sram_bank_size, errp); g_free(ramname); - if (err) { - error_propagate(errp, err); + if (*errp) { return; } object_property_set_link(OBJECT(&s->mpc[i]), "downstream", -- 2.20.1 Currently we model the ITCM in the AN547's RAMInfo list. This is incorrect because this RAM is really a part of the SSE-300. We can't just delete it from the RAMInfo list, though, because this would make boot_ram_size() assert because it wouldn't be able to find an entry in the list covering guest address 0. Allow a board to specify a boot RAM size manually if it doesn't have any RAM itself at address 0 and is relying on the SSE for that, and set the correct value for the AN547. The other boards can continue to use the "look it up from the RAMInfo list" logic. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210510190844.17799-6-peter.maydell@linaro.org --- hw/arm/mps2-tz.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/hw/arm/mps2-tz.c b/hw/arm/mps2-tz.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/mps2-tz.c +++ b/hw/arm/mps2-tz.c @@ -XXX,XX +XXX,XX @@ struct MPS2TZMachineClass { uint32_t sram_addr_width; /* SRAM_ADDR_WIDTH setting for SSE */ const RAMInfo *raminfo; const char *armsse_type; + uint32_t boot_ram_size; /* size of ram at address 0; 0 == find in raminfo */ }; struct MPS2TZMachineState { @@ -XXX,XX +XXX,XX @@ static uint32_t boot_ram_size(MPS2TZMachineState *mms) const RAMInfo *p; MPS2TZMachineClass *mmc = MPS2TZ_MACHINE_GET_CLASS(mms); + /* + * Use a per-board specification (for when the boot RAM is in + * the SSE and so doesn't have a RAMInfo list entry) + */ + if (mmc->boot_ram_size) { + return mmc->boot_ram_size; + } + for (p = mmc->raminfo; p->name; p++) { if (p->base == boot_mem_base(mms)) { return p->size; @@ -XXX,XX +XXX,XX @@ static void mps2tz_an505_class_init(ObjectClass *oc, void *data) mmc->sram_addr_width = 15; mmc->raminfo = an505_raminfo; mmc->armsse_type = TYPE_IOTKIT; + mmc->boot_ram_size = 0; mps2tz_set_default_ram_info(mmc); } @@ -XXX,XX +XXX,XX @@ static void mps2tz_an521_class_init(ObjectClass *oc, void *data) mmc->sram_addr_width = 15; mmc->raminfo = an505_raminfo; /* AN521 is the same as AN505 here */ mmc->armsse_type = TYPE_SSE200; + mmc->boot_ram_size = 0; mps2tz_set_default_ram_info(mmc); } @@ -XXX,XX +XXX,XX @@ static void mps3tz_an524_class_init(ObjectClass *oc, void *data) mmc->sram_addr_width = 15; mmc->raminfo = an524_raminfo; mmc->armsse_type = TYPE_SSE200; + mmc->boot_ram_size = 0; mps2tz_set_default_ram_info(mmc); object_class_property_add_str(oc, "remap", mps2_get_remap, mps2_set_remap); @@ -XXX,XX +XXX,XX @@ static void mps3tz_an547_class_init(ObjectClass *oc, void *data) mmc->sram_addr_width = 21; mmc->raminfo = an547_raminfo; mmc->armsse_type = TYPE_SSE300; + mmc->boot_ram_size = 512 * KiB; mps2tz_set_default_ram_info(mmc); } -- 2.20.1 The SSE-300 has an ITCM at 0x0000_0000 and a DTCM at 0x2000_0000. Currently we model these in the AN547 board, but this is conceptually wrong, because they are a part of the SSE-300 itself. Move the modelling of the TCMs out of mps2-tz.c into sse300.c. This has no guest-visible effects. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210510190844.17799-7-peter.maydell@linaro.org --- include/hw/arm/armsse.h | 2 ++ hw/arm/armsse.c | 19 +++++++++++++++++++ hw/arm/mps2-tz.c | 12 ------------ 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/include/hw/arm/armsse.h b/include/hw/arm/armsse.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/armsse.h +++ b/include/hw/arm/armsse.h @@ -XXX,XX +XXX,XX @@ struct ARMSSE { MemoryRegion alias2; MemoryRegion alias3[SSE_MAX_CPUS]; MemoryRegion sram[MAX_SRAM_BANKS]; + MemoryRegion itcm; + MemoryRegion dtcm; qemu_irq *exp_irqs[SSE_MAX_CPUS]; qemu_irq ppc0_irq; diff --git a/hw/arm/armsse.c b/hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armsse.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ #include "qemu/log.h" #include "qemu/module.h" #include "qemu/bitops.h" +#include "qemu/units.h" #include "qapi/error.h" #include "trace.h" #include "hw/sysbus.h" @@ -XXX,XX +XXX,XX @@ struct ARMSSEInfo { bool has_cpuid; bool has_cpu_pwrctrl; bool has_sse_counter; + bool has_tcms; Property *props; const ARMSSEDeviceInfo *devinfo; const bool *irq_is_common; @@ -XXX,XX +XXX,XX @@ static const ARMSSEInfo armsse_variants[] = { .has_cpuid = false, .has_cpu_pwrctrl = false, .has_sse_counter = false, + .has_tcms = false, .props = iotkit_properties, .devinfo = iotkit_devices, .irq_is_common = sse200_irq_is_common, @@ -XXX,XX +XXX,XX @@ static const ARMSSEInfo armsse_variants[] = { .has_cpuid = true, .has_cpu_pwrctrl = false, .has_sse_counter = false, + .has_tcms = false, .props = sse200_properties, .devinfo = sse200_devices, .irq_is_common = sse200_irq_is_common, @@ -XXX,XX +XXX,XX @@ static const ARMSSEInfo armsse_variants[] = { .has_cpuid = true, .has_cpu_pwrctrl = true, .has_sse_counter = true, + .has_tcms = true, .props = sse300_properties, .devinfo = sse300_devices, .irq_is_common = sse300_irq_is_common, @@ -XXX,XX +XXX,XX @@ static void armsse_realize(DeviceState *dev, Error **errp) sysbus_mmio_get_region(sbd, 1)); } + if (info->has_tcms) { + /* The SSE-300 has an ITCM at 0x0000_0000 and a DTCM at 0x2000_0000 */ + memory_region_init_ram(&s->itcm, NULL, "sse300-itcm", 512 * KiB, errp); + if (*errp) { + return; + } + memory_region_init_ram(&s->dtcm, NULL, "sse300-dtcm", 512 * KiB, errp); + if (*errp) { + return; + } + memory_region_add_subregion(&s->container, 0x00000000, &s->itcm); + memory_region_add_subregion(&s->container, 0x20000000, &s->dtcm); + } + /* Devices behind APB PPC0: * 0x40000000: timer0 * 0x40001000: timer1 diff --git a/hw/arm/mps2-tz.c b/hw/arm/mps2-tz.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/mps2-tz.c +++ b/hw/arm/mps2-tz.c @@ -XXX,XX +XXX,XX @@ static const RAMInfo an524_raminfo[] = { { }; static const RAMInfo an547_raminfo[] = { { - .name = "itcm", - .base = 0x00000000, - .size = 512 * KiB, - .mpc = -1, - .mrindex = 0, - }, { .name = "sram", .base = 0x01000000, .size = 2 * MiB, .mpc = 0, .mrindex = 1, - }, { - .name = "dtcm", - .base = 0x20000000, - .size = 4 * 128 * KiB, - .mpc = -1, - .mrindex = 2, }, { .name = "sram 2", .base = 0x21000000, -- 2.20.1 When an M-profile CPU is restoring registers from the stack on exception return, the stack pointer to use is determined based on bits in the magic exception return type value. We were not getting this logic entirely correct. Whether we use one of the Secure stack pointers or one of the Non-Secure stack pointers depends on the EXCRET.S bit. However, whether we use the MSP or the PSP then depends on the SPSEL bit in either the CONTROL_S or CONTROL_NS register. We were incorrectly selecting MSP vs PSP based on the EXCRET.SPSEL bit. (In the pseudocode this is in the PopStack() function, which calls LookUpSp_with_security_mode() which in turn looks at the relevant CONTROL.SPSEL bit.) The buggy behaviour wasn't noticeable in most cases, because we write EXCRET.SPSEL to the CONTROL.SPSEL bit for the S/NS register selected by EXCRET.ES, so we only do the wrong thing when EXCRET.S and EXCRET.ES are different. This will happen when secure code takes a secure exception, which then tail-chains to a non-secure exception which finally returns to the original secure code. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210520130905.2049-1-peter.maydell@linaro.org --- target/arm/m_helper.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/target/arm/m_helper.c b/target/arm/m_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/m_helper.c +++ b/target/arm/m_helper.c @@ -XXX,XX +XXX,XX @@ static void do_v7m_exception_exit(ARMCPU *cpu) * We use this limited C variable scope so we don't accidentally * use 'frame_sp_p' after we do something that makes it invalid. */ + bool spsel = env->v7m.control[return_to_secure] & R_V7M_CONTROL_SPSEL_MASK; uint32_t *frame_sp_p = get_v7m_sp_ptr(env, return_to_secure, !return_to_handler, - return_to_sp_process); + spsel); uint32_t frameptr = *frame_sp_p; bool pop_ok = true; ARMMMUIdx mmu_idx; -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Using g_memdup is a bit more compact than g_new + memcpy. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Message-id: 20210509151618.2331764-2-f4bug@amsat.org Message-Id: <20210508201640.1045808-1-richard.henderson@linaro.org> [PMD: Split from bigger patch] Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- accel/tcg/cputlb.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, target_ulong addr, } else if (encode_pbm_to_runon(&runon, d)) { async_run_on_cpu(cpu, tlb_flush_page_bits_by_mmuidx_async_1, runon); } else { - TLBFlushPageBitsByMMUIdxData *p - = g_new(TLBFlushPageBitsByMMUIdxData, 1); - /* Otherwise allocate a structure, freed by the worker. */ - *p = d; + TLBFlushPageBitsByMMUIdxData *p = g_memdup(&d, sizeof(d)); async_run_on_cpu(cpu, tlb_flush_page_bits_by_mmuidx_async_2, RUN_ON_CPU_HOST_PTR(p)); } @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *src_cpu, flush_all_helper(src_cpu, tlb_flush_page_bits_by_mmuidx_async_1, runon); } else { CPUState *dst_cpu; - TLBFlushPageBitsByMMUIdxData *p; /* Allocate a separate data block for each destination cpu. */ CPU_FOREACH(dst_cpu) { if (dst_cpu != src_cpu) { - p = g_new(TLBFlushPageBitsByMMUIdxData, 1); - *p = d; + TLBFlushPageBitsByMMUIdxData *p = g_memdup(&d, sizeof(d)); async_run_on_cpu(dst_cpu, tlb_flush_page_bits_by_mmuidx_async_2, RUN_ON_CPU_HOST_PTR(p)); @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu, /* Allocate a separate data block for each destination cpu. */ CPU_FOREACH(dst_cpu) { if (dst_cpu != src_cpu) { - p = g_new(TLBFlushPageBitsByMMUIdxData, 1); - *p = d; + p = g_memdup(&d, sizeof(d)); async_run_on_cpu(dst_cpu, tlb_flush_page_bits_by_mmuidx_async_2, RUN_ON_CPU_HOST_PTR(p)); } } - p = g_new(TLBFlushPageBitsByMMUIdxData, 1); - *p = d; + p = g_memdup(&d, sizeof(d)); async_safe_run_on_cpu(src_cpu, tlb_flush_page_bits_by_mmuidx_async_2, RUN_ON_CPU_HOST_PTR(p)); } -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Rename tlb_flush_page_bits_locked() -> tlb_flush_range_locked(), and have callers pass a length argument (currently TARGET_PAGE_SIZE) via the TLBFlushPageBitsByMMUIdxData structure. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Message-id: 20210509151618.2331764-3-f4bug@amsat.org Message-Id: <20210508201640.1045808-1-richard.henderson@linaro.org> [PMD: Split from bigger patch] Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- accel/tcg/cputlb.c | 48 +++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_all_cpus_synced(CPUState *src, target_ulong addr) tlb_flush_page_by_mmuidx_all_cpus_synced(src, addr, ALL_MMUIDX_BITS); } -static void tlb_flush_page_bits_locked(CPUArchState *env, int midx, - target_ulong page, unsigned bits) +static void tlb_flush_range_locked(CPUArchState *env, int midx, + target_ulong addr, target_ulong len, + unsigned bits) { CPUTLBDesc *d = &env_tlb(env)->d[midx]; CPUTLBDescFast *f = &env_tlb(env)->f[midx]; @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_bits_locked(CPUArchState *env, int midx, * If @bits is smaller than the tlb size, there may be multiple entries * within the TLB; otherwise all addresses that match under @mask hit * the same TLB entry. - * * TODO: Perhaps allow bits to be a few bits less than the size. * For now, just flush the entire TLB. + * + * If @len is larger than the tlb size, then it will take longer to + * test all of the entries in the TLB than it will to flush it all. */ - if (mask < f->mask) { + if (mask < f->mask || len > f->mask) { tlb_debug("forcing full flush midx %d (" - TARGET_FMT_lx "/" TARGET_FMT_lx ")\n", - midx, page, mask); + TARGET_FMT_lx "/" TARGET_FMT_lx "+" TARGET_FMT_lx ")\n", + midx, addr, mask, len); tlb_flush_one_mmuidx_locked(env, midx, get_clock_realtime()); return; } - /* Check if we need to flush due to large pages. */ - if ((page & d->large_page_mask) == d->large_page_addr) { + /* + * Check if we need to flush due to large pages. + * Because large_page_mask contains all 1's from the msb, + * we only need to test the end of the range. + */ + if (((addr + len - 1) & d->large_page_mask) == d->large_page_addr) { tlb_debug("forcing full flush midx %d (" TARGET_FMT_lx "/" TARGET_FMT_lx ")\n", midx, d->large_page_addr, d->large_page_mask); @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_bits_locked(CPUArchState *env, int midx, return; } - if (tlb_flush_entry_mask_locked(tlb_entry(env, midx, page), page, mask)) { - tlb_n_used_entries_dec(env, midx); + for (target_ulong i = 0; i < len; i += TARGET_PAGE_SIZE) { + target_ulong page = addr + i; + CPUTLBEntry *entry = tlb_entry(env, midx, page); + + if (tlb_flush_entry_mask_locked(entry, page, mask)) { + tlb_n_used_entries_dec(env, midx); + } + tlb_flush_vtlb_page_mask_locked(env, midx, page, mask); } - tlb_flush_vtlb_page_mask_locked(env, midx, page, mask); } typedef struct { target_ulong addr; + target_ulong len; uint16_t idxmap; uint16_t bits; } TLBFlushPageBitsByMMUIdxData; @@ -XXX,XX +XXX,XX @@ tlb_flush_page_bits_by_mmuidx_async_0(CPUState *cpu, assert_cpu_is_self(cpu); - tlb_debug("page addr:" TARGET_FMT_lx "/%u mmu_map:0x%x\n", - d.addr, d.bits, d.idxmap); + tlb_debug("range:" TARGET_FMT_lx "/%u+" TARGET_FMT_lx " mmu_map:0x%x\n", + d.addr, d.bits, d.len, d.idxmap); qemu_spin_lock(&env_tlb(env)->c.lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { if ((d.idxmap >> mmu_idx) & 1) { - tlb_flush_page_bits_locked(env, mmu_idx, d.addr, d.bits); + tlb_flush_range_locked(env, mmu_idx, d.addr, d.len, d.bits); } } qemu_spin_unlock(&env_tlb(env)->c.lock); - tb_flush_jmp_cache(cpu, d.addr); + for (target_ulong i = 0; i < d.len; i += TARGET_PAGE_SIZE) { + tb_flush_jmp_cache(cpu, d.addr + i); + } } static bool encode_pbm_to_runon(run_on_cpu_data *out, @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, target_ulong addr, /* This should already be page aligned */ d.addr = addr & TARGET_PAGE_MASK; + d.len = TARGET_PAGE_SIZE; d.idxmap = idxmap; d.bits = bits; @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *src_cpu, /* This should already be page aligned */ d.addr = addr & TARGET_PAGE_MASK; + d.len = TARGET_PAGE_SIZE; d.idxmap = idxmap; d.bits = bits; @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu, /* This should already be page aligned */ d.addr = addr & TARGET_PAGE_MASK; + d.len = TARGET_PAGE_SIZE; d.idxmap = idxmap; d.bits = bits; -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Rename the structure to match the rename of tlb_flush_range_locked. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Message-id: 20210509151618.2331764-4-f4bug@amsat.org Message-Id: <20210508201640.1045808-1-richard.henderson@linaro.org> [PMD: Split from bigger patch] Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- accel/tcg/cputlb.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ typedef struct { target_ulong len; uint16_t idxmap; uint16_t bits; -} TLBFlushPageBitsByMMUIdxData; +} TLBFlushRangeData; static void tlb_flush_page_bits_by_mmuidx_async_0(CPUState *cpu, - TLBFlushPageBitsByMMUIdxData d) + TLBFlushRangeData d) { CPUArchState *env = cpu->env_ptr; int mmu_idx; @@ -XXX,XX +XXX,XX @@ tlb_flush_page_bits_by_mmuidx_async_0(CPUState *cpu, } static bool encode_pbm_to_runon(run_on_cpu_data *out, - TLBFlushPageBitsByMMUIdxData d) + TLBFlushRangeData d) { /* We need 6 bits to hold to hold @bits up to 63. */ if (d.idxmap <= MAKE_64BIT_MASK(0, TARGET_PAGE_BITS - 6)) { @@ -XXX,XX +XXX,XX @@ static bool encode_pbm_to_runon(run_on_cpu_data *out, return false; } -static TLBFlushPageBitsByMMUIdxData +static TLBFlushRangeData decode_runon_to_pbm(run_on_cpu_data data) { target_ulong addr_map_bits = (target_ulong) data.target_ptr; - return (TLBFlushPageBitsByMMUIdxData){ + return (TLBFlushRangeData){ .addr = addr_map_bits & TARGET_PAGE_MASK, .idxmap = (addr_map_bits & ~TARGET_PAGE_MASK) >> 6, .bits = addr_map_bits & 0x3f @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_bits_by_mmuidx_async_1(CPUState *cpu, static void tlb_flush_page_bits_by_mmuidx_async_2(CPUState *cpu, run_on_cpu_data data) { - TLBFlushPageBitsByMMUIdxData *d = data.host_ptr; + TLBFlushRangeData *d = data.host_ptr; tlb_flush_page_bits_by_mmuidx_async_0(cpu, *d); g_free(d); } @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_bits_by_mmuidx_async_2(CPUState *cpu, void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, target_ulong addr, uint16_t idxmap, unsigned bits) { - TLBFlushPageBitsByMMUIdxData d; + TLBFlushRangeData d; run_on_cpu_data runon; /* If all bits are significant, this devolves to tlb_flush_page. */ @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, target_ulong addr, async_run_on_cpu(cpu, tlb_flush_page_bits_by_mmuidx_async_1, runon); } else { /* Otherwise allocate a structure, freed by the worker. */ - TLBFlushPageBitsByMMUIdxData *p = g_memdup(&d, sizeof(d)); + TLBFlushRangeData *p = g_memdup(&d, sizeof(d)); async_run_on_cpu(cpu, tlb_flush_page_bits_by_mmuidx_async_2, RUN_ON_CPU_HOST_PTR(p)); } @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *src_cpu, uint16_t idxmap, unsigned bits) { - TLBFlushPageBitsByMMUIdxData d; + TLBFlushRangeData d; run_on_cpu_data runon; /* If all bits are significant, this devolves to tlb_flush_page. */ @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *src_cpu, /* Allocate a separate data block for each destination cpu. */ CPU_FOREACH(dst_cpu) { if (dst_cpu != src_cpu) { - TLBFlushPageBitsByMMUIdxData *p = g_memdup(&d, sizeof(d)); + TLBFlushRangeData *p = g_memdup(&d, sizeof(d)); async_run_on_cpu(dst_cpu, tlb_flush_page_bits_by_mmuidx_async_2, RUN_ON_CPU_HOST_PTR(p)); @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu, uint16_t idxmap, unsigned bits) { - TLBFlushPageBitsByMMUIdxData d; + TLBFlushRangeData d; run_on_cpu_data runon; /* If all bits are significant, this devolves to tlb_flush_page. */ @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu, runon); } else { CPUState *dst_cpu; - TLBFlushPageBitsByMMUIdxData *p; + TLBFlushRangeData *p; /* Allocate a separate data block for each destination cpu. */ CPU_FOREACH(dst_cpu) { -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> We will not be able to fit address + length into a 64-bit packet. Drop this optimization before re-organizing this code. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Message-id: 20210509151618.2331764-10-f4bug@amsat.org Message-Id: <20210508201640.1045808-1-richard.henderson@linaro.org> [PMD: Split from bigger patch] Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> [PMM: Moved patch earlier in the series] Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- accel/tcg/cputlb.c | 86 +++++++++++----------------------------------- 1 file changed, 20 insertions(+), 66 deletions(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ tlb_flush_page_bits_by_mmuidx_async_0(CPUState *cpu, } } -static bool encode_pbm_to_runon(run_on_cpu_data *out, - TLBFlushRangeData d) -{ - /* We need 6 bits to hold to hold @bits up to 63. */ - if (d.idxmap <= MAKE_64BIT_MASK(0, TARGET_PAGE_BITS - 6)) { - *out = RUN_ON_CPU_TARGET_PTR(d.addr | (d.idxmap << 6) | d.bits); - return true; - } - return false; -} - -static TLBFlushRangeData -decode_runon_to_pbm(run_on_cpu_data data) -{ - target_ulong addr_map_bits = (target_ulong) data.target_ptr; - return (TLBFlushRangeData){ - .addr = addr_map_bits & TARGET_PAGE_MASK, - .idxmap = (addr_map_bits & ~TARGET_PAGE_MASK) >> 6, - .bits = addr_map_bits & 0x3f - }; -} - -static void tlb_flush_page_bits_by_mmuidx_async_1(CPUState *cpu, - run_on_cpu_data runon) -{ - tlb_flush_page_bits_by_mmuidx_async_0(cpu, decode_runon_to_pbm(runon)); -} - static void tlb_flush_page_bits_by_mmuidx_async_2(CPUState *cpu, run_on_cpu_data data) { @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, target_ulong addr, uint16_t idxmap, unsigned bits) { TLBFlushRangeData d; - run_on_cpu_data runon; /* If all bits are significant, this devolves to tlb_flush_page. */ if (bits >= TARGET_LONG_BITS) { @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, target_ulong addr, if (qemu_cpu_is_self(cpu)) { tlb_flush_page_bits_by_mmuidx_async_0(cpu, d); - } else if (encode_pbm_to_runon(&runon, d)) { - async_run_on_cpu(cpu, tlb_flush_page_bits_by_mmuidx_async_1, runon); } else { /* Otherwise allocate a structure, freed by the worker. */ TLBFlushRangeData *p = g_memdup(&d, sizeof(d)); @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *src_cpu, unsigned bits) { TLBFlushRangeData d; - run_on_cpu_data runon; + CPUState *dst_cpu; /* If all bits are significant, this devolves to tlb_flush_page. */ if (bits >= TARGET_LONG_BITS) { @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *src_cpu, d.idxmap = idxmap; d.bits = bits; - if (encode_pbm_to_runon(&runon, d)) { - flush_all_helper(src_cpu, tlb_flush_page_bits_by_mmuidx_async_1, runon); - } else { - CPUState *dst_cpu; - - /* Allocate a separate data block for each destination cpu. */ - CPU_FOREACH(dst_cpu) { - if (dst_cpu != src_cpu) { - TLBFlushRangeData *p = g_memdup(&d, sizeof(d)); - async_run_on_cpu(dst_cpu, - tlb_flush_page_bits_by_mmuidx_async_2, - RUN_ON_CPU_HOST_PTR(p)); - } + /* Allocate a separate data block for each destination cpu. */ + CPU_FOREACH(dst_cpu) { + if (dst_cpu != src_cpu) { + TLBFlushRangeData *p = g_memdup(&d, sizeof(d)); + async_run_on_cpu(dst_cpu, + tlb_flush_page_bits_by_mmuidx_async_2, + RUN_ON_CPU_HOST_PTR(p)); } } @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu, uint16_t idxmap, unsigned bits) { - TLBFlushRangeData d; - run_on_cpu_data runon; + TLBFlushRangeData d, *p; + CPUState *dst_cpu; /* If all bits are significant, this devolves to tlb_flush_page. */ if (bits >= TARGET_LONG_BITS) { @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu, d.idxmap = idxmap; d.bits = bits; - if (encode_pbm_to_runon(&runon, d)) { - flush_all_helper(src_cpu, tlb_flush_page_bits_by_mmuidx_async_1, runon); - async_safe_run_on_cpu(src_cpu, tlb_flush_page_bits_by_mmuidx_async_1, - runon); - } else { - CPUState *dst_cpu; - TLBFlushRangeData *p; - - /* Allocate a separate data block for each destination cpu. */ - CPU_FOREACH(dst_cpu) { - if (dst_cpu != src_cpu) { - p = g_memdup(&d, sizeof(d)); - async_run_on_cpu(dst_cpu, tlb_flush_page_bits_by_mmuidx_async_2, - RUN_ON_CPU_HOST_PTR(p)); - } + /* Allocate a separate data block for each destination cpu. */ + CPU_FOREACH(dst_cpu) { + if (dst_cpu != src_cpu) { + p = g_memdup(&d, sizeof(d)); + async_run_on_cpu(dst_cpu, tlb_flush_page_bits_by_mmuidx_async_2, + RUN_ON_CPU_HOST_PTR(p)); } - - p = g_memdup(&d, sizeof(d)); - async_safe_run_on_cpu(src_cpu, tlb_flush_page_bits_by_mmuidx_async_2, - RUN_ON_CPU_HOST_PTR(p)); } + + p = g_memdup(&d, sizeof(d)); + async_safe_run_on_cpu(src_cpu, tlb_flush_page_bits_by_mmuidx_async_2, + RUN_ON_CPU_HOST_PTR(p)); } /* update the TLBs so that writes to code in the virtual page 'addr' -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Forward tlb_flush_page_bits_by_mmuidx to tlb_flush_range_by_mmuidx passing TARGET_PAGE_SIZE. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Message-id: 20210509151618.2331764-5-f4bug@amsat.org Message-Id: <20210508201640.1045808-1-richard.henderson@linaro.org> [PMD: Split from bigger patch] Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- include/exec/exec-all.h | 19 +++++++++++++++++++ accel/tcg/cputlb.c | 20 +++++++++++++++----- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *cpu, target_ulong addr, void tlb_flush_page_bits_by_mmuidx_all_cpus_synced (CPUState *cpu, target_ulong addr, uint16_t idxmap, unsigned bits); +/** + * tlb_flush_range_by_mmuidx + * @cpu: CPU whose TLB should be flushed + * @addr: virtual address of the start of the range to be flushed + * @len: length of range to be flushed + * @idxmap: bitmap of mmu indexes to flush + * @bits: number of significant bits in address + * + * For each mmuidx in @idxmap, flush all pages within [@addr,@addr+@len), + * comparing only the low @bits worth of each virtual page. + */ +void tlb_flush_range_by_mmuidx(CPUState *cpu, target_ulong addr, + target_ulong len, uint16_t idxmap, + unsigned bits); /** * tlb_set_page_with_attrs: * @cpu: CPU to add this TLB entry for @@ -XXX,XX +XXX,XX @@ tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *cpu, target_ulong addr, uint16_t idxmap, unsigned bits) { } +static inline void tlb_flush_range_by_mmuidx(CPUState *cpu, target_ulong addr, + target_ulong len, uint16_t idxmap, + unsigned bits) +{ +} #endif /** * probe_access: diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_bits_by_mmuidx_async_2(CPUState *cpu, g_free(d); } -void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, target_ulong addr, - uint16_t idxmap, unsigned bits) +void tlb_flush_range_by_mmuidx(CPUState *cpu, target_ulong addr, + target_ulong len, uint16_t idxmap, + unsigned bits) { TLBFlushRangeData d; - /* If all bits are significant, this devolves to tlb_flush_page. */ - if (bits >= TARGET_LONG_BITS) { + /* + * If all bits are significant, and len is small, + * this devolves to tlb_flush_page. + */ + if (bits >= TARGET_LONG_BITS && len <= TARGET_PAGE_SIZE) { tlb_flush_page_by_mmuidx(cpu, addr, idxmap); return; } @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, target_ulong addr, /* This should already be page aligned */ d.addr = addr & TARGET_PAGE_MASK; - d.len = TARGET_PAGE_SIZE; + d.len = len; d.idxmap = idxmap; d.bits = bits; @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, target_ulong addr, } } +void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, target_ulong addr, + uint16_t idxmap, unsigned bits) +{ + tlb_flush_range_by_mmuidx(cpu, addr, TARGET_PAGE_SIZE, idxmap, bits); +} + void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *src_cpu, target_ulong addr, uint16_t idxmap, -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Forward tlb_flush_page_bits_by_mmuidx_all_cpus to tlb_flush_range_by_mmuidx_all_cpus passing TARGET_PAGE_SIZE. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Message-id: 20210509151618.2331764-6-f4bug@amsat.org Message-Id: <20210508201640.1045808-1-richard.henderson@linaro.org> [PMD: Split from bigger patch] Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- include/exec/exec-all.h | 13 +++++++++++++ accel/tcg/cputlb.c | 24 +++++++++++++++++------- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus_synced void tlb_flush_range_by_mmuidx(CPUState *cpu, target_ulong addr, target_ulong len, uint16_t idxmap, unsigned bits); + +/* Similarly, with broadcast and syncing. */ +void tlb_flush_range_by_mmuidx_all_cpus(CPUState *cpu, target_ulong addr, + target_ulong len, uint16_t idxmap, + unsigned bits); + /** * tlb_set_page_with_attrs: * @cpu: CPU to add this TLB entry for @@ -XXX,XX +XXX,XX @@ static inline void tlb_flush_range_by_mmuidx(CPUState *cpu, target_ulong addr, unsigned bits) { } +static inline void tlb_flush_range_by_mmuidx_all_cpus(CPUState *cpu, + target_ulong addr, + target_ulong len, + uint16_t idxmap, + unsigned bits) +{ +} #endif /** * probe_access: diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, target_ulong addr, tlb_flush_range_by_mmuidx(cpu, addr, TARGET_PAGE_SIZE, idxmap, bits); } -void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *src_cpu, - target_ulong addr, - uint16_t idxmap, - unsigned bits) +void tlb_flush_range_by_mmuidx_all_cpus(CPUState *src_cpu, + target_ulong addr, target_ulong len, + uint16_t idxmap, unsigned bits) { TLBFlushRangeData d; CPUState *dst_cpu; - /* If all bits are significant, this devolves to tlb_flush_page. */ - if (bits >= TARGET_LONG_BITS) { + /* + * If all bits are significant, and len is small, + * this devolves to tlb_flush_page. + */ + if (bits >= TARGET_LONG_BITS && len <= TARGET_PAGE_SIZE) { tlb_flush_page_by_mmuidx_all_cpus(src_cpu, addr, idxmap); return; } @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *src_cpu, /* This should already be page aligned */ d.addr = addr & TARGET_PAGE_MASK; - d.len = TARGET_PAGE_SIZE; + d.len = len; d.idxmap = idxmap; d.bits = bits; @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *src_cpu, tlb_flush_page_bits_by_mmuidx_async_0(src_cpu, d); } +void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *src_cpu, + target_ulong addr, + uint16_t idxmap, unsigned bits) +{ + tlb_flush_range_by_mmuidx_all_cpus(src_cpu, addr, TARGET_PAGE_SIZE, + idxmap, bits); +} + void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu, target_ulong addr, uint16_t idxmap, -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Forward tlb_flush_page_bits_by_mmuidx_all_cpus_synced to tlb_flush_range_by_mmuidx_all_cpus_synced passing TARGET_PAGE_SIZE. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Message-id: 20210509151618.2331764-7-f4bug@amsat.org Message-Id: <20210508201640.1045808-1-richard.henderson@linaro.org> [PMD: Split from bigger patch] Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- include/exec/exec-all.h | 12 ++++++++++++ accel/tcg/cputlb.c | 27 ++++++++++++++++++++------- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ void tlb_flush_range_by_mmuidx(CPUState *cpu, target_ulong addr, void tlb_flush_range_by_mmuidx_all_cpus(CPUState *cpu, target_ulong addr, target_ulong len, uint16_t idxmap, unsigned bits); +void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *cpu, + target_ulong addr, + target_ulong len, + uint16_t idxmap, + unsigned bits); /** * tlb_set_page_with_attrs: @@ -XXX,XX +XXX,XX @@ static inline void tlb_flush_range_by_mmuidx_all_cpus(CPUState *cpu, unsigned bits) { } +static inline void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *cpu, + target_ulong addr, + target_long len, + uint16_t idxmap, + unsigned bits) +{ +} #endif /** * probe_access: diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *src_cpu, idxmap, bits); } -void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu, - target_ulong addr, - uint16_t idxmap, - unsigned bits) +void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *src_cpu, + target_ulong addr, + target_ulong len, + uint16_t idxmap, + unsigned bits) { TLBFlushRangeData d, *p; CPUState *dst_cpu; - /* If all bits are significant, this devolves to tlb_flush_page. */ - if (bits >= TARGET_LONG_BITS) { + /* + * If all bits are significant, and len is small, + * this devolves to tlb_flush_page. + */ + if (bits >= TARGET_LONG_BITS && len <= TARGET_PAGE_SIZE) { tlb_flush_page_by_mmuidx_all_cpus_synced(src_cpu, addr, idxmap); return; } @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu, /* This should already be page aligned */ d.addr = addr & TARGET_PAGE_MASK; - d.len = TARGET_PAGE_SIZE; + d.len = len; d.idxmap = idxmap; d.bits = bits; @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu, RUN_ON_CPU_HOST_PTR(p)); } +void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu, + target_ulong addr, + uint16_t idxmap, + unsigned bits) +{ + tlb_flush_range_by_mmuidx_all_cpus_synced(src_cpu, addr, TARGET_PAGE_SIZE, + idxmap, bits); +} + /* update the TLBs so that writes to code in the virtual page 'addr' can be detected */ void tlb_protect_code(ram_addr_t ram_addr) -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Rename to match tlb_flush_range_locked. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Message-id: 20210509151618.2331764-8-f4bug@amsat.org Message-Id: <20210508201640.1045808-1-richard.henderson@linaro.org> [PMD: Split from bigger patch] Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- accel/tcg/cputlb.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ typedef struct { uint16_t bits; } TLBFlushRangeData; -static void -tlb_flush_page_bits_by_mmuidx_async_0(CPUState *cpu, - TLBFlushRangeData d) +static void tlb_flush_range_by_mmuidx_async_0(CPUState *cpu, + TLBFlushRangeData d) { CPUArchState *env = cpu->env_ptr; int mmu_idx; @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_bits_by_mmuidx_async_2(CPUState *cpu, run_on_cpu_data data) { TLBFlushRangeData *d = data.host_ptr; - tlb_flush_page_bits_by_mmuidx_async_0(cpu, *d); + tlb_flush_range_by_mmuidx_async_0(cpu, *d); g_free(d); } @@ -XXX,XX +XXX,XX @@ void tlb_flush_range_by_mmuidx(CPUState *cpu, target_ulong addr, d.bits = bits; if (qemu_cpu_is_self(cpu)) { - tlb_flush_page_bits_by_mmuidx_async_0(cpu, d); + tlb_flush_range_by_mmuidx_async_0(cpu, d); } else { /* Otherwise allocate a structure, freed by the worker. */ TLBFlushRangeData *p = g_memdup(&d, sizeof(d)); @@ -XXX,XX +XXX,XX @@ void tlb_flush_range_by_mmuidx_all_cpus(CPUState *src_cpu, } } - tlb_flush_page_bits_by_mmuidx_async_0(src_cpu, d); + tlb_flush_range_by_mmuidx_async_0(src_cpu, d); } void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *src_cpu, -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Rename to match tlb_flush_range_locked. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Message-id: 20210509151618.2331764-9-f4bug@amsat.org Message-Id: <20210508201640.1045808-1-richard.henderson@linaro.org> [PMD: Split from bigger patch] Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- accel/tcg/cputlb.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -XXX,XX +XXX,XX @@ static void tlb_flush_range_by_mmuidx_async_0(CPUState *cpu, } } -static void tlb_flush_page_bits_by_mmuidx_async_2(CPUState *cpu, - run_on_cpu_data data) +static void tlb_flush_range_by_mmuidx_async_1(CPUState *cpu, + run_on_cpu_data data) { TLBFlushRangeData *d = data.host_ptr; tlb_flush_range_by_mmuidx_async_0(cpu, *d); @@ -XXX,XX +XXX,XX @@ void tlb_flush_range_by_mmuidx(CPUState *cpu, target_ulong addr, } else { /* Otherwise allocate a structure, freed by the worker. */ TLBFlushRangeData *p = g_memdup(&d, sizeof(d)); - async_run_on_cpu(cpu, tlb_flush_page_bits_by_mmuidx_async_2, + async_run_on_cpu(cpu, tlb_flush_range_by_mmuidx_async_1, RUN_ON_CPU_HOST_PTR(p)); } } @@ -XXX,XX +XXX,XX @@ void tlb_flush_range_by_mmuidx_all_cpus(CPUState *src_cpu, if (dst_cpu != src_cpu) { TLBFlushRangeData *p = g_memdup(&d, sizeof(d)); async_run_on_cpu(dst_cpu, - tlb_flush_page_bits_by_mmuidx_async_2, + tlb_flush_range_by_mmuidx_async_1, RUN_ON_CPU_HOST_PTR(p)); } } @@ -XXX,XX +XXX,XX @@ void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *src_cpu, CPU_FOREACH(dst_cpu) { if (dst_cpu != src_cpu) { p = g_memdup(&d, sizeof(d)); - async_run_on_cpu(dst_cpu, tlb_flush_page_bits_by_mmuidx_async_2, + async_run_on_cpu(dst_cpu, tlb_flush_range_by_mmuidx_async_1, RUN_ON_CPU_HOST_PTR(p)); } } p = g_memdup(&d, sizeof(d)); - async_safe_run_on_cpu(src_cpu, tlb_flush_page_bits_by_mmuidx_async_2, + async_safe_run_on_cpu(src_cpu, tlb_flush_range_by_mmuidx_async_1, RUN_ON_CPU_HOST_PTR(p)); } -- 2.20.1 From: Rebecca Cran <rebecca@nuviainc.com> ARMv8.4 adds the mandatory FEAT_TLBIRANGE. It provides TLBI maintenance instructions that apply to a range of input addresses. Signed-off-by: Rebecca Cran <rebecca@nuviainc.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210512182337.18563-2-rebecca@nuviainc.com Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/cpu.h | 5 + target/arm/helper.c | 281 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 286 insertions(+) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -XXX,XX +XXX,XX @@ static inline bool isar_feature_aa64_pauth_arch(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64isar1, ID_AA64ISAR1, APA) != 0; } +static inline bool isar_feature_aa64_tlbirange(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64isar0, ID_AA64ISAR0, TLB) == 2; +} + static inline bool isar_feature_aa64_sb(const ARMISARegisters *id) { return FIELD_EX64(id->id_aa64isar1, ID_AA64ISAR1, SB) != 0; diff --git a/target/arm/helper.c b/target/arm/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -XXX,XX +XXX,XX @@ static void tlbi_aa64_vae3is_write(CPUARMState *env, const ARMCPRegInfo *ri, ARMMMUIdxBit_SE3, bits); } +#ifdef TARGET_AARCH64 +static uint64_t tlbi_aa64_range_get_length(CPUARMState *env, + uint64_t value) +{ + unsigned int page_shift; + unsigned int page_size_granule; + uint64_t num; + uint64_t scale; + uint64_t exponent; + uint64_t length; + + num = extract64(value, 39, 4); + scale = extract64(value, 44, 2); + page_size_granule = extract64(value, 46, 2); + + page_shift = page_size_granule * 2 + 12; + + if (page_size_granule == 0) { + qemu_log_mask(LOG_GUEST_ERROR, "Invalid page size granule %d\n", + page_size_granule); + return 0; + } + + exponent = (5 * scale) + 1; + length = (num + 1) << (exponent + page_shift); + + return length; +} + +static uint64_t tlbi_aa64_range_get_base(CPUARMState *env, uint64_t value, + bool two_ranges) +{ + /* TODO: ARMv8.7 FEAT_LPA2 */ + uint64_t pageaddr; + + if (two_ranges) { + pageaddr = sextract64(value, 0, 37) << TARGET_PAGE_BITS; + } else { + pageaddr = extract64(value, 0, 37) << TARGET_PAGE_BITS; + } + + return pageaddr; +} + +static void do_rvae_write(CPUARMState *env, uint64_t value, + int idxmap, bool synced) +{ + ARMMMUIdx one_idx = ARM_MMU_IDX_A | ctz32(idxmap); + bool two_ranges = regime_has_2_ranges(one_idx); + uint64_t baseaddr, length; + int bits; + + baseaddr = tlbi_aa64_range_get_base(env, value, two_ranges); + length = tlbi_aa64_range_get_length(env, value); + bits = tlbbits_for_regime(env, one_idx, baseaddr); + + if (synced) { + tlb_flush_range_by_mmuidx_all_cpus_synced(env_cpu(env), + baseaddr, + length, + idxmap, + bits); + } else { + tlb_flush_range_by_mmuidx(env_cpu(env), baseaddr, + length, idxmap, bits); + } +} + +static void tlbi_aa64_rvae1_write(CPUARMState *env, + const ARMCPRegInfo *ri, + uint64_t value) +{ + /* + * Invalidate by VA range, EL1&0. + * Currently handles all of RVAE1, RVAAE1, RVAALE1 and RVALE1, + * since we don't support flush-for-specific-ASID-only or + * flush-last-level-only. + */ + + do_rvae_write(env, value, vae1_tlbmask(env), + tlb_force_broadcast(env)); +} + +static void tlbi_aa64_rvae1is_write(CPUARMState *env, + const ARMCPRegInfo *ri, + uint64_t value) +{ + /* + * Invalidate by VA range, Inner/Outer Shareable EL1&0. + * Currently handles all of RVAE1IS, RVAE1OS, RVAAE1IS, RVAAE1OS, + * RVAALE1IS, RVAALE1OS, RVALE1IS and RVALE1OS, since we don't support + * flush-for-specific-ASID-only, flush-last-level-only or inner/outer + * shareable specific flushes. + */ + + do_rvae_write(env, value, vae1_tlbmask(env), true); +} + +static int vae2_tlbmask(CPUARMState *env) +{ + return (arm_is_secure_below_el3(env) + ? ARMMMUIdxBit_SE2 : ARMMMUIdxBit_E2); +} + +static void tlbi_aa64_rvae2_write(CPUARMState *env, + const ARMCPRegInfo *ri, + uint64_t value) +{ + /* + * Invalidate by VA range, EL2. + * Currently handles all of RVAE2 and RVALE2, + * since we don't support flush-for-specific-ASID-only or + * flush-last-level-only. + */ + + do_rvae_write(env, value, vae2_tlbmask(env), + tlb_force_broadcast(env)); + + +} + +static void tlbi_aa64_rvae2is_write(CPUARMState *env, + const ARMCPRegInfo *ri, + uint64_t value) +{ + /* + * Invalidate by VA range, Inner/Outer Shareable, EL2. + * Currently handles all of RVAE2IS, RVAE2OS, RVALE2IS and RVALE2OS, + * since we don't support flush-for-specific-ASID-only, + * flush-last-level-only or inner/outer shareable specific flushes. + */ + + do_rvae_write(env, value, vae2_tlbmask(env), true); + +} + +static void tlbi_aa64_rvae3_write(CPUARMState *env, + const ARMCPRegInfo *ri, + uint64_t value) +{ + /* + * Invalidate by VA range, EL3. + * Currently handles all of RVAE3 and RVALE3, + * since we don't support flush-for-specific-ASID-only or + * flush-last-level-only. + */ + + do_rvae_write(env, value, ARMMMUIdxBit_SE3, + tlb_force_broadcast(env)); +} + +static void tlbi_aa64_rvae3is_write(CPUARMState *env, + const ARMCPRegInfo *ri, + uint64_t value) +{ + /* + * Invalidate by VA range, EL3, Inner/Outer Shareable. + * Currently handles all of RVAE3IS, RVAE3OS, RVALE3IS and RVALE3OS, + * since we don't support flush-for-specific-ASID-only, + * flush-last-level-only or inner/outer specific flushes. + */ + + do_rvae_write(env, value, ARMMMUIdxBit_SE3, true); +} +#endif + static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri, bool isread) { @@ -XXX,XX +XXX,XX @@ static const ARMCPRegInfo pauth_reginfo[] = { REGINFO_SENTINEL }; +static const ARMCPRegInfo tlbirange_reginfo[] = { + { .name = "TLBI_RVAE1IS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 1, + .access = PL1_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae1is_write }, + { .name = "TLBI_RVAAE1IS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 3, + .access = PL1_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae1is_write }, + { .name = "TLBI_RVALE1IS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 5, + .access = PL1_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae1is_write }, + { .name = "TLBI_RVAALE1IS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 7, + .access = PL1_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae1is_write }, + { .name = "TLBI_RVAE1OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 1, + .access = PL1_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae1is_write }, + { .name = "TLBI_RVAAE1OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 3, + .access = PL1_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae1is_write }, + { .name = "TLBI_RVALE1OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 5, + .access = PL1_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae1is_write }, + { .name = "TLBI_RVAALE1OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 7, + .access = PL1_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae1is_write }, + { .name = "TLBI_RVAE1", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 1, + .access = PL1_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae1_write }, + { .name = "TLBI_RVAAE1", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 3, + .access = PL1_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae1_write }, + { .name = "TLBI_RVALE1", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 5, + .access = PL1_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae1_write }, + { .name = "TLBI_RVAALE1", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 7, + .access = PL1_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae1_write }, + { .name = "TLBI_RIPAS2E1IS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 2, + .access = PL2_W, .type = ARM_CP_NOP }, + { .name = "TLBI_RIPAS2LE1IS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 6, + .access = PL2_W, .type = ARM_CP_NOP }, + { .name = "TLBI_RVAE2IS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 2, .opc2 = 1, + .access = PL2_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae2is_write }, + { .name = "TLBI_RVALE2IS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 2, .opc2 = 5, + .access = PL2_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae2is_write }, + { .name = "TLBI_RIPAS2E1", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 2, + .access = PL2_W, .type = ARM_CP_NOP }, + { .name = "TLBI_RIPAS2LE1", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 6, + .access = PL2_W, .type = ARM_CP_NOP }, + { .name = "TLBI_RVAE2OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 5, .opc2 = 1, + .access = PL2_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae2is_write }, + { .name = "TLBI_RVALE2OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 5, .opc2 = 5, + .access = PL2_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae2is_write }, + { .name = "TLBI_RVAE2", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 6, .opc2 = 1, + .access = PL2_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae2_write }, + { .name = "TLBI_RVALE2", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 6, .opc2 = 5, + .access = PL2_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae2_write }, + { .name = "TLBI_RVAE3IS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 2, .opc2 = 1, + .access = PL3_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae3is_write }, + { .name = "TLBI_RVALE3IS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 2, .opc2 = 5, + .access = PL3_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae3is_write }, + { .name = "TLBI_RVAE3OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 5, .opc2 = 1, + .access = PL3_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae3is_write }, + { .name = "TLBI_RVALE3OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 5, .opc2 = 5, + .access = PL3_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae3is_write }, + { .name = "TLBI_RVAE3", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 6, .opc2 = 1, + .access = PL3_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae3_write }, + { .name = "TLBI_RVALE3", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 6, .opc2 = 5, + .access = PL3_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_rvae3_write }, + REGINFO_SENTINEL +}; + static uint64_t rndr_readfn(CPUARMState *env, const ARMCPRegInfo *ri) { Error *err = NULL; @@ -XXX,XX +XXX,XX @@ void register_cp_regs_for_features(ARMCPU *cpu) if (cpu_isar_feature(aa64_rndr, cpu)) { define_arm_cp_regs(cpu, rndr_reginfo); } + if (cpu_isar_feature(aa64_tlbirange, cpu)) { + define_arm_cp_regs(cpu, tlbirange_reginfo); + } #ifndef CONFIG_USER_ONLY /* Data Cache clean instructions up to PoP */ if (cpu_isar_feature(aa64_dcpop, cpu)) { -- 2.20.1 From: Rebecca Cran <rebecca@nuviainc.com> ARMv8.4 adds the mandatory FEAT_TLBIOS. It provides TLBI maintenance instructions that extend to the Outer Shareable domain. Signed-off-by: Rebecca Cran <rebecca@nuviainc.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210512182337.18563-3-rebecca@nuviainc.com Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/cpu.h | 5 +++++ target/arm/helper.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -XXX,XX +XXX,XX @@ static inline bool isar_feature_aa64_tlbirange(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64isar0, ID_AA64ISAR0, TLB) == 2; } +static inline bool isar_feature_aa64_tlbios(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64isar0, ID_AA64ISAR0, TLB) != 0; +} + static inline bool isar_feature_aa64_sb(const ARMISARegisters *id) { return FIELD_EX64(id->id_aa64isar1, ID_AA64ISAR1, SB) != 0; diff --git a/target/arm/helper.c b/target/arm/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -XXX,XX +XXX,XX @@ static const ARMCPRegInfo tlbirange_reginfo[] = { REGINFO_SENTINEL }; +static const ARMCPRegInfo tlbios_reginfo[] = { + { .name = "TLBI_VMALLE1OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 0, + .access = PL1_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_vmalle1is_write }, + { .name = "TLBI_ASIDE1OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 2, + .access = PL1_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_vmalle1is_write }, + { .name = "TLBI_ALLE2OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 1, .opc2 = 0, + .access = PL2_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_alle2is_write }, + { .name = "TLBI_ALLE1OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 1, .opc2 = 4, + .access = PL2_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_alle1is_write }, + { .name = "TLBI_VMALLS12E1OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 1, .opc2 = 6, + .access = PL2_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_alle1is_write }, + { .name = "TLBI_IPAS2E1OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 0, + .access = PL2_W, .type = ARM_CP_NOP }, + { .name = "TLBI_RIPAS2E1OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 3, + .access = PL2_W, .type = ARM_CP_NOP }, + { .name = "TLBI_IPAS2LE1OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 4, + .access = PL2_W, .type = ARM_CP_NOP }, + { .name = "TLBI_RIPAS2LE1OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 7, + .access = PL2_W, .type = ARM_CP_NOP }, + { .name = "TLBI_ALLE3OS", .state = ARM_CP_STATE_AA64, + .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 1, .opc2 = 0, + .access = PL3_W, .type = ARM_CP_NO_RAW, + .writefn = tlbi_aa64_alle3is_write }, + REGINFO_SENTINEL +}; + static uint64_t rndr_readfn(CPUARMState *env, const ARMCPRegInfo *ri) { Error *err = NULL; @@ -XXX,XX +XXX,XX @@ void register_cp_regs_for_features(ARMCPU *cpu) if (cpu_isar_feature(aa64_tlbirange, cpu)) { define_arm_cp_regs(cpu, tlbirange_reginfo); } + if (cpu_isar_feature(aa64_tlbios, cpu)) { + define_arm_cp_regs(cpu, tlbios_reginfo); + } #ifndef CONFIG_USER_ONLY /* Data Cache clean instructions up to PoP */ if (cpu_isar_feature(aa64_dcpop, cpu)) { -- 2.20.1 From: Rebecca Cran <rebecca@nuviainc.com> Indicate support for FEAT_TLBIOS and FEAT_TLBIRANGE by setting ID_AA64ISAR0.TLB to 2 for the max AARCH64 CPU type. Signed-off-by: Rebecca Cran <rebecca@nuviainc.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210512182337.18563-4-rebecca@nuviainc.com Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/cpu64.c | 1 + 1 file changed, 1 insertion(+) diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu64.c +++ b/target/arm/cpu64.c @@ -XXX,XX +XXX,XX @@ static void aarch64_max_initfn(Object *obj) t = FIELD_DP64(t, ID_AA64ISAR0, DP, 1); t = FIELD_DP64(t, ID_AA64ISAR0, FHM, 1); t = FIELD_DP64(t, ID_AA64ISAR0, TS, 2); /* v8.5-CondM */ + t = FIELD_DP64(t, ID_AA64ISAR0, TLB, 2); /* FEAT_TLBIRANGE */ t = FIELD_DP64(t, ID_AA64ISAR0, RNDR, 1); cpu->isar.id_aa64isar0 = t; -- 2.20.1 From: Philippe Mathieu-Daudé <f4bug@amsat.org> When selecting an ARM target on Debian unstable, we get: Compiling C++ object libcommon.fa.p/disas_libvixl_vixl_utils.cc.o FAILED: libcommon.fa.p/disas_libvixl_vixl_utils.cc.o c++ -Ilibcommon.fa.p -I. -I.. [...] -o libcommon.fa.p/disas_libvixl_vixl_utils.cc.o -c ../disas/libvixl/vixl/utils.cc In file included from /home/philmd/qemu/disas/libvixl/vixl/utils.h:30, from ../disas/libvixl/vixl/utils.cc:27: /usr/include/string.h:36:43: error: missing binary operator before token "(" 36 | #if defined __cplusplus && (__GNUC_PREREQ (4, 4) \ | ^ /usr/include/string.h:53:62: error: missing binary operator before token "(" 53 | #if defined __USE_MISC || defined __USE_XOPEN || __GLIBC_USE (ISOC2X) | ^ /usr/include/string.h:165:21: error: missing binary operator before token "(" 165 | || __GLIBC_USE (LIB_EXT2) || __GLIBC_USE (ISOC2X)) | ^ /usr/include/string.h:174:43: error: missing binary operator before token "(" 174 | #if defined __USE_XOPEN2K8 || __GLIBC_USE (LIB_EXT2) || __GLIBC_USE (ISOC2X) | ^ /usr/include/string.h:492:19: error: missing binary operator before token "(" 492 | #if __GNUC_PREREQ (3,4) | ^ Relevant information from the host: $ lsb_release -d Description: Debian GNU/Linux 11 (bullseye) $ gcc --version gcc (Debian 10.2.1-6) 10.2.1 20210110 $ dpkg -S /usr/include/string.h libc6-dev: /usr/include/string.h $ apt-cache show libc6-dev Package: libc6-dev Version: 2.31-11 Partially cherry-pick vixl commit 78973f258039f6e96 [*]: Refactor VIXL to use `extern` block when including C header that do not have a C++ counterpart. which is similar to commit 875df03b221 ('osdep: protect qemu/osdep.h with extern "C"'). [*] https://git.linaro.org/arm/vixl.git/commit/?id=78973f258039f6e96 Buglink: https://bugs.launchpad.net/qemu/+bug/1914870 Suggested-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Thomas Huth <thuth@redhat.com> Message-id: 20210516171023.510778-1-f4bug@amsat.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- disas/libvixl/vixl/code-buffer.h | 2 +- disas/libvixl/vixl/globals.h | 16 +++++++++------- disas/libvixl/vixl/invalset.h | 2 +- disas/libvixl/vixl/platform.h | 2 ++ disas/libvixl/vixl/utils.h | 2 +- disas/libvixl/vixl/utils.cc | 2 +- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/disas/libvixl/vixl/code-buffer.h b/disas/libvixl/vixl/code-buffer.h index XXXXXXX..XXXXXXX 100644 --- a/disas/libvixl/vixl/code-buffer.h +++ b/disas/libvixl/vixl/code-buffer.h @@ -XXX,XX +XXX,XX @@ #ifndef VIXL_CODE_BUFFER_H #define VIXL_CODE_BUFFER_H -#include <string.h> +#include <cstring> #include "vixl/globals.h" namespace vixl { diff --git a/disas/libvixl/vixl/globals.h b/disas/libvixl/vixl/globals.h index XXXXXXX..XXXXXXX 100644 --- a/disas/libvixl/vixl/globals.h +++ b/disas/libvixl/vixl/globals.h @@ -XXX,XX +XXX,XX @@ #define __STDC_FORMAT_MACROS #endif -#include <stdint.h> +extern "C" { #include <inttypes.h> - -#include <assert.h> -#include <stdarg.h> -#include <stdio.h> #include <stdint.h> -#include <stdlib.h> -#include <stddef.h> +} + +#include <cassert> +#include <cstdarg> +#include <cstddef> +#include <cstdio> +#include <cstdlib> + #include "vixl/platform.h" diff --git a/disas/libvixl/vixl/invalset.h b/disas/libvixl/vixl/invalset.h index XXXXXXX..XXXXXXX 100644 --- a/disas/libvixl/vixl/invalset.h +++ b/disas/libvixl/vixl/invalset.h @@ -XXX,XX +XXX,XX @@ #ifndef VIXL_INVALSET_H_ #define VIXL_INVALSET_H_ -#include <string.h> +#include <cstring> #include <algorithm> #include <vector> diff --git a/disas/libvixl/vixl/platform.h b/disas/libvixl/vixl/platform.h index XXXXXXX..XXXXXXX 100644 --- a/disas/libvixl/vixl/platform.h +++ b/disas/libvixl/vixl/platform.h @@ -XXX,XX +XXX,XX @@ #define PLATFORM_H // Define platform specific functionalities. +extern "C" { #include <signal.h> +} namespace vixl { inline void HostBreakpoint() { raise(SIGINT); } diff --git a/disas/libvixl/vixl/utils.h b/disas/libvixl/vixl/utils.h index XXXXXXX..XXXXXXX 100644 --- a/disas/libvixl/vixl/utils.h +++ b/disas/libvixl/vixl/utils.h @@ -XXX,XX +XXX,XX @@ #ifndef VIXL_UTILS_H #define VIXL_UTILS_H -#include <string.h> #include <cmath> +#include <cstring> #include "vixl/globals.h" #include "vixl/compiler-intrinsics.h" diff --git a/disas/libvixl/vixl/utils.cc b/disas/libvixl/vixl/utils.cc index XXXXXXX..XXXXXXX 100644 --- a/disas/libvixl/vixl/utils.cc +++ b/disas/libvixl/vixl/utils.cc @@ -XXX,XX +XXX,XX @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "vixl/utils.h" -#include <stdio.h> +#include <cstdio> namespace vixl { -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Will be used for SVE2 isa subset enablement. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-2-richard.henderson@linaro.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/cpu.h | 16 ++++++++++++++++ target/arm/helper.c | 3 +-- target/arm/kvm64.c | 21 +++++++++++++++------ 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -XXX,XX +XXX,XX @@ struct ARMCPU { uint64_t id_aa64mmfr2; uint64_t id_aa64dfr0; uint64_t id_aa64dfr1; + uint64_t id_aa64zfr0; } isar; uint64_t midr; uint32_t revidr; @@ -XXX,XX +XXX,XX @@ FIELD(ID_AA64DFR0, DOUBLELOCK, 36, 4) FIELD(ID_AA64DFR0, TRACEFILT, 40, 4) FIELD(ID_AA64DFR0, MTPMU, 48, 4) +FIELD(ID_AA64ZFR0, SVEVER, 0, 4) +FIELD(ID_AA64ZFR0, AES, 4, 4) +FIELD(ID_AA64ZFR0, BITPERM, 16, 4) +FIELD(ID_AA64ZFR0, BFLOAT16, 20, 4) +FIELD(ID_AA64ZFR0, SHA3, 32, 4) +FIELD(ID_AA64ZFR0, SM4, 40, 4) +FIELD(ID_AA64ZFR0, I8MM, 44, 4) +FIELD(ID_AA64ZFR0, F32MM, 52, 4) +FIELD(ID_AA64ZFR0, F64MM, 56, 4) + FIELD(ID_DFR0, COPDBG, 0, 4) FIELD(ID_DFR0, COPSDBG, 4, 4) FIELD(ID_DFR0, MMAPDBG, 8, 4) @@ -XXX,XX +XXX,XX @@ static inline bool isar_feature_aa64_ssbs(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64pfr1, ID_AA64PFR1, SSBS) != 0; } +static inline bool isar_feature_aa64_sve2(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, SVEVER) != 0; +} + /* * Feature tests for "does this exist in either 32-bit or 64-bit?" */ diff --git a/target/arm/helper.c b/target/arm/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -XXX,XX +XXX,XX @@ void register_cp_regs_for_features(ARMCPU *cpu) .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 4, .access = PL1_R, .type = ARM_CP_CONST, .accessfn = access_aa64_tid3, - /* At present, only SVEver == 0 is defined anyway. */ - .resetvalue = 0 }, + .resetvalue = cpu->isar.id_aa64zfr0 }, { .name = "ID_AA64PFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 5, .access = PL1_R, .type = ARM_CP_CONST, diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/kvm64.c +++ b/target/arm/kvm64.c @@ -XXX,XX +XXX,XX @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf) sve_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_SVE) > 0; - kvm_arm_destroy_scratch_host_vcpu(fdarray); - - if (err < 0) { - return false; - } - /* Add feature bits that can't appear until after VCPU init. */ if (sve_supported) { t = ahcf->isar.id_aa64pfr0; t = FIELD_DP64(t, ID_AA64PFR0, SVE, 1); ahcf->isar.id_aa64pfr0 = t; + + /* + * Before v5.1, KVM did not support SVE and did not expose + * ID_AA64ZFR0_EL1 even as RAZ. After v5.1, KVM still does + * not expose the register to "user" requests like this + * unless the host supports SVE. + */ + err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64zfr0, + ARM64_SYS_REG(3, 0, 0, 4, 4)); + } + + kvm_arm_destroy_scratch_host_vcpu(fdarray); + + if (err < 0) { + return false; } /* -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> For MUL, we can rely on generic support. For SMULH and UMULH, create some trivial helpers. For PMUL, back in a21bb78e5817, we organized helper_gvec_pmul_b in preparation for this use. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-3-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper.h | 10 ++++ target/arm/sve.decode | 10 ++++ target/arm/translate-sve.c | 50 ++++++++++++++++++++ target/arm/vec_helper.c | 96 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 166 insertions(+) diff --git a/target/arm/helper.h b/target/arm/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.h +++ b/target/arm/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_3(gvec_cgt0_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(gvec_cge0_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(gvec_cge0_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(gvec_smulh_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(gvec_smulh_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(gvec_smulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(gvec_smulh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(gvec_umulh_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(gvec_umulh_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(gvec_umulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(gvec_umulh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_4(gvec_sshl_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(gvec_sshl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(gvec_ushl_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ ST1_zprz 1110010 .. 00 ..... 100 ... ..... ..... \ @rprr_scatter_store xs=0 esz=3 scale=0 ST1_zprz 1110010 .. 00 ..... 110 ... ..... ..... \ @rprr_scatter_store xs=1 esz=3 scale=0 + +#### SVE2 Support + +### SVE2 Integer Multiply - Unpredicated + +# SVE2 integer multiply vectors (unpredicated) +MUL_zzz 00000100 .. 1 ..... 0110 00 ..... ..... @rd_rn_rm +SMULH_zzz 00000100 .. 1 ..... 0110 10 ..... ..... @rd_rn_rm +UMULH_zzz 00000100 .. 1 ..... 0110 11 ..... ..... @rd_rn_rm +PMUL_zzz 00000100 00 1 ..... 0110 01 ..... ..... @rd_rn_rm_e0 diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_MOVPRFX_z(DisasContext *s, arg_rpr_esz *a) { return do_movz_zpz(s, a->rd, a->rn, a->pg, a->esz, false); } + +/* + * SVE2 Integer Multiply - Unpredicated + */ + +static bool trans_MUL_zzz(DisasContext *s, arg_rrr_esz *a) +{ + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (sve_access_check(s)) { + gen_gvec_fn_zzz(s, tcg_gen_gvec_mul, a->esz, a->rd, a->rn, a->rm); + } + return true; +} + +static bool do_sve2_zzz_ool(DisasContext *s, arg_rrr_esz *a, + gen_helper_gvec_3 *fn) +{ + if (fn == NULL || !dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (sve_access_check(s)) { + gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, 0); + } + return true; +} + +static bool trans_SMULH_zzz(DisasContext *s, arg_rrr_esz *a) +{ + static gen_helper_gvec_3 * const fns[4] = { + gen_helper_gvec_smulh_b, gen_helper_gvec_smulh_h, + gen_helper_gvec_smulh_s, gen_helper_gvec_smulh_d, + }; + return do_sve2_zzz_ool(s, a, fns[a->esz]); +} + +static bool trans_UMULH_zzz(DisasContext *s, arg_rrr_esz *a) +{ + static gen_helper_gvec_3 * const fns[4] = { + gen_helper_gvec_umulh_b, gen_helper_gvec_umulh_h, + gen_helper_gvec_umulh_s, gen_helper_gvec_umulh_d, + }; + return do_sve2_zzz_ool(s, a, fns[a->esz]); +} + +static bool trans_PMUL_zzz(DisasContext *s, arg_rrr_esz *a) +{ + return do_sve2_zzz_ool(s, a, gen_helper_gvec_pmul_b); +} diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(simd_tblx)(void *vd, void *vm, void *venv, uint32_t desc) clear_tail(vd, oprsz, simd_maxsz(desc)); } #endif + +/* + * NxN -> N highpart multiply + * + * TODO: expose this as a generic vector operation. + */ + +void HELPER(gvec_smulh_b)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int8_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz; ++i) { + d[i] = ((int32_t)n[i] * m[i]) >> 8; + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +void HELPER(gvec_smulh_h)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int16_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz / 2; ++i) { + d[i] = ((int32_t)n[i] * m[i]) >> 16; + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +void HELPER(gvec_smulh_s)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int32_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz / 4; ++i) { + d[i] = ((int64_t)n[i] * m[i]) >> 32; + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +void HELPER(gvec_smulh_d)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + uint64_t *d = vd, *n = vn, *m = vm; + uint64_t discard; + + for (i = 0; i < opr_sz / 8; ++i) { + muls64(&discard, &d[i], n[i], m[i]); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +void HELPER(gvec_umulh_b)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + uint8_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz; ++i) { + d[i] = ((uint32_t)n[i] * m[i]) >> 8; + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +void HELPER(gvec_umulh_h)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + uint16_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz / 2; ++i) { + d[i] = ((uint32_t)n[i] * m[i]) >> 16; + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +void HELPER(gvec_umulh_s)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + uint32_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz / 4; ++i) { + d[i] = ((uint64_t)n[i] * m[i]) >> 32; + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +void HELPER(gvec_umulh_d)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + uint64_t *d = vd, *n = vn, *m = vm; + uint64_t discard; + + for (i = 0; i < opr_sz / 8; ++i) { + mulu64(&discard, &d[i], n[i], m[i]); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-4-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 14 ++++++++++++ target/arm/sve.decode | 5 +++++ target/arm/sve_helper.c | 44 ++++++++++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 39 +++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve_umulh_zpzz_s, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_5(sve_umulh_zpzz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sadalp_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sadalp_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sadalp_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_uadalp_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uadalp_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uadalp_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_5(sve_sdiv_zpzz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve_sdiv_zpzz_d, TCG_CALL_NO_RWG, diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ MUL_zzz 00000100 .. 1 ..... 0110 00 ..... ..... @rd_rn_rm SMULH_zzz 00000100 .. 1 ..... 0110 10 ..... ..... @rd_rn_rm UMULH_zzz 00000100 .. 1 ..... 0110 11 ..... ..... @rd_rn_rm PMUL_zzz 00000100 00 1 ..... 0110 01 ..... ..... @rd_rn_rm_e0 + +### SVE2 Integer - Predicated + +SADALP_zpzz 01000100 .. 000 100 101 ... ..... ..... @rdm_pg_rn +UADALP_zpzz 01000100 .. 000 101 101 ... ..... ..... @rdm_pg_rn diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZPZZ_D(sve_asr_zpzz_d, int64_t, DO_ASR) DO_ZPZZ_D(sve_lsr_zpzz_d, uint64_t, DO_LSR) DO_ZPZZ_D(sve_lsl_zpzz_d, uint64_t, DO_LSL) +static inline uint16_t do_sadalp_h(int16_t n, int16_t m) +{ + int8_t n1 = n, n2 = n >> 8; + return m + n1 + n2; +} + +static inline uint32_t do_sadalp_s(int32_t n, int32_t m) +{ + int16_t n1 = n, n2 = n >> 16; + return m + n1 + n2; +} + +static inline uint64_t do_sadalp_d(int64_t n, int64_t m) +{ + int32_t n1 = n, n2 = n >> 32; + return m + n1 + n2; +} + +DO_ZPZZ(sve2_sadalp_zpzz_h, int16_t, H1_2, do_sadalp_h) +DO_ZPZZ(sve2_sadalp_zpzz_s, int32_t, H1_4, do_sadalp_s) +DO_ZPZZ_D(sve2_sadalp_zpzz_d, int64_t, do_sadalp_d) + +static inline uint16_t do_uadalp_h(uint16_t n, uint16_t m) +{ + uint8_t n1 = n, n2 = n >> 8; + return m + n1 + n2; +} + +static inline uint32_t do_uadalp_s(uint32_t n, uint32_t m) +{ + uint16_t n1 = n, n2 = n >> 16; + return m + n1 + n2; +} + +static inline uint64_t do_uadalp_d(uint64_t n, uint64_t m) +{ + uint32_t n1 = n, n2 = n >> 32; + return m + n1 + n2; +} + +DO_ZPZZ(sve2_uadalp_zpzz_h, uint16_t, H1_2, do_uadalp_h) +DO_ZPZZ(sve2_uadalp_zpzz_s, uint32_t, H1_4, do_uadalp_s) +DO_ZPZZ_D(sve2_uadalp_zpzz_d, uint64_t, do_uadalp_d) + #undef DO_ZPZZ #undef DO_ZPZZ_D diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_PMUL_zzz(DisasContext *s, arg_rrr_esz *a) { return do_sve2_zzz_ool(s, a, gen_helper_gvec_pmul_b); } + +/* + * SVE2 Integer - Predicated + */ + +static bool do_sve2_zpzz_ool(DisasContext *s, arg_rprr_esz *a, + gen_helper_gvec_4 *fn) +{ + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + return do_zpzz_ool(s, a, fn); +} + +static bool trans_SADALP_zpzz(DisasContext *s, arg_rprr_esz *a) +{ + static gen_helper_gvec_4 * const fns[3] = { + gen_helper_sve2_sadalp_zpzz_h, + gen_helper_sve2_sadalp_zpzz_s, + gen_helper_sve2_sadalp_zpzz_d, + }; + if (a->esz == 0) { + return false; + } + return do_sve2_zpzz_ool(s, a, fns[a->esz - 1]); +} + +static bool trans_UADALP_zpzz(DisasContext *s, arg_rprr_esz *a) +{ + static gen_helper_gvec_4 * const fns[3] = { + gen_helper_sve2_uadalp_zpzz_h, + gen_helper_sve2_uadalp_zpzz_s, + gen_helper_sve2_uadalp_zpzz_d, + }; + if (a->esz == 0) { + return false; + } + return do_sve2_zpzz_ool(s, a, fns[a->esz - 1]); +} -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-5-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 13 +++++++++++ target/arm/sve.decode | 7 ++++++ target/arm/sve_helper.c | 21 +++++++++++++++++ target/arm/translate-sve.c | 47 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_4(sve_rbit_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve_rbit_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve_rbit_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqabs_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqabs_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqabs_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqabs_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_sqneg_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqneg_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqneg_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqneg_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_urecpe_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_ursqrte_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_5(sve_splice, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve_cmpeq_ppzz_b, TCG_CALL_NO_RWG, diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ PMUL_zzz 00000100 00 1 ..... 0110 01 ..... ..... @rd_rn_rm_e0 SADALP_zpzz 01000100 .. 000 100 101 ... ..... ..... @rdm_pg_rn UADALP_zpzz 01000100 .. 000 101 101 ... ..... ..... @rdm_pg_rn + +### SVE2 integer unary operations (predicated) + +URECPE 01000100 .. 000 000 101 ... ..... ..... @rd_pg_rn +URSQRTE 01000100 .. 000 001 101 ... ..... ..... @rd_pg_rn +SQABS 01000100 .. 001 000 101 ... ..... ..... @rd_pg_rn +SQNEG 01000100 .. 001 001 101 ... ..... ..... @rd_pg_rn diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZPZ(sve_rbit_h, uint16_t, H1_2, revbit16) DO_ZPZ(sve_rbit_s, uint32_t, H1_4, revbit32) DO_ZPZ_D(sve_rbit_d, uint64_t, revbit64) +#define DO_SQABS(X) \ + ({ __typeof(X) x_ = (X), min_ = 1ull << (sizeof(X) * 8 - 1); \ + x_ >= 0 ? x_ : x_ == min_ ? -min_ - 1 : -x_; }) + +DO_ZPZ(sve2_sqabs_b, int8_t, H1, DO_SQABS) +DO_ZPZ(sve2_sqabs_h, int16_t, H1_2, DO_SQABS) +DO_ZPZ(sve2_sqabs_s, int32_t, H1_4, DO_SQABS) +DO_ZPZ_D(sve2_sqabs_d, int64_t, DO_SQABS) + +#define DO_SQNEG(X) \ + ({ __typeof(X) x_ = (X), min_ = 1ull << (sizeof(X) * 8 - 1); \ + x_ == min_ ? -min_ - 1 : -x_; }) + +DO_ZPZ(sve2_sqneg_b, uint8_t, H1, DO_SQNEG) +DO_ZPZ(sve2_sqneg_h, uint16_t, H1_2, DO_SQNEG) +DO_ZPZ(sve2_sqneg_s, uint32_t, H1_4, DO_SQNEG) +DO_ZPZ_D(sve2_sqneg_d, uint64_t, DO_SQNEG) + +DO_ZPZ(sve2_urecpe_s, uint32_t, H1_4, helper_recpe_u32) +DO_ZPZ(sve2_ursqrte_s, uint32_t, H1_4, helper_rsqrte_u32) + /* Three-operand expander, unpredicated, in which the third operand is "wide". */ #define DO_ZZW(NAME, TYPE, TYPEW, H, OP) \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_UADALP_zpzz(DisasContext *s, arg_rprr_esz *a) } return do_sve2_zpzz_ool(s, a, fns[a->esz - 1]); } + +/* + * SVE2 integer unary operations (predicated) + */ + +static bool do_sve2_zpz_ool(DisasContext *s, arg_rpr_esz *a, + gen_helper_gvec_3 *fn) +{ + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + return do_zpz_ool(s, a, fn); +} + +static bool trans_URECPE(DisasContext *s, arg_rpr_esz *a) +{ + if (a->esz != 2) { + return false; + } + return do_sve2_zpz_ool(s, a, gen_helper_sve2_urecpe_s); +} + +static bool trans_URSQRTE(DisasContext *s, arg_rpr_esz *a) +{ + if (a->esz != 2) { + return false; + } + return do_sve2_zpz_ool(s, a, gen_helper_sve2_ursqrte_s); +} + +static bool trans_SQABS(DisasContext *s, arg_rpr_esz *a) +{ + static gen_helper_gvec_3 * const fns[4] = { + gen_helper_sve2_sqabs_b, gen_helper_sve2_sqabs_h, + gen_helper_sve2_sqabs_s, gen_helper_sve2_sqabs_d, + }; + return do_sve2_zpz_ool(s, a, fns[a->esz]); +} + +static bool trans_SQNEG(DisasContext *s, arg_rpr_esz *a) +{ + static gen_helper_gvec_3 * const fns[4] = { + gen_helper_sve2_sqneg_b, gen_helper_sve2_sqneg_h, + gen_helper_sve2_sqneg_s, gen_helper_sve2_sqneg_d, + }; + return do_sve2_zpz_ool(s, a, fns[a->esz]); +} -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Split these operations out into a header that can be shared between neon and sve. The "sat" pointer acts both as a boolean for control of saturating behavior and controls the difference in behavior between neon and sve -- QC bit or no QC bit. Widen the shift operand in the new helpers, as the SVE2 insns treat the whole input element as significant. For the neon uses, truncate the shift to int8_t while passing the parameter. Implement right-shift rounding as tmp = src >> (shift - 1); dst = (tmp >> 1) + (tmp & 1); This is the same number of instructions as the current tmp = 1 << (shift - 1); dst = (src + tmp) >> shift; without any possibility of intermediate overflow. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-6-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/vec_internal.h | 138 +++++++++++ target/arm/neon_helper.c | 507 +++++++------------------------------- 2 files changed, 221 insertions(+), 424 deletions(-) diff --git a/target/arm/vec_internal.h b/target/arm/vec_internal.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_internal.h +++ b/target/arm/vec_internal.h @@ -XXX,XX +XXX,XX @@ static inline void clear_tail(void *vd, uintptr_t opr_sz, uintptr_t max_sz) } } +static inline int32_t do_sqrshl_bhs(int32_t src, int32_t shift, int bits, + bool round, uint32_t *sat) +{ + if (shift <= -bits) { + /* Rounding the sign bit always produces 0. */ + if (round) { + return 0; + } + return src >> 31; + } else if (shift < 0) { + if (round) { + src >>= -shift - 1; + return (src >> 1) + (src & 1); + } + return src >> -shift; + } else if (shift < bits) { + int32_t val = src << shift; + if (bits == 32) { + if (!sat || val >> shift == src) { + return val; + } + } else { + int32_t extval = sextract32(val, 0, bits); + if (!sat || val == extval) { + return extval; + } + } + } else if (!sat || src == 0) { + return 0; + } + + *sat = 1; + return (1u << (bits - 1)) - (src >= 0); +} + +static inline uint32_t do_uqrshl_bhs(uint32_t src, int32_t shift, int bits, + bool round, uint32_t *sat) +{ + if (shift <= -(bits + round)) { + return 0; + } else if (shift < 0) { + if (round) { + src >>= -shift - 1; + return (src >> 1) + (src & 1); + } + return src >> -shift; + } else if (shift < bits) { + uint32_t val = src << shift; + if (bits == 32) { + if (!sat || val >> shift == src) { + return val; + } + } else { + uint32_t extval = extract32(val, 0, bits); + if (!sat || val == extval) { + return extval; + } + } + } else if (!sat || src == 0) { + return 0; + } + + *sat = 1; + return MAKE_64BIT_MASK(0, bits); +} + +static inline int32_t do_suqrshl_bhs(int32_t src, int32_t shift, int bits, + bool round, uint32_t *sat) +{ + if (sat && src < 0) { + *sat = 1; + return 0; + } + return do_uqrshl_bhs(src, shift, bits, round, sat); +} + +static inline int64_t do_sqrshl_d(int64_t src, int64_t shift, + bool round, uint32_t *sat) +{ + if (shift <= -64) { + /* Rounding the sign bit always produces 0. */ + if (round) { + return 0; + } + return src >> 63; + } else if (shift < 0) { + if (round) { + src >>= -shift - 1; + return (src >> 1) + (src & 1); + } + return src >> -shift; + } else if (shift < 64) { + int64_t val = src << shift; + if (!sat || val >> shift == src) { + return val; + } + } else if (!sat || src == 0) { + return 0; + } + + *sat = 1; + return src < 0 ? INT64_MIN : INT64_MAX; +} + +static inline uint64_t do_uqrshl_d(uint64_t src, int64_t shift, + bool round, uint32_t *sat) +{ + if (shift <= -(64 + round)) { + return 0; + } else if (shift < 0) { + if (round) { + src >>= -shift - 1; + return (src >> 1) + (src & 1); + } + return src >> -shift; + } else if (shift < 64) { + uint64_t val = src << shift; + if (!sat || val >> shift == src) { + return val; + } + } else if (!sat || src == 0) { + return 0; + } + + *sat = 1; + return UINT64_MAX; +} + +static inline int64_t do_suqrshl_d(int64_t src, int64_t shift, + bool round, uint32_t *sat) +{ + if (sat && src < 0) { + *sat = 1; + return 0; + } + return do_uqrshl_d(src, shift, round, sat); +} + #endif /* TARGET_ARM_VEC_INTERNALS_H */ diff --git a/target/arm/neon_helper.c b/target/arm/neon_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/neon_helper.c +++ b/target/arm/neon_helper.c @@ -XXX,XX +XXX,XX @@ #include "cpu.h" #include "exec/helper-proto.h" #include "fpu/softfloat.h" +#include "vec_internal.h" #define SIGNBIT (uint32_t)0x80000000 #define SIGNBIT64 ((uint64_t)1 << 63) @@ -XXX,XX +XXX,XX @@ NEON_POP(pmax_s16, neon_s16, 2) NEON_POP(pmax_u16, neon_u16, 2) #undef NEON_FN -#define NEON_FN(dest, src1, src2) do { \ - int8_t tmp; \ - tmp = (int8_t)src2; \ - if (tmp >= (ssize_t)sizeof(src1) * 8 || \ - tmp <= -(ssize_t)sizeof(src1) * 8) { \ - dest = 0; \ - } else if (tmp < 0) { \ - dest = src1 >> -tmp; \ - } else { \ - dest = src1 << tmp; \ - }} while (0) +#define NEON_FN(dest, src1, src2) \ + (dest = do_uqrshl_bhs(src1, (int8_t)src2, 16, false, NULL)) NEON_VOP(shl_u16, neon_u16, 2) #undef NEON_FN -#define NEON_FN(dest, src1, src2) do { \ - int8_t tmp; \ - tmp = (int8_t)src2; \ - if (tmp >= (ssize_t)sizeof(src1) * 8) { \ - dest = 0; \ - } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \ - dest = src1 >> (sizeof(src1) * 8 - 1); \ - } else if (tmp < 0) { \ - dest = src1 >> -tmp; \ - } else { \ - dest = src1 << tmp; \ - }} while (0) +#define NEON_FN(dest, src1, src2) \ + (dest = do_sqrshl_bhs(src1, (int8_t)src2, 16, false, NULL)) NEON_VOP(shl_s16, neon_s16, 2) #undef NEON_FN -#define NEON_FN(dest, src1, src2) do { \ - int8_t tmp; \ - tmp = (int8_t)src2; \ - if ((tmp >= (ssize_t)sizeof(src1) * 8) \ - || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \ - dest = 0; \ - } else if (tmp < 0) { \ - dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \ - } else { \ - dest = src1 << tmp; \ - }} while (0) +#define NEON_FN(dest, src1, src2) \ + (dest = do_sqrshl_bhs(src1, (int8_t)src2, 8, true, NULL)) NEON_VOP(rshl_s8, neon_s8, 4) +#undef NEON_FN + +#define NEON_FN(dest, src1, src2) \ + (dest = do_sqrshl_bhs(src1, (int8_t)src2, 16, true, NULL)) NEON_VOP(rshl_s16, neon_s16, 2) #undef NEON_FN -/* The addition of the rounding constant may overflow, so we use an - * intermediate 64 bit accumulator. */ -uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop) +uint32_t HELPER(neon_rshl_s32)(uint32_t val, uint32_t shift) { - int32_t dest; - int32_t val = (int32_t)valop; - int8_t shift = (int8_t)shiftop; - if ((shift >= 32) || (shift <= -32)) { - dest = 0; - } else if (shift < 0) { - int64_t big_dest = ((int64_t)val + (1 << (-1 - shift))); - dest = big_dest >> -shift; - } else { - dest = val << shift; - } - return dest; + return do_sqrshl_bhs(val, (int8_t)shift, 32, true, NULL); } -/* Handling addition overflow with 64 bit input values is more - * tricky than with 32 bit values. */ -uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop) +uint64_t HELPER(neon_rshl_s64)(uint64_t val, uint64_t shift) { - int8_t shift = (int8_t)shiftop; - int64_t val = valop; - if ((shift >= 64) || (shift <= -64)) { - val = 0; - } else if (shift < 0) { - val >>= (-shift - 1); - if (val == INT64_MAX) { - /* In this case, it means that the rounding constant is 1, - * and the addition would overflow. Return the actual - * result directly. */ - val = 0x4000000000000000LL; - } else { - val++; - val >>= 1; - } - } else { - val <<= shift; - } - return val; + return do_sqrshl_d(val, (int8_t)shift, true, NULL); } -#define NEON_FN(dest, src1, src2) do { \ - int8_t tmp; \ - tmp = (int8_t)src2; \ - if (tmp >= (ssize_t)sizeof(src1) * 8 || \ - tmp < -(ssize_t)sizeof(src1) * 8) { \ - dest = 0; \ - } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \ - dest = src1 >> (-tmp - 1); \ - } else if (tmp < 0) { \ - dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \ - } else { \ - dest = src1 << tmp; \ - }} while (0) +#define NEON_FN(dest, src1, src2) \ + (dest = do_uqrshl_bhs(src1, (int8_t)src2, 8, true, NULL)) NEON_VOP(rshl_u8, neon_u8, 4) +#undef NEON_FN + +#define NEON_FN(dest, src1, src2) \ + (dest = do_uqrshl_bhs(src1, (int8_t)src2, 16, true, NULL)) NEON_VOP(rshl_u16, neon_u16, 2) #undef NEON_FN -/* The addition of the rounding constant may overflow, so we use an - * intermediate 64 bit accumulator. */ -uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop) +uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shift) { - uint32_t dest; - int8_t shift = (int8_t)shiftop; - if (shift >= 32 || shift < -32) { - dest = 0; - } else if (shift == -32) { - dest = val >> 31; - } else if (shift < 0) { - uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift))); - dest = big_dest >> -shift; - } else { - dest = val << shift; - } - return dest; + return do_uqrshl_bhs(val, (int8_t)shift, 32, true, NULL); } -/* Handling addition overflow with 64 bit input values is more - * tricky than with 32 bit values. */ -uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop) +uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shift) { - int8_t shift = (uint8_t)shiftop; - if (shift >= 64 || shift < -64) { - val = 0; - } else if (shift == -64) { - /* Rounding a 1-bit result just preserves that bit. */ - val >>= 63; - } else if (shift < 0) { - val >>= (-shift - 1); - if (val == UINT64_MAX) { - /* In this case, it means that the rounding constant is 1, - * and the addition would overflow. Return the actual - * result directly. */ - val = 0x8000000000000000ULL; - } else { - val++; - val >>= 1; - } - } else { - val <<= shift; - } - return val; + return do_uqrshl_d(val, (int8_t)shift, true, NULL); } -#define NEON_FN(dest, src1, src2) do { \ - int8_t tmp; \ - tmp = (int8_t)src2; \ - if (tmp >= (ssize_t)sizeof(src1) * 8) { \ - if (src1) { \ - SET_QC(); \ - dest = ~0; \ - } else { \ - dest = 0; \ - } \ - } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \ - dest = 0; \ - } else if (tmp < 0) { \ - dest = src1 >> -tmp; \ - } else { \ - dest = src1 << tmp; \ - if ((dest >> tmp) != src1) { \ - SET_QC(); \ - dest = ~0; \ - } \ - }} while (0) +#define NEON_FN(dest, src1, src2) \ + (dest = do_uqrshl_bhs(src1, (int8_t)src2, 8, false, env->vfp.qc)) NEON_VOP_ENV(qshl_u8, neon_u8, 4) +#undef NEON_FN + +#define NEON_FN(dest, src1, src2) \ + (dest = do_uqrshl_bhs(src1, (int8_t)src2, 16, false, env->vfp.qc)) NEON_VOP_ENV(qshl_u16, neon_u16, 2) -NEON_VOP_ENV(qshl_u32, neon_u32, 1) #undef NEON_FN -uint64_t HELPER(neon_qshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop) +uint32_t HELPER(neon_qshl_u32)(CPUARMState *env, uint32_t val, uint32_t shift) { - int8_t shift = (int8_t)shiftop; - if (shift >= 64) { - if (val) { - val = ~(uint64_t)0; - SET_QC(); - } - } else if (shift <= -64) { - val = 0; - } else if (shift < 0) { - val >>= -shift; - } else { - uint64_t tmp = val; - val <<= shift; - if ((val >> shift) != tmp) { - SET_QC(); - val = ~(uint64_t)0; - } - } - return val; + return do_uqrshl_bhs(val, (int8_t)shift, 32, false, env->vfp.qc); } -#define NEON_FN(dest, src1, src2) do { \ - int8_t tmp; \ - tmp = (int8_t)src2; \ - if (tmp >= (ssize_t)sizeof(src1) * 8) { \ - if (src1) { \ - SET_QC(); \ - dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \ - if (src1 > 0) { \ - dest--; \ - } \ - } else { \ - dest = src1; \ - } \ - } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \ - dest = src1 >> 31; \ - } else if (tmp < 0) { \ - dest = src1 >> -tmp; \ - } else { \ - dest = src1 << tmp; \ - if ((dest >> tmp) != src1) { \ - SET_QC(); \ - dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \ - if (src1 > 0) { \ - dest--; \ - } \ - } \ - }} while (0) +uint64_t HELPER(neon_qshl_u64)(CPUARMState *env, uint64_t val, uint64_t shift) +{ + return do_uqrshl_d(val, (int8_t)shift, false, env->vfp.qc); +} + +#define NEON_FN(dest, src1, src2) \ + (dest = do_sqrshl_bhs(src1, (int8_t)src2, 8, false, env->vfp.qc)) NEON_VOP_ENV(qshl_s8, neon_s8, 4) +#undef NEON_FN + +#define NEON_FN(dest, src1, src2) \ + (dest = do_sqrshl_bhs(src1, (int8_t)src2, 16, false, env->vfp.qc)) NEON_VOP_ENV(qshl_s16, neon_s16, 2) -NEON_VOP_ENV(qshl_s32, neon_s32, 1) #undef NEON_FN -uint64_t HELPER(neon_qshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop) +uint32_t HELPER(neon_qshl_s32)(CPUARMState *env, uint32_t val, uint32_t shift) { - int8_t shift = (uint8_t)shiftop; - int64_t val = valop; - if (shift >= 64) { - if (val) { - SET_QC(); - val = (val >> 63) ^ ~SIGNBIT64; - } - } else if (shift <= -64) { - val >>= 63; - } else if (shift < 0) { - val >>= -shift; - } else { - int64_t tmp = val; - val <<= shift; - if ((val >> shift) != tmp) { - SET_QC(); - val = (tmp >> 63) ^ ~SIGNBIT64; - } - } - return val; + return do_sqrshl_bhs(val, (int8_t)shift, 32, false, env->vfp.qc); } -#define NEON_FN(dest, src1, src2) do { \ - if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \ - SET_QC(); \ - dest = 0; \ - } else { \ - int8_t tmp; \ - tmp = (int8_t)src2; \ - if (tmp >= (ssize_t)sizeof(src1) * 8) { \ - if (src1) { \ - SET_QC(); \ - dest = ~0; \ - } else { \ - dest = 0; \ - } \ - } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \ - dest = 0; \ - } else if (tmp < 0) { \ - dest = src1 >> -tmp; \ - } else { \ - dest = src1 << tmp; \ - if ((dest >> tmp) != src1) { \ - SET_QC(); \ - dest = ~0; \ - } \ - } \ - }} while (0) -NEON_VOP_ENV(qshlu_s8, neon_u8, 4) -NEON_VOP_ENV(qshlu_s16, neon_u16, 2) +uint64_t HELPER(neon_qshl_s64)(CPUARMState *env, uint64_t val, uint64_t shift) +{ + return do_sqrshl_d(val, (int8_t)shift, false, env->vfp.qc); +} + +#define NEON_FN(dest, src1, src2) \ + (dest = do_suqrshl_bhs(src1, (int8_t)src2, 8, false, env->vfp.qc)) +NEON_VOP_ENV(qshlu_s8, neon_s8, 4) #undef NEON_FN -uint32_t HELPER(neon_qshlu_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop) +#define NEON_FN(dest, src1, src2) \ + (dest = do_suqrshl_bhs(src1, (int8_t)src2, 16, false, env->vfp.qc)) +NEON_VOP_ENV(qshlu_s16, neon_s16, 2) +#undef NEON_FN + +uint32_t HELPER(neon_qshlu_s32)(CPUARMState *env, uint32_t val, uint32_t shift) { - if ((int32_t)valop < 0) { - SET_QC(); - return 0; - } - return helper_neon_qshl_u32(env, valop, shiftop); + return do_suqrshl_bhs(val, (int8_t)shift, 32, false, env->vfp.qc); } -uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop) +uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t val, uint64_t shift) { - if ((int64_t)valop < 0) { - SET_QC(); - return 0; - } - return helper_neon_qshl_u64(env, valop, shiftop); + return do_suqrshl_d(val, (int8_t)shift, false, env->vfp.qc); } -#define NEON_FN(dest, src1, src2) do { \ - int8_t tmp; \ - tmp = (int8_t)src2; \ - if (tmp >= (ssize_t)sizeof(src1) * 8) { \ - if (src1) { \ - SET_QC(); \ - dest = ~0; \ - } else { \ - dest = 0; \ - } \ - } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \ - dest = 0; \ - } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \ - dest = src1 >> (sizeof(src1) * 8 - 1); \ - } else if (tmp < 0) { \ - dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \ - } else { \ - dest = src1 << tmp; \ - if ((dest >> tmp) != src1) { \ - SET_QC(); \ - dest = ~0; \ - } \ - }} while (0) +#define NEON_FN(dest, src1, src2) \ + (dest = do_uqrshl_bhs(src1, (int8_t)src2, 8, true, env->vfp.qc)) NEON_VOP_ENV(qrshl_u8, neon_u8, 4) +#undef NEON_FN + +#define NEON_FN(dest, src1, src2) \ + (dest = do_uqrshl_bhs(src1, (int8_t)src2, 16, true, env->vfp.qc)) NEON_VOP_ENV(qrshl_u16, neon_u16, 2) #undef NEON_FN -/* The addition of the rounding constant may overflow, so we use an - * intermediate 64 bit accumulator. */ -uint32_t HELPER(neon_qrshl_u32)(CPUARMState *env, uint32_t val, uint32_t shiftop) +uint32_t HELPER(neon_qrshl_u32)(CPUARMState *env, uint32_t val, uint32_t shift) { - uint32_t dest; - int8_t shift = (int8_t)shiftop; - if (shift >= 32) { - if (val) { - SET_QC(); - dest = ~0; - } else { - dest = 0; - } - } else if (shift < -32) { - dest = 0; - } else if (shift == -32) { - dest = val >> 31; - } else if (shift < 0) { - uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift))); - dest = big_dest >> -shift; - } else { - dest = val << shift; - if ((dest >> shift) != val) { - SET_QC(); - dest = ~0; - } - } - return dest; + return do_uqrshl_bhs(val, (int8_t)shift, 32, true, env->vfp.qc); } -/* Handling addition overflow with 64 bit input values is more - * tricky than with 32 bit values. */ -uint64_t HELPER(neon_qrshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop) +uint64_t HELPER(neon_qrshl_u64)(CPUARMState *env, uint64_t val, uint64_t shift) { - int8_t shift = (int8_t)shiftop; - if (shift >= 64) { - if (val) { - SET_QC(); - val = ~0; - } - } else if (shift < -64) { - val = 0; - } else if (shift == -64) { - val >>= 63; - } else if (shift < 0) { - val >>= (-shift - 1); - if (val == UINT64_MAX) { - /* In this case, it means that the rounding constant is 1, - * and the addition would overflow. Return the actual - * result directly. */ - val = 0x8000000000000000ULL; - } else { - val++; - val >>= 1; - } - } else { \ - uint64_t tmp = val; - val <<= shift; - if ((val >> shift) != tmp) { - SET_QC(); - val = ~0; - } - } - return val; + return do_uqrshl_d(val, (int8_t)shift, true, env->vfp.qc); } -#define NEON_FN(dest, src1, src2) do { \ - int8_t tmp; \ - tmp = (int8_t)src2; \ - if (tmp >= (ssize_t)sizeof(src1) * 8) { \ - if (src1) { \ - SET_QC(); \ - dest = (typeof(dest))(1 << (sizeof(src1) * 8 - 1)); \ - if (src1 > 0) { \ - dest--; \ - } \ - } else { \ - dest = 0; \ - } \ - } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \ - dest = 0; \ - } else if (tmp < 0) { \ - dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \ - } else { \ - dest = src1 << tmp; \ - if ((dest >> tmp) != src1) { \ - SET_QC(); \ - dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \ - if (src1 > 0) { \ - dest--; \ - } \ - } \ - }} while (0) +#define NEON_FN(dest, src1, src2) \ + (dest = do_sqrshl_bhs(src1, (int8_t)src2, 8, true, env->vfp.qc)) NEON_VOP_ENV(qrshl_s8, neon_s8, 4) +#undef NEON_FN + +#define NEON_FN(dest, src1, src2) \ + (dest = do_sqrshl_bhs(src1, (int8_t)src2, 16, true, env->vfp.qc)) NEON_VOP_ENV(qrshl_s16, neon_s16, 2) #undef NEON_FN -/* The addition of the rounding constant may overflow, so we use an - * intermediate 64 bit accumulator. */ -uint32_t HELPER(neon_qrshl_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop) +uint32_t HELPER(neon_qrshl_s32)(CPUARMState *env, uint32_t val, uint32_t shift) { - int32_t dest; - int32_t val = (int32_t)valop; - int8_t shift = (int8_t)shiftop; - if (shift >= 32) { - if (val) { - SET_QC(); - dest = (val >> 31) ^ ~SIGNBIT; - } else { - dest = 0; - } - } else if (shift <= -32) { - dest = 0; - } else if (shift < 0) { - int64_t big_dest = ((int64_t)val + (1 << (-1 - shift))); - dest = big_dest >> -shift; - } else { - dest = val << shift; - if ((dest >> shift) != val) { - SET_QC(); - dest = (val >> 31) ^ ~SIGNBIT; - } - } - return dest; + return do_sqrshl_bhs(val, (int8_t)shift, 32, true, env->vfp.qc); } -/* Handling addition overflow with 64 bit input values is more - * tricky than with 32 bit values. */ -uint64_t HELPER(neon_qrshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop) +uint64_t HELPER(neon_qrshl_s64)(CPUARMState *env, uint64_t val, uint64_t shift) { - int8_t shift = (uint8_t)shiftop; - int64_t val = valop; - - if (shift >= 64) { - if (val) { - SET_QC(); - val = (val >> 63) ^ ~SIGNBIT64; - } - } else if (shift <= -64) { - val = 0; - } else if (shift < 0) { - val >>= (-shift - 1); - if (val == INT64_MAX) { - /* In this case, it means that the rounding constant is 1, - * and the addition would overflow. Return the actual - * result directly. */ - val = 0x4000000000000000ULL; - } else { - val++; - val >>= 1; - } - } else { - int64_t tmp = val; - val <<= shift; - if ((val >> shift) != tmp) { - SET_QC(); - val = (tmp >> 63) ^ ~SIGNBIT64; - } - } - return val; + return do_sqrshl_d(val, (int8_t)shift, true, env->vfp.qc); } uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b) -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-7-richard.henderson@linaro.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 54 +++++++++++++++++++++++ target/arm/sve.decode | 17 ++++++++ target/arm/sve_helper.c | 87 ++++++++++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 18 ++++++++ 4 files changed, 176 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_uadalp_zpzz_s, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_5(sve2_uadalp_zpzz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_srshl_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_srshl_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_srshl_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_srshl_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_urshl_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_urshl_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_urshl_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_urshl_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_sqshl_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqshl_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqshl_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqshl_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_uqshl_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uqshl_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uqshl_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uqshl_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_sqrshl_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrshl_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrshl_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrshl_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_5(sve_sdiv_zpzz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve_sdiv_zpzz_d, TCG_CALL_NO_RWG, diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ URECPE 01000100 .. 000 000 101 ... ..... ..... @rd_pg_rn URSQRTE 01000100 .. 000 001 101 ... ..... ..... @rd_pg_rn SQABS 01000100 .. 001 000 101 ... ..... ..... @rd_pg_rn SQNEG 01000100 .. 001 001 101 ... ..... ..... @rd_pg_rn + +### SVE2 saturating/rounding bitwise shift left (predicated) + +SRSHL 01000100 .. 000 010 100 ... ..... ..... @rdn_pg_rm +URSHL 01000100 .. 000 011 100 ... ..... ..... @rdn_pg_rm +SRSHL 01000100 .. 000 110 100 ... ..... ..... @rdm_pg_rn # SRSHLR +URSHL 01000100 .. 000 111 100 ... ..... ..... @rdm_pg_rn # URSHLR + +SQSHL 01000100 .. 001 000 100 ... ..... ..... @rdn_pg_rm +UQSHL 01000100 .. 001 001 100 ... ..... ..... @rdn_pg_rm +SQSHL 01000100 .. 001 100 100 ... ..... ..... @rdm_pg_rn # SQSHLR +UQSHL 01000100 .. 001 101 100 ... ..... ..... @rdm_pg_rn # UQSHLR + +SQRSHL 01000100 .. 001 010 100 ... ..... ..... @rdn_pg_rm +UQRSHL 01000100 .. 001 011 100 ... ..... ..... @rdn_pg_rm +SQRSHL 01000100 .. 001 110 100 ... ..... ..... @rdm_pg_rn # SQRSHLR +UQRSHL 01000100 .. 001 111 100 ... ..... ..... @rdm_pg_rn # UQRSHLR diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ #include "tcg/tcg-gvec-desc.h" #include "fpu/softfloat.h" #include "tcg/tcg.h" +#include "vec_internal.h" /* Note that vector data is stored in host-endian 64-bit chunks, @@ -XXX,XX +XXX,XX @@ DO_ZPZZ(sve2_uadalp_zpzz_h, uint16_t, H1_2, do_uadalp_h) DO_ZPZZ(sve2_uadalp_zpzz_s, uint32_t, H1_4, do_uadalp_s) DO_ZPZZ_D(sve2_uadalp_zpzz_d, uint64_t, do_uadalp_d) +#define do_srshl_b(n, m) do_sqrshl_bhs(n, m, 8, true, NULL) +#define do_srshl_h(n, m) do_sqrshl_bhs(n, m, 16, true, NULL) +#define do_srshl_s(n, m) do_sqrshl_bhs(n, m, 32, true, NULL) +#define do_srshl_d(n, m) do_sqrshl_d(n, m, true, NULL) + +DO_ZPZZ(sve2_srshl_zpzz_b, int8_t, H1, do_srshl_b) +DO_ZPZZ(sve2_srshl_zpzz_h, int16_t, H1_2, do_srshl_h) +DO_ZPZZ(sve2_srshl_zpzz_s, int32_t, H1_4, do_srshl_s) +DO_ZPZZ_D(sve2_srshl_zpzz_d, int64_t, do_srshl_d) + +#define do_urshl_b(n, m) do_uqrshl_bhs(n, (int8_t)m, 8, true, NULL) +#define do_urshl_h(n, m) do_uqrshl_bhs(n, (int16_t)m, 16, true, NULL) +#define do_urshl_s(n, m) do_uqrshl_bhs(n, m, 32, true, NULL) +#define do_urshl_d(n, m) do_uqrshl_d(n, m, true, NULL) + +DO_ZPZZ(sve2_urshl_zpzz_b, uint8_t, H1, do_urshl_b) +DO_ZPZZ(sve2_urshl_zpzz_h, uint16_t, H1_2, do_urshl_h) +DO_ZPZZ(sve2_urshl_zpzz_s, uint32_t, H1_4, do_urshl_s) +DO_ZPZZ_D(sve2_urshl_zpzz_d, uint64_t, do_urshl_d) + +/* + * Unlike the NEON and AdvSIMD versions, there is no QC bit to set. + * We pass in a pointer to a dummy saturation field to trigger + * the saturating arithmetic but discard the information about + * whether it has occurred. + */ +#define do_sqshl_b(n, m) \ + ({ uint32_t discard; do_sqrshl_bhs(n, m, 8, false, &discard); }) +#define do_sqshl_h(n, m) \ + ({ uint32_t discard; do_sqrshl_bhs(n, m, 16, false, &discard); }) +#define do_sqshl_s(n, m) \ + ({ uint32_t discard; do_sqrshl_bhs(n, m, 32, false, &discard); }) +#define do_sqshl_d(n, m) \ + ({ uint32_t discard; do_sqrshl_d(n, m, false, &discard); }) + +DO_ZPZZ(sve2_sqshl_zpzz_b, int8_t, H1_2, do_sqshl_b) +DO_ZPZZ(sve2_sqshl_zpzz_h, int16_t, H1_2, do_sqshl_h) +DO_ZPZZ(sve2_sqshl_zpzz_s, int32_t, H1_4, do_sqshl_s) +DO_ZPZZ_D(sve2_sqshl_zpzz_d, int64_t, do_sqshl_d) + +#define do_uqshl_b(n, m) \ + ({ uint32_t discard; do_uqrshl_bhs(n, (int8_t)m, 8, false, &discard); }) +#define do_uqshl_h(n, m) \ + ({ uint32_t discard; do_uqrshl_bhs(n, (int16_t)m, 16, false, &discard); }) +#define do_uqshl_s(n, m) \ + ({ uint32_t discard; do_uqrshl_bhs(n, m, 32, false, &discard); }) +#define do_uqshl_d(n, m) \ + ({ uint32_t discard; do_uqrshl_d(n, m, false, &discard); }) + +DO_ZPZZ(sve2_uqshl_zpzz_b, uint8_t, H1_2, do_uqshl_b) +DO_ZPZZ(sve2_uqshl_zpzz_h, uint16_t, H1_2, do_uqshl_h) +DO_ZPZZ(sve2_uqshl_zpzz_s, uint32_t, H1_4, do_uqshl_s) +DO_ZPZZ_D(sve2_uqshl_zpzz_d, uint64_t, do_uqshl_d) + +#define do_sqrshl_b(n, m) \ + ({ uint32_t discard; do_sqrshl_bhs(n, m, 8, true, &discard); }) +#define do_sqrshl_h(n, m) \ + ({ uint32_t discard; do_sqrshl_bhs(n, m, 16, true, &discard); }) +#define do_sqrshl_s(n, m) \ + ({ uint32_t discard; do_sqrshl_bhs(n, m, 32, true, &discard); }) +#define do_sqrshl_d(n, m) \ + ({ uint32_t discard; do_sqrshl_d(n, m, true, &discard); }) + +DO_ZPZZ(sve2_sqrshl_zpzz_b, int8_t, H1_2, do_sqrshl_b) +DO_ZPZZ(sve2_sqrshl_zpzz_h, int16_t, H1_2, do_sqrshl_h) +DO_ZPZZ(sve2_sqrshl_zpzz_s, int32_t, H1_4, do_sqrshl_s) +DO_ZPZZ_D(sve2_sqrshl_zpzz_d, int64_t, do_sqrshl_d) + +#undef do_sqrshl_d + +#define do_uqrshl_b(n, m) \ + ({ uint32_t discard; do_uqrshl_bhs(n, (int8_t)m, 8, true, &discard); }) +#define do_uqrshl_h(n, m) \ + ({ uint32_t discard; do_uqrshl_bhs(n, (int16_t)m, 16, true, &discard); }) +#define do_uqrshl_s(n, m) \ + ({ uint32_t discard; do_uqrshl_bhs(n, m, 32, true, &discard); }) +#define do_uqrshl_d(n, m) \ + ({ uint32_t discard; do_uqrshl_d(n, m, true, &discard); }) + +DO_ZPZZ(sve2_uqrshl_zpzz_b, uint8_t, H1_2, do_uqrshl_b) +DO_ZPZZ(sve2_uqrshl_zpzz_h, uint16_t, H1_2, do_uqrshl_h) +DO_ZPZZ(sve2_uqrshl_zpzz_s, uint32_t, H1_4, do_uqrshl_s) +DO_ZPZZ_D(sve2_uqrshl_zpzz_d, uint64_t, do_uqrshl_d) + +#undef do_uqrshl_d + #undef DO_ZPZZ #undef DO_ZPZZ_D diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_SQNEG(DisasContext *s, arg_rpr_esz *a) }; return do_sve2_zpz_ool(s, a, fns[a->esz]); } + +#define DO_SVE2_ZPZZ(NAME, name) \ +static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \ +{ \ + static gen_helper_gvec_4 * const fns[4] = { \ + gen_helper_sve2_##name##_zpzz_b, gen_helper_sve2_##name##_zpzz_h, \ + gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d, \ + }; \ + return do_sve2_zpzz_ool(s, a, fns[a->esz]); \ +} + +DO_SVE2_ZPZZ(SQSHL, sqshl) +DO_SVE2_ZPZZ(SQRSHL, sqrshl) +DO_SVE2_ZPZZ(SRSHL, srshl) + +DO_SVE2_ZPZZ(UQSHL, uqshl) +DO_SVE2_ZPZZ(UQRSHL, uqrshl) +DO_SVE2_ZPZZ(URSHL, urshl) -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-8-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 54 ++++++++++++++++++++++++++++++++++++++ target/arm/sve.decode | 11 ++++++++ target/arm/sve_helper.c | 39 +++++++++++++++++++++++++++ target/arm/translate-sve.c | 8 ++++++ 4 files changed, 112 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_s, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_shadd_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_shadd_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_shadd_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_shadd_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_uhadd_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uhadd_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uhadd_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uhadd_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_srhadd_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_srhadd_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_srhadd_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_srhadd_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_urhadd_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_urhadd_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_urhadd_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_urhadd_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_shsub_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_shsub_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_shsub_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_shsub_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_uhsub_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uhsub_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uhsub_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uhsub_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_5(sve_sdiv_zpzz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve_sdiv_zpzz_d, TCG_CALL_NO_RWG, diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SQRSHL 01000100 .. 001 010 100 ... ..... ..... @rdn_pg_rm UQRSHL 01000100 .. 001 011 100 ... ..... ..... @rdn_pg_rm SQRSHL 01000100 .. 001 110 100 ... ..... ..... @rdm_pg_rn # SQRSHLR UQRSHL 01000100 .. 001 111 100 ... ..... ..... @rdm_pg_rn # UQRSHLR + +### SVE2 integer halving add/subtract (predicated) + +SHADD 01000100 .. 010 000 100 ... ..... ..... @rdn_pg_rm +UHADD 01000100 .. 010 001 100 ... ..... ..... @rdn_pg_rm +SHSUB 01000100 .. 010 010 100 ... ..... ..... @rdn_pg_rm +UHSUB 01000100 .. 010 011 100 ... ..... ..... @rdn_pg_rm +SRHADD 01000100 .. 010 100 100 ... ..... ..... @rdn_pg_rm +URHADD 01000100 .. 010 101 100 ... ..... ..... @rdn_pg_rm +SHSUB 01000100 .. 010 110 100 ... ..... ..... @rdm_pg_rn # SHSUBR +UHSUB 01000100 .. 010 111 100 ... ..... ..... @rdm_pg_rn # UHSUBR diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZPZZ_D(sve2_uqrshl_zpzz_d, uint64_t, do_uqrshl_d) #undef do_uqrshl_d +#define DO_HADD_BHS(n, m) (((int64_t)n + m) >> 1) +#define DO_HADD_D(n, m) ((n >> 1) + (m >> 1) + (n & m & 1)) + +DO_ZPZZ(sve2_shadd_zpzz_b, int8_t, H1, DO_HADD_BHS) +DO_ZPZZ(sve2_shadd_zpzz_h, int16_t, H1_2, DO_HADD_BHS) +DO_ZPZZ(sve2_shadd_zpzz_s, int32_t, H1_4, DO_HADD_BHS) +DO_ZPZZ_D(sve2_shadd_zpzz_d, int64_t, DO_HADD_D) + +DO_ZPZZ(sve2_uhadd_zpzz_b, uint8_t, H1, DO_HADD_BHS) +DO_ZPZZ(sve2_uhadd_zpzz_h, uint16_t, H1_2, DO_HADD_BHS) +DO_ZPZZ(sve2_uhadd_zpzz_s, uint32_t, H1_4, DO_HADD_BHS) +DO_ZPZZ_D(sve2_uhadd_zpzz_d, uint64_t, DO_HADD_D) + +#define DO_RHADD_BHS(n, m) (((int64_t)n + m + 1) >> 1) +#define DO_RHADD_D(n, m) ((n >> 1) + (m >> 1) + ((n | m) & 1)) + +DO_ZPZZ(sve2_srhadd_zpzz_b, int8_t, H1, DO_RHADD_BHS) +DO_ZPZZ(sve2_srhadd_zpzz_h, int16_t, H1_2, DO_RHADD_BHS) +DO_ZPZZ(sve2_srhadd_zpzz_s, int32_t, H1_4, DO_RHADD_BHS) +DO_ZPZZ_D(sve2_srhadd_zpzz_d, int64_t, DO_RHADD_D) + +DO_ZPZZ(sve2_urhadd_zpzz_b, uint8_t, H1, DO_RHADD_BHS) +DO_ZPZZ(sve2_urhadd_zpzz_h, uint16_t, H1_2, DO_RHADD_BHS) +DO_ZPZZ(sve2_urhadd_zpzz_s, uint32_t, H1_4, DO_RHADD_BHS) +DO_ZPZZ_D(sve2_urhadd_zpzz_d, uint64_t, DO_RHADD_D) + +#define DO_HSUB_BHS(n, m) (((int64_t)n - m) >> 1) +#define DO_HSUB_D(n, m) ((n >> 1) - (m >> 1) - (~n & m & 1)) + +DO_ZPZZ(sve2_shsub_zpzz_b, int8_t, H1, DO_HSUB_BHS) +DO_ZPZZ(sve2_shsub_zpzz_h, int16_t, H1_2, DO_HSUB_BHS) +DO_ZPZZ(sve2_shsub_zpzz_s, int32_t, H1_4, DO_HSUB_BHS) +DO_ZPZZ_D(sve2_shsub_zpzz_d, int64_t, DO_HSUB_D) + +DO_ZPZZ(sve2_uhsub_zpzz_b, uint8_t, H1, DO_HSUB_BHS) +DO_ZPZZ(sve2_uhsub_zpzz_h, uint16_t, H1_2, DO_HSUB_BHS) +DO_ZPZZ(sve2_uhsub_zpzz_s, uint32_t, H1_4, DO_HSUB_BHS) +DO_ZPZZ_D(sve2_uhsub_zpzz_d, uint64_t, DO_HSUB_D) + #undef DO_ZPZZ #undef DO_ZPZZ_D diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_ZPZZ(SRSHL, srshl) DO_SVE2_ZPZZ(UQSHL, uqshl) DO_SVE2_ZPZZ(UQRSHL, uqrshl) DO_SVE2_ZPZZ(URSHL, urshl) + +DO_SVE2_ZPZZ(SHADD, shadd) +DO_SVE2_ZPZZ(SRHADD, srhadd) +DO_SVE2_ZPZZ(SHSUB, shsub) + +DO_SVE2_ZPZZ(UHADD, uhadd) +DO_SVE2_ZPZZ(URHADD, urhadd) +DO_SVE2_ZPZZ(UHSUB, uhsub) -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-9-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 45 ++++++++++++++++++++++ target/arm/sve.decode | 8 ++++ target/arm/sve_helper.c | 76 ++++++++++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 6 +++ 4 files changed, 135 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve_sel_zpzz_s, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_5(sve_sel_zpzz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_addp_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_addp_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_addp_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_addp_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_smaxp_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_smaxp_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_smaxp_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_smaxp_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_umaxp_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_umaxp_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_umaxp_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_umaxp_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_sminp_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sminp_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sminp_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sminp_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_5(sve_asr_zpzw_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve_asr_zpzw_h, TCG_CALL_NO_RWG, diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SRHADD 01000100 .. 010 100 100 ... ..... ..... @rdn_pg_rm URHADD 01000100 .. 010 101 100 ... ..... ..... @rdn_pg_rm SHSUB 01000100 .. 010 110 100 ... ..... ..... @rdm_pg_rn # SHSUBR UHSUB 01000100 .. 010 111 100 ... ..... ..... @rdm_pg_rn # UHSUBR + +### SVE2 integer pairwise arithmetic + +ADDP 01000100 .. 010 001 101 ... ..... ..... @rdn_pg_rm +SMAXP 01000100 .. 010 100 101 ... ..... ..... @rdn_pg_rm +UMAXP 01000100 .. 010 101 101 ... ..... ..... @rdn_pg_rm +SMINP 01000100 .. 010 110 101 ... ..... ..... @rdn_pg_rm +UMINP 01000100 .. 010 111 101 ... ..... ..... @rdn_pg_rm diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZPZZ_D(sve2_uhsub_zpzz_d, uint64_t, DO_HSUB_D) #undef DO_ZPZZ #undef DO_ZPZZ_D +/* + * Three operand expander, operating on element pairs. + * If the slot I is even, the elements from from VN {I, I+1}. + * If the slot I is odd, the elements from from VM {I-1, I}. + * Load all of the input elements in each pair before overwriting output. + */ +#define DO_ZPZZ_PAIR(NAME, TYPE, H, OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc); \ + for (i = 0; i < opr_sz; ) { \ + uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \ + do { \ + TYPE n0 = *(TYPE *)(vn + H(i)); \ + TYPE m0 = *(TYPE *)(vm + H(i)); \ + TYPE n1 = *(TYPE *)(vn + H(i + sizeof(TYPE))); \ + TYPE m1 = *(TYPE *)(vm + H(i + sizeof(TYPE))); \ + if (pg & 1) { \ + *(TYPE *)(vd + H(i)) = OP(n0, n1); \ + } \ + i += sizeof(TYPE), pg >>= sizeof(TYPE); \ + if (pg & 1) { \ + *(TYPE *)(vd + H(i)) = OP(m0, m1); \ + } \ + i += sizeof(TYPE), pg >>= sizeof(TYPE); \ + } while (i & 15); \ + } \ +} + +/* Similarly, specialized for 64-bit operands. */ +#define DO_ZPZZ_PAIR_D(NAME, TYPE, OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc) / 8; \ + TYPE *d = vd, *n = vn, *m = vm; \ + uint8_t *pg = vg; \ + for (i = 0; i < opr_sz; i += 2) { \ + TYPE n0 = n[i], n1 = n[i + 1]; \ + TYPE m0 = m[i], m1 = m[i + 1]; \ + if (pg[H1(i)] & 1) { \ + d[i] = OP(n0, n1); \ + } \ + if (pg[H1(i + 1)] & 1) { \ + d[i + 1] = OP(m0, m1); \ + } \ + } \ +} + +DO_ZPZZ_PAIR(sve2_addp_zpzz_b, uint8_t, H1, DO_ADD) +DO_ZPZZ_PAIR(sve2_addp_zpzz_h, uint16_t, H1_2, DO_ADD) +DO_ZPZZ_PAIR(sve2_addp_zpzz_s, uint32_t, H1_4, DO_ADD) +DO_ZPZZ_PAIR_D(sve2_addp_zpzz_d, uint64_t, DO_ADD) + +DO_ZPZZ_PAIR(sve2_umaxp_zpzz_b, uint8_t, H1, DO_MAX) +DO_ZPZZ_PAIR(sve2_umaxp_zpzz_h, uint16_t, H1_2, DO_MAX) +DO_ZPZZ_PAIR(sve2_umaxp_zpzz_s, uint32_t, H1_4, DO_MAX) +DO_ZPZZ_PAIR_D(sve2_umaxp_zpzz_d, uint64_t, DO_MAX) + +DO_ZPZZ_PAIR(sve2_uminp_zpzz_b, uint8_t, H1, DO_MIN) +DO_ZPZZ_PAIR(sve2_uminp_zpzz_h, uint16_t, H1_2, DO_MIN) +DO_ZPZZ_PAIR(sve2_uminp_zpzz_s, uint32_t, H1_4, DO_MIN) +DO_ZPZZ_PAIR_D(sve2_uminp_zpzz_d, uint64_t, DO_MIN) + +DO_ZPZZ_PAIR(sve2_smaxp_zpzz_b, int8_t, H1, DO_MAX) +DO_ZPZZ_PAIR(sve2_smaxp_zpzz_h, int16_t, H1_2, DO_MAX) +DO_ZPZZ_PAIR(sve2_smaxp_zpzz_s, int32_t, H1_4, DO_MAX) +DO_ZPZZ_PAIR_D(sve2_smaxp_zpzz_d, int64_t, DO_MAX) + +DO_ZPZZ_PAIR(sve2_sminp_zpzz_b, int8_t, H1, DO_MIN) +DO_ZPZZ_PAIR(sve2_sminp_zpzz_h, int16_t, H1_2, DO_MIN) +DO_ZPZZ_PAIR(sve2_sminp_zpzz_s, int32_t, H1_4, DO_MIN) +DO_ZPZZ_PAIR_D(sve2_sminp_zpzz_d, int64_t, DO_MIN) + +#undef DO_ZPZZ_PAIR +#undef DO_ZPZZ_PAIR_D + /* Three-operand expander, controlled by a predicate, in which the * third operand is "wide". That is, for D = N op M, the same 64-bit * value of M is used with all of the narrower values of N. diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_ZPZZ(SHSUB, shsub) DO_SVE2_ZPZZ(UHADD, uhadd) DO_SVE2_ZPZZ(URHADD, urhadd) DO_SVE2_ZPZZ(UHSUB, uhsub) + +DO_SVE2_ZPZZ(ADDP, addp) +DO_SVE2_ZPZZ(SMAXP, smaxp) +DO_SVE2_ZPZZ(UMAXP, umaxp) +DO_SVE2_ZPZZ(SMINP, sminp) +DO_SVE2_ZPZZ(UMINP, uminp) -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-10-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 54 +++++++++++ target/arm/sve.decode | 11 +++ target/arm/sve_helper.c | 194 ++++++++++++++++++++++++++----------- target/arm/translate-sve.c | 7 ++ 4 files changed, 210 insertions(+), 56 deletions(-) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_s, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqadd_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqadd_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqadd_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqadd_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_uqadd_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uqadd_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uqadd_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uqadd_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_sqsub_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqsub_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqsub_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqsub_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_uqsub_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uqsub_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uqsub_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uqsub_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_suqadd_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_suqadd_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_suqadd_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_suqadd_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_usqadd_zpzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_usqadd_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_usqadd_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_usqadd_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_5(sve_asr_zpzw_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve_asr_zpzw_h, TCG_CALL_NO_RWG, diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SMAXP 01000100 .. 010 100 101 ... ..... ..... @rdn_pg_rm UMAXP 01000100 .. 010 101 101 ... ..... ..... @rdn_pg_rm SMINP 01000100 .. 010 110 101 ... ..... ..... @rdn_pg_rm UMINP 01000100 .. 010 111 101 ... ..... ..... @rdn_pg_rm + +### SVE2 saturating add/subtract (predicated) + +SQADD_zpzz 01000100 .. 011 000 100 ... ..... ..... @rdn_pg_rm +UQADD_zpzz 01000100 .. 011 001 100 ... ..... ..... @rdn_pg_rm +SQSUB_zpzz 01000100 .. 011 010 100 ... ..... ..... @rdn_pg_rm +UQSUB_zpzz 01000100 .. 011 011 100 ... ..... ..... @rdn_pg_rm +SUQADD 01000100 .. 011 100 100 ... ..... ..... @rdn_pg_rm +USQADD 01000100 .. 011 101 100 ... ..... ..... @rdn_pg_rm +SQSUB_zpzz 01000100 .. 011 110 100 ... ..... ..... @rdm_pg_rn # SQSUBR +UQSUB_zpzz 01000100 .. 011 111 100 ... ..... ..... @rdm_pg_rn # UQSUBR diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZPZZ(sve2_uhsub_zpzz_h, uint16_t, H1_2, DO_HSUB_BHS) DO_ZPZZ(sve2_uhsub_zpzz_s, uint32_t, H1_4, DO_HSUB_BHS) DO_ZPZZ_D(sve2_uhsub_zpzz_d, uint64_t, DO_HSUB_D) +static inline int32_t do_sat_bhs(int64_t val, int64_t min, int64_t max) +{ + return val >= max ? max : val <= min ? min : val; +} + +#define DO_SQADD_B(n, m) do_sat_bhs((int64_t)n + m, INT8_MIN, INT8_MAX) +#define DO_SQADD_H(n, m) do_sat_bhs((int64_t)n + m, INT16_MIN, INT16_MAX) +#define DO_SQADD_S(n, m) do_sat_bhs((int64_t)n + m, INT32_MIN, INT32_MAX) + +static inline int64_t do_sqadd_d(int64_t n, int64_t m) +{ + int64_t r = n + m; + if (((r ^ n) & ~(n ^ m)) < 0) { + /* Signed overflow. */ + return r < 0 ? INT64_MAX : INT64_MIN; + } + return r; +} + +DO_ZPZZ(sve2_sqadd_zpzz_b, int8_t, H1, DO_SQADD_B) +DO_ZPZZ(sve2_sqadd_zpzz_h, int16_t, H1_2, DO_SQADD_H) +DO_ZPZZ(sve2_sqadd_zpzz_s, int32_t, H1_4, DO_SQADD_S) +DO_ZPZZ_D(sve2_sqadd_zpzz_d, int64_t, do_sqadd_d) + +#define DO_UQADD_B(n, m) do_sat_bhs((int64_t)n + m, 0, UINT8_MAX) +#define DO_UQADD_H(n, m) do_sat_bhs((int64_t)n + m, 0, UINT16_MAX) +#define DO_UQADD_S(n, m) do_sat_bhs((int64_t)n + m, 0, UINT32_MAX) + +static inline uint64_t do_uqadd_d(uint64_t n, uint64_t m) +{ + uint64_t r = n + m; + return r < n ? UINT64_MAX : r; +} + +DO_ZPZZ(sve2_uqadd_zpzz_b, uint8_t, H1, DO_UQADD_B) +DO_ZPZZ(sve2_uqadd_zpzz_h, uint16_t, H1_2, DO_UQADD_H) +DO_ZPZZ(sve2_uqadd_zpzz_s, uint32_t, H1_4, DO_UQADD_S) +DO_ZPZZ_D(sve2_uqadd_zpzz_d, uint64_t, do_uqadd_d) + +#define DO_SQSUB_B(n, m) do_sat_bhs((int64_t)n - m, INT8_MIN, INT8_MAX) +#define DO_SQSUB_H(n, m) do_sat_bhs((int64_t)n - m, INT16_MIN, INT16_MAX) +#define DO_SQSUB_S(n, m) do_sat_bhs((int64_t)n - m, INT32_MIN, INT32_MAX) + +static inline int64_t do_sqsub_d(int64_t n, int64_t m) +{ + int64_t r = n - m; + if (((r ^ n) & (n ^ m)) < 0) { + /* Signed overflow. */ + return r < 0 ? INT64_MAX : INT64_MIN; + } + return r; +} + +DO_ZPZZ(sve2_sqsub_zpzz_b, int8_t, H1, DO_SQSUB_B) +DO_ZPZZ(sve2_sqsub_zpzz_h, int16_t, H1_2, DO_SQSUB_H) +DO_ZPZZ(sve2_sqsub_zpzz_s, int32_t, H1_4, DO_SQSUB_S) +DO_ZPZZ_D(sve2_sqsub_zpzz_d, int64_t, do_sqsub_d) + +#define DO_UQSUB_B(n, m) do_sat_bhs((int64_t)n - m, 0, UINT8_MAX) +#define DO_UQSUB_H(n, m) do_sat_bhs((int64_t)n - m, 0, UINT16_MAX) +#define DO_UQSUB_S(n, m) do_sat_bhs((int64_t)n - m, 0, UINT32_MAX) + +static inline uint64_t do_uqsub_d(uint64_t n, uint64_t m) +{ + return n > m ? n - m : 0; +} + +DO_ZPZZ(sve2_uqsub_zpzz_b, uint8_t, H1, DO_UQSUB_B) +DO_ZPZZ(sve2_uqsub_zpzz_h, uint16_t, H1_2, DO_UQSUB_H) +DO_ZPZZ(sve2_uqsub_zpzz_s, uint32_t, H1_4, DO_UQSUB_S) +DO_ZPZZ_D(sve2_uqsub_zpzz_d, uint64_t, do_uqsub_d) + +#define DO_SUQADD_B(n, m) \ + do_sat_bhs((int64_t)(int8_t)n + m, INT8_MIN, INT8_MAX) +#define DO_SUQADD_H(n, m) \ + do_sat_bhs((int64_t)(int16_t)n + m, INT16_MIN, INT16_MAX) +#define DO_SUQADD_S(n, m) \ + do_sat_bhs((int64_t)(int32_t)n + m, INT32_MIN, INT32_MAX) + +static inline int64_t do_suqadd_d(int64_t n, uint64_t m) +{ + uint64_t r = n + m; + + if (n < 0) { + /* Note that m - abs(n) cannot underflow. */ + if (r > INT64_MAX) { + /* Result is either very large positive or negative. */ + if (m > -n) { + /* m > abs(n), so r is a very large positive. */ + return INT64_MAX; + } + /* Result is negative. */ + } + } else { + /* Both inputs are positive: check for overflow. */ + if (r < m || r > INT64_MAX) { + return INT64_MAX; + } + } + return r; +} + +DO_ZPZZ(sve2_suqadd_zpzz_b, uint8_t, H1, DO_SUQADD_B) +DO_ZPZZ(sve2_suqadd_zpzz_h, uint16_t, H1_2, DO_SUQADD_H) +DO_ZPZZ(sve2_suqadd_zpzz_s, uint32_t, H1_4, DO_SUQADD_S) +DO_ZPZZ_D(sve2_suqadd_zpzz_d, uint64_t, do_suqadd_d) + +#define DO_USQADD_B(n, m) \ + do_sat_bhs((int64_t)n + (int8_t)m, 0, UINT8_MAX) +#define DO_USQADD_H(n, m) \ + do_sat_bhs((int64_t)n + (int16_t)m, 0, UINT16_MAX) +#define DO_USQADD_S(n, m) \ + do_sat_bhs((int64_t)n + (int32_t)m, 0, UINT32_MAX) + +static inline uint64_t do_usqadd_d(uint64_t n, int64_t m) +{ + uint64_t r = n + m; + + if (m < 0) { + return n < -m ? 0 : r; + } + return r < n ? UINT64_MAX : r; +} + +DO_ZPZZ(sve2_usqadd_zpzz_b, uint8_t, H1, DO_USQADD_B) +DO_ZPZZ(sve2_usqadd_zpzz_h, uint16_t, H1_2, DO_USQADD_H) +DO_ZPZZ(sve2_usqadd_zpzz_s, uint32_t, H1_4, DO_USQADD_S) +DO_ZPZZ_D(sve2_usqadd_zpzz_d, uint64_t, do_usqadd_d) + #undef DO_ZPZZ #undef DO_ZPZZ_D @@ -XXX,XX +XXX,XX @@ void HELPER(sve_sqaddi_b)(void *d, void *a, int32_t b, uint32_t desc) intptr_t i, oprsz = simd_oprsz(desc); for (i = 0; i < oprsz; i += sizeof(int8_t)) { - int r = *(int8_t *)(a + i) + b; - if (r > INT8_MAX) { - r = INT8_MAX; - } else if (r < INT8_MIN) { - r = INT8_MIN; - } - *(int8_t *)(d + i) = r; + *(int8_t *)(d + i) = DO_SQADD_B(b, *(int8_t *)(a + i)); } } @@ -XXX,XX +XXX,XX @@ void HELPER(sve_sqaddi_h)(void *d, void *a, int32_t b, uint32_t desc) intptr_t i, oprsz = simd_oprsz(desc); for (i = 0; i < oprsz; i += sizeof(int16_t)) { - int r = *(int16_t *)(a + i) + b; - if (r > INT16_MAX) { - r = INT16_MAX; - } else if (r < INT16_MIN) { - r = INT16_MIN; - } - *(int16_t *)(d + i) = r; + *(int16_t *)(d + i) = DO_SQADD_H(b, *(int16_t *)(a + i)); } } @@ -XXX,XX +XXX,XX @@ void HELPER(sve_sqaddi_s)(void *d, void *a, int64_t b, uint32_t desc) intptr_t i, oprsz = simd_oprsz(desc); for (i = 0; i < oprsz; i += sizeof(int32_t)) { - int64_t r = *(int32_t *)(a + i) + b; - if (r > INT32_MAX) { - r = INT32_MAX; - } else if (r < INT32_MIN) { - r = INT32_MIN; - } - *(int32_t *)(d + i) = r; + *(int32_t *)(d + i) = DO_SQADD_S(b, *(int32_t *)(a + i)); } } @@ -XXX,XX +XXX,XX @@ void HELPER(sve_sqaddi_d)(void *d, void *a, int64_t b, uint32_t desc) intptr_t i, oprsz = simd_oprsz(desc); for (i = 0; i < oprsz; i += sizeof(int64_t)) { - int64_t ai = *(int64_t *)(a + i); - int64_t r = ai + b; - if (((r ^ ai) & ~(ai ^ b)) < 0) { - /* Signed overflow. */ - r = (r < 0 ? INT64_MAX : INT64_MIN); - } - *(int64_t *)(d + i) = r; + *(int64_t *)(d + i) = do_sqadd_d(b, *(int64_t *)(a + i)); } } @@ -XXX,XX +XXX,XX @@ void HELPER(sve_uqaddi_b)(void *d, void *a, int32_t b, uint32_t desc) intptr_t i, oprsz = simd_oprsz(desc); for (i = 0; i < oprsz; i += sizeof(uint8_t)) { - int r = *(uint8_t *)(a + i) + b; - if (r > UINT8_MAX) { - r = UINT8_MAX; - } else if (r < 0) { - r = 0; - } - *(uint8_t *)(d + i) = r; + *(uint8_t *)(d + i) = DO_UQADD_B(b, *(uint8_t *)(a + i)); } } @@ -XXX,XX +XXX,XX @@ void HELPER(sve_uqaddi_h)(void *d, void *a, int32_t b, uint32_t desc) intptr_t i, oprsz = simd_oprsz(desc); for (i = 0; i < oprsz; i += sizeof(uint16_t)) { - int r = *(uint16_t *)(a + i) + b; - if (r > UINT16_MAX) { - r = UINT16_MAX; - } else if (r < 0) { - r = 0; - } - *(uint16_t *)(d + i) = r; + *(uint16_t *)(d + i) = DO_UQADD_H(b, *(uint16_t *)(a + i)); } } @@ -XXX,XX +XXX,XX @@ void HELPER(sve_uqaddi_s)(void *d, void *a, int64_t b, uint32_t desc) intptr_t i, oprsz = simd_oprsz(desc); for (i = 0; i < oprsz; i += sizeof(uint32_t)) { - int64_t r = *(uint32_t *)(a + i) + b; - if (r > UINT32_MAX) { - r = UINT32_MAX; - } else if (r < 0) { - r = 0; - } - *(uint32_t *)(d + i) = r; + *(uint32_t *)(d + i) = DO_UQADD_S(b, *(uint32_t *)(a + i)); } } @@ -XXX,XX +XXX,XX @@ void HELPER(sve_uqaddi_d)(void *d, void *a, uint64_t b, uint32_t desc) intptr_t i, oprsz = simd_oprsz(desc); for (i = 0; i < oprsz; i += sizeof(uint64_t)) { - uint64_t r = *(uint64_t *)(a + i) + b; - if (r < b) { - r = UINT64_MAX; - } - *(uint64_t *)(d + i) = r; + *(uint64_t *)(d + i) = do_uqadd_d(b, *(uint64_t *)(a + i)); } } @@ -XXX,XX +XXX,XX @@ void HELPER(sve_uqsubi_d)(void *d, void *a, uint64_t b, uint32_t desc) intptr_t i, oprsz = simd_oprsz(desc); for (i = 0; i < oprsz; i += sizeof(uint64_t)) { - uint64_t ai = *(uint64_t *)(a + i); - *(uint64_t *)(d + i) = (ai < b ? 0 : ai - b); + *(uint64_t *)(d + i) = do_uqsub_d(*(uint64_t *)(a + i), b); } } diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_ZPZZ(SMAXP, smaxp) DO_SVE2_ZPZZ(UMAXP, umaxp) DO_SVE2_ZPZZ(SMINP, sminp) DO_SVE2_ZPZZ(UMINP, uminp) + +DO_SVE2_ZPZZ(SQADD_zpzz, sqadd) +DO_SVE2_ZPZZ(UQADD_zpzz, uqadd) +DO_SVE2_ZPZZ(SQSUB_zpzz, sqsub) +DO_SVE2_ZPZZ(UQSUB_zpzz, uqsub) +DO_SVE2_ZPZZ(SUQADD, suqadd) +DO_SVE2_ZPZZ(USQADD, usqadd) -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-11-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 24 ++++++++++++++++++++ target/arm/sve.decode | 19 ++++++++++++++++ target/arm/sve_helper.c | 43 +++++++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 46 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 132 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve_ftmad_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve_ftmad_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve_ftmad_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_saddl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_saddl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_saddl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_ssubl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_ssubl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_ssubl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_sabdl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sabdl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sabdl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_uaddl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_uaddl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_uaddl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_usubl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_usubl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_usubl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_uabdl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_uabdl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_uabdl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_4(sve_ld1bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32) DEF_HELPER_FLAGS_4(sve_ld2bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32) DEF_HELPER_FLAGS_4(sve_ld3bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SUQADD 01000100 .. 011 100 100 ... ..... ..... @rdn_pg_rm USQADD 01000100 .. 011 101 100 ... ..... ..... @rdn_pg_rm SQSUB_zpzz 01000100 .. 011 110 100 ... ..... ..... @rdm_pg_rn # SQSUBR UQSUB_zpzz 01000100 .. 011 111 100 ... ..... ..... @rdm_pg_rn # UQSUBR + +#### SVE2 Widening Integer Arithmetic + +## SVE2 integer add/subtract long + +SADDLB 01000101 .. 0 ..... 00 0000 ..... ..... @rd_rn_rm +SADDLT 01000101 .. 0 ..... 00 0001 ..... ..... @rd_rn_rm +UADDLB 01000101 .. 0 ..... 00 0010 ..... ..... @rd_rn_rm +UADDLT 01000101 .. 0 ..... 00 0011 ..... ..... @rd_rn_rm + +SSUBLB 01000101 .. 0 ..... 00 0100 ..... ..... @rd_rn_rm +SSUBLT 01000101 .. 0 ..... 00 0101 ..... ..... @rd_rn_rm +USUBLB 01000101 .. 0 ..... 00 0110 ..... ..... @rd_rn_rm +USUBLT 01000101 .. 0 ..... 00 0111 ..... ..... @rd_rn_rm + +SABDLB 01000101 .. 0 ..... 00 1100 ..... ..... @rd_rn_rm +SABDLT 01000101 .. 0 ..... 00 1101 ..... ..... @rd_rn_rm +UABDLB 01000101 .. 0 ..... 00 1110 ..... ..... @rd_rn_rm +UABDLT 01000101 .. 0 ..... 00 1111 ..... ..... @rd_rn_rm diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZZW(sve_lsl_zzw_s, uint32_t, uint64_t, H1_4, DO_LSL) #undef DO_ZPZ #undef DO_ZPZ_D +/* + * Three-operand expander, unpredicated, in which the two inputs are + * selected from the top or bottom half of the wide column. + */ +#define DO_ZZZ_TB(NAME, TYPEW, TYPEN, HW, HN, OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc); \ + int sel1 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN); \ + int sel2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(TYPEN); \ + for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \ + TYPEW nn = *(TYPEN *)(vn + HN(i + sel1)); \ + TYPEW mm = *(TYPEN *)(vm + HN(i + sel2)); \ + *(TYPEW *)(vd + HW(i)) = OP(nn, mm); \ + } \ +} + +DO_ZZZ_TB(sve2_saddl_h, int16_t, int8_t, H1_2, H1, DO_ADD) +DO_ZZZ_TB(sve2_saddl_s, int32_t, int16_t, H1_4, H1_2, DO_ADD) +DO_ZZZ_TB(sve2_saddl_d, int64_t, int32_t, , H1_4, DO_ADD) + +DO_ZZZ_TB(sve2_ssubl_h, int16_t, int8_t, H1_2, H1, DO_SUB) +DO_ZZZ_TB(sve2_ssubl_s, int32_t, int16_t, H1_4, H1_2, DO_SUB) +DO_ZZZ_TB(sve2_ssubl_d, int64_t, int32_t, , H1_4, DO_SUB) + +DO_ZZZ_TB(sve2_sabdl_h, int16_t, int8_t, H1_2, H1, DO_ABD) +DO_ZZZ_TB(sve2_sabdl_s, int32_t, int16_t, H1_4, H1_2, DO_ABD) +DO_ZZZ_TB(sve2_sabdl_d, int64_t, int32_t, , H1_4, DO_ABD) + +DO_ZZZ_TB(sve2_uaddl_h, uint16_t, uint8_t, H1_2, H1, DO_ADD) +DO_ZZZ_TB(sve2_uaddl_s, uint32_t, uint16_t, H1_4, H1_2, DO_ADD) +DO_ZZZ_TB(sve2_uaddl_d, uint64_t, uint32_t, , H1_4, DO_ADD) + +DO_ZZZ_TB(sve2_usubl_h, uint16_t, uint8_t, H1_2, H1, DO_SUB) +DO_ZZZ_TB(sve2_usubl_s, uint32_t, uint16_t, H1_4, H1_2, DO_SUB) +DO_ZZZ_TB(sve2_usubl_d, uint64_t, uint32_t, , H1_4, DO_SUB) + +DO_ZZZ_TB(sve2_uabdl_h, uint16_t, uint8_t, H1_2, H1, DO_ABD) +DO_ZZZ_TB(sve2_uabdl_s, uint32_t, uint16_t, H1_4, H1_2, DO_ABD) +DO_ZZZ_TB(sve2_uabdl_d, uint64_t, uint32_t, , H1_4, DO_ABD) + +#undef DO_ZZZ_TB + /* Two-operand reduction expander, controlled by a predicate. * The difference between TYPERED and TYPERET has to do with * sign-extension. E.g. for SMAX, TYPERED must be signed, diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_ZPZZ(SQSUB_zpzz, sqsub) DO_SVE2_ZPZZ(UQSUB_zpzz, uqsub) DO_SVE2_ZPZZ(SUQADD, suqadd) DO_SVE2_ZPZZ(USQADD, usqadd) + +/* + * SVE2 Widening Integer Arithmetic + */ + +static bool do_sve2_zzw_ool(DisasContext *s, arg_rrr_esz *a, + gen_helper_gvec_3 *fn, int data) +{ + if (fn == NULL || !dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (sve_access_check(s)) { + unsigned vsz = vec_full_reg_size(s); + tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd), + vec_full_reg_offset(s, a->rn), + vec_full_reg_offset(s, a->rm), + vsz, vsz, data, fn); + } + return true; +} + +#define DO_SVE2_ZZZ_TB(NAME, name, SEL1, SEL2) \ +static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \ +{ \ + static gen_helper_gvec_3 * const fns[4] = { \ + NULL, gen_helper_sve2_##name##_h, \ + gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \ + }; \ + return do_sve2_zzw_ool(s, a, fns[a->esz], (SEL2 << 1) | SEL1); \ +} + +DO_SVE2_ZZZ_TB(SADDLB, saddl, false, false) +DO_SVE2_ZZZ_TB(SSUBLB, ssubl, false, false) +DO_SVE2_ZZZ_TB(SABDLB, sabdl, false, false) + +DO_SVE2_ZZZ_TB(UADDLB, uaddl, false, false) +DO_SVE2_ZZZ_TB(USUBLB, usubl, false, false) +DO_SVE2_ZZZ_TB(UABDLB, uabdl, false, false) + +DO_SVE2_ZZZ_TB(SADDLT, saddl, true, true) +DO_SVE2_ZZZ_TB(SSUBLT, ssubl, true, true) +DO_SVE2_ZZZ_TB(SABDLT, sabdl, true, true) + +DO_SVE2_ZZZ_TB(UADDLT, uaddl, true, true) +DO_SVE2_ZZZ_TB(USUBLT, usubl, true, true) +DO_SVE2_ZZZ_TB(UABDLT, uabdl, true, true) -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-12-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/sve.decode | 6 ++++++ target/arm/translate-sve.c | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SABDLB 01000101 .. 0 ..... 00 1100 ..... ..... @rd_rn_rm SABDLT 01000101 .. 0 ..... 00 1101 ..... ..... @rd_rn_rm UABDLB 01000101 .. 0 ..... 00 1110 ..... ..... @rd_rn_rm UABDLT 01000101 .. 0 ..... 00 1111 ..... ..... @rd_rn_rm + +## SVE2 integer add/subtract interleaved long + +SADDLBT 01000101 .. 0 ..... 1000 00 ..... ..... @rd_rn_rm +SSUBLBT 01000101 .. 0 ..... 1000 10 ..... ..... @rd_rn_rm +SSUBLTB 01000101 .. 0 ..... 1000 11 ..... ..... @rd_rn_rm diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_ZZZ_TB(SABDLT, sabdl, true, true) DO_SVE2_ZZZ_TB(UADDLT, uaddl, true, true) DO_SVE2_ZZZ_TB(USUBLT, usubl, true, true) DO_SVE2_ZZZ_TB(UABDLT, uabdl, true, true) + +DO_SVE2_ZZZ_TB(SADDLBT, saddl, false, true) +DO_SVE2_ZZZ_TB(SSUBLBT, ssubl, false, true) +DO_SVE2_ZZZ_TB(SSUBLTB, ssubl, true, false) -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-13-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 16 ++++++++++++++++ target/arm/sve.decode | 12 ++++++++++++ target/arm/sve_helper.c | 30 ++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 20 ++++++++++++++++++++ 4 files changed, 78 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_4(sve2_uabdl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_uabdl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_uabdl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_saddw_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_saddw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_saddw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_ssubw_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_ssubw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_ssubw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_uaddw_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_uaddw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_uaddw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_usubw_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_usubw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_usubw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_4(sve_ld1bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32) DEF_HELPER_FLAGS_4(sve_ld2bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32) DEF_HELPER_FLAGS_4(sve_ld3bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ UABDLT 01000101 .. 0 ..... 00 1111 ..... ..... @rd_rn_rm SADDLBT 01000101 .. 0 ..... 1000 00 ..... ..... @rd_rn_rm SSUBLBT 01000101 .. 0 ..... 1000 10 ..... ..... @rd_rn_rm SSUBLTB 01000101 .. 0 ..... 1000 11 ..... ..... @rd_rn_rm + +## SVE2 integer add/subtract wide + +SADDWB 01000101 .. 0 ..... 010 000 ..... ..... @rd_rn_rm +SADDWT 01000101 .. 0 ..... 010 001 ..... ..... @rd_rn_rm +UADDWB 01000101 .. 0 ..... 010 010 ..... ..... @rd_rn_rm +UADDWT 01000101 .. 0 ..... 010 011 ..... ..... @rd_rn_rm + +SSUBWB 01000101 .. 0 ..... 010 100 ..... ..... @rd_rn_rm +SSUBWT 01000101 .. 0 ..... 010 101 ..... ..... @rd_rn_rm +USUBWB 01000101 .. 0 ..... 010 110 ..... ..... @rd_rn_rm +USUBWT 01000101 .. 0 ..... 010 111 ..... ..... @rd_rn_rm diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZZZ_TB(sve2_uabdl_d, uint64_t, uint32_t, , H1_4, DO_ABD) #undef DO_ZZZ_TB +#define DO_ZZZ_WTB(NAME, TYPEW, TYPEN, HW, HN, OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc); \ + int sel2 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN); \ + for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \ + TYPEW nn = *(TYPEW *)(vn + HW(i)); \ + TYPEW mm = *(TYPEN *)(vm + HN(i + sel2)); \ + *(TYPEW *)(vd + HW(i)) = OP(nn, mm); \ + } \ +} + +DO_ZZZ_WTB(sve2_saddw_h, int16_t, int8_t, H1_2, H1, DO_ADD) +DO_ZZZ_WTB(sve2_saddw_s, int32_t, int16_t, H1_4, H1_2, DO_ADD) +DO_ZZZ_WTB(sve2_saddw_d, int64_t, int32_t, , H1_4, DO_ADD) + +DO_ZZZ_WTB(sve2_ssubw_h, int16_t, int8_t, H1_2, H1, DO_SUB) +DO_ZZZ_WTB(sve2_ssubw_s, int32_t, int16_t, H1_4, H1_2, DO_SUB) +DO_ZZZ_WTB(sve2_ssubw_d, int64_t, int32_t, , H1_4, DO_SUB) + +DO_ZZZ_WTB(sve2_uaddw_h, uint16_t, uint8_t, H1_2, H1, DO_ADD) +DO_ZZZ_WTB(sve2_uaddw_s, uint32_t, uint16_t, H1_4, H1_2, DO_ADD) +DO_ZZZ_WTB(sve2_uaddw_d, uint64_t, uint32_t, , H1_4, DO_ADD) + +DO_ZZZ_WTB(sve2_usubw_h, uint16_t, uint8_t, H1_2, H1, DO_SUB) +DO_ZZZ_WTB(sve2_usubw_s, uint32_t, uint16_t, H1_4, H1_2, DO_SUB) +DO_ZZZ_WTB(sve2_usubw_d, uint64_t, uint32_t, , H1_4, DO_SUB) + +#undef DO_ZZZ_WTB + /* Two-operand reduction expander, controlled by a predicate. * The difference between TYPERED and TYPERET has to do with * sign-extension. E.g. for SMAX, TYPERED must be signed, diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_ZZZ_TB(UABDLT, uabdl, true, true) DO_SVE2_ZZZ_TB(SADDLBT, saddl, false, true) DO_SVE2_ZZZ_TB(SSUBLBT, ssubl, false, true) DO_SVE2_ZZZ_TB(SSUBLTB, ssubl, true, false) + +#define DO_SVE2_ZZZ_WTB(NAME, name, SEL2) \ +static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \ +{ \ + static gen_helper_gvec_3 * const fns[4] = { \ + NULL, gen_helper_sve2_##name##_h, \ + gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \ + }; \ + return do_sve2_zzw_ool(s, a, fns[a->esz], SEL2); \ +} + +DO_SVE2_ZZZ_WTB(SADDWB, saddw, false) +DO_SVE2_ZZZ_WTB(SADDWT, saddw, true) +DO_SVE2_ZZZ_WTB(SSUBWB, ssubw, false) +DO_SVE2_ZZZ_WTB(SSUBWT, ssubw, true) + +DO_SVE2_ZZZ_WTB(UADDWB, uaddw, false) +DO_SVE2_ZZZ_WTB(UADDWT, uaddw, true) +DO_SVE2_ZZZ_WTB(USUBWB, usubw, false) +DO_SVE2_ZZZ_WTB(USUBWT, usubw, true) -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Exclude PMULL from this category for the moment. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-14-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 15 +++++++++++++++ target/arm/sve.decode | 9 +++++++++ target/arm/sve_helper.c | 31 +++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 9 +++++++++ 4 files changed, 64 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_6(sve_stdd_le_zd_mte, TCG_CALL_NO_WG, DEF_HELPER_FLAGS_6(sve_stdd_be_zd_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve2_sqdmull_zzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqdmull_zzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqdmull_zzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_smull_zzz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_smull_zzz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_smull_zzz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_umull_zzz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_umull_zzz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_umull_zzz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_4(sve2_pmull_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SSUBWB 01000101 .. 0 ..... 010 100 ..... ..... @rd_rn_rm SSUBWT 01000101 .. 0 ..... 010 101 ..... ..... @rd_rn_rm USUBWB 01000101 .. 0 ..... 010 110 ..... ..... @rd_rn_rm USUBWT 01000101 .. 0 ..... 010 111 ..... ..... @rd_rn_rm + +## SVE2 integer multiply long + +SQDMULLB_zzz 01000101 .. 0 ..... 011 000 ..... ..... @rd_rn_rm +SQDMULLT_zzz 01000101 .. 0 ..... 011 001 ..... ..... @rd_rn_rm +SMULLB_zzz 01000101 .. 0 ..... 011 100 ..... ..... @rd_rn_rm +SMULLT_zzz 01000101 .. 0 ..... 011 101 ..... ..... @rd_rn_rm +UMULLB_zzz 01000101 .. 0 ..... 011 110 ..... ..... @rd_rn_rm +UMULLT_zzz 01000101 .. 0 ..... 011 111 ..... ..... @rd_rn_rm diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZZZ_TB(sve2_uabdl_h, uint16_t, uint8_t, H1_2, H1, DO_ABD) DO_ZZZ_TB(sve2_uabdl_s, uint32_t, uint16_t, H1_4, H1_2, DO_ABD) DO_ZZZ_TB(sve2_uabdl_d, uint64_t, uint32_t, , H1_4, DO_ABD) +DO_ZZZ_TB(sve2_smull_zzz_h, int16_t, int8_t, H1_2, H1, DO_MUL) +DO_ZZZ_TB(sve2_smull_zzz_s, int32_t, int16_t, H1_4, H1_2, DO_MUL) +DO_ZZZ_TB(sve2_smull_zzz_d, int64_t, int32_t, , H1_4, DO_MUL) + +DO_ZZZ_TB(sve2_umull_zzz_h, uint16_t, uint8_t, H1_2, H1, DO_MUL) +DO_ZZZ_TB(sve2_umull_zzz_s, uint32_t, uint16_t, H1_4, H1_2, DO_MUL) +DO_ZZZ_TB(sve2_umull_zzz_d, uint64_t, uint32_t, , H1_4, DO_MUL) + +/* Note that the multiply cannot overflow, but the doubling can. */ +static inline int16_t do_sqdmull_h(int16_t n, int16_t m) +{ + int16_t val = n * m; + return DO_SQADD_H(val, val); +} + +static inline int32_t do_sqdmull_s(int32_t n, int32_t m) +{ + int32_t val = n * m; + return DO_SQADD_S(val, val); +} + +static inline int64_t do_sqdmull_d(int64_t n, int64_t m) +{ + int64_t val = n * m; + return do_sqadd_d(val, val); +} + +DO_ZZZ_TB(sve2_sqdmull_zzz_h, int16_t, int8_t, H1_2, H1, do_sqdmull_h) +DO_ZZZ_TB(sve2_sqdmull_zzz_s, int32_t, int16_t, H1_4, H1_2, do_sqdmull_s) +DO_ZZZ_TB(sve2_sqdmull_zzz_d, int64_t, int32_t, , H1_4, do_sqdmull_d) + #undef DO_ZZZ_TB #define DO_ZZZ_WTB(NAME, TYPEW, TYPEN, HW, HN, OP) \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_ZZZ_TB(SADDLBT, saddl, false, true) DO_SVE2_ZZZ_TB(SSUBLBT, ssubl, false, true) DO_SVE2_ZZZ_TB(SSUBLTB, ssubl, true, false) +DO_SVE2_ZZZ_TB(SQDMULLB_zzz, sqdmull_zzz, false, false) +DO_SVE2_ZZZ_TB(SQDMULLT_zzz, sqdmull_zzz, true, true) + +DO_SVE2_ZZZ_TB(SMULLB_zzz, smull_zzz, false, false) +DO_SVE2_ZZZ_TB(SMULLT_zzz, smull_zzz, true, true) + +DO_SVE2_ZZZ_TB(UMULLB_zzz, umull_zzz, false, false) +DO_SVE2_ZZZ_TB(UMULLT_zzz, umull_zzz, true, true) + #define DO_SVE2_ZZZ_WTB(NAME, name, SEL2) \ static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \ { \ -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-15-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/cpu.h | 10 ++++++++++ target/arm/helper-sve.h | 1 + target/arm/sve.decode | 2 ++ target/arm/translate-sve.c | 22 ++++++++++++++++++++++ target/arm/vec_helper.c | 24 ++++++++++++++++++++++++ 5 files changed, 59 insertions(+) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -XXX,XX +XXX,XX @@ static inline bool isar_feature_aa64_sve2(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, SVEVER) != 0; } +static inline bool isar_feature_aa64_sve2_aes(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, AES) != 0; +} + +static inline bool isar_feature_aa64_sve2_pmull128(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, AES) >= 2; +} + /* * Feature tests for "does this exist in either 32-bit or 64-bit?" */ diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_4(sve2_umull_zzz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_umull_zzz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_pmull_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_pmull_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ USUBWT 01000101 .. 0 ..... 010 111 ..... ..... @rd_rn_rm SQDMULLB_zzz 01000101 .. 0 ..... 011 000 ..... ..... @rd_rn_rm SQDMULLT_zzz 01000101 .. 0 ..... 011 001 ..... ..... @rd_rn_rm +PMULLB 01000101 .. 0 ..... 011 010 ..... ..... @rd_rn_rm +PMULLT 01000101 .. 0 ..... 011 011 ..... ..... @rd_rn_rm SMULLB_zzz 01000101 .. 0 ..... 011 100 ..... ..... @rd_rn_rm SMULLT_zzz 01000101 .. 0 ..... 011 101 ..... ..... @rd_rn_rm UMULLB_zzz 01000101 .. 0 ..... 011 110 ..... ..... @rd_rn_rm diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_ZZZ_TB(SMULLT_zzz, smull_zzz, true, true) DO_SVE2_ZZZ_TB(UMULLB_zzz, umull_zzz, false, false) DO_SVE2_ZZZ_TB(UMULLT_zzz, umull_zzz, true, true) +static bool do_trans_pmull(DisasContext *s, arg_rrr_esz *a, bool sel) +{ + static gen_helper_gvec_3 * const fns[4] = { + gen_helper_gvec_pmull_q, gen_helper_sve2_pmull_h, + NULL, gen_helper_sve2_pmull_d, + }; + if (a->esz == 0 && !dc_isar_feature(aa64_sve2_pmull128, s)) { + return false; + } + return do_sve2_zzw_ool(s, a, fns[a->esz], sel); +} + +static bool trans_PMULLB(DisasContext *s, arg_rrr_esz *a) +{ + return do_trans_pmull(s, a, false); +} + +static bool trans_PMULLT(DisasContext *s, arg_rrr_esz *a) +{ + return do_trans_pmull(s, a, true); +} + #define DO_SVE2_ZZZ_WTB(NAME, name, SEL2) \ static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \ { \ diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(sve2_pmull_h)(void *vd, void *vn, void *vm, uint32_t desc) d[i] = pmull_h(nn, mm); } } + +static uint64_t pmull_d(uint64_t op1, uint64_t op2) +{ + uint64_t result = 0; + int i; + + for (i = 0; i < 32; ++i) { + uint64_t mask = -((op1 >> i) & 1); + result ^= (op2 << i) & mask; + } + return result; +} + +void HELPER(sve2_pmull_d)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t sel = H4(simd_data(desc)); + intptr_t i, opr_sz = simd_oprsz(desc); + uint32_t *n = vn, *m = vm; + uint64_t *d = vd; + + for (i = 0; i < opr_sz / 8; ++i) { + d[i] = pmull_d(n[2 * i + sel], m[2 * i + sel]); + } +} #endif #define DO_CMP0(NAME, TYPE, OP) \ -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-16-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 8 ++ target/arm/sve.decode | 8 ++ target/arm/sve_helper.c | 22 +++++ target/arm/translate-sve.c | 159 +++++++++++++++++++++++++++++++++++++ 4 files changed, 197 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_4(sve2_umull_zzz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_pmull_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_pmull_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_sshll_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sshll_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sshll_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_ushll_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_ushll_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_ushll_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SMULLB_zzz 01000101 .. 0 ..... 011 100 ..... ..... @rd_rn_rm SMULLT_zzz 01000101 .. 0 ..... 011 101 ..... ..... @rd_rn_rm UMULLB_zzz 01000101 .. 0 ..... 011 110 ..... ..... @rd_rn_rm UMULLT_zzz 01000101 .. 0 ..... 011 111 ..... ..... @rd_rn_rm + +## SVE2 bitwise shift left long + +# Note bit23 == 0 is handled by esz > 0 in do_sve2_shll_tb. +SSHLLB 01000101 .. 0 ..... 1010 00 ..... ..... @rd_rn_tszimm_shl +SSHLLT 01000101 .. 0 ..... 1010 01 ..... ..... @rd_rn_tszimm_shl +USHLLB 01000101 .. 0 ..... 1010 10 ..... ..... @rd_rn_tszimm_shl +USHLLT 01000101 .. 0 ..... 1010 11 ..... ..... @rd_rn_tszimm_shl diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZZZ_WTB(sve2_usubw_d, uint64_t, uint32_t, , H1_4, DO_SUB) #undef DO_ZZZ_WTB +#define DO_ZZI_SHLL(NAME, TYPEW, TYPEN, HW, HN) \ +void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc); \ + intptr_t sel = (simd_data(desc) & 1) * sizeof(TYPEN); \ + int shift = simd_data(desc) >> 1; \ + for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \ + TYPEW nn = *(TYPEN *)(vn + HN(i + sel)); \ + *(TYPEW *)(vd + HW(i)) = nn << shift; \ + } \ +} + +DO_ZZI_SHLL(sve2_sshll_h, int16_t, int8_t, H1_2, H1) +DO_ZZI_SHLL(sve2_sshll_s, int32_t, int16_t, H1_4, H1_2) +DO_ZZI_SHLL(sve2_sshll_d, int64_t, int32_t, , H1_4) + +DO_ZZI_SHLL(sve2_ushll_h, uint16_t, uint8_t, H1_2, H1) +DO_ZZI_SHLL(sve2_ushll_s, uint32_t, uint16_t, H1_4, H1_2) +DO_ZZI_SHLL(sve2_ushll_d, uint64_t, uint32_t, , H1_4) + +#undef DO_ZZI_SHLL + /* Two-operand reduction expander, controlled by a predicate. * The difference between TYPERED and TYPERET has to do with * sign-extension. E.g. for SMAX, TYPERED must be signed, diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_ZZZ_WTB(UADDWB, uaddw, false) DO_SVE2_ZZZ_WTB(UADDWT, uaddw, true) DO_SVE2_ZZZ_WTB(USUBWB, usubw, false) DO_SVE2_ZZZ_WTB(USUBWT, usubw, true) + +static void gen_sshll_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t imm) +{ + int top = imm & 1; + int shl = imm >> 1; + int halfbits = 4 << vece; + + if (top) { + if (shl == halfbits) { + TCGv_vec t = tcg_temp_new_vec_matching(d); + tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(halfbits, halfbits)); + tcg_gen_and_vec(vece, d, n, t); + tcg_temp_free_vec(t); + } else { + tcg_gen_sari_vec(vece, d, n, halfbits); + tcg_gen_shli_vec(vece, d, d, shl); + } + } else { + tcg_gen_shli_vec(vece, d, n, halfbits); + tcg_gen_sari_vec(vece, d, d, halfbits - shl); + } +} + +static void gen_ushll_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int imm) +{ + int halfbits = 4 << vece; + int top = imm & 1; + int shl = (imm >> 1); + int shift; + uint64_t mask; + + mask = MAKE_64BIT_MASK(0, halfbits); + mask <<= shl; + mask = dup_const(vece, mask); + + shift = shl - top * halfbits; + if (shift < 0) { + tcg_gen_shri_i64(d, n, -shift); + } else { + tcg_gen_shli_i64(d, n, shift); + } + tcg_gen_andi_i64(d, d, mask); +} + +static void gen_ushll16_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm) +{ + gen_ushll_i64(MO_16, d, n, imm); +} + +static void gen_ushll32_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm) +{ + gen_ushll_i64(MO_32, d, n, imm); +} + +static void gen_ushll64_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm) +{ + gen_ushll_i64(MO_64, d, n, imm); +} + +static void gen_ushll_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t imm) +{ + int halfbits = 4 << vece; + int top = imm & 1; + int shl = imm >> 1; + + if (top) { + if (shl == halfbits) { + TCGv_vec t = tcg_temp_new_vec_matching(d); + tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(halfbits, halfbits)); + tcg_gen_and_vec(vece, d, n, t); + tcg_temp_free_vec(t); + } else { + tcg_gen_shri_vec(vece, d, n, halfbits); + tcg_gen_shli_vec(vece, d, d, shl); + } + } else { + if (shl == 0) { + TCGv_vec t = tcg_temp_new_vec_matching(d); + tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits)); + tcg_gen_and_vec(vece, d, n, t); + tcg_temp_free_vec(t); + } else { + tcg_gen_shli_vec(vece, d, n, halfbits); + tcg_gen_shri_vec(vece, d, d, halfbits - shl); + } + } +} + +static bool do_sve2_shll_tb(DisasContext *s, arg_rri_esz *a, + bool sel, bool uns) +{ + static const TCGOpcode sshll_list[] = { + INDEX_op_shli_vec, INDEX_op_sari_vec, 0 + }; + static const TCGOpcode ushll_list[] = { + INDEX_op_shli_vec, INDEX_op_shri_vec, 0 + }; + static const GVecGen2i ops[2][3] = { + { { .fniv = gen_sshll_vec, + .opt_opc = sshll_list, + .fno = gen_helper_sve2_sshll_h, + .vece = MO_16 }, + { .fniv = gen_sshll_vec, + .opt_opc = sshll_list, + .fno = gen_helper_sve2_sshll_s, + .vece = MO_32 }, + { .fniv = gen_sshll_vec, + .opt_opc = sshll_list, + .fno = gen_helper_sve2_sshll_d, + .vece = MO_64 } }, + { { .fni8 = gen_ushll16_i64, + .fniv = gen_ushll_vec, + .opt_opc = ushll_list, + .fno = gen_helper_sve2_ushll_h, + .vece = MO_16 }, + { .fni8 = gen_ushll32_i64, + .fniv = gen_ushll_vec, + .opt_opc = ushll_list, + .fno = gen_helper_sve2_ushll_s, + .vece = MO_32 }, + { .fni8 = gen_ushll64_i64, + .fniv = gen_ushll_vec, + .opt_opc = ushll_list, + .fno = gen_helper_sve2_ushll_d, + .vece = MO_64 } }, + }; + + if (a->esz < 0 || a->esz > 2 || !dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (sve_access_check(s)) { + unsigned vsz = vec_full_reg_size(s); + tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd), + vec_full_reg_offset(s, a->rn), + vsz, vsz, (a->imm << 1) | sel, + &ops[uns][a->esz]); + } + return true; +} + +static bool trans_SSHLLB(DisasContext *s, arg_rri_esz *a) +{ + return do_sve2_shll_tb(s, a, false, false); +} + +static bool trans_SSHLLT(DisasContext *s, arg_rri_esz *a) +{ + return do_sve2_shll_tb(s, a, true, false); +} + +static bool trans_USHLLB(DisasContext *s, arg_rri_esz *a) +{ + return do_sve2_shll_tb(s, a, false, true); +} + +static bool trans_USHLLT(DisasContext *s, arg_rri_esz *a) +{ + return do_sve2_shll_tb(s, a, true, true); +} -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-17-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 5 +++++ target/arm/sve.decode | 5 +++++ target/arm/sve_helper.c | 20 ++++++++++++++++++++ target/arm/translate-sve.c | 19 +++++++++++++++++++ 4 files changed, 49 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_3(sve2_sshll_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_ushll_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_ushll_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_ushll_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_eoril_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_eoril_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_eoril_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_eoril_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SSHLLB 01000101 .. 0 ..... 1010 00 ..... ..... @rd_rn_tszimm_shl SSHLLT 01000101 .. 0 ..... 1010 01 ..... ..... @rd_rn_tszimm_shl USHLLB 01000101 .. 0 ..... 1010 10 ..... ..... @rd_rn_tszimm_shl USHLLT 01000101 .. 0 ..... 1010 11 ..... ..... @rd_rn_tszimm_shl + +## SVE2 bitwise exclusive-or interleaved + +EORBT 01000101 .. 0 ..... 10010 0 ..... ..... @rd_rn_rm +EORTB 01000101 .. 0 ..... 10010 1 ..... ..... @rd_rn_rm diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZZZ_WTB(sve2_usubw_d, uint64_t, uint32_t, , H1_4, DO_SUB) #undef DO_ZZZ_WTB +#define DO_ZZZ_NTB(NAME, TYPE, H, OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc); \ + intptr_t sel1 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPE); \ + intptr_t sel2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(TYPE); \ + for (i = 0; i < opr_sz; i += 2 * sizeof(TYPE)) { \ + TYPE nn = *(TYPE *)(vn + H(i + sel1)); \ + TYPE mm = *(TYPE *)(vm + H(i + sel2)); \ + *(TYPE *)(vd + H(i + sel1)) = OP(nn, mm); \ + } \ +} + +DO_ZZZ_NTB(sve2_eoril_b, uint8_t, H1, DO_EOR) +DO_ZZZ_NTB(sve2_eoril_h, uint16_t, H1_2, DO_EOR) +DO_ZZZ_NTB(sve2_eoril_s, uint32_t, H1_4, DO_EOR) +DO_ZZZ_NTB(sve2_eoril_d, uint64_t, , DO_EOR) + +#undef DO_ZZZ_NTB + #define DO_ZZI_SHLL(NAME, TYPEW, TYPEN, HW, HN) \ void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \ { \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_ZZZ_TB(SMULLT_zzz, smull_zzz, true, true) DO_SVE2_ZZZ_TB(UMULLB_zzz, umull_zzz, false, false) DO_SVE2_ZZZ_TB(UMULLT_zzz, umull_zzz, true, true) +static bool do_eor_tb(DisasContext *s, arg_rrr_esz *a, bool sel1) +{ + static gen_helper_gvec_3 * const fns[4] = { + gen_helper_sve2_eoril_b, gen_helper_sve2_eoril_h, + gen_helper_sve2_eoril_s, gen_helper_sve2_eoril_d, + }; + return do_sve2_zzw_ool(s, a, fns[a->esz], (!sel1 << 1) | sel1); +} + +static bool trans_EORBT(DisasContext *s, arg_rrr_esz *a) +{ + return do_eor_tb(s, a, false); +} + +static bool trans_EORTB(DisasContext *s, arg_rrr_esz *a) +{ + return do_eor_tb(s, a, true); +} + static bool do_trans_pmull(DisasContext *s, arg_rrr_esz *a, bool sel) { static gen_helper_gvec_3 * const fns[4] = { -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-18-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/cpu.h | 5 +++ target/arm/helper-sve.h | 15 ++++++++ target/arm/sve.decode | 6 ++++ target/arm/sve_helper.c | 73 ++++++++++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 36 +++++++++++++++++++ 5 files changed, 135 insertions(+) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -XXX,XX +XXX,XX @@ static inline bool isar_feature_aa64_sve2_pmull128(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, AES) >= 2; } +static inline bool isar_feature_aa64_sve2_bitperm(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, BITPERM) != 0; +} + /* * Feature tests for "does this exist in either 32-bit or 64-bit?" */ diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_4(sve2_eoril_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_eoril_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_eoril_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_eoril_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_bext_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_bext_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_bext_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_bext_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_bdep_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_bdep_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_bdep_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_bdep_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_bgrp_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_bgrp_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_bgrp_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_bgrp_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ USHLLT 01000101 .. 0 ..... 1010 11 ..... ..... @rd_rn_tszimm_shl EORBT 01000101 .. 0 ..... 10010 0 ..... ..... @rd_rn_rm EORTB 01000101 .. 0 ..... 10010 1 ..... ..... @rd_rn_rm + +## SVE2 bitwise permute + +BEXT 01000101 .. 0 ..... 1011 00 ..... ..... @rd_rn_rm +BDEP 01000101 .. 0 ..... 1011 01 ..... ..... @rd_rn_rm +BGRP 01000101 .. 0 ..... 1011 10 ..... ..... @rd_rn_rm diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZZZ_NTB(sve2_eoril_d, uint64_t, , DO_EOR) #undef DO_ZZZ_NTB +#define DO_BITPERM(NAME, TYPE, OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc); \ + for (i = 0; i < opr_sz; i += sizeof(TYPE)) { \ + TYPE nn = *(TYPE *)(vn + i); \ + TYPE mm = *(TYPE *)(vm + i); \ + *(TYPE *)(vd + i) = OP(nn, mm, sizeof(TYPE) * 8); \ + } \ +} + +static uint64_t bitextract(uint64_t data, uint64_t mask, int n) +{ + uint64_t res = 0; + int db, rb = 0; + + for (db = 0; db < n; ++db) { + if ((mask >> db) & 1) { + res |= ((data >> db) & 1) << rb; + ++rb; + } + } + return res; +} + +DO_BITPERM(sve2_bext_b, uint8_t, bitextract) +DO_BITPERM(sve2_bext_h, uint16_t, bitextract) +DO_BITPERM(sve2_bext_s, uint32_t, bitextract) +DO_BITPERM(sve2_bext_d, uint64_t, bitextract) + +static uint64_t bitdeposit(uint64_t data, uint64_t mask, int n) +{ + uint64_t res = 0; + int rb, db = 0; + + for (rb = 0; rb < n; ++rb) { + if ((mask >> rb) & 1) { + res |= ((data >> db) & 1) << rb; + ++db; + } + } + return res; +} + +DO_BITPERM(sve2_bdep_b, uint8_t, bitdeposit) +DO_BITPERM(sve2_bdep_h, uint16_t, bitdeposit) +DO_BITPERM(sve2_bdep_s, uint32_t, bitdeposit) +DO_BITPERM(sve2_bdep_d, uint64_t, bitdeposit) + +static uint64_t bitgroup(uint64_t data, uint64_t mask, int n) +{ + uint64_t resm = 0, resu = 0; + int db, rbm = 0, rbu = 0; + + for (db = 0; db < n; ++db) { + uint64_t val = (data >> db) & 1; + if ((mask >> db) & 1) { + resm |= val << rbm++; + } else { + resu |= val << rbu++; + } + } + + return resm | (resu << rbm); +} + +DO_BITPERM(sve2_bgrp_b, uint8_t, bitgroup) +DO_BITPERM(sve2_bgrp_h, uint16_t, bitgroup) +DO_BITPERM(sve2_bgrp_s, uint32_t, bitgroup) +DO_BITPERM(sve2_bgrp_d, uint64_t, bitgroup) + +#undef DO_BITPERM + #define DO_ZZI_SHLL(NAME, TYPEW, TYPEN, HW, HN) \ void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \ { \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_USHLLT(DisasContext *s, arg_rri_esz *a) { return do_sve2_shll_tb(s, a, true, true); } + +static bool trans_BEXT(DisasContext *s, arg_rrr_esz *a) +{ + static gen_helper_gvec_3 * const fns[4] = { + gen_helper_sve2_bext_b, gen_helper_sve2_bext_h, + gen_helper_sve2_bext_s, gen_helper_sve2_bext_d, + }; + if (!dc_isar_feature(aa64_sve2_bitperm, s)) { + return false; + } + return do_sve2_zzw_ool(s, a, fns[a->esz], 0); +} + +static bool trans_BDEP(DisasContext *s, arg_rrr_esz *a) +{ + static gen_helper_gvec_3 * const fns[4] = { + gen_helper_sve2_bdep_b, gen_helper_sve2_bdep_h, + gen_helper_sve2_bdep_s, gen_helper_sve2_bdep_d, + }; + if (!dc_isar_feature(aa64_sve2_bitperm, s)) { + return false; + } + return do_sve2_zzw_ool(s, a, fns[a->esz], 0); +} + +static bool trans_BGRP(DisasContext *s, arg_rrr_esz *a) +{ + static gen_helper_gvec_3 * const fns[4] = { + gen_helper_sve2_bgrp_b, gen_helper_sve2_bgrp_h, + gen_helper_sve2_bgrp_s, gen_helper_sve2_bgrp_d, + }; + if (!dc_isar_feature(aa64_sve2_bitperm, s)) { + return false; + } + return do_sve2_zzw_ool(s, a, fns[a->esz], 0); +} -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-19-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 10 +++++++++ target/arm/sve.decode | 9 ++++++++ target/arm/sve_helper.c | 42 ++++++++++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 31 ++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_4(sve2_bgrp_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_bgrp_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_bgrp_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_bgrp_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_cadd_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_cadd_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_cadd_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_cadd_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_sqcadd_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqcadd_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqcadd_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqcadd_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ EORTB 01000101 .. 0 ..... 10010 1 ..... ..... @rd_rn_rm BEXT 01000101 .. 0 ..... 1011 00 ..... ..... @rd_rn_rm BDEP 01000101 .. 0 ..... 1011 01 ..... ..... @rd_rn_rm BGRP 01000101 .. 0 ..... 1011 10 ..... ..... @rd_rn_rm + +#### SVE2 Accumulate + +## SVE2 complex integer add + +CADD_rot90 01000101 .. 00000 0 11011 0 ..... ..... @rdn_rm +CADD_rot270 01000101 .. 00000 0 11011 1 ..... ..... @rdn_rm +SQCADD_rot90 01000101 .. 00000 1 11011 0 ..... ..... @rdn_rm +SQCADD_rot270 01000101 .. 00000 1 11011 1 ..... ..... @rdn_rm diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_BITPERM(sve2_bgrp_d, uint64_t, bitgroup) #undef DO_BITPERM +#define DO_CADD(NAME, TYPE, H, ADD_OP, SUB_OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc); \ + int sub_r = simd_data(desc); \ + if (sub_r) { \ + for (i = 0; i < opr_sz; i += 2 * sizeof(TYPE)) { \ + TYPE acc_r = *(TYPE *)(vn + H(i)); \ + TYPE acc_i = *(TYPE *)(vn + H(i + sizeof(TYPE))); \ + TYPE el2_r = *(TYPE *)(vm + H(i)); \ + TYPE el2_i = *(TYPE *)(vm + H(i + sizeof(TYPE))); \ + acc_r = ADD_OP(acc_r, el2_i); \ + acc_i = SUB_OP(acc_i, el2_r); \ + *(TYPE *)(vd + H(i)) = acc_r; \ + *(TYPE *)(vd + H(i + sizeof(TYPE))) = acc_i; \ + } \ + } else { \ + for (i = 0; i < opr_sz; i += 2 * sizeof(TYPE)) { \ + TYPE acc_r = *(TYPE *)(vn + H(i)); \ + TYPE acc_i = *(TYPE *)(vn + H(i + sizeof(TYPE))); \ + TYPE el2_r = *(TYPE *)(vm + H(i)); \ + TYPE el2_i = *(TYPE *)(vm + H(i + sizeof(TYPE))); \ + acc_r = SUB_OP(acc_r, el2_i); \ + acc_i = ADD_OP(acc_i, el2_r); \ + *(TYPE *)(vd + H(i)) = acc_r; \ + *(TYPE *)(vd + H(i + sizeof(TYPE))) = acc_i; \ + } \ + } \ +} + +DO_CADD(sve2_cadd_b, int8_t, H1, DO_ADD, DO_SUB) +DO_CADD(sve2_cadd_h, int16_t, H1_2, DO_ADD, DO_SUB) +DO_CADD(sve2_cadd_s, int32_t, H1_4, DO_ADD, DO_SUB) +DO_CADD(sve2_cadd_d, int64_t, , DO_ADD, DO_SUB) + +DO_CADD(sve2_sqcadd_b, int8_t, H1, DO_SQADD_B, DO_SQSUB_B) +DO_CADD(sve2_sqcadd_h, int16_t, H1_2, DO_SQADD_H, DO_SQSUB_H) +DO_CADD(sve2_sqcadd_s, int32_t, H1_4, DO_SQADD_S, DO_SQSUB_S) +DO_CADD(sve2_sqcadd_d, int64_t, , do_sqadd_d, do_sqsub_d) + +#undef DO_CADD + #define DO_ZZI_SHLL(NAME, TYPEW, TYPEN, HW, HN) \ void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \ { \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_BGRP(DisasContext *s, arg_rrr_esz *a) } return do_sve2_zzw_ool(s, a, fns[a->esz], 0); } + +static bool do_cadd(DisasContext *s, arg_rrr_esz *a, bool sq, bool rot) +{ + static gen_helper_gvec_3 * const fns[2][4] = { + { gen_helper_sve2_cadd_b, gen_helper_sve2_cadd_h, + gen_helper_sve2_cadd_s, gen_helper_sve2_cadd_d }, + { gen_helper_sve2_sqcadd_b, gen_helper_sve2_sqcadd_h, + gen_helper_sve2_sqcadd_s, gen_helper_sve2_sqcadd_d }, + }; + return do_sve2_zzw_ool(s, a, fns[sq][a->esz], rot); +} + +static bool trans_CADD_rot90(DisasContext *s, arg_rrr_esz *a) +{ + return do_cadd(s, a, false, false); +} + +static bool trans_CADD_rot270(DisasContext *s, arg_rrr_esz *a) +{ + return do_cadd(s, a, false, true); +} + +static bool trans_SQCADD_rot90(DisasContext *s, arg_rrr_esz *a) +{ + return do_cadd(s, a, true, false); +} + +static bool trans_SQCADD_rot270(DisasContext *s, arg_rrr_esz *a) +{ + return do_cadd(s, a, true, true); +} -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-20-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 14 ++++++++++ target/arm/sve.decode | 12 +++++++++ target/arm/sve_helper.c | 23 ++++++++++++++++ target/arm/translate-sve.c | 55 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_4(sve2_sqcadd_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_sqcadd_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_sqcadd_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_sqcadd_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_sabal_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sabal_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sabal_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_uabal_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uabal_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_uabal_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ &rpr_s rd pg rn s &rprr_s rd pg rn rm s &rprr_esz rd pg rn rm esz +&rrrr_esz rd ra rn rm esz &rprrr_esz rd pg rn rm ra esz &rpri_esz rd pg rn imm esz &ptrue rd esz pat s @@ -XXX,XX +XXX,XX @@ @rdn_i8s ........ esz:2 ...... ... imm:s8 rd:5 \ &rri_esz rn=%reg_movprfx +# Four operand, vector element size +@rda_rn_rm ........ esz:2 . rm:5 ... ... rn:5 rd:5 \ + &rrrr_esz ra=%reg_movprfx + # Three operand with "memory" size, aka immediate left shift @rd_rn_msz_rm ........ ... rm:5 .... imm:2 rn:5 rd:5 &rrri @@ -XXX,XX +XXX,XX @@ CADD_rot90 01000101 .. 00000 0 11011 0 ..... ..... @rdn_rm CADD_rot270 01000101 .. 00000 0 11011 1 ..... ..... @rdn_rm SQCADD_rot90 01000101 .. 00000 1 11011 0 ..... ..... @rdn_rm SQCADD_rot270 01000101 .. 00000 1 11011 1 ..... ..... @rdn_rm + +## SVE2 integer absolute difference and accumulate long + +SABALB 01000101 .. 0 ..... 1100 00 ..... ..... @rda_rn_rm +SABALT 01000101 .. 0 ..... 1100 01 ..... ..... @rda_rn_rm +UABALB 01000101 .. 0 ..... 1100 10 ..... ..... @rda_rn_rm +UABALT 01000101 .. 0 ..... 1100 11 ..... ..... @rda_rn_rm diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZZZ_NTB(sve2_eoril_d, uint64_t, , DO_EOR) #undef DO_ZZZ_NTB +#define DO_ZZZW_ACC(NAME, TYPEW, TYPEN, HW, HN, OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc); \ + intptr_t sel1 = simd_data(desc) * sizeof(TYPEN); \ + for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \ + TYPEW nn = *(TYPEN *)(vn + HN(i + sel1)); \ + TYPEW mm = *(TYPEN *)(vm + HN(i + sel1)); \ + TYPEW aa = *(TYPEW *)(va + HW(i)); \ + *(TYPEW *)(vd + HW(i)) = OP(nn, mm) + aa; \ + } \ +} + +DO_ZZZW_ACC(sve2_sabal_h, int16_t, int8_t, H1_2, H1, DO_ABD) +DO_ZZZW_ACC(sve2_sabal_s, int32_t, int16_t, H1_4, H1_2, DO_ABD) +DO_ZZZW_ACC(sve2_sabal_d, int64_t, int32_t, , H1_4, DO_ABD) + +DO_ZZZW_ACC(sve2_uabal_h, uint16_t, uint8_t, H1_2, H1, DO_ABD) +DO_ZZZW_ACC(sve2_uabal_s, uint32_t, uint16_t, H1_4, H1_2, DO_ABD) +DO_ZZZW_ACC(sve2_uabal_d, uint64_t, uint32_t, , H1_4, DO_ABD) + +#undef DO_ZZZW_ACC + #define DO_BITPERM(NAME, TYPE, OP) \ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ { \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static void gen_gvec_ool_zzz(DisasContext *s, gen_helper_gvec_3 *fn, vsz, vsz, data, fn); } +/* Invoke an out-of-line helper on 4 Zregs. */ +static void gen_gvec_ool_zzzz(DisasContext *s, gen_helper_gvec_4 *fn, + int rd, int rn, int rm, int ra, int data) +{ + unsigned vsz = vec_full_reg_size(s); + tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + vec_full_reg_offset(s, rm), + vec_full_reg_offset(s, ra), + vsz, vsz, data, fn); +} + /* Invoke an out-of-line helper on 2 Zregs and a predicate. */ static void gen_gvec_ool_zzp(DisasContext *s, gen_helper_gvec_3 *fn, int rd, int rn, int pg, int data) @@ -XXX,XX +XXX,XX @@ static bool trans_SQCADD_rot270(DisasContext *s, arg_rrr_esz *a) { return do_cadd(s, a, true, true); } + +static bool do_sve2_zzzz_ool(DisasContext *s, arg_rrrr_esz *a, + gen_helper_gvec_4 *fn, int data) +{ + if (fn == NULL || !dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (sve_access_check(s)) { + gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, data); + } + return true; +} + +static bool do_abal(DisasContext *s, arg_rrrr_esz *a, bool uns, bool sel) +{ + static gen_helper_gvec_4 * const fns[2][4] = { + { NULL, gen_helper_sve2_sabal_h, + gen_helper_sve2_sabal_s, gen_helper_sve2_sabal_d }, + { NULL, gen_helper_sve2_uabal_h, + gen_helper_sve2_uabal_s, gen_helper_sve2_uabal_d }, + }; + return do_sve2_zzzz_ool(s, a, fns[uns][a->esz], sel); +} + +static bool trans_SABALB(DisasContext *s, arg_rrrr_esz *a) +{ + return do_abal(s, a, false, false); +} + +static bool trans_SABALT(DisasContext *s, arg_rrrr_esz *a) +{ + return do_abal(s, a, false, true); +} + +static bool trans_UABALB(DisasContext *s, arg_rrrr_esz *a) +{ + return do_abal(s, a, true, false); +} + +static bool trans_UABALT(DisasContext *s, arg_rrrr_esz *a) +{ + return do_abal(s, a, true, true); +} -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-21-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 3 +++ target/arm/sve.decode | 6 ++++++ target/arm/sve_helper.c | 34 ++++++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 23 +++++++++++++++++++++++ 4 files changed, 66 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_uabal_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve2_uabal_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_adcl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_adcl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SABALB 01000101 .. 0 ..... 1100 00 ..... ..... @rda_rn_rm SABALT 01000101 .. 0 ..... 1100 01 ..... ..... @rda_rn_rm UABALB 01000101 .. 0 ..... 1100 10 ..... ..... @rda_rn_rm UABALT 01000101 .. 0 ..... 1100 11 ..... ..... @rda_rn_rm + +## SVE2 integer add/subtract long with carry + +# ADC and SBC decoded via size in helper dispatch. +ADCLB 01000101 .. 0 ..... 11010 0 ..... ..... @rda_rn_rm +ADCLT 01000101 .. 0 ..... 11010 1 ..... ..... @rda_rn_rm diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZZZW_ACC(sve2_uabal_d, uint64_t, uint32_t, , H1_4, DO_ABD) #undef DO_ZZZW_ACC +void HELPER(sve2_adcl_s)(void *vd, void *vn, void *vm, void *va, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int sel = H4(extract32(desc, SIMD_DATA_SHIFT, 1)); + uint32_t inv = -extract32(desc, SIMD_DATA_SHIFT + 1, 1); + uint32_t *a = va, *n = vn; + uint64_t *d = vd, *m = vm; + + for (i = 0; i < opr_sz / 8; ++i) { + uint32_t e1 = a[2 * i + H4(0)]; + uint32_t e2 = n[2 * i + sel] ^ inv; + uint64_t c = extract64(m[i], 32, 1); + /* Compute and store the entire 33-bit result at once. */ + d[i] = c + e1 + e2; + } +} + +void HELPER(sve2_adcl_d)(void *vd, void *vn, void *vm, void *va, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int sel = extract32(desc, SIMD_DATA_SHIFT, 1); + uint64_t inv = -(uint64_t)extract32(desc, SIMD_DATA_SHIFT + 1, 1); + uint64_t *d = vd, *a = va, *n = vn, *m = vm; + + for (i = 0; i < opr_sz / 8; i += 2) { + Int128 e1 = int128_make64(a[i]); + Int128 e2 = int128_make64(n[i + sel] ^ inv); + Int128 c = int128_make64(m[i + 1] & 1); + Int128 r = int128_add(int128_add(e1, e2), c); + d[i + 0] = int128_getlo(r); + d[i + 1] = int128_gethi(r); + } +} + #define DO_BITPERM(NAME, TYPE, OP) \ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ { \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_UABALT(DisasContext *s, arg_rrrr_esz *a) { return do_abal(s, a, true, true); } + +static bool do_adcl(DisasContext *s, arg_rrrr_esz *a, bool sel) +{ + static gen_helper_gvec_4 * const fns[2] = { + gen_helper_sve2_adcl_s, + gen_helper_sve2_adcl_d, + }; + /* + * Note that in this case the ESZ field encodes both size and sign. + * Split out 'subtract' into bit 1 of the data field for the helper. + */ + return do_sve2_zzzz_ool(s, a, fns[a->esz & 1], (a->esz & 2) | sel); +} + +static bool trans_ADCLB(DisasContext *s, arg_rrrr_esz *a) +{ + return do_adcl(s, a, false); +} + +static bool trans_ADCLT(DisasContext *s, arg_rrrr_esz *a) +{ + return do_adcl(s, a, true); +} -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-22-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/sve.decode | 8 ++++++++ target/arm/translate-sve.c | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ UABALT 01000101 .. 0 ..... 1100 11 ..... ..... @rda_rn_rm # ADC and SBC decoded via size in helper dispatch. ADCLB 01000101 .. 0 ..... 11010 0 ..... ..... @rda_rn_rm ADCLT 01000101 .. 0 ..... 11010 1 ..... ..... @rda_rn_rm + +## SVE2 bitwise shift right and accumulate + +# TODO: Use @rda and %reg_movprfx here. +SSRA 01000101 .. 0 ..... 1110 00 ..... ..... @rd_rn_tszimm_shr +USRA 01000101 .. 0 ..... 1110 01 ..... ..... @rd_rn_tszimm_shr +SRSRA 01000101 .. 0 ..... 1110 10 ..... ..... @rd_rn_tszimm_shr +URSRA 01000101 .. 0 ..... 1110 11 ..... ..... @rd_rn_tszimm_shr diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_ADCLT(DisasContext *s, arg_rrrr_esz *a) { return do_adcl(s, a, true); } + +static bool do_sve2_fn2i(DisasContext *s, arg_rri_esz *a, GVecGen2iFn *fn) +{ + if (a->esz < 0 || !dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (sve_access_check(s)) { + unsigned vsz = vec_full_reg_size(s); + unsigned rd_ofs = vec_full_reg_offset(s, a->rd); + unsigned rn_ofs = vec_full_reg_offset(s, a->rn); + fn(a->esz, rd_ofs, rn_ofs, a->imm, vsz, vsz); + } + return true; +} + +static bool trans_SSRA(DisasContext *s, arg_rri_esz *a) +{ + return do_sve2_fn2i(s, a, gen_gvec_ssra); +} + +static bool trans_USRA(DisasContext *s, arg_rri_esz *a) +{ + return do_sve2_fn2i(s, a, gen_gvec_usra); +} + +static bool trans_SRSRA(DisasContext *s, arg_rri_esz *a) +{ + return do_sve2_fn2i(s, a, gen_gvec_srsra); +} + +static bool trans_URSRA(DisasContext *s, arg_rri_esz *a) +{ + return do_sve2_fn2i(s, a, gen_gvec_ursra); +} -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-23-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/sve.decode | 5 +++++ target/arm/translate-sve.c | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SSRA 01000101 .. 0 ..... 1110 00 ..... ..... @rd_rn_tszimm_shr USRA 01000101 .. 0 ..... 1110 01 ..... ..... @rd_rn_tszimm_shr SRSRA 01000101 .. 0 ..... 1110 10 ..... ..... @rd_rn_tszimm_shr URSRA 01000101 .. 0 ..... 1110 11 ..... ..... @rd_rn_tszimm_shr + +## SVE2 bitwise shift and insert + +SRI 01000101 .. 0 ..... 11110 0 ..... ..... @rd_rn_tszimm_shr +SLI 01000101 .. 0 ..... 11110 1 ..... ..... @rd_rn_tszimm_shl diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_URSRA(DisasContext *s, arg_rri_esz *a) { return do_sve2_fn2i(s, a, gen_gvec_ursra); } + +static bool trans_SRI(DisasContext *s, arg_rri_esz *a) +{ + return do_sve2_fn2i(s, a, gen_gvec_sri); +} + +static bool trans_SLI(DisasContext *s, arg_rri_esz *a) +{ + return do_sve2_fn2i(s, a, gen_gvec_sli); +} -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-24-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/sve.decode | 6 ++++++ target/arm/translate-sve.c | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ URSRA 01000101 .. 0 ..... 1110 11 ..... ..... @rd_rn_tszimm_shr SRI 01000101 .. 0 ..... 11110 0 ..... ..... @rd_rn_tszimm_shr SLI 01000101 .. 0 ..... 11110 1 ..... ..... @rd_rn_tszimm_shl + +## SVE2 integer absolute difference and accumulate + +# TODO: Use @rda and %reg_movprfx here. +SABA 01000101 .. 0 ..... 11111 0 ..... ..... @rd_rn_rm +UABA 01000101 .. 0 ..... 11111 1 ..... ..... @rd_rn_rm diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_SLI(DisasContext *s, arg_rri_esz *a) { return do_sve2_fn2i(s, a, gen_gvec_sli); } + +static bool do_sve2_fn_zzz(DisasContext *s, arg_rrr_esz *a, GVecGen3Fn *fn) +{ + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (sve_access_check(s)) { + gen_gvec_fn_zzz(s, fn, a->esz, a->rd, a->rn, a->rm); + } + return true; +} + +static bool trans_SABA(DisasContext *s, arg_rrr_esz *a) +{ + return do_sve2_fn_zzz(s, a, gen_gvec_saba); +} + +static bool trans_UABA(DisasContext *s, arg_rrr_esz *a) +{ + return do_sve2_fn_zzz(s, a, gen_gvec_uaba); +} -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-25-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 24 ++++ target/arm/sve.decode | 12 ++ target/arm/sve_helper.c | 56 +++++++++ target/arm/translate-sve.c | 238 +++++++++++++++++++++++++++++++++++++ 4 files changed, 330 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_uabal_d, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_5(sve2_adcl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve2_adcl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_sqxtnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqxtnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqxtnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_uqxtnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_uqxtnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_uqxtnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_sqxtunb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqxtunb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqxtunb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_sqxtnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqxtnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqxtnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_uqxtnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_uqxtnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_uqxtnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_sqxtunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqxtunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqxtunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SLI 01000101 .. 0 ..... 11110 1 ..... ..... @rd_rn_tszimm_shl # TODO: Use @rda and %reg_movprfx here. SABA 01000101 .. 0 ..... 11111 0 ..... ..... @rd_rn_rm UABA 01000101 .. 0 ..... 11111 1 ..... ..... @rd_rn_rm + +#### SVE2 Narrowing + +## SVE2 saturating extract narrow + +# Bits 23, 18-16 are zero, limited in the translator via esz < 3 & imm == 0. +SQXTNB 01000101 .. 1 ..... 010 000 ..... ..... @rd_rn_tszimm_shl +SQXTNT 01000101 .. 1 ..... 010 001 ..... ..... @rd_rn_tszimm_shl +UQXTNB 01000101 .. 1 ..... 010 010 ..... ..... @rd_rn_tszimm_shl +UQXTNT 01000101 .. 1 ..... 010 011 ..... ..... @rd_rn_tszimm_shl +SQXTUNB 01000101 .. 1 ..... 010 100 ..... ..... @rd_rn_tszimm_shl +SQXTUNT 01000101 .. 1 ..... 010 101 ..... ..... @rd_rn_tszimm_shl diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZZZW_ACC(sve2_uabal_d, uint64_t, uint32_t, , H1_4, DO_ABD) #undef DO_ZZZW_ACC +#define DO_XTNB(NAME, TYPE, OP) \ +void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc); \ + for (i = 0; i < opr_sz; i += sizeof(TYPE)) { \ + TYPE nn = *(TYPE *)(vn + i); \ + nn = OP(nn) & MAKE_64BIT_MASK(0, sizeof(TYPE) * 4); \ + *(TYPE *)(vd + i) = nn; \ + } \ +} + +#define DO_XTNT(NAME, TYPE, TYPEN, H, OP) \ +void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc), odd = H(sizeof(TYPEN)); \ + for (i = 0; i < opr_sz; i += sizeof(TYPE)) { \ + TYPE nn = *(TYPE *)(vn + i); \ + *(TYPEN *)(vd + i + odd) = OP(nn); \ + } \ +} + +#define DO_SQXTN_H(n) do_sat_bhs(n, INT8_MIN, INT8_MAX) +#define DO_SQXTN_S(n) do_sat_bhs(n, INT16_MIN, INT16_MAX) +#define DO_SQXTN_D(n) do_sat_bhs(n, INT32_MIN, INT32_MAX) + +DO_XTNB(sve2_sqxtnb_h, int16_t, DO_SQXTN_H) +DO_XTNB(sve2_sqxtnb_s, int32_t, DO_SQXTN_S) +DO_XTNB(sve2_sqxtnb_d, int64_t, DO_SQXTN_D) + +DO_XTNT(sve2_sqxtnt_h, int16_t, int8_t, H1, DO_SQXTN_H) +DO_XTNT(sve2_sqxtnt_s, int32_t, int16_t, H1_2, DO_SQXTN_S) +DO_XTNT(sve2_sqxtnt_d, int64_t, int32_t, H1_4, DO_SQXTN_D) + +#define DO_UQXTN_H(n) do_sat_bhs(n, 0, UINT8_MAX) +#define DO_UQXTN_S(n) do_sat_bhs(n, 0, UINT16_MAX) +#define DO_UQXTN_D(n) do_sat_bhs(n, 0, UINT32_MAX) + +DO_XTNB(sve2_uqxtnb_h, uint16_t, DO_UQXTN_H) +DO_XTNB(sve2_uqxtnb_s, uint32_t, DO_UQXTN_S) +DO_XTNB(sve2_uqxtnb_d, uint64_t, DO_UQXTN_D) + +DO_XTNT(sve2_uqxtnt_h, uint16_t, uint8_t, H1, DO_UQXTN_H) +DO_XTNT(sve2_uqxtnt_s, uint32_t, uint16_t, H1_2, DO_UQXTN_S) +DO_XTNT(sve2_uqxtnt_d, uint64_t, uint32_t, H1_4, DO_UQXTN_D) + +DO_XTNB(sve2_sqxtunb_h, int16_t, DO_UQXTN_H) +DO_XTNB(sve2_sqxtunb_s, int32_t, DO_UQXTN_S) +DO_XTNB(sve2_sqxtunb_d, int64_t, DO_UQXTN_D) + +DO_XTNT(sve2_sqxtunt_h, int16_t, int8_t, H1, DO_UQXTN_H) +DO_XTNT(sve2_sqxtunt_s, int32_t, int16_t, H1_2, DO_UQXTN_S) +DO_XTNT(sve2_sqxtunt_d, int64_t, int32_t, H1_4, DO_UQXTN_D) + +#undef DO_XTNB +#undef DO_XTNT + void HELPER(sve2_adcl_s)(void *vd, void *vn, void *vm, void *va, uint32_t desc) { intptr_t i, opr_sz = simd_oprsz(desc); diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_UABA(DisasContext *s, arg_rrr_esz *a) { return do_sve2_fn_zzz(s, a, gen_gvec_uaba); } + +static bool do_sve2_narrow_extract(DisasContext *s, arg_rri_esz *a, + const GVecGen2 ops[3]) +{ + if (a->esz < 0 || a->esz > MO_32 || a->imm != 0 || + !dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (sve_access_check(s)) { + unsigned vsz = vec_full_reg_size(s); + tcg_gen_gvec_2(vec_full_reg_offset(s, a->rd), + vec_full_reg_offset(s, a->rn), + vsz, vsz, &ops[a->esz]); + } + return true; +} + +static const TCGOpcode sqxtn_list[] = { + INDEX_op_shli_vec, INDEX_op_smin_vec, INDEX_op_smax_vec, 0 +}; + +static void gen_sqxtnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n) +{ + TCGv_vec t = tcg_temp_new_vec_matching(d); + int halfbits = 4 << vece; + int64_t mask = (1ull << halfbits) - 1; + int64_t min = -1ull << (halfbits - 1); + int64_t max = -min - 1; + + tcg_gen_dupi_vec(vece, t, min); + tcg_gen_smax_vec(vece, d, n, t); + tcg_gen_dupi_vec(vece, t, max); + tcg_gen_smin_vec(vece, d, d, t); + tcg_gen_dupi_vec(vece, t, mask); + tcg_gen_and_vec(vece, d, d, t); + tcg_temp_free_vec(t); +} + +static bool trans_SQXTNB(DisasContext *s, arg_rri_esz *a) +{ + static const GVecGen2 ops[3] = { + { .fniv = gen_sqxtnb_vec, + .opt_opc = sqxtn_list, + .fno = gen_helper_sve2_sqxtnb_h, + .vece = MO_16 }, + { .fniv = gen_sqxtnb_vec, + .opt_opc = sqxtn_list, + .fno = gen_helper_sve2_sqxtnb_s, + .vece = MO_32 }, + { .fniv = gen_sqxtnb_vec, + .opt_opc = sqxtn_list, + .fno = gen_helper_sve2_sqxtnb_d, + .vece = MO_64 }, + }; + return do_sve2_narrow_extract(s, a, ops); +} + +static void gen_sqxtnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n) +{ + TCGv_vec t = tcg_temp_new_vec_matching(d); + int halfbits = 4 << vece; + int64_t mask = (1ull << halfbits) - 1; + int64_t min = -1ull << (halfbits - 1); + int64_t max = -min - 1; + + tcg_gen_dupi_vec(vece, t, min); + tcg_gen_smax_vec(vece, n, n, t); + tcg_gen_dupi_vec(vece, t, max); + tcg_gen_smin_vec(vece, n, n, t); + tcg_gen_shli_vec(vece, n, n, halfbits); + tcg_gen_dupi_vec(vece, t, mask); + tcg_gen_bitsel_vec(vece, d, t, d, n); + tcg_temp_free_vec(t); +} + +static bool trans_SQXTNT(DisasContext *s, arg_rri_esz *a) +{ + static const GVecGen2 ops[3] = { + { .fniv = gen_sqxtnt_vec, + .opt_opc = sqxtn_list, + .load_dest = true, + .fno = gen_helper_sve2_sqxtnt_h, + .vece = MO_16 }, + { .fniv = gen_sqxtnt_vec, + .opt_opc = sqxtn_list, + .load_dest = true, + .fno = gen_helper_sve2_sqxtnt_s, + .vece = MO_32 }, + { .fniv = gen_sqxtnt_vec, + .opt_opc = sqxtn_list, + .load_dest = true, + .fno = gen_helper_sve2_sqxtnt_d, + .vece = MO_64 }, + }; + return do_sve2_narrow_extract(s, a, ops); +} + +static const TCGOpcode uqxtn_list[] = { + INDEX_op_shli_vec, INDEX_op_umin_vec, 0 +}; + +static void gen_uqxtnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n) +{ + TCGv_vec t = tcg_temp_new_vec_matching(d); + int halfbits = 4 << vece; + int64_t max = (1ull << halfbits) - 1; + + tcg_gen_dupi_vec(vece, t, max); + tcg_gen_umin_vec(vece, d, n, t); + tcg_temp_free_vec(t); +} + +static bool trans_UQXTNB(DisasContext *s, arg_rri_esz *a) +{ + static const GVecGen2 ops[3] = { + { .fniv = gen_uqxtnb_vec, + .opt_opc = uqxtn_list, + .fno = gen_helper_sve2_uqxtnb_h, + .vece = MO_16 }, + { .fniv = gen_uqxtnb_vec, + .opt_opc = uqxtn_list, + .fno = gen_helper_sve2_uqxtnb_s, + .vece = MO_32 }, + { .fniv = gen_uqxtnb_vec, + .opt_opc = uqxtn_list, + .fno = gen_helper_sve2_uqxtnb_d, + .vece = MO_64 }, + }; + return do_sve2_narrow_extract(s, a, ops); +} + +static void gen_uqxtnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n) +{ + TCGv_vec t = tcg_temp_new_vec_matching(d); + int halfbits = 4 << vece; + int64_t max = (1ull << halfbits) - 1; + + tcg_gen_dupi_vec(vece, t, max); + tcg_gen_umin_vec(vece, n, n, t); + tcg_gen_shli_vec(vece, n, n, halfbits); + tcg_gen_bitsel_vec(vece, d, t, d, n); + tcg_temp_free_vec(t); +} + +static bool trans_UQXTNT(DisasContext *s, arg_rri_esz *a) +{ + static const GVecGen2 ops[3] = { + { .fniv = gen_uqxtnt_vec, + .opt_opc = uqxtn_list, + .load_dest = true, + .fno = gen_helper_sve2_uqxtnt_h, + .vece = MO_16 }, + { .fniv = gen_uqxtnt_vec, + .opt_opc = uqxtn_list, + .load_dest = true, + .fno = gen_helper_sve2_uqxtnt_s, + .vece = MO_32 }, + { .fniv = gen_uqxtnt_vec, + .opt_opc = uqxtn_list, + .load_dest = true, + .fno = gen_helper_sve2_uqxtnt_d, + .vece = MO_64 }, + }; + return do_sve2_narrow_extract(s, a, ops); +} + +static const TCGOpcode sqxtun_list[] = { + INDEX_op_shli_vec, INDEX_op_umin_vec, INDEX_op_smax_vec, 0 +}; + +static void gen_sqxtunb_vec(unsigned vece, TCGv_vec d, TCGv_vec n) +{ + TCGv_vec t = tcg_temp_new_vec_matching(d); + int halfbits = 4 << vece; + int64_t max = (1ull << halfbits) - 1; + + tcg_gen_dupi_vec(vece, t, 0); + tcg_gen_smax_vec(vece, d, n, t); + tcg_gen_dupi_vec(vece, t, max); + tcg_gen_umin_vec(vece, d, d, t); + tcg_temp_free_vec(t); +} + +static bool trans_SQXTUNB(DisasContext *s, arg_rri_esz *a) +{ + static const GVecGen2 ops[3] = { + { .fniv = gen_sqxtunb_vec, + .opt_opc = sqxtun_list, + .fno = gen_helper_sve2_sqxtunb_h, + .vece = MO_16 }, + { .fniv = gen_sqxtunb_vec, + .opt_opc = sqxtun_list, + .fno = gen_helper_sve2_sqxtunb_s, + .vece = MO_32 }, + { .fniv = gen_sqxtunb_vec, + .opt_opc = sqxtun_list, + .fno = gen_helper_sve2_sqxtunb_d, + .vece = MO_64 }, + }; + return do_sve2_narrow_extract(s, a, ops); +} + +static void gen_sqxtunt_vec(unsigned vece, TCGv_vec d, TCGv_vec n) +{ + TCGv_vec t = tcg_temp_new_vec_matching(d); + int halfbits = 4 << vece; + int64_t max = (1ull << halfbits) - 1; + + tcg_gen_dupi_vec(vece, t, 0); + tcg_gen_smax_vec(vece, n, n, t); + tcg_gen_dupi_vec(vece, t, max); + tcg_gen_umin_vec(vece, n, n, t); + tcg_gen_shli_vec(vece, n, n, halfbits); + tcg_gen_bitsel_vec(vece, d, t, d, n); + tcg_temp_free_vec(t); +} + +static bool trans_SQXTUNT(DisasContext *s, arg_rri_esz *a) +{ + static const GVecGen2 ops[3] = { + { .fniv = gen_sqxtunt_vec, + .opt_opc = sqxtun_list, + .load_dest = true, + .fno = gen_helper_sve2_sqxtunt_h, + .vece = MO_16 }, + { .fniv = gen_sqxtunt_vec, + .opt_opc = sqxtun_list, + .load_dest = true, + .fno = gen_helper_sve2_sqxtunt_s, + .vece = MO_32 }, + { .fniv = gen_sqxtunt_vec, + .opt_opc = sqxtun_list, + .load_dest = true, + .fno = gen_helper_sve2_sqxtunt_d, + .vece = MO_64 }, + }; + return do_sve2_narrow_extract(s, a, ops); +} -- 2.20.1 From: Stephen Long <steplong@quicinc.com> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Stephen Long <steplong@quicinc.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-26-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 35 +++++++++++++++++++++++++++++ target/arm/sve.decode | 8 +++++++ target/arm/sve_helper.c | 46 ++++++++++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 25 +++++++++++++++++++++ 4 files changed, 114 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_3(sve2_uqxtnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_sqxtunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_sqxtunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_sqxtunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_6(sve2_fmaxnmp_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(sve2_fmaxnmp_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(sve2_fmaxnmp_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_6(sve2_fminnmp_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(sve2_fminnmp_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(sve2_fminnmp_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_6(sve2_fmaxp_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(sve2_fmaxp_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(sve2_fmaxp_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_6(sve2_fminp_zpzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(sve2_fminp_zpzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(sve2_fminp_zpzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ UQXTNB 01000101 .. 1 ..... 010 010 ..... ..... @rd_rn_tszimm_shl UQXTNT 01000101 .. 1 ..... 010 011 ..... ..... @rd_rn_tszimm_shl SQXTUNB 01000101 .. 1 ..... 010 100 ..... ..... @rd_rn_tszimm_shl SQXTUNT 01000101 .. 1 ..... 010 101 ..... ..... @rd_rn_tszimm_shl + +## SVE2 floating-point pairwise operations + +FADDP 01100100 .. 010 00 0 100 ... ..... ..... @rdn_pg_rm +FMAXNMP 01100100 .. 010 10 0 100 ... ..... ..... @rdn_pg_rm +FMINNMP 01100100 .. 010 10 1 100 ... ..... ..... @rdn_pg_rm +FMAXP 01100100 .. 010 11 0 100 ... ..... ..... @rdn_pg_rm +FMINP 01100100 .. 010 11 1 100 ... ..... ..... @rdn_pg_rm diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZPZZ_PAIR_D(sve2_sminp_zpzz_d, int64_t, DO_MIN) #undef DO_ZPZZ_PAIR #undef DO_ZPZZ_PAIR_D +#define DO_ZPZZ_PAIR_FP(NAME, TYPE, H, OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, \ + void *status, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc); \ + for (i = 0; i < opr_sz; ) { \ + uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \ + do { \ + TYPE n0 = *(TYPE *)(vn + H(i)); \ + TYPE m0 = *(TYPE *)(vm + H(i)); \ + TYPE n1 = *(TYPE *)(vn + H(i + sizeof(TYPE))); \ + TYPE m1 = *(TYPE *)(vm + H(i + sizeof(TYPE))); \ + if (pg & 1) { \ + *(TYPE *)(vd + H(i)) = OP(n0, n1, status); \ + } \ + i += sizeof(TYPE), pg >>= sizeof(TYPE); \ + if (pg & 1) { \ + *(TYPE *)(vd + H(i)) = OP(m0, m1, status); \ + } \ + i += sizeof(TYPE), pg >>= sizeof(TYPE); \ + } while (i & 15); \ + } \ +} + +DO_ZPZZ_PAIR_FP(sve2_faddp_zpzz_h, float16, H1_2, float16_add) +DO_ZPZZ_PAIR_FP(sve2_faddp_zpzz_s, float32, H1_4, float32_add) +DO_ZPZZ_PAIR_FP(sve2_faddp_zpzz_d, float64, , float64_add) + +DO_ZPZZ_PAIR_FP(sve2_fmaxnmp_zpzz_h, float16, H1_2, float16_maxnum) +DO_ZPZZ_PAIR_FP(sve2_fmaxnmp_zpzz_s, float32, H1_4, float32_maxnum) +DO_ZPZZ_PAIR_FP(sve2_fmaxnmp_zpzz_d, float64, , float64_maxnum) + +DO_ZPZZ_PAIR_FP(sve2_fminnmp_zpzz_h, float16, H1_2, float16_minnum) +DO_ZPZZ_PAIR_FP(sve2_fminnmp_zpzz_s, float32, H1_4, float32_minnum) +DO_ZPZZ_PAIR_FP(sve2_fminnmp_zpzz_d, float64, , float64_minnum) + +DO_ZPZZ_PAIR_FP(sve2_fmaxp_zpzz_h, float16, H1_2, float16_max) +DO_ZPZZ_PAIR_FP(sve2_fmaxp_zpzz_s, float32, H1_4, float32_max) +DO_ZPZZ_PAIR_FP(sve2_fmaxp_zpzz_d, float64, , float64_max) + +DO_ZPZZ_PAIR_FP(sve2_fminp_zpzz_h, float16, H1_2, float16_min) +DO_ZPZZ_PAIR_FP(sve2_fminp_zpzz_s, float32, H1_4, float32_min) +DO_ZPZZ_PAIR_FP(sve2_fminp_zpzz_d, float64, , float64_min) + +#undef DO_ZPZZ_PAIR_FP + /* Three-operand expander, controlled by a predicate, in which the * third operand is "wide". That is, for D = N op M, the same 64-bit * value of M is used with all of the narrower values of N. diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_SQXTUNT(DisasContext *s, arg_rri_esz *a) }; return do_sve2_narrow_extract(s, a, ops); } + +static bool do_sve2_zpzz_fp(DisasContext *s, arg_rprr_esz *a, + gen_helper_gvec_4_ptr *fn) +{ + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + return do_zpzz_fp(s, a, fn); +} + +#define DO_SVE2_ZPZZ_FP(NAME, name) \ +static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \ +{ \ + static gen_helper_gvec_4_ptr * const fns[4] = { \ + NULL, gen_helper_sve2_##name##_zpzz_h, \ + gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d \ + }; \ + return do_sve2_zpzz_fp(s, a, fns[a->esz]); \ +} + +DO_SVE2_ZPZZ_FP(FADDP, faddp) +DO_SVE2_ZPZZ_FP(FMAXNMP, fmaxnmp) +DO_SVE2_ZPZZ_FP(FMINNMP, fminnmp) +DO_SVE2_ZPZZ_FP(FMAXP, fmaxp) +DO_SVE2_ZPZZ_FP(FMINP, fminp) -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-27-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 16 ++++ target/arm/sve.decode | 8 ++ target/arm/sve_helper.c | 54 ++++++++++++- target/arm/translate-sve.c | 160 +++++++++++++++++++++++++++++++++++++ 4 files changed, 236 insertions(+), 2 deletions(-) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_3(sve2_sqxtunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_sqxtunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_sqxtunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_shrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_shrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_shrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_shrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_shrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_shrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_rshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_rshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_rshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_rshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_rshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_rshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG, diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ UQXTNT 01000101 .. 1 ..... 010 011 ..... ..... @rd_rn_tszimm_shl SQXTUNB 01000101 .. 1 ..... 010 100 ..... ..... @rd_rn_tszimm_shl SQXTUNT 01000101 .. 1 ..... 010 101 ..... ..... @rd_rn_tszimm_shl +## SVE2 bitwise shift right narrow + +# Bit 23 == 0 is handled by esz > 0 in the translator. +SHRNB 01000101 .. 1 ..... 00 0100 ..... ..... @rd_rn_tszimm_shr +SHRNT 01000101 .. 1 ..... 00 0101 ..... ..... @rd_rn_tszimm_shr +RSHRNB 01000101 .. 1 ..... 00 0110 ..... ..... @rd_rn_tszimm_shr +RSHRNT 01000101 .. 1 ..... 00 0111 ..... ..... @rd_rn_tszimm_shr + ## SVE2 floating-point pairwise operations FADDP 01100100 .. 010 00 0 100 ... ..... ..... @rdn_pg_rm diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc) \ when N is negative, add 2**M-1. */ #define DO_ASRD(N, M) ((N + (N < 0 ? ((__typeof(N))1 << M) - 1 : 0)) >> M) +static inline uint64_t do_urshr(uint64_t x, unsigned sh) +{ + if (likely(sh < 64)) { + return (x >> sh) + ((x >> (sh - 1)) & 1); + } else if (sh == 64) { + return x >> 63; + } else { + return 0; + } +} + DO_ZPZI(sve_asr_zpzi_b, int8_t, H1, DO_SHR) DO_ZPZI(sve_asr_zpzi_h, int16_t, H1_2, DO_SHR) DO_ZPZI(sve_asr_zpzi_s, int32_t, H1_4, DO_SHR) @@ -XXX,XX +XXX,XX @@ DO_ZPZI(sve_asrd_h, int16_t, H1_2, DO_ASRD) DO_ZPZI(sve_asrd_s, int32_t, H1_4, DO_ASRD) DO_ZPZI_D(sve_asrd_d, int64_t, DO_ASRD) -#undef DO_SHR -#undef DO_SHL #undef DO_ASRD #undef DO_ZPZI #undef DO_ZPZI_D +#define DO_SHRNB(NAME, TYPEW, TYPEN, OP) \ +void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc); \ + int shift = simd_data(desc); \ + for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \ + TYPEW nn = *(TYPEW *)(vn + i); \ + *(TYPEW *)(vd + i) = (TYPEN)OP(nn, shift); \ + } \ +} + +#define DO_SHRNT(NAME, TYPEW, TYPEN, HW, HN, OP) \ +void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc); \ + int shift = simd_data(desc); \ + for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \ + TYPEW nn = *(TYPEW *)(vn + HW(i)); \ + *(TYPEN *)(vd + HN(i + sizeof(TYPEN))) = OP(nn, shift); \ + } \ +} + +DO_SHRNB(sve2_shrnb_h, uint16_t, uint8_t, DO_SHR) +DO_SHRNB(sve2_shrnb_s, uint32_t, uint16_t, DO_SHR) +DO_SHRNB(sve2_shrnb_d, uint64_t, uint32_t, DO_SHR) + +DO_SHRNT(sve2_shrnt_h, uint16_t, uint8_t, H1_2, H1, DO_SHR) +DO_SHRNT(sve2_shrnt_s, uint32_t, uint16_t, H1_4, H1_2, DO_SHR) +DO_SHRNT(sve2_shrnt_d, uint64_t, uint32_t, , H1_4, DO_SHR) + +DO_SHRNB(sve2_rshrnb_h, uint16_t, uint8_t, do_urshr) +DO_SHRNB(sve2_rshrnb_s, uint32_t, uint16_t, do_urshr) +DO_SHRNB(sve2_rshrnb_d, uint64_t, uint32_t, do_urshr) + +DO_SHRNT(sve2_rshrnt_h, uint16_t, uint8_t, H1_2, H1, do_urshr) +DO_SHRNT(sve2_rshrnt_s, uint32_t, uint16_t, H1_4, H1_2, do_urshr) +DO_SHRNT(sve2_rshrnt_d, uint64_t, uint32_t, , H1_4, do_urshr) + +#undef DO_SHRNB +#undef DO_SHRNT + /* Fully general four-operand expander, controlled by a predicate. */ #define DO_ZPZZZ(NAME, TYPE, H, OP) \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_SQXTUNT(DisasContext *s, arg_rri_esz *a) return do_sve2_narrow_extract(s, a, ops); } +static bool do_sve2_shr_narrow(DisasContext *s, arg_rri_esz *a, + const GVecGen2i ops[3]) +{ + if (a->esz < 0 || a->esz > MO_32 || !dc_isar_feature(aa64_sve2, s)) { + return false; + } + assert(a->imm > 0 && a->imm <= (8 << a->esz)); + if (sve_access_check(s)) { + unsigned vsz = vec_full_reg_size(s); + tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd), + vec_full_reg_offset(s, a->rn), + vsz, vsz, a->imm, &ops[a->esz]); + } + return true; +} + +static void gen_shrnb_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int shr) +{ + int halfbits = 4 << vece; + uint64_t mask = dup_const(vece, MAKE_64BIT_MASK(0, halfbits)); + + tcg_gen_shri_i64(d, n, shr); + tcg_gen_andi_i64(d, d, mask); +} + +static void gen_shrnb16_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) +{ + gen_shrnb_i64(MO_16, d, n, shr); +} + +static void gen_shrnb32_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) +{ + gen_shrnb_i64(MO_32, d, n, shr); +} + +static void gen_shrnb64_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) +{ + gen_shrnb_i64(MO_64, d, n, shr); +} + +static void gen_shrnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t shr) +{ + TCGv_vec t = tcg_temp_new_vec_matching(d); + int halfbits = 4 << vece; + uint64_t mask = MAKE_64BIT_MASK(0, halfbits); + + tcg_gen_shri_vec(vece, n, n, shr); + tcg_gen_dupi_vec(vece, t, mask); + tcg_gen_and_vec(vece, d, n, t); + tcg_temp_free_vec(t); +} + +static bool trans_SHRNB(DisasContext *s, arg_rri_esz *a) +{ + static const TCGOpcode vec_list[] = { INDEX_op_shri_vec, 0 }; + static const GVecGen2i ops[3] = { + { .fni8 = gen_shrnb16_i64, + .fniv = gen_shrnb_vec, + .opt_opc = vec_list, + .fno = gen_helper_sve2_shrnb_h, + .vece = MO_16 }, + { .fni8 = gen_shrnb32_i64, + .fniv = gen_shrnb_vec, + .opt_opc = vec_list, + .fno = gen_helper_sve2_shrnb_s, + .vece = MO_32 }, + { .fni8 = gen_shrnb64_i64, + .fniv = gen_shrnb_vec, + .opt_opc = vec_list, + .fno = gen_helper_sve2_shrnb_d, + .vece = MO_64 }, + }; + return do_sve2_shr_narrow(s, a, ops); +} + +static void gen_shrnt_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int shr) +{ + int halfbits = 4 << vece; + uint64_t mask = dup_const(vece, MAKE_64BIT_MASK(0, halfbits)); + + tcg_gen_shli_i64(n, n, halfbits - shr); + tcg_gen_andi_i64(n, n, ~mask); + tcg_gen_andi_i64(d, d, mask); + tcg_gen_or_i64(d, d, n); +} + +static void gen_shrnt16_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) +{ + gen_shrnt_i64(MO_16, d, n, shr); +} + +static void gen_shrnt32_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) +{ + gen_shrnt_i64(MO_32, d, n, shr); +} + +static void gen_shrnt64_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) +{ + tcg_gen_shri_i64(n, n, shr); + tcg_gen_deposit_i64(d, d, n, 32, 32); +} + +static void gen_shrnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t shr) +{ + TCGv_vec t = tcg_temp_new_vec_matching(d); + int halfbits = 4 << vece; + uint64_t mask = MAKE_64BIT_MASK(0, halfbits); + + tcg_gen_shli_vec(vece, n, n, halfbits - shr); + tcg_gen_dupi_vec(vece, t, mask); + tcg_gen_bitsel_vec(vece, d, t, d, n); + tcg_temp_free_vec(t); +} + +static bool trans_SHRNT(DisasContext *s, arg_rri_esz *a) +{ + static const TCGOpcode vec_list[] = { INDEX_op_shli_vec, 0 }; + static const GVecGen2i ops[3] = { + { .fni8 = gen_shrnt16_i64, + .fniv = gen_shrnt_vec, + .opt_opc = vec_list, + .load_dest = true, + .fno = gen_helper_sve2_shrnt_h, + .vece = MO_16 }, + { .fni8 = gen_shrnt32_i64, + .fniv = gen_shrnt_vec, + .opt_opc = vec_list, + .load_dest = true, + .fno = gen_helper_sve2_shrnt_s, + .vece = MO_32 }, + { .fni8 = gen_shrnt64_i64, + .fniv = gen_shrnt_vec, + .opt_opc = vec_list, + .load_dest = true, + .fno = gen_helper_sve2_shrnt_d, + .vece = MO_64 }, + }; + return do_sve2_shr_narrow(s, a, ops); +} + +static bool trans_RSHRNB(DisasContext *s, arg_rri_esz *a) +{ + static const GVecGen2i ops[3] = { + { .fno = gen_helper_sve2_rshrnb_h }, + { .fno = gen_helper_sve2_rshrnb_s }, + { .fno = gen_helper_sve2_rshrnb_d }, + }; + return do_sve2_shr_narrow(s, a, ops); +} + +static bool trans_RSHRNT(DisasContext *s, arg_rri_esz *a) +{ + static const GVecGen2i ops[3] = { + { .fno = gen_helper_sve2_rshrnt_h }, + { .fno = gen_helper_sve2_rshrnt_s }, + { .fno = gen_helper_sve2_rshrnt_d }, + }; + return do_sve2_shr_narrow(s, a, ops); +} + static bool do_sve2_zpzz_fp(DisasContext *s, arg_rprr_esz *a, gen_helper_gvec_4_ptr *fn) { -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-28-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 16 +++++++ target/arm/sve.decode | 4 ++ target/arm/sve_helper.c | 35 ++++++++++++++ target/arm/translate-sve.c | 98 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 153 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_3(sve2_rshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_rshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_rshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqshrunb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqshrunb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqshrunb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_sqshrunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqshrunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqshrunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_sqrshrunb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqrshrunb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqrshrunb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_sqrshrunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqrshrunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqrshrunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG, diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SQXTUNT 01000101 .. 1 ..... 010 101 ..... ..... @rd_rn_tszimm_shl ## SVE2 bitwise shift right narrow # Bit 23 == 0 is handled by esz > 0 in the translator. +SQSHRUNB 01000101 .. 1 ..... 00 0000 ..... ..... @rd_rn_tszimm_shr +SQSHRUNT 01000101 .. 1 ..... 00 0001 ..... ..... @rd_rn_tszimm_shr +SQRSHRUNB 01000101 .. 1 ..... 00 0010 ..... ..... @rd_rn_tszimm_shr +SQRSHRUNT 01000101 .. 1 ..... 00 0011 ..... ..... @rd_rn_tszimm_shr SHRNB 01000101 .. 1 ..... 00 0100 ..... ..... @rd_rn_tszimm_shr SHRNT 01000101 .. 1 ..... 00 0101 ..... ..... @rd_rn_tszimm_shr RSHRNB 01000101 .. 1 ..... 00 0110 ..... ..... @rd_rn_tszimm_shr diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ static inline uint64_t do_urshr(uint64_t x, unsigned sh) } } +static inline int64_t do_srshr(int64_t x, unsigned sh) +{ + if (likely(sh < 64)) { + return (x >> sh) + ((x >> (sh - 1)) & 1); + } else { + /* Rounding the sign bit always produces 0. */ + return 0; + } +} + DO_ZPZI(sve_asr_zpzi_b, int8_t, H1, DO_SHR) DO_ZPZI(sve_asr_zpzi_h, int16_t, H1_2, DO_SHR) DO_ZPZI(sve_asr_zpzi_s, int32_t, H1_4, DO_SHR) @@ -XXX,XX +XXX,XX @@ DO_SHRNT(sve2_rshrnt_h, uint16_t, uint8_t, H1_2, H1, do_urshr) DO_SHRNT(sve2_rshrnt_s, uint32_t, uint16_t, H1_4, H1_2, do_urshr) DO_SHRNT(sve2_rshrnt_d, uint64_t, uint32_t, , H1_4, do_urshr) +#define DO_SQSHRUN_H(x, sh) do_sat_bhs((int64_t)(x) >> sh, 0, UINT8_MAX) +#define DO_SQSHRUN_S(x, sh) do_sat_bhs((int64_t)(x) >> sh, 0, UINT16_MAX) +#define DO_SQSHRUN_D(x, sh) \ + do_sat_bhs((int64_t)(x) >> (sh < 64 ? sh : 63), 0, UINT32_MAX) + +DO_SHRNB(sve2_sqshrunb_h, int16_t, uint8_t, DO_SQSHRUN_H) +DO_SHRNB(sve2_sqshrunb_s, int32_t, uint16_t, DO_SQSHRUN_S) +DO_SHRNB(sve2_sqshrunb_d, int64_t, uint32_t, DO_SQSHRUN_D) + +DO_SHRNT(sve2_sqshrunt_h, int16_t, uint8_t, H1_2, H1, DO_SQSHRUN_H) +DO_SHRNT(sve2_sqshrunt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQSHRUN_S) +DO_SHRNT(sve2_sqshrunt_d, int64_t, uint32_t, , H1_4, DO_SQSHRUN_D) + +#define DO_SQRSHRUN_H(x, sh) do_sat_bhs(do_srshr(x, sh), 0, UINT8_MAX) +#define DO_SQRSHRUN_S(x, sh) do_sat_bhs(do_srshr(x, sh), 0, UINT16_MAX) +#define DO_SQRSHRUN_D(x, sh) do_sat_bhs(do_srshr(x, sh), 0, UINT32_MAX) + +DO_SHRNB(sve2_sqrshrunb_h, int16_t, uint8_t, DO_SQRSHRUN_H) +DO_SHRNB(sve2_sqrshrunb_s, int32_t, uint16_t, DO_SQRSHRUN_S) +DO_SHRNB(sve2_sqrshrunb_d, int64_t, uint32_t, DO_SQRSHRUN_D) + +DO_SHRNT(sve2_sqrshrunt_h, int16_t, uint8_t, H1_2, H1, DO_SQRSHRUN_H) +DO_SHRNT(sve2_sqrshrunt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQRSHRUN_S) +DO_SHRNT(sve2_sqrshrunt_d, int64_t, uint32_t, , H1_4, DO_SQRSHRUN_D) + #undef DO_SHRNB #undef DO_SHRNT diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_RSHRNT(DisasContext *s, arg_rri_esz *a) return do_sve2_shr_narrow(s, a, ops); } +static void gen_sqshrunb_vec(unsigned vece, TCGv_vec d, + TCGv_vec n, int64_t shr) +{ + TCGv_vec t = tcg_temp_new_vec_matching(d); + int halfbits = 4 << vece; + + tcg_gen_sari_vec(vece, n, n, shr); + tcg_gen_dupi_vec(vece, t, 0); + tcg_gen_smax_vec(vece, n, n, t); + tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits)); + tcg_gen_umin_vec(vece, d, n, t); + tcg_temp_free_vec(t); +} + +static bool trans_SQSHRUNB(DisasContext *s, arg_rri_esz *a) +{ + static const TCGOpcode vec_list[] = { + INDEX_op_sari_vec, INDEX_op_smax_vec, INDEX_op_umin_vec, 0 + }; + static const GVecGen2i ops[3] = { + { .fniv = gen_sqshrunb_vec, + .opt_opc = vec_list, + .fno = gen_helper_sve2_sqshrunb_h, + .vece = MO_16 }, + { .fniv = gen_sqshrunb_vec, + .opt_opc = vec_list, + .fno = gen_helper_sve2_sqshrunb_s, + .vece = MO_32 }, + { .fniv = gen_sqshrunb_vec, + .opt_opc = vec_list, + .fno = gen_helper_sve2_sqshrunb_d, + .vece = MO_64 }, + }; + return do_sve2_shr_narrow(s, a, ops); +} + +static void gen_sqshrunt_vec(unsigned vece, TCGv_vec d, + TCGv_vec n, int64_t shr) +{ + TCGv_vec t = tcg_temp_new_vec_matching(d); + int halfbits = 4 << vece; + + tcg_gen_sari_vec(vece, n, n, shr); + tcg_gen_dupi_vec(vece, t, 0); + tcg_gen_smax_vec(vece, n, n, t); + tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits)); + tcg_gen_umin_vec(vece, n, n, t); + tcg_gen_shli_vec(vece, n, n, halfbits); + tcg_gen_bitsel_vec(vece, d, t, d, n); + tcg_temp_free_vec(t); +} + +static bool trans_SQSHRUNT(DisasContext *s, arg_rri_esz *a) +{ + static const TCGOpcode vec_list[] = { + INDEX_op_shli_vec, INDEX_op_sari_vec, + INDEX_op_smax_vec, INDEX_op_umin_vec, 0 + }; + static const GVecGen2i ops[3] = { + { .fniv = gen_sqshrunt_vec, + .opt_opc = vec_list, + .load_dest = true, + .fno = gen_helper_sve2_sqshrunt_h, + .vece = MO_16 }, + { .fniv = gen_sqshrunt_vec, + .opt_opc = vec_list, + .load_dest = true, + .fno = gen_helper_sve2_sqshrunt_s, + .vece = MO_32 }, + { .fniv = gen_sqshrunt_vec, + .opt_opc = vec_list, + .load_dest = true, + .fno = gen_helper_sve2_sqshrunt_d, + .vece = MO_64 }, + }; + return do_sve2_shr_narrow(s, a, ops); +} + +static bool trans_SQRSHRUNB(DisasContext *s, arg_rri_esz *a) +{ + static const GVecGen2i ops[3] = { + { .fno = gen_helper_sve2_sqrshrunb_h }, + { .fno = gen_helper_sve2_sqrshrunb_s }, + { .fno = gen_helper_sve2_sqrshrunb_d }, + }; + return do_sve2_shr_narrow(s, a, ops); +} + +static bool trans_SQRSHRUNT(DisasContext *s, arg_rri_esz *a) +{ + static const GVecGen2i ops[3] = { + { .fno = gen_helper_sve2_sqrshrunt_h }, + { .fno = gen_helper_sve2_sqrshrunt_s }, + { .fno = gen_helper_sve2_sqrshrunt_d }, + }; + return do_sve2_shr_narrow(s, a, ops); +} + static bool do_sve2_zpzz_fp(DisasContext *s, arg_rprr_esz *a, gen_helper_gvec_4_ptr *fn) { -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-29-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 16 +++++++ target/arm/sve.decode | 4 ++ target/arm/sve_helper.c | 24 ++++++++++ target/arm/translate-sve.c | 93 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 137 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_3(sve2_sqrshrunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_sqrshrunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_sqrshrunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_uqshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_uqshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_uqshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_uqshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_uqshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_uqshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_uqrshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_uqrshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_uqrshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_uqrshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_uqrshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_uqrshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG, diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SHRNB 01000101 .. 1 ..... 00 0100 ..... ..... @rd_rn_tszimm_shr SHRNT 01000101 .. 1 ..... 00 0101 ..... ..... @rd_rn_tszimm_shr RSHRNB 01000101 .. 1 ..... 00 0110 ..... ..... @rd_rn_tszimm_shr RSHRNT 01000101 .. 1 ..... 00 0111 ..... ..... @rd_rn_tszimm_shr +UQSHRNB 01000101 .. 1 ..... 00 1100 ..... ..... @rd_rn_tszimm_shr +UQSHRNT 01000101 .. 1 ..... 00 1101 ..... ..... @rd_rn_tszimm_shr +UQRSHRNB 01000101 .. 1 ..... 00 1110 ..... ..... @rd_rn_tszimm_shr +UQRSHRNT 01000101 .. 1 ..... 00 1111 ..... ..... @rd_rn_tszimm_shr ## SVE2 floating-point pairwise operations diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_SHRNT(sve2_sqrshrunt_h, int16_t, uint8_t, H1_2, H1, DO_SQRSHRUN_H) DO_SHRNT(sve2_sqrshrunt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQRSHRUN_S) DO_SHRNT(sve2_sqrshrunt_d, int64_t, uint32_t, , H1_4, DO_SQRSHRUN_D) +#define DO_UQSHRN_H(x, sh) MIN(x >> sh, UINT8_MAX) +#define DO_UQSHRN_S(x, sh) MIN(x >> sh, UINT16_MAX) +#define DO_UQSHRN_D(x, sh) MIN(x >> sh, UINT32_MAX) + +DO_SHRNB(sve2_uqshrnb_h, uint16_t, uint8_t, DO_UQSHRN_H) +DO_SHRNB(sve2_uqshrnb_s, uint32_t, uint16_t, DO_UQSHRN_S) +DO_SHRNB(sve2_uqshrnb_d, uint64_t, uint32_t, DO_UQSHRN_D) + +DO_SHRNT(sve2_uqshrnt_h, uint16_t, uint8_t, H1_2, H1, DO_UQSHRN_H) +DO_SHRNT(sve2_uqshrnt_s, uint32_t, uint16_t, H1_4, H1_2, DO_UQSHRN_S) +DO_SHRNT(sve2_uqshrnt_d, uint64_t, uint32_t, , H1_4, DO_UQSHRN_D) + +#define DO_UQRSHRN_H(x, sh) MIN(do_urshr(x, sh), UINT8_MAX) +#define DO_UQRSHRN_S(x, sh) MIN(do_urshr(x, sh), UINT16_MAX) +#define DO_UQRSHRN_D(x, sh) MIN(do_urshr(x, sh), UINT32_MAX) + +DO_SHRNB(sve2_uqrshrnb_h, uint16_t, uint8_t, DO_UQRSHRN_H) +DO_SHRNB(sve2_uqrshrnb_s, uint32_t, uint16_t, DO_UQRSHRN_S) +DO_SHRNB(sve2_uqrshrnb_d, uint64_t, uint32_t, DO_UQRSHRN_D) + +DO_SHRNT(sve2_uqrshrnt_h, uint16_t, uint8_t, H1_2, H1, DO_UQRSHRN_H) +DO_SHRNT(sve2_uqrshrnt_s, uint32_t, uint16_t, H1_4, H1_2, DO_UQRSHRN_S) +DO_SHRNT(sve2_uqrshrnt_d, uint64_t, uint32_t, , H1_4, DO_UQRSHRN_D) + #undef DO_SHRNB #undef DO_SHRNT diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_SQRSHRUNT(DisasContext *s, arg_rri_esz *a) return do_sve2_shr_narrow(s, a, ops); } +static void gen_uqshrnb_vec(unsigned vece, TCGv_vec d, + TCGv_vec n, int64_t shr) +{ + TCGv_vec t = tcg_temp_new_vec_matching(d); + int halfbits = 4 << vece; + + tcg_gen_shri_vec(vece, n, n, shr); + tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits)); + tcg_gen_umin_vec(vece, d, n, t); + tcg_temp_free_vec(t); +} + +static bool trans_UQSHRNB(DisasContext *s, arg_rri_esz *a) +{ + static const TCGOpcode vec_list[] = { + INDEX_op_shri_vec, INDEX_op_umin_vec, 0 + }; + static const GVecGen2i ops[3] = { + { .fniv = gen_uqshrnb_vec, + .opt_opc = vec_list, + .fno = gen_helper_sve2_uqshrnb_h, + .vece = MO_16 }, + { .fniv = gen_uqshrnb_vec, + .opt_opc = vec_list, + .fno = gen_helper_sve2_uqshrnb_s, + .vece = MO_32 }, + { .fniv = gen_uqshrnb_vec, + .opt_opc = vec_list, + .fno = gen_helper_sve2_uqshrnb_d, + .vece = MO_64 }, + }; + return do_sve2_shr_narrow(s, a, ops); +} + +static void gen_uqshrnt_vec(unsigned vece, TCGv_vec d, + TCGv_vec n, int64_t shr) +{ + TCGv_vec t = tcg_temp_new_vec_matching(d); + int halfbits = 4 << vece; + + tcg_gen_shri_vec(vece, n, n, shr); + tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits)); + tcg_gen_umin_vec(vece, n, n, t); + tcg_gen_shli_vec(vece, n, n, halfbits); + tcg_gen_bitsel_vec(vece, d, t, d, n); + tcg_temp_free_vec(t); +} + +static bool trans_UQSHRNT(DisasContext *s, arg_rri_esz *a) +{ + static const TCGOpcode vec_list[] = { + INDEX_op_shli_vec, INDEX_op_shri_vec, INDEX_op_umin_vec, 0 + }; + static const GVecGen2i ops[3] = { + { .fniv = gen_uqshrnt_vec, + .opt_opc = vec_list, + .load_dest = true, + .fno = gen_helper_sve2_uqshrnt_h, + .vece = MO_16 }, + { .fniv = gen_uqshrnt_vec, + .opt_opc = vec_list, + .load_dest = true, + .fno = gen_helper_sve2_uqshrnt_s, + .vece = MO_32 }, + { .fniv = gen_uqshrnt_vec, + .opt_opc = vec_list, + .load_dest = true, + .fno = gen_helper_sve2_uqshrnt_d, + .vece = MO_64 }, + }; + return do_sve2_shr_narrow(s, a, ops); +} + +static bool trans_UQRSHRNB(DisasContext *s, arg_rri_esz *a) +{ + static const GVecGen2i ops[3] = { + { .fno = gen_helper_sve2_uqrshrnb_h }, + { .fno = gen_helper_sve2_uqrshrnb_s }, + { .fno = gen_helper_sve2_uqrshrnb_d }, + }; + return do_sve2_shr_narrow(s, a, ops); +} + +static bool trans_UQRSHRNT(DisasContext *s, arg_rri_esz *a) +{ + static const GVecGen2i ops[3] = { + { .fno = gen_helper_sve2_uqrshrnt_h }, + { .fno = gen_helper_sve2_uqrshrnt_s }, + { .fno = gen_helper_sve2_uqrshrnt_d }, + }; + return do_sve2_shr_narrow(s, a, ops); +} + static bool do_sve2_zpzz_fp(DisasContext *s, arg_rprr_esz *a, gen_helper_gvec_4_ptr *fn) { -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> This completes the section "SVE2 bitwise shift right narrow". Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-30-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 16 ++++++ target/arm/sve.decode | 4 ++ target/arm/sve_helper.c | 24 +++++++++ target/arm/translate-sve.c | 105 +++++++++++++++++++++++++++++++++++++ 4 files changed, 149 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_3(sve2_sqrshrunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_sqrshrunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_sqrshrunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_sqshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_sqrshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqrshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqrshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(sve2_sqrshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqrshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(sve2_sqrshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + DEF_HELPER_FLAGS_3(sve2_uqshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_uqshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_uqshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SHRNB 01000101 .. 1 ..... 00 0100 ..... ..... @rd_rn_tszimm_shr SHRNT 01000101 .. 1 ..... 00 0101 ..... ..... @rd_rn_tszimm_shr RSHRNB 01000101 .. 1 ..... 00 0110 ..... ..... @rd_rn_tszimm_shr RSHRNT 01000101 .. 1 ..... 00 0111 ..... ..... @rd_rn_tszimm_shr +SQSHRNB 01000101 .. 1 ..... 00 1000 ..... ..... @rd_rn_tszimm_shr +SQSHRNT 01000101 .. 1 ..... 00 1001 ..... ..... @rd_rn_tszimm_shr +SQRSHRNB 01000101 .. 1 ..... 00 1010 ..... ..... @rd_rn_tszimm_shr +SQRSHRNT 01000101 .. 1 ..... 00 1011 ..... ..... @rd_rn_tszimm_shr UQSHRNB 01000101 .. 1 ..... 00 1100 ..... ..... @rd_rn_tszimm_shr UQSHRNT 01000101 .. 1 ..... 00 1101 ..... ..... @rd_rn_tszimm_shr UQRSHRNB 01000101 .. 1 ..... 00 1110 ..... ..... @rd_rn_tszimm_shr diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_SHRNT(sve2_sqrshrunt_h, int16_t, uint8_t, H1_2, H1, DO_SQRSHRUN_H) DO_SHRNT(sve2_sqrshrunt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQRSHRUN_S) DO_SHRNT(sve2_sqrshrunt_d, int64_t, uint32_t, , H1_4, DO_SQRSHRUN_D) +#define DO_SQSHRN_H(x, sh) do_sat_bhs(x >> sh, INT8_MIN, INT8_MAX) +#define DO_SQSHRN_S(x, sh) do_sat_bhs(x >> sh, INT16_MIN, INT16_MAX) +#define DO_SQSHRN_D(x, sh) do_sat_bhs(x >> sh, INT32_MIN, INT32_MAX) + +DO_SHRNB(sve2_sqshrnb_h, int16_t, uint8_t, DO_SQSHRN_H) +DO_SHRNB(sve2_sqshrnb_s, int32_t, uint16_t, DO_SQSHRN_S) +DO_SHRNB(sve2_sqshrnb_d, int64_t, uint32_t, DO_SQSHRN_D) + +DO_SHRNT(sve2_sqshrnt_h, int16_t, uint8_t, H1_2, H1, DO_SQSHRN_H) +DO_SHRNT(sve2_sqshrnt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQSHRN_S) +DO_SHRNT(sve2_sqshrnt_d, int64_t, uint32_t, , H1_4, DO_SQSHRN_D) + +#define DO_SQRSHRN_H(x, sh) do_sat_bhs(do_srshr(x, sh), INT8_MIN, INT8_MAX) +#define DO_SQRSHRN_S(x, sh) do_sat_bhs(do_srshr(x, sh), INT16_MIN, INT16_MAX) +#define DO_SQRSHRN_D(x, sh) do_sat_bhs(do_srshr(x, sh), INT32_MIN, INT32_MAX) + +DO_SHRNB(sve2_sqrshrnb_h, int16_t, uint8_t, DO_SQRSHRN_H) +DO_SHRNB(sve2_sqrshrnb_s, int32_t, uint16_t, DO_SQRSHRN_S) +DO_SHRNB(sve2_sqrshrnb_d, int64_t, uint32_t, DO_SQRSHRN_D) + +DO_SHRNT(sve2_sqrshrnt_h, int16_t, uint8_t, H1_2, H1, DO_SQRSHRN_H) +DO_SHRNT(sve2_sqrshrnt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQRSHRN_S) +DO_SHRNT(sve2_sqrshrnt_d, int64_t, uint32_t, , H1_4, DO_SQRSHRN_D) + #define DO_UQSHRN_H(x, sh) MIN(x >> sh, UINT8_MAX) #define DO_UQSHRN_S(x, sh) MIN(x >> sh, UINT16_MAX) #define DO_UQSHRN_D(x, sh) MIN(x >> sh, UINT32_MAX) diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_SQRSHRUNT(DisasContext *s, arg_rri_esz *a) return do_sve2_shr_narrow(s, a, ops); } +static void gen_sqshrnb_vec(unsigned vece, TCGv_vec d, + TCGv_vec n, int64_t shr) +{ + TCGv_vec t = tcg_temp_new_vec_matching(d); + int halfbits = 4 << vece; + int64_t max = MAKE_64BIT_MASK(0, halfbits - 1); + int64_t min = -max - 1; + + tcg_gen_sari_vec(vece, n, n, shr); + tcg_gen_dupi_vec(vece, t, min); + tcg_gen_smax_vec(vece, n, n, t); + tcg_gen_dupi_vec(vece, t, max); + tcg_gen_smin_vec(vece, n, n, t); + tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits)); + tcg_gen_and_vec(vece, d, n, t); + tcg_temp_free_vec(t); +} + +static bool trans_SQSHRNB(DisasContext *s, arg_rri_esz *a) +{ + static const TCGOpcode vec_list[] = { + INDEX_op_sari_vec, INDEX_op_smax_vec, INDEX_op_smin_vec, 0 + }; + static const GVecGen2i ops[3] = { + { .fniv = gen_sqshrnb_vec, + .opt_opc = vec_list, + .fno = gen_helper_sve2_sqshrnb_h, + .vece = MO_16 }, + { .fniv = gen_sqshrnb_vec, + .opt_opc = vec_list, + .fno = gen_helper_sve2_sqshrnb_s, + .vece = MO_32 }, + { .fniv = gen_sqshrnb_vec, + .opt_opc = vec_list, + .fno = gen_helper_sve2_sqshrnb_d, + .vece = MO_64 }, + }; + return do_sve2_shr_narrow(s, a, ops); +} + +static void gen_sqshrnt_vec(unsigned vece, TCGv_vec d, + TCGv_vec n, int64_t shr) +{ + TCGv_vec t = tcg_temp_new_vec_matching(d); + int halfbits = 4 << vece; + int64_t max = MAKE_64BIT_MASK(0, halfbits - 1); + int64_t min = -max - 1; + + tcg_gen_sari_vec(vece, n, n, shr); + tcg_gen_dupi_vec(vece, t, min); + tcg_gen_smax_vec(vece, n, n, t); + tcg_gen_dupi_vec(vece, t, max); + tcg_gen_smin_vec(vece, n, n, t); + tcg_gen_shli_vec(vece, n, n, halfbits); + tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits)); + tcg_gen_bitsel_vec(vece, d, t, d, n); + tcg_temp_free_vec(t); +} + +static bool trans_SQSHRNT(DisasContext *s, arg_rri_esz *a) +{ + static const TCGOpcode vec_list[] = { + INDEX_op_shli_vec, INDEX_op_sari_vec, + INDEX_op_smax_vec, INDEX_op_smin_vec, 0 + }; + static const GVecGen2i ops[3] = { + { .fniv = gen_sqshrnt_vec, + .opt_opc = vec_list, + .load_dest = true, + .fno = gen_helper_sve2_sqshrnt_h, + .vece = MO_16 }, + { .fniv = gen_sqshrnt_vec, + .opt_opc = vec_list, + .load_dest = true, + .fno = gen_helper_sve2_sqshrnt_s, + .vece = MO_32 }, + { .fniv = gen_sqshrnt_vec, + .opt_opc = vec_list, + .load_dest = true, + .fno = gen_helper_sve2_sqshrnt_d, + .vece = MO_64 }, + }; + return do_sve2_shr_narrow(s, a, ops); +} + +static bool trans_SQRSHRNB(DisasContext *s, arg_rri_esz *a) +{ + static const GVecGen2i ops[3] = { + { .fno = gen_helper_sve2_sqrshrnb_h }, + { .fno = gen_helper_sve2_sqrshrnb_s }, + { .fno = gen_helper_sve2_sqrshrnb_d }, + }; + return do_sve2_shr_narrow(s, a, ops); +} + +static bool trans_SQRSHRNT(DisasContext *s, arg_rri_esz *a) +{ + static const GVecGen2i ops[3] = { + { .fno = gen_helper_sve2_sqrshrnt_h }, + { .fno = gen_helper_sve2_sqrshrnt_s }, + { .fno = gen_helper_sve2_sqrshrnt_d }, + }; + return do_sve2_shr_narrow(s, a, ops); +} + static void gen_uqshrnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t shr) { -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Rename the existing sve_while (less-than) helper to sve_whilel to make room for a new sve_whileg helper for greater-than. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-31-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 3 +- target/arm/sve.decode | 2 +- target/arm/sve_helper.c | 38 +++++++++++++++++++++++++- target/arm/translate-sve.c | 56 ++++++++++++++++++++++++++++---------- 4 files changed, 82 insertions(+), 17 deletions(-) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_4(sve_brkns, TCG_CALL_NO_RWG, i32, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve_cntp, TCG_CALL_NO_RWG, i64, ptr, ptr, i32) -DEF_HELPER_FLAGS_3(sve_while, TCG_CALL_NO_RWG, i32, ptr, i32, i32) +DEF_HELPER_FLAGS_3(sve_whilel, TCG_CALL_NO_RWG, i32, ptr, i32, i32) +DEF_HELPER_FLAGS_3(sve_whileg, TCG_CALL_NO_RWG, i32, ptr, i32, i32) DEF_HELPER_FLAGS_4(sve_subri_b, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) DEF_HELPER_FLAGS_4(sve_subri_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SINCDECP_z 00100101 .. 1010 d:1 u:1 10000 00 .... ..... @incdec2_pred CTERM 00100101 1 sf:1 1 rm:5 001000 rn:5 ne:1 0000 # SVE integer compare scalar count and limit -WHILE 00100101 esz:2 1 rm:5 000 sf:1 u:1 1 rn:5 eq:1 rd:4 +WHILE 00100101 esz:2 1 rm:5 000 sf:1 u:1 lt:1 rn:5 eq:1 rd:4 ### SVE Integer Wide Immediate - Unpredicated Group diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ uint64_t HELPER(sve_cntp)(void *vn, void *vg, uint32_t pred_desc) return sum; } -uint32_t HELPER(sve_while)(void *vd, uint32_t count, uint32_t pred_desc) +uint32_t HELPER(sve_whilel)(void *vd, uint32_t count, uint32_t pred_desc) { intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ); intptr_t esz = FIELD_EX32(pred_desc, PREDDESC, ESZ); @@ -XXX,XX +XXX,XX @@ uint32_t HELPER(sve_while)(void *vd, uint32_t count, uint32_t pred_desc) return predtest_ones(d, oprsz, esz_mask); } +uint32_t HELPER(sve_whileg)(void *vd, uint32_t count, uint32_t pred_desc) +{ + intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ); + intptr_t esz = FIELD_EX32(pred_desc, PREDDESC, ESZ); + uint64_t esz_mask = pred_esz_masks[esz]; + ARMPredicateReg *d = vd; + intptr_t i, invcount, oprbits; + uint64_t bits; + + if (count == 0) { + return do_zero(d, oprsz); + } + + oprbits = oprsz * 8; + tcg_debug_assert(count <= oprbits); + + bits = esz_mask; + if (oprbits & 63) { + bits &= MAKE_64BIT_MASK(0, oprbits & 63); + } + + invcount = oprbits - count; + for (i = (oprsz - 1) / 8; i > invcount / 64; --i) { + d->p[i] = bits; + bits = esz_mask; + } + + d->p[i] = bits & MAKE_64BIT_MASK(invcount & 63, 64); + + while (--i >= 0) { + d->p[i] = 0; + } + + return predtest_ones(d, oprsz, esz_mask); +} + /* Recursive reduction on a function; * C.f. the ARM ARM function ReducePredicated. * diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_WHILE(DisasContext *s, arg_WHILE *a) unsigned vsz = vec_full_reg_size(s); unsigned desc = 0; TCGCond cond; + uint64_t maxval; + /* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */ + bool eq = a->eq == a->lt; + /* The greater-than conditions are all SVE2. */ + if (!a->lt && !dc_isar_feature(aa64_sve2, s)) { + return false; + } if (!sve_access_check(s)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool trans_WHILE(DisasContext *s, arg_WHILE *a) */ t0 = tcg_temp_new_i64(); t1 = tcg_temp_new_i64(); - tcg_gen_sub_i64(t0, op1, op0); + + if (a->lt) { + tcg_gen_sub_i64(t0, op1, op0); + if (a->u) { + maxval = a->sf ? UINT64_MAX : UINT32_MAX; + cond = eq ? TCG_COND_LEU : TCG_COND_LTU; + } else { + maxval = a->sf ? INT64_MAX : INT32_MAX; + cond = eq ? TCG_COND_LE : TCG_COND_LT; + } + } else { + tcg_gen_sub_i64(t0, op0, op1); + if (a->u) { + maxval = 0; + cond = eq ? TCG_COND_GEU : TCG_COND_GTU; + } else { + maxval = a->sf ? INT64_MIN : INT32_MIN; + cond = eq ? TCG_COND_GE : TCG_COND_GT; + } + } tmax = tcg_const_i64(vsz >> a->esz); - if (a->eq) { + if (eq) { /* Equality means one more iteration. */ tcg_gen_addi_i64(t0, t0, 1); - /* If op1 is max (un)signed integer (and the only time the addition - * above could overflow), then we produce an all-true predicate by - * setting the count to the vector length. This is because the - * pseudocode is described as an increment + compare loop, and the - * max integer would always compare true. + /* + * For the less-than while, if op1 is maxval (and the only time + * the addition above could overflow), then we produce an all-true + * predicate by setting the count to the vector length. This is + * because the pseudocode is described as an increment + compare + * loop, and the maximum integer would always compare true. + * Similarly, the greater-than while has the same issue with the + * minimum integer due to the decrement + compare loop. */ - tcg_gen_movi_i64(t1, (a->sf - ? (a->u ? UINT64_MAX : INT64_MAX) - : (a->u ? UINT32_MAX : INT32_MAX))); + tcg_gen_movi_i64(t1, maxval); tcg_gen_movcond_i64(TCG_COND_EQ, t0, op1, t1, tmax, t0); } @@ -XXX,XX +XXX,XX @@ static bool trans_WHILE(DisasContext *s, arg_WHILE *a) tcg_temp_free_i64(tmax); /* Set the count to zero if the condition is false. */ - cond = (a->u - ? (a->eq ? TCG_COND_LEU : TCG_COND_LTU) - : (a->eq ? TCG_COND_LE : TCG_COND_LT)); tcg_gen_movi_i64(t1, 0); tcg_gen_movcond_i64(cond, t0, op0, op1, t0, t1); tcg_temp_free_i64(t1); @@ -XXX,XX +XXX,XX @@ static bool trans_WHILE(DisasContext *s, arg_WHILE *a) ptr = tcg_temp_new_ptr(); tcg_gen_addi_ptr(ptr, cpu_env, pred_full_reg_offset(s, a->rd)); - gen_helper_sve_while(t2, ptr, t2, t3); + if (a->lt) { + gen_helper_sve_whilel(t2, ptr, t2, t3); + } else { + gen_helper_sve_whileg(t2, ptr, t2, t3); + } do_pred_flags(t2); tcg_temp_free_ptr(ptr); -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-32-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/sve.decode | 3 ++ target/arm/translate-sve.c | 67 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ CTERM 00100101 1 sf:1 1 rm:5 001000 rn:5 ne:1 0000 # SVE integer compare scalar count and limit WHILE 00100101 esz:2 1 rm:5 000 sf:1 u:1 lt:1 rn:5 eq:1 rd:4 +# SVE2 pointer conflict compare +WHILE_ptr 00100101 esz:2 1 rm:5 001 100 rn:5 rw:1 rd:4 + ### SVE Integer Wide Immediate - Unpredicated Group # SVE broadcast floating-point immediate (unpredicated) diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_WHILE(DisasContext *s, arg_WHILE *a) return true; } +static bool trans_WHILE_ptr(DisasContext *s, arg_WHILE_ptr *a) +{ + TCGv_i64 op0, op1, diff, t1, tmax; + TCGv_i32 t2, t3; + TCGv_ptr ptr; + unsigned vsz = vec_full_reg_size(s); + unsigned desc = 0; + + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (!sve_access_check(s)) { + return true; + } + + op0 = read_cpu_reg(s, a->rn, 1); + op1 = read_cpu_reg(s, a->rm, 1); + + tmax = tcg_const_i64(vsz); + diff = tcg_temp_new_i64(); + + if (a->rw) { + /* WHILERW */ + /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */ + t1 = tcg_temp_new_i64(); + tcg_gen_sub_i64(diff, op0, op1); + tcg_gen_sub_i64(t1, op1, op0); + tcg_gen_movcond_i64(TCG_COND_GEU, diff, op0, op1, diff, t1); + tcg_temp_free_i64(t1); + /* Round down to a multiple of ESIZE. */ + tcg_gen_andi_i64(diff, diff, -1 << a->esz); + /* If op1 == op0, diff == 0, and the condition is always true. */ + tcg_gen_movcond_i64(TCG_COND_EQ, diff, op0, op1, tmax, diff); + } else { + /* WHILEWR */ + tcg_gen_sub_i64(diff, op1, op0); + /* Round down to a multiple of ESIZE. */ + tcg_gen_andi_i64(diff, diff, -1 << a->esz); + /* If op0 >= op1, diff <= 0, the condition is always true. */ + tcg_gen_movcond_i64(TCG_COND_GEU, diff, op0, op1, tmax, diff); + } + + /* Bound to the maximum. */ + tcg_gen_umin_i64(diff, diff, tmax); + tcg_temp_free_i64(tmax); + + /* Since we're bounded, pass as a 32-bit type. */ + t2 = tcg_temp_new_i32(); + tcg_gen_extrl_i64_i32(t2, diff); + tcg_temp_free_i64(diff); + + desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz / 8); + desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); + t3 = tcg_const_i32(desc); + + ptr = tcg_temp_new_ptr(); + tcg_gen_addi_ptr(ptr, cpu_env, pred_full_reg_offset(s, a->rd)); + + gen_helper_sve_whilel(t2, ptr, t2, t3); + do_pred_flags(t2); + + tcg_temp_free_ptr(ptr); + tcg_temp_free_i32(t2); + tcg_temp_free_i32(t3); + return true; +} + /* *** SVE Integer Wide Immediate - Unpredicated Group */ -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-33-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 6 ++ target/arm/sve.decode | 12 +++ target/arm/sve_helper.c | 50 +++++++++ target/arm/translate-sve.c | 213 +++++++++++++++++++++++++++++++++++++ 4 files changed, 281 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_6(sve2_fminp_zpzz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_6(sve2_fminp_zpzz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_eor3, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_bcax, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_bsl1n, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_bsl2n, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_nbsl, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ @rda_rn_rm ........ esz:2 . rm:5 ... ... rn:5 rd:5 \ &rrrr_esz ra=%reg_movprfx +# Four operand with unused vector element size +@rdn_ra_rm_e0 ........ ... rm:5 ... ... ra:5 rd:5 \ + &rrrr_esz esz=0 rn=%reg_movprfx + # Three operand with "memory" size, aka immediate left shift @rd_rn_msz_rm ........ ... rm:5 .... imm:2 rn:5 rd:5 &rrri @@ -XXX,XX +XXX,XX @@ ORR_zzz 00000100 01 1 ..... 001 100 ..... ..... @rd_rn_rm_e0 EOR_zzz 00000100 10 1 ..... 001 100 ..... ..... @rd_rn_rm_e0 BIC_zzz 00000100 11 1 ..... 001 100 ..... ..... @rd_rn_rm_e0 +# SVE2 bitwise ternary operations +EOR3 00000100 00 1 ..... 001 110 ..... ..... @rdn_ra_rm_e0 +BSL 00000100 00 1 ..... 001 111 ..... ..... @rdn_ra_rm_e0 +BCAX 00000100 01 1 ..... 001 110 ..... ..... @rdn_ra_rm_e0 +BSL1N 00000100 01 1 ..... 001 111 ..... ..... @rdn_ra_rm_e0 +BSL2N 00000100 10 1 ..... 001 111 ..... ..... @rdn_ra_rm_e0 +NBSL 00000100 11 1 ..... 001 111 ..... ..... @rdn_ra_rm_e0 + ### SVE Index Generation Group # SVE index generation (immediate start, immediate increment) diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ST1_ZPZ_D(dd_be, zd, MO_64) #undef DO_ST1_ZPZ_S #undef DO_ST1_ZPZ_D + +void HELPER(sve2_eor3)(void *vd, void *vn, void *vm, void *vk, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc) / 8; + uint64_t *d = vd, *n = vn, *m = vm, *k = vk; + + for (i = 0; i < opr_sz; ++i) { + d[i] = n[i] ^ m[i] ^ k[i]; + } +} + +void HELPER(sve2_bcax)(void *vd, void *vn, void *vm, void *vk, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc) / 8; + uint64_t *d = vd, *n = vn, *m = vm, *k = vk; + + for (i = 0; i < opr_sz; ++i) { + d[i] = n[i] ^ (m[i] & ~k[i]); + } +} + +void HELPER(sve2_bsl1n)(void *vd, void *vn, void *vm, void *vk, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc) / 8; + uint64_t *d = vd, *n = vn, *m = vm, *k = vk; + + for (i = 0; i < opr_sz; ++i) { + d[i] = (~n[i] & k[i]) | (m[i] & ~k[i]); + } +} + +void HELPER(sve2_bsl2n)(void *vd, void *vn, void *vm, void *vk, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc) / 8; + uint64_t *d = vd, *n = vn, *m = vm, *k = vk; + + for (i = 0; i < opr_sz; ++i) { + d[i] = (n[i] & k[i]) | (~m[i] & ~k[i]); + } +} + +void HELPER(sve2_nbsl)(void *vd, void *vn, void *vm, void *vk, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc) / 8; + uint64_t *d = vd, *n = vn, *m = vm, *k = vk; + + for (i = 0; i < opr_sz; ++i) { + d[i] = ~((n[i] & k[i]) | (m[i] & ~k[i])); + } +} diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static void gen_gvec_fn_zzz(DisasContext *s, GVecGen3Fn *gvec_fn, vec_full_reg_offset(s, rm), vsz, vsz); } +/* Invoke a vector expander on four Zregs. */ +static void gen_gvec_fn_zzzz(DisasContext *s, GVecGen4Fn *gvec_fn, + int esz, int rd, int rn, int rm, int ra) +{ + unsigned vsz = vec_full_reg_size(s); + gvec_fn(esz, vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + vec_full_reg_offset(s, rm), + vec_full_reg_offset(s, ra), vsz, vsz); +} + /* Invoke a vector move on two Zregs. */ static bool do_mov_z(DisasContext *s, int rd, int rn) { @@ -XXX,XX +XXX,XX @@ static bool trans_BIC_zzz(DisasContext *s, arg_rrr_esz *a) return do_zzz_fn(s, a, tcg_gen_gvec_andc); } +static bool do_sve2_zzzz_fn(DisasContext *s, arg_rrrr_esz *a, GVecGen4Fn *fn) +{ + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (sve_access_check(s)) { + gen_gvec_fn_zzzz(s, fn, a->esz, a->rd, a->rn, a->rm, a->ra); + } + return true; +} + +static void gen_eor3_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k) +{ + tcg_gen_xor_i64(d, n, m); + tcg_gen_xor_i64(d, d, k); +} + +static void gen_eor3_vec(unsigned vece, TCGv_vec d, TCGv_vec n, + TCGv_vec m, TCGv_vec k) +{ + tcg_gen_xor_vec(vece, d, n, m); + tcg_gen_xor_vec(vece, d, d, k); +} + +static void gen_eor3(unsigned vece, uint32_t d, uint32_t n, uint32_t m, + uint32_t a, uint32_t oprsz, uint32_t maxsz) +{ + static const GVecGen4 op = { + .fni8 = gen_eor3_i64, + .fniv = gen_eor3_vec, + .fno = gen_helper_sve2_eor3, + .vece = MO_64, + .prefer_i64 = TCG_TARGET_REG_BITS == 64, + }; + tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op); +} + +static bool trans_EOR3(DisasContext *s, arg_rrrr_esz *a) +{ + return do_sve2_zzzz_fn(s, a, gen_eor3); +} + +static void gen_bcax_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k) +{ + tcg_gen_andc_i64(d, m, k); + tcg_gen_xor_i64(d, d, n); +} + +static void gen_bcax_vec(unsigned vece, TCGv_vec d, TCGv_vec n, + TCGv_vec m, TCGv_vec k) +{ + tcg_gen_andc_vec(vece, d, m, k); + tcg_gen_xor_vec(vece, d, d, n); +} + +static void gen_bcax(unsigned vece, uint32_t d, uint32_t n, uint32_t m, + uint32_t a, uint32_t oprsz, uint32_t maxsz) +{ + static const GVecGen4 op = { + .fni8 = gen_bcax_i64, + .fniv = gen_bcax_vec, + .fno = gen_helper_sve2_bcax, + .vece = MO_64, + .prefer_i64 = TCG_TARGET_REG_BITS == 64, + }; + tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op); +} + +static bool trans_BCAX(DisasContext *s, arg_rrrr_esz *a) +{ + return do_sve2_zzzz_fn(s, a, gen_bcax); +} + +static void gen_bsl(unsigned vece, uint32_t d, uint32_t n, uint32_t m, + uint32_t a, uint32_t oprsz, uint32_t maxsz) +{ + /* BSL differs from the generic bitsel in argument ordering. */ + tcg_gen_gvec_bitsel(vece, d, a, n, m, oprsz, maxsz); +} + +static bool trans_BSL(DisasContext *s, arg_rrrr_esz *a) +{ + return do_sve2_zzzz_fn(s, a, gen_bsl); +} + +static void gen_bsl1n_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k) +{ + tcg_gen_andc_i64(n, k, n); + tcg_gen_andc_i64(m, m, k); + tcg_gen_or_i64(d, n, m); +} + +static void gen_bsl1n_vec(unsigned vece, TCGv_vec d, TCGv_vec n, + TCGv_vec m, TCGv_vec k) +{ + if (TCG_TARGET_HAS_bitsel_vec) { + tcg_gen_not_vec(vece, n, n); + tcg_gen_bitsel_vec(vece, d, k, n, m); + } else { + tcg_gen_andc_vec(vece, n, k, n); + tcg_gen_andc_vec(vece, m, m, k); + tcg_gen_or_vec(vece, d, n, m); + } +} + +static void gen_bsl1n(unsigned vece, uint32_t d, uint32_t n, uint32_t m, + uint32_t a, uint32_t oprsz, uint32_t maxsz) +{ + static const GVecGen4 op = { + .fni8 = gen_bsl1n_i64, + .fniv = gen_bsl1n_vec, + .fno = gen_helper_sve2_bsl1n, + .vece = MO_64, + .prefer_i64 = TCG_TARGET_REG_BITS == 64, + }; + tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op); +} + +static bool trans_BSL1N(DisasContext *s, arg_rrrr_esz *a) +{ + return do_sve2_zzzz_fn(s, a, gen_bsl1n); +} + +static void gen_bsl2n_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k) +{ + /* + * Z[dn] = (n & k) | (~m & ~k) + * = | ~(m | k) + */ + tcg_gen_and_i64(n, n, k); + if (TCG_TARGET_HAS_orc_i64) { + tcg_gen_or_i64(m, m, k); + tcg_gen_orc_i64(d, n, m); + } else { + tcg_gen_nor_i64(m, m, k); + tcg_gen_or_i64(d, n, m); + } +} + +static void gen_bsl2n_vec(unsigned vece, TCGv_vec d, TCGv_vec n, + TCGv_vec m, TCGv_vec k) +{ + if (TCG_TARGET_HAS_bitsel_vec) { + tcg_gen_not_vec(vece, m, m); + tcg_gen_bitsel_vec(vece, d, k, n, m); + } else { + tcg_gen_and_vec(vece, n, n, k); + tcg_gen_or_vec(vece, m, m, k); + tcg_gen_orc_vec(vece, d, n, m); + } +} + +static void gen_bsl2n(unsigned vece, uint32_t d, uint32_t n, uint32_t m, + uint32_t a, uint32_t oprsz, uint32_t maxsz) +{ + static const GVecGen4 op = { + .fni8 = gen_bsl2n_i64, + .fniv = gen_bsl2n_vec, + .fno = gen_helper_sve2_bsl2n, + .vece = MO_64, + .prefer_i64 = TCG_TARGET_REG_BITS == 64, + }; + tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op); +} + +static bool trans_BSL2N(DisasContext *s, arg_rrrr_esz *a) +{ + return do_sve2_zzzz_fn(s, a, gen_bsl2n); +} + +static void gen_nbsl_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k) +{ + tcg_gen_and_i64(n, n, k); + tcg_gen_andc_i64(m, m, k); + tcg_gen_nor_i64(d, n, m); +} + +static void gen_nbsl_vec(unsigned vece, TCGv_vec d, TCGv_vec n, + TCGv_vec m, TCGv_vec k) +{ + tcg_gen_bitsel_vec(vece, d, k, n, m); + tcg_gen_not_vec(vece, d, d); +} + +static void gen_nbsl(unsigned vece, uint32_t d, uint32_t n, uint32_t m, + uint32_t a, uint32_t oprsz, uint32_t maxsz) +{ + static const GVecGen4 op = { + .fni8 = gen_nbsl_i64, + .fniv = gen_nbsl_vec, + .fno = gen_helper_sve2_nbsl, + .vece = MO_64, + .prefer_i64 = TCG_TARGET_REG_BITS == 64, + }; + tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op); +} + +static bool trans_NBSL(DisasContext *s, arg_rrrr_esz *a) +{ + return do_sve2_zzzz_fn(s, a, gen_nbsl); +} + /* *** SVE Integer Arithmetic - Unpredicated Group */ -- 2.20.1 From: Stephen Long <steplong@quicinc.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Stephen Long <steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-34-richard.henderson@linaro.org Message-Id: <20200415145915.2859-1-steplong@quicinc.com> [rth: Expanded comment for do_match2] Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 10 ++++++ target/arm/sve.decode | 5 +++ target/arm/sve_helper.c | 64 ++++++++++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 22 +++++++++++++ 4 files changed, 101 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_3(sve2_uqrshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_uqrshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_uqrshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_match_ppzz_b, TCG_CALL_NO_RWG, + i32, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_match_ppzz_h, TCG_CALL_NO_RWG, + i32, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_nmatch_ppzz_b, TCG_CALL_NO_RWG, + i32, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_nmatch_ppzz_h, TCG_CALL_NO_RWG, + i32, ptr, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG, diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ UQSHRNT 01000101 .. 1 ..... 00 1101 ..... ..... @rd_rn_tszimm_shr UQRSHRNB 01000101 .. 1 ..... 00 1110 ..... ..... @rd_rn_tszimm_shr UQRSHRNT 01000101 .. 1 ..... 00 1111 ..... ..... @rd_rn_tszimm_shr +### SVE2 Character Match + +MATCH 01000101 .. 1 ..... 100 ... ..... 0 .... @pd_pg_rn_rm +NMATCH 01000101 .. 1 ..... 100 ... ..... 1 .... @pd_pg_rn_rm + ## SVE2 floating-point pairwise operations FADDP 01100100 .. 010 00 0 100 ... ..... ..... @rdn_pg_rm diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(sve2_nbsl)(void *vd, void *vn, void *vm, void *vk, uint32_t desc) d[i] = ~((n[i] & k[i]) | (m[i] & ~k[i])); } } + +/* + * Returns true if m0 or m1 contains the low uint8_t/uint16_t in n. + * See hasless(v,1) from + * https://graphics.stanford.edu/~seander/bithacks.html#ZeroInWord + */ +static inline bool do_match2(uint64_t n, uint64_t m0, uint64_t m1, int esz) +{ + int bits = 8 << esz; + uint64_t ones = dup_const(esz, 1); + uint64_t signs = ones << (bits - 1); + uint64_t cmp0, cmp1; + + cmp1 = dup_const(esz, n); + cmp0 = cmp1 ^ m0; + cmp1 = cmp1 ^ m1; + cmp0 = (cmp0 - ones) & ~cmp0; + cmp1 = (cmp1 - ones) & ~cmp1; + return (cmp0 | cmp1) & signs; +} + +static inline uint32_t do_match(void *vd, void *vn, void *vm, void *vg, + uint32_t desc, int esz, bool nmatch) +{ + uint16_t esz_mask = pred_esz_masks[esz]; + intptr_t opr_sz = simd_oprsz(desc); + uint32_t flags = PREDTEST_INIT; + intptr_t i, j, k; + + for (i = 0; i < opr_sz; i += 16) { + uint64_t m0 = *(uint64_t *)(vm + i); + uint64_t m1 = *(uint64_t *)(vm + i + 8); + uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)) & esz_mask; + uint16_t out = 0; + + for (j = 0; j < 16; j += 8) { + uint64_t n = *(uint64_t *)(vn + i + j); + + for (k = 0; k < 8; k += 1 << esz) { + if (pg & (1 << (j + k))) { + bool o = do_match2(n >> (k * 8), m0, m1, esz); + out |= (o ^ nmatch) << (j + k); + } + } + } + *(uint16_t *)(vd + H1_2(i >> 3)) = out; + flags = iter_predtest_fwd(out, pg, flags); + } + return flags; +} + +#define DO_PPZZ_MATCH(NAME, ESZ, INV) \ +uint32_t HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \ +{ \ + return do_match(vd, vn, vm, vg, desc, ESZ, INV); \ +} + +DO_PPZZ_MATCH(sve2_match_ppzz_b, MO_8, false) +DO_PPZZ_MATCH(sve2_match_ppzz_h, MO_16, false) + +DO_PPZZ_MATCH(sve2_nmatch_ppzz_b, MO_8, true) +DO_PPZZ_MATCH(sve2_nmatch_ppzz_h, MO_16, true) + +#undef DO_PPZZ_MATCH diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_UQRSHRNT(DisasContext *s, arg_rri_esz *a) return do_sve2_shr_narrow(s, a, ops); } +static bool do_sve2_ppzz_flags(DisasContext *s, arg_rprr_esz *a, + gen_helper_gvec_flags_4 *fn) +{ + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + return do_ppzz_flags(s, a, fn); +} + +#define DO_SVE2_PPZZ_MATCH(NAME, name) \ +static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \ +{ \ + static gen_helper_gvec_flags_4 * const fns[4] = { \ + gen_helper_sve2_##name##_ppzz_b, gen_helper_sve2_##name##_ppzz_h, \ + NULL, NULL \ + }; \ + return do_sve2_ppzz_flags(s, a, fns[a->esz]); \ +} + +DO_SVE2_PPZZ_MATCH(MATCH, match) +DO_SVE2_PPZZ_MATCH(NMATCH, nmatch) + static bool do_sve2_zpzz_fp(DisasContext *s, arg_rprr_esz *a, gen_helper_gvec_4_ptr *fn) { -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-35-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 14 ++++++++++ target/arm/sve.decode | 14 ++++++++++ target/arm/sve_helper.c | 30 +++++++++++++++++++++ target/arm/translate-sve.c | 54 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_bcax, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve2_bsl1n, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve2_bsl2n, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve2_nbsl, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_sqdmlal_zzzw_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqdmlal_zzzw_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqdmlal_zzzw_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_sqdmlsl_zzzw_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqdmlsl_zzzw_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqdmlsl_zzzw_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ FMAXNMP 01100100 .. 010 10 0 100 ... ..... ..... @rdn_pg_rm FMINNMP 01100100 .. 010 10 1 100 ... ..... ..... @rdn_pg_rm FMAXP 01100100 .. 010 11 0 100 ... ..... ..... @rdn_pg_rm FMINP 01100100 .. 010 11 1 100 ... ..... ..... @rdn_pg_rm + +#### SVE Integer Multiply-Add (unpredicated) + +## SVE2 saturating multiply-add long + +SQDMLALB_zzzw 01000100 .. 0 ..... 0110 00 ..... ..... @rda_rn_rm +SQDMLALT_zzzw 01000100 .. 0 ..... 0110 01 ..... ..... @rda_rn_rm +SQDMLSLB_zzzw 01000100 .. 0 ..... 0110 10 ..... ..... @rda_rn_rm +SQDMLSLT_zzzw 01000100 .. 0 ..... 0110 11 ..... ..... @rda_rn_rm + +## SVE2 saturating multiply-add interleaved long + +SQDMLALBT 01000100 .. 0 ..... 00001 0 ..... ..... @rda_rn_rm +SQDMLSLBT 01000100 .. 0 ..... 00001 1 ..... ..... @rda_rn_rm diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(sve2_adcl_d)(void *vd, void *vn, void *vm, void *va, uint32_t desc) } } +#define DO_SQDMLAL(NAME, TYPEW, TYPEN, HW, HN, DMUL_OP, SUM_OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc); \ + int sel1 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN); \ + int sel2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(TYPEN); \ + for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \ + TYPEW nn = *(TYPEN *)(vn + HN(i + sel1)); \ + TYPEW mm = *(TYPEN *)(vm + HN(i + sel2)); \ + TYPEW aa = *(TYPEW *)(va + HW(i)); \ + *(TYPEW *)(vd + HW(i)) = SUM_OP(aa, DMUL_OP(nn, mm)); \ + } \ +} + +DO_SQDMLAL(sve2_sqdmlal_zzzw_h, int16_t, int8_t, H1_2, H1, + do_sqdmull_h, DO_SQADD_H) +DO_SQDMLAL(sve2_sqdmlal_zzzw_s, int32_t, int16_t, H1_4, H1_2, + do_sqdmull_s, DO_SQADD_S) +DO_SQDMLAL(sve2_sqdmlal_zzzw_d, int64_t, int32_t, , H1_4, + do_sqdmull_d, do_sqadd_d) + +DO_SQDMLAL(sve2_sqdmlsl_zzzw_h, int16_t, int8_t, H1_2, H1, + do_sqdmull_h, DO_SQSUB_H) +DO_SQDMLAL(sve2_sqdmlsl_zzzw_s, int32_t, int16_t, H1_4, H1_2, + do_sqdmull_s, DO_SQSUB_S) +DO_SQDMLAL(sve2_sqdmlsl_zzzw_d, int64_t, int32_t, , H1_4, + do_sqdmull_d, do_sqsub_d) + +#undef DO_SQDMLAL + #define DO_BITPERM(NAME, TYPE, OP) \ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ { \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_ZPZZ_FP(FMAXNMP, fmaxnmp) DO_SVE2_ZPZZ_FP(FMINNMP, fminnmp) DO_SVE2_ZPZZ_FP(FMAXP, fmaxp) DO_SVE2_ZPZZ_FP(FMINP, fminp) + +/* + * SVE Integer Multiply-Add (unpredicated) + */ + +static bool do_sqdmlal_zzzw(DisasContext *s, arg_rrrr_esz *a, + bool sel1, bool sel2) +{ + static gen_helper_gvec_4 * const fns[] = { + NULL, gen_helper_sve2_sqdmlal_zzzw_h, + gen_helper_sve2_sqdmlal_zzzw_s, gen_helper_sve2_sqdmlal_zzzw_d, + }; + return do_sve2_zzzz_ool(s, a, fns[a->esz], (sel2 << 1) | sel1); +} + +static bool do_sqdmlsl_zzzw(DisasContext *s, arg_rrrr_esz *a, + bool sel1, bool sel2) +{ + static gen_helper_gvec_4 * const fns[] = { + NULL, gen_helper_sve2_sqdmlsl_zzzw_h, + gen_helper_sve2_sqdmlsl_zzzw_s, gen_helper_sve2_sqdmlsl_zzzw_d, + }; + return do_sve2_zzzz_ool(s, a, fns[a->esz], (sel2 << 1) | sel1); +} + +static bool trans_SQDMLALB_zzzw(DisasContext *s, arg_rrrr_esz *a) +{ + return do_sqdmlal_zzzw(s, a, false, false); +} + +static bool trans_SQDMLALT_zzzw(DisasContext *s, arg_rrrr_esz *a) +{ + return do_sqdmlal_zzzw(s, a, true, true); +} + +static bool trans_SQDMLALBT(DisasContext *s, arg_rrrr_esz *a) +{ + return do_sqdmlal_zzzw(s, a, false, true); +} + +static bool trans_SQDMLSLB_zzzw(DisasContext *s, arg_rrrr_esz *a) +{ + return do_sqdmlsl_zzzw(s, a, false, false); +} + +static bool trans_SQDMLSLT_zzzw(DisasContext *s, arg_rrrr_esz *a) +{ + return do_sqdmlsl_zzzw(s, a, true, true); +} + +static bool trans_SQDMLSLBT(DisasContext *s, arg_rrrr_esz *a) +{ + return do_sqdmlsl_zzzw(s, a, false, true); +} -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> SVE2 has two additional sizes of the operation and unlike NEON, there is no saturation flag. Create new entry points for SVE2 that do not set QC. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-36-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper.h | 17 ++++ target/arm/sve.decode | 5 ++ target/arm/translate-sve.c | 18 +++++ target/arm/vec_helper.c | 161 +++++++++++++++++++++++++++++++++++-- 4 files changed, 195 insertions(+), 6 deletions(-) diff --git a/target/arm/helper.h b/target/arm/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.h +++ b/target/arm/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(gvec_qrdmlah_s32, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_5(gvec_qrdmlsh_s32, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrdmlah_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrdmlah_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrdmlah_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrdmlah_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_4(gvec_sdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(gvec_udot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(gvec_sdot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SQDMLSLT_zzzw 01000100 .. 0 ..... 0110 11 ..... ..... @rda_rn_rm SQDMLALBT 01000100 .. 0 ..... 00001 0 ..... ..... @rda_rn_rm SQDMLSLBT 01000100 .. 0 ..... 00001 1 ..... ..... @rda_rn_rm + +## SVE2 saturating multiply-add high + +SQRDMLAH_zzzz 01000100 .. 0 ..... 01110 0 ..... ..... @rda_rn_rm +SQRDMLSH_zzzz 01000100 .. 0 ..... 01110 1 ..... ..... @rda_rn_rm diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_SQDMLSLBT(DisasContext *s, arg_rrrr_esz *a) { return do_sqdmlsl_zzzw(s, a, false, true); } + +static bool trans_SQRDMLAH_zzzz(DisasContext *s, arg_rrrr_esz *a) +{ + static gen_helper_gvec_4 * const fns[] = { + gen_helper_sve2_sqrdmlah_b, gen_helper_sve2_sqrdmlah_h, + gen_helper_sve2_sqrdmlah_s, gen_helper_sve2_sqrdmlah_d, + }; + return do_sve2_zzzz_ool(s, a, fns[a->esz], 0); +} + +static bool trans_SQRDMLSH_zzzz(DisasContext *s, arg_rrrr_esz *a) +{ + static gen_helper_gvec_4 * const fns[] = { + gen_helper_sve2_sqrdmlsh_b, gen_helper_sve2_sqrdmlsh_h, + gen_helper_sve2_sqrdmlsh_s, gen_helper_sve2_sqrdmlsh_d, + }; + return do_sve2_zzzz_ool(s, a, fns[a->esz], 0); +} diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ #include "exec/helper-proto.h" #include "tcg/tcg-gvec-desc.h" #include "fpu/softfloat.h" +#include "qemu/int128.h" #include "vec_internal.h" /* Note that vector data is stored in host-endian 64-bit chunks, @@ -XXX,XX +XXX,XX @@ #define H4(x) (x) #endif +/* Signed saturating rounding doubling multiply-accumulate high half, 8-bit */ +static int8_t do_sqrdmlah_b(int8_t src1, int8_t src2, int8_t src3, + bool neg, bool round) +{ + /* + * Simplify: + * = ((a3 << 8) + ((e1 * e2) << 1) + (round << 7)) >> 8 + * = ((a3 << 7) + (e1 * e2) + (round << 6)) >> 7 + */ + int32_t ret = (int32_t)src1 * src2; + if (neg) { + ret = -ret; + } + ret += ((int32_t)src3 << 7) + (round << 6); + ret >>= 7; + + if (ret != (int8_t)ret) { + ret = (ret < 0 ? INT8_MIN : INT8_MAX); + } + return ret; +} + +void HELPER(sve2_sqrdmlah_b)(void *vd, void *vn, void *vm, + void *va, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int8_t *d = vd, *n = vn, *m = vm, *a = va; + + for (i = 0; i < opr_sz; ++i) { + d[i] = do_sqrdmlah_b(n[i], m[i], a[i], false, true); + } +} + +void HELPER(sve2_sqrdmlsh_b)(void *vd, void *vn, void *vm, + void *va, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int8_t *d = vd, *n = vn, *m = vm, *a = va; + + for (i = 0; i < opr_sz; ++i) { + d[i] = do_sqrdmlah_b(n[i], m[i], a[i], true, true); + } +} + /* Signed saturating rounding doubling multiply-accumulate high half, 16-bit */ static int16_t do_sqrdmlah_h(int16_t src1, int16_t src2, int16_t src3, bool neg, bool round, uint32_t *sat) { - /* - * Simplify: - * = ((a3 << 16) + ((e1 * e2) << 1) + (1 << 15)) >> 16 - * = ((a3 << 15) + (e1 * e2) + (1 << 14)) >> 15 - */ + /* Simplify similarly to do_sqrdmlah_b above. */ int32_t ret = (int32_t)src1 * src2; if (neg) { ret = -ret; @@ -XXX,XX +XXX,XX @@ void HELPER(neon_sqrdmulh_h)(void *vd, void *vn, void *vm, clear_tail(d, opr_sz, simd_maxsz(desc)); } +void HELPER(sve2_sqrdmlah_h)(void *vd, void *vn, void *vm, + void *va, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int16_t *d = vd, *n = vn, *m = vm, *a = va; + uint32_t discard; + + for (i = 0; i < opr_sz / 2; ++i) { + d[i] = do_sqrdmlah_h(n[i], m[i], a[i], false, true, &discard); + } +} + +void HELPER(sve2_sqrdmlsh_h)(void *vd, void *vn, void *vm, + void *va, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int16_t *d = vd, *n = vn, *m = vm, *a = va; + uint32_t discard; + + for (i = 0; i < opr_sz / 2; ++i) { + d[i] = do_sqrdmlah_h(n[i], m[i], a[i], true, true, &discard); + } +} + /* Signed saturating rounding doubling multiply-accumulate high half, 32-bit */ static int32_t do_sqrdmlah_s(int32_t src1, int32_t src2, int32_t src3, bool neg, bool round, uint32_t *sat) { - /* Simplify similarly to int_qrdmlah_s16 above. */ + /* Simplify similarly to do_sqrdmlah_b above. */ int64_t ret = (int64_t)src1 * src2; if (neg) { ret = -ret; @@ -XXX,XX +XXX,XX @@ void HELPER(neon_sqrdmulh_s)(void *vd, void *vn, void *vm, clear_tail(d, opr_sz, simd_maxsz(desc)); } +void HELPER(sve2_sqrdmlah_s)(void *vd, void *vn, void *vm, + void *va, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int32_t *d = vd, *n = vn, *m = vm, *a = va; + uint32_t discard; + + for (i = 0; i < opr_sz / 4; ++i) { + d[i] = do_sqrdmlah_s(n[i], m[i], a[i], false, true, &discard); + } +} + +void HELPER(sve2_sqrdmlsh_s)(void *vd, void *vn, void *vm, + void *va, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int32_t *d = vd, *n = vn, *m = vm, *a = va; + uint32_t discard; + + for (i = 0; i < opr_sz / 4; ++i) { + d[i] = do_sqrdmlah_s(n[i], m[i], a[i], true, true, &discard); + } +} + +/* Signed saturating rounding doubling multiply-accumulate high half, 64-bit */ +static int64_t do_sat128_d(Int128 r) +{ + int64_t ls = int128_getlo(r); + int64_t hs = int128_gethi(r); + + if (unlikely(hs != (ls >> 63))) { + return hs < 0 ? INT64_MIN : INT64_MAX; + } + return ls; +} + +static int64_t do_sqrdmlah_d(int64_t n, int64_t m, int64_t a, + bool neg, bool round) +{ + uint64_t l, h; + Int128 r, t; + + /* As in do_sqrdmlah_b, but with 128-bit arithmetic. */ + muls64(&l, &h, m, n); + r = int128_make128(l, h); + if (neg) { + r = int128_neg(r); + } + if (a) { + t = int128_exts64(a); + t = int128_lshift(t, 63); + r = int128_add(r, t); + } + if (round) { + t = int128_exts64(1ll << 62); + r = int128_add(r, t); + } + r = int128_rshift(r, 63); + + return do_sat128_d(r); +} + +void HELPER(sve2_sqrdmlah_d)(void *vd, void *vn, void *vm, + void *va, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int64_t *d = vd, *n = vn, *m = vm, *a = va; + + for (i = 0; i < opr_sz / 8; ++i) { + d[i] = do_sqrdmlah_d(n[i], m[i], a[i], false, true); + } +} + +void HELPER(sve2_sqrdmlsh_d)(void *vd, void *vn, void *vm, + void *va, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int64_t *d = vd, *n = vn, *m = vm, *a = va; + + for (i = 0; i < opr_sz / 8; ++i) { + d[i] = do_sqrdmlah_d(n[i], m[i], a[i], true, true); + } +} + /* Integer 8 and 16-bit dot-product. * * Note that for the loops herein, host endianness does not matter -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-37-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 28 ++++++++++++++ target/arm/sve.decode | 11 ++++++ target/arm/sve_helper.c | 18 +++++++++ target/arm/translate-sve.c | 76 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_sqdmlsl_zzzw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve2_sqdmlsl_zzzw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_smlal_zzzw_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_smlal_zzzw_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_smlal_zzzw_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_umlal_zzzw_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_umlal_zzzw_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_umlal_zzzw_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_smlsl_zzzw_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_smlsl_zzzw_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_smlsl_zzzw_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_umlsl_zzzw_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_umlsl_zzzw_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_umlsl_zzzw_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SQDMLSLBT 01000100 .. 0 ..... 00001 1 ..... ..... @rda_rn_rm SQRDMLAH_zzzz 01000100 .. 0 ..... 01110 0 ..... ..... @rda_rn_rm SQRDMLSH_zzzz 01000100 .. 0 ..... 01110 1 ..... ..... @rda_rn_rm + +## SVE2 integer multiply-add long + +SMLALB_zzzw 01000100 .. 0 ..... 010 000 ..... ..... @rda_rn_rm +SMLALT_zzzw 01000100 .. 0 ..... 010 001 ..... ..... @rda_rn_rm +UMLALB_zzzw 01000100 .. 0 ..... 010 010 ..... ..... @rda_rn_rm +UMLALT_zzzw 01000100 .. 0 ..... 010 011 ..... ..... @rda_rn_rm +SMLSLB_zzzw 01000100 .. 0 ..... 010 100 ..... ..... @rda_rn_rm +SMLSLT_zzzw 01000100 .. 0 ..... 010 101 ..... ..... @rda_rn_rm +UMLSLB_zzzw 01000100 .. 0 ..... 010 110 ..... ..... @rda_rn_rm +UMLSLT_zzzw 01000100 .. 0 ..... 010 111 ..... ..... @rda_rn_rm diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZZZW_ACC(sve2_uabal_h, uint16_t, uint8_t, H1_2, H1, DO_ABD) DO_ZZZW_ACC(sve2_uabal_s, uint32_t, uint16_t, H1_4, H1_2, DO_ABD) DO_ZZZW_ACC(sve2_uabal_d, uint64_t, uint32_t, , H1_4, DO_ABD) +DO_ZZZW_ACC(sve2_smlal_zzzw_h, int16_t, int8_t, H1_2, H1, DO_MUL) +DO_ZZZW_ACC(sve2_smlal_zzzw_s, int32_t, int16_t, H1_4, H1_2, DO_MUL) +DO_ZZZW_ACC(sve2_smlal_zzzw_d, int64_t, int32_t, , H1_4, DO_MUL) + +DO_ZZZW_ACC(sve2_umlal_zzzw_h, uint16_t, uint8_t, H1_2, H1, DO_MUL) +DO_ZZZW_ACC(sve2_umlal_zzzw_s, uint32_t, uint16_t, H1_4, H1_2, DO_MUL) +DO_ZZZW_ACC(sve2_umlal_zzzw_d, uint64_t, uint32_t, , H1_4, DO_MUL) + +#define DO_NMUL(N, M) -(N * M) + +DO_ZZZW_ACC(sve2_smlsl_zzzw_h, int16_t, int8_t, H1_2, H1, DO_NMUL) +DO_ZZZW_ACC(sve2_smlsl_zzzw_s, int32_t, int16_t, H1_4, H1_2, DO_NMUL) +DO_ZZZW_ACC(sve2_smlsl_zzzw_d, int64_t, int32_t, , H1_4, DO_NMUL) + +DO_ZZZW_ACC(sve2_umlsl_zzzw_h, uint16_t, uint8_t, H1_2, H1, DO_NMUL) +DO_ZZZW_ACC(sve2_umlsl_zzzw_s, uint32_t, uint16_t, H1_4, H1_2, DO_NMUL) +DO_ZZZW_ACC(sve2_umlsl_zzzw_d, uint64_t, uint32_t, , H1_4, DO_NMUL) + #undef DO_ZZZW_ACC #define DO_XTNB(NAME, TYPE, OP) \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_SQRDMLSH_zzzz(DisasContext *s, arg_rrrr_esz *a) }; return do_sve2_zzzz_ool(s, a, fns[a->esz], 0); } + +static bool do_smlal_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel) +{ + static gen_helper_gvec_4 * const fns[] = { + NULL, gen_helper_sve2_smlal_zzzw_h, + gen_helper_sve2_smlal_zzzw_s, gen_helper_sve2_smlal_zzzw_d, + }; + return do_sve2_zzzz_ool(s, a, fns[a->esz], sel); +} + +static bool trans_SMLALB_zzzw(DisasContext *s, arg_rrrr_esz *a) +{ + return do_smlal_zzzw(s, a, false); +} + +static bool trans_SMLALT_zzzw(DisasContext *s, arg_rrrr_esz *a) +{ + return do_smlal_zzzw(s, a, true); +} + +static bool do_umlal_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel) +{ + static gen_helper_gvec_4 * const fns[] = { + NULL, gen_helper_sve2_umlal_zzzw_h, + gen_helper_sve2_umlal_zzzw_s, gen_helper_sve2_umlal_zzzw_d, + }; + return do_sve2_zzzz_ool(s, a, fns[a->esz], sel); +} + +static bool trans_UMLALB_zzzw(DisasContext *s, arg_rrrr_esz *a) +{ + return do_umlal_zzzw(s, a, false); +} + +static bool trans_UMLALT_zzzw(DisasContext *s, arg_rrrr_esz *a) +{ + return do_umlal_zzzw(s, a, true); +} + +static bool do_smlsl_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel) +{ + static gen_helper_gvec_4 * const fns[] = { + NULL, gen_helper_sve2_smlsl_zzzw_h, + gen_helper_sve2_smlsl_zzzw_s, gen_helper_sve2_smlsl_zzzw_d, + }; + return do_sve2_zzzz_ool(s, a, fns[a->esz], sel); +} + +static bool trans_SMLSLB_zzzw(DisasContext *s, arg_rrrr_esz *a) +{ + return do_smlsl_zzzw(s, a, false); +} + +static bool trans_SMLSLT_zzzw(DisasContext *s, arg_rrrr_esz *a) +{ + return do_smlsl_zzzw(s, a, true); +} + +static bool do_umlsl_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel) +{ + static gen_helper_gvec_4 * const fns[] = { + NULL, gen_helper_sve2_umlsl_zzzw_h, + gen_helper_sve2_umlsl_zzzw_s, gen_helper_sve2_umlsl_zzzw_d, + }; + return do_sve2_zzzz_ool(s, a, fns[a->esz], sel); +} + +static bool trans_UMLSLB_zzzw(DisasContext *s, arg_rrrr_esz *a) +{ + return do_umlsl_zzzw(s, a, false); +} + +static bool trans_UMLSLT_zzzw(DisasContext *s, arg_rrrr_esz *a) +{ + return do_umlsl_zzzw(s, a, true); +} -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-38-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 18 +++++++++++++++ target/arm/vec_internal.h | 5 +++++ target/arm/sve.decode | 5 +++++ target/arm/sve_helper.c | 46 ++++++++++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 32 ++++++++++++++++++++++++++ target/arm/vec_helper.c | 15 ++++++------- 6 files changed, 113 insertions(+), 8 deletions(-) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_umlsl_zzzw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve2_umlsl_zzzw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_cmla_zzzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_cmla_zzzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_cmla_zzzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_cmla_zzzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/vec_internal.h b/target/arm/vec_internal.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_internal.h +++ b/target/arm/vec_internal.h @@ -XXX,XX +XXX,XX @@ static inline int64_t do_suqrshl_d(int64_t src, int64_t shift, return do_uqrshl_d(src, shift, round, sat); } +int8_t do_sqrdmlah_b(int8_t, int8_t, int8_t, bool, bool); +int16_t do_sqrdmlah_h(int16_t, int16_t, int16_t, bool, bool, uint32_t *); +int32_t do_sqrdmlah_s(int32_t, int32_t, int32_t, bool, bool, uint32_t *); +int64_t do_sqrdmlah_d(int64_t, int64_t, int64_t, bool, bool); + #endif /* TARGET_ARM_VEC_INTERNALS_H */ diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SMLSLB_zzzw 01000100 .. 0 ..... 010 100 ..... ..... @rda_rn_rm SMLSLT_zzzw 01000100 .. 0 ..... 010 101 ..... ..... @rda_rn_rm UMLSLB_zzzw 01000100 .. 0 ..... 010 110 ..... ..... @rda_rn_rm UMLSLT_zzzw 01000100 .. 0 ..... 010 111 ..... ..... @rda_rn_rm + +## SVE2 complex integer multiply-add + +CMLA_zzzz 01000100 esz:2 0 rm:5 0010 rot:2 rn:5 rd:5 ra=%reg_movprfx +SQRDCMLAH_zzzz 01000100 esz:2 0 rm:5 0011 rot:2 rn:5 rd:5 ra=%reg_movprfx diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_SQDMLAL(sve2_sqdmlsl_zzzw_d, int64_t, int32_t, , H1_4, #undef DO_SQDMLAL +#define DO_CMLA_FUNC(NAME, TYPE, H, OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc) / sizeof(TYPE); \ + int rot = simd_data(desc); \ + int sel_a = rot & 1, sel_b = sel_a ^ 1; \ + bool sub_r = rot == 1 || rot == 2; \ + bool sub_i = rot >= 2; \ + TYPE *d = vd, *n = vn, *m = vm, *a = va; \ + for (i = 0; i < opr_sz; i += 2) { \ + TYPE elt1_a = n[H(i + sel_a)]; \ + TYPE elt2_a = m[H(i + sel_a)]; \ + TYPE elt2_b = m[H(i + sel_b)]; \ + d[H(i)] = OP(elt1_a, elt2_a, a[H(i)], sub_r); \ + d[H(i + 1)] = OP(elt1_a, elt2_b, a[H(i + 1)], sub_i); \ + } \ +} + +#define DO_CMLA(N, M, A, S) (A + (N * M) * (S ? -1 : 1)) + +DO_CMLA_FUNC(sve2_cmla_zzzz_b, uint8_t, H1, DO_CMLA) +DO_CMLA_FUNC(sve2_cmla_zzzz_h, uint16_t, H2, DO_CMLA) +DO_CMLA_FUNC(sve2_cmla_zzzz_s, uint32_t, H4, DO_CMLA) +DO_CMLA_FUNC(sve2_cmla_zzzz_d, uint64_t, , DO_CMLA) + +#define DO_SQRDMLAH_B(N, M, A, S) \ + do_sqrdmlah_b(N, M, A, S, true) +#define DO_SQRDMLAH_H(N, M, A, S) \ + ({ uint32_t discard; do_sqrdmlah_h(N, M, A, S, true, &discard); }) +#define DO_SQRDMLAH_S(N, M, A, S) \ + ({ uint32_t discard; do_sqrdmlah_s(N, M, A, S, true, &discard); }) +#define DO_SQRDMLAH_D(N, M, A, S) \ + do_sqrdmlah_d(N, M, A, S, true) + +DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_b, int8_t, H1, DO_SQRDMLAH_B) +DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_h, int16_t, H2, DO_SQRDMLAH_H) +DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_s, int32_t, H4, DO_SQRDMLAH_S) +DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_d, int64_t, , DO_SQRDMLAH_D) + +#undef DO_CMLA +#undef DO_CMLA_FUNC +#undef DO_SQRDMLAH_B +#undef DO_SQRDMLAH_H +#undef DO_SQRDMLAH_S +#undef DO_SQRDMLAH_D + #define DO_BITPERM(NAME, TYPE, OP) \ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ { \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_UMLSLT_zzzw(DisasContext *s, arg_rrrr_esz *a) { return do_umlsl_zzzw(s, a, true); } + +static bool trans_CMLA_zzzz(DisasContext *s, arg_CMLA_zzzz *a) +{ + static gen_helper_gvec_4 * const fns[] = { + gen_helper_sve2_cmla_zzzz_b, gen_helper_sve2_cmla_zzzz_h, + gen_helper_sve2_cmla_zzzz_s, gen_helper_sve2_cmla_zzzz_d, + }; + + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (sve_access_check(s)) { + gen_gvec_ool_zzzz(s, fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot); + } + return true; +} + +static bool trans_SQRDCMLAH_zzzz(DisasContext *s, arg_SQRDCMLAH_zzzz *a) +{ + static gen_helper_gvec_4 * const fns[] = { + gen_helper_sve2_sqrdcmlah_zzzz_b, gen_helper_sve2_sqrdcmlah_zzzz_h, + gen_helper_sve2_sqrdcmlah_zzzz_s, gen_helper_sve2_sqrdcmlah_zzzz_d, + }; + + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (sve_access_check(s)) { + gen_gvec_ool_zzzz(s, fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot); + } + return true; +} diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ #endif /* Signed saturating rounding doubling multiply-accumulate high half, 8-bit */ -static int8_t do_sqrdmlah_b(int8_t src1, int8_t src2, int8_t src3, - bool neg, bool round) +int8_t do_sqrdmlah_b(int8_t src1, int8_t src2, int8_t src3, + bool neg, bool round) { /* * Simplify: @@ -XXX,XX +XXX,XX @@ void HELPER(sve2_sqrdmlsh_b)(void *vd, void *vn, void *vm, } /* Signed saturating rounding doubling multiply-accumulate high half, 16-bit */ -static int16_t do_sqrdmlah_h(int16_t src1, int16_t src2, int16_t src3, - bool neg, bool round, uint32_t *sat) +int16_t do_sqrdmlah_h(int16_t src1, int16_t src2, int16_t src3, + bool neg, bool round, uint32_t *sat) { /* Simplify similarly to do_sqrdmlah_b above. */ int32_t ret = (int32_t)src1 * src2; @@ -XXX,XX +XXX,XX @@ void HELPER(sve2_sqrdmlsh_h)(void *vd, void *vn, void *vm, } /* Signed saturating rounding doubling multiply-accumulate high half, 32-bit */ -static int32_t do_sqrdmlah_s(int32_t src1, int32_t src2, int32_t src3, - bool neg, bool round, uint32_t *sat) +int32_t do_sqrdmlah_s(int32_t src1, int32_t src2, int32_t src3, + bool neg, bool round, uint32_t *sat) { /* Simplify similarly to do_sqrdmlah_b above. */ int64_t ret = (int64_t)src1 * src2; @@ -XXX,XX +XXX,XX @@ static int64_t do_sat128_d(Int128 r) return ls; } -static int64_t do_sqrdmlah_d(int64_t n, int64_t m, int64_t a, - bool neg, bool round) +int64_t do_sqrdmlah_d(int64_t n, int64_t m, int64_t a, bool neg, bool round) { uint64_t l, h; Int128 r, t; -- 2.20.1 From: Stephen Long <steplong@quicinc.com> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Stephen Long <steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-39-richard.henderson@linaro.org Message-Id: <20200417162231.10374-2-steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 8 ++++++++ target/arm/sve.decode | 5 +++++ target/arm/sve_helper.c | 36 ++++++++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 13 +++++++++++++ 4 files changed, 62 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_3(sve2_uqrshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_uqrshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve2_uqrshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_addhnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_addhnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_addhnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_addhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_addhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_addhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_5(sve2_match_ppzz_b, TCG_CALL_NO_RWG, i32, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve2_match_ppzz_h, TCG_CALL_NO_RWG, diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ UQSHRNT 01000101 .. 1 ..... 00 1101 ..... ..... @rd_rn_tszimm_shr UQRSHRNB 01000101 .. 1 ..... 00 1110 ..... ..... @rd_rn_tszimm_shr UQRSHRNT 01000101 .. 1 ..... 00 1111 ..... ..... @rd_rn_tszimm_shr +## SVE2 integer add/subtract narrow high part + +ADDHNB 01000101 .. 1 ..... 011 000 ..... ..... @rd_rn_rm +ADDHNT 01000101 .. 1 ..... 011 001 ..... ..... @rd_rn_rm + ### SVE2 Character Match MATCH 01000101 .. 1 ..... 100 ... ..... 0 .... @pd_pg_rn_rm diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_SHRNT(sve2_uqrshrnt_d, uint64_t, uint32_t, , H1_4, DO_UQRSHRN_D) #undef DO_SHRNB #undef DO_SHRNT +#define DO_BINOPNB(NAME, TYPEW, TYPEN, SHIFT, OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc); \ + for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \ + TYPEW nn = *(TYPEW *)(vn + i); \ + TYPEW mm = *(TYPEW *)(vm + i); \ + *(TYPEW *)(vd + i) = (TYPEN)OP(nn, mm, SHIFT); \ + } \ +} + +#define DO_BINOPNT(NAME, TYPEW, TYPEN, SHIFT, HW, HN, OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc); \ + for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \ + TYPEW nn = *(TYPEW *)(vn + HW(i)); \ + TYPEW mm = *(TYPEW *)(vm + HW(i)); \ + *(TYPEN *)(vd + HN(i + sizeof(TYPEN))) = OP(nn, mm, SHIFT); \ + } \ +} + +#define DO_ADDHN(N, M, SH) ((N + M) >> SH) + +DO_BINOPNB(sve2_addhnb_h, uint16_t, uint8_t, 8, DO_ADDHN) +DO_BINOPNB(sve2_addhnb_s, uint32_t, uint16_t, 16, DO_ADDHN) +DO_BINOPNB(sve2_addhnb_d, uint64_t, uint32_t, 32, DO_ADDHN) + +DO_BINOPNT(sve2_addhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_ADDHN) +DO_BINOPNT(sve2_addhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_ADDHN) +DO_BINOPNT(sve2_addhnt_d, uint64_t, uint32_t, 32, , H1_4, DO_ADDHN) + +#undef DO_ADDHN + +#undef DO_BINOPNB + /* Fully general four-operand expander, controlled by a predicate. */ #define DO_ZPZZZ(NAME, TYPE, H, OP) \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_UQRSHRNT(DisasContext *s, arg_rri_esz *a) return do_sve2_shr_narrow(s, a, ops); } +#define DO_SVE2_ZZZ_NARROW(NAME, name) \ +static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \ +{ \ + static gen_helper_gvec_3 * const fns[4] = { \ + NULL, gen_helper_sve2_##name##_h, \ + gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \ + }; \ + return do_sve2_zzz_ool(s, a, fns[a->esz]); \ +} + +DO_SVE2_ZZZ_NARROW(ADDHNB, addhnb) +DO_SVE2_ZZZ_NARROW(ADDHNT, addhnt) + static bool do_sve2_ppzz_flags(DisasContext *s, arg_rprr_esz *a, gen_helper_gvec_flags_4 *fn) { -- 2.20.1 From: Stephen Long <steplong@quicinc.com> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Stephen Long <steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-40-richard.henderson@linaro.org Message-Id: <20200417162231.10374-3-steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 8 ++++++++ target/arm/sve.decode | 2 ++ target/arm/sve_helper.c | 10 ++++++++++ target/arm/translate-sve.c | 2 ++ 4 files changed, 22 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_4(sve2_addhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_addhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_addhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_raddhnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_raddhnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_raddhnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_raddhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_raddhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_raddhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_5(sve2_match_ppzz_b, TCG_CALL_NO_RWG, i32, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve2_match_ppzz_h, TCG_CALL_NO_RWG, diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ UQRSHRNT 01000101 .. 1 ..... 00 1111 ..... ..... @rd_rn_tszimm_shr ADDHNB 01000101 .. 1 ..... 011 000 ..... ..... @rd_rn_rm ADDHNT 01000101 .. 1 ..... 011 001 ..... ..... @rd_rn_rm +RADDHNB 01000101 .. 1 ..... 011 010 ..... ..... @rd_rn_rm +RADDHNT 01000101 .. 1 ..... 011 011 ..... ..... @rd_rn_rm ### SVE2 Character Match diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ } #define DO_ADDHN(N, M, SH) ((N + M) >> SH) +#define DO_RADDHN(N, M, SH) ((N + M + ((__typeof(N))1 << (SH - 1))) >> SH) DO_BINOPNB(sve2_addhnb_h, uint16_t, uint8_t, 8, DO_ADDHN) DO_BINOPNB(sve2_addhnb_s, uint32_t, uint16_t, 16, DO_ADDHN) @@ -XXX,XX +XXX,XX @@ DO_BINOPNT(sve2_addhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_ADDHN) DO_BINOPNT(sve2_addhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_ADDHN) DO_BINOPNT(sve2_addhnt_d, uint64_t, uint32_t, 32, , H1_4, DO_ADDHN) +DO_BINOPNB(sve2_raddhnb_h, uint16_t, uint8_t, 8, DO_RADDHN) +DO_BINOPNB(sve2_raddhnb_s, uint32_t, uint16_t, 16, DO_RADDHN) +DO_BINOPNB(sve2_raddhnb_d, uint64_t, uint32_t, 32, DO_RADDHN) + +DO_BINOPNT(sve2_raddhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_RADDHN) +DO_BINOPNT(sve2_raddhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_RADDHN) +DO_BINOPNT(sve2_raddhnt_d, uint64_t, uint32_t, 32, , H1_4, DO_RADDHN) + +#undef DO_RADDHN #undef DO_ADDHN #undef DO_BINOPNB diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \ DO_SVE2_ZZZ_NARROW(ADDHNB, addhnb) DO_SVE2_ZZZ_NARROW(ADDHNT, addhnt) +DO_SVE2_ZZZ_NARROW(RADDHNB, raddhnb) +DO_SVE2_ZZZ_NARROW(RADDHNT, raddhnt) static bool do_sve2_ppzz_flags(DisasContext *s, arg_rprr_esz *a, gen_helper_gvec_flags_4 *fn) -- 2.20.1 From: Stephen Long <steplong@quicinc.com> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Stephen Long <steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-41-richard.henderson@linaro.org Message-Id: <20200417162231.10374-4-steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 8 ++++++++ target/arm/sve.decode | 2 ++ target/arm/sve_helper.c | 10 ++++++++++ target/arm/translate-sve.c | 3 +++ 4 files changed, 23 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_4(sve2_raddhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_raddhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_raddhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_subhnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_subhnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_subhnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_subhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_subhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_subhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_5(sve2_match_ppzz_b, TCG_CALL_NO_RWG, i32, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve2_match_ppzz_h, TCG_CALL_NO_RWG, diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ ADDHNB 01000101 .. 1 ..... 011 000 ..... ..... @rd_rn_rm ADDHNT 01000101 .. 1 ..... 011 001 ..... ..... @rd_rn_rm RADDHNB 01000101 .. 1 ..... 011 010 ..... ..... @rd_rn_rm RADDHNT 01000101 .. 1 ..... 011 011 ..... ..... @rd_rn_rm +SUBHNB 01000101 .. 1 ..... 011 100 ..... ..... @rd_rn_rm +SUBHNT 01000101 .. 1 ..... 011 101 ..... ..... @rd_rn_rm ### SVE2 Character Match diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ #define DO_ADDHN(N, M, SH) ((N + M) >> SH) #define DO_RADDHN(N, M, SH) ((N + M + ((__typeof(N))1 << (SH - 1))) >> SH) +#define DO_SUBHN(N, M, SH) ((N - M) >> SH) DO_BINOPNB(sve2_addhnb_h, uint16_t, uint8_t, 8, DO_ADDHN) DO_BINOPNB(sve2_addhnb_s, uint32_t, uint16_t, 16, DO_ADDHN) @@ -XXX,XX +XXX,XX @@ DO_BINOPNT(sve2_raddhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_RADDHN) DO_BINOPNT(sve2_raddhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_RADDHN) DO_BINOPNT(sve2_raddhnt_d, uint64_t, uint32_t, 32, , H1_4, DO_RADDHN) +DO_BINOPNB(sve2_subhnb_h, uint16_t, uint8_t, 8, DO_SUBHN) +DO_BINOPNB(sve2_subhnb_s, uint32_t, uint16_t, 16, DO_SUBHN) +DO_BINOPNB(sve2_subhnb_d, uint64_t, uint32_t, 32, DO_SUBHN) + +DO_BINOPNT(sve2_subhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_SUBHN) +DO_BINOPNT(sve2_subhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_SUBHN) +DO_BINOPNT(sve2_subhnt_d, uint64_t, uint32_t, 32, , H1_4, DO_SUBHN) + +#undef DO_SUBHN #undef DO_RADDHN #undef DO_ADDHN diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_ZZZ_NARROW(ADDHNT, addhnt) DO_SVE2_ZZZ_NARROW(RADDHNB, raddhnb) DO_SVE2_ZZZ_NARROW(RADDHNT, raddhnt) +DO_SVE2_ZZZ_NARROW(SUBHNB, subhnb) +DO_SVE2_ZZZ_NARROW(SUBHNT, subhnt) + static bool do_sve2_ppzz_flags(DisasContext *s, arg_rprr_esz *a, gen_helper_gvec_flags_4 *fn) { -- 2.20.1 From: Stephen Long <steplong@quicinc.com> This completes the section 'SVE2 integer add/subtract narrow high part' Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Stephen Long <steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-42-richard.henderson@linaro.org Message-Id: <20200417162231.10374-5-steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 8 ++++++++ target/arm/sve.decode | 2 ++ target/arm/sve_helper.c | 10 ++++++++++ target/arm/translate-sve.c | 2 ++ 4 files changed, 22 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_4(sve2_subhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_subhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_subhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_rsubhnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_rsubhnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_rsubhnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_rsubhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_rsubhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_rsubhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_5(sve2_match_ppzz_b, TCG_CALL_NO_RWG, i32, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve2_match_ppzz_h, TCG_CALL_NO_RWG, diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ RADDHNB 01000101 .. 1 ..... 011 010 ..... ..... @rd_rn_rm RADDHNT 01000101 .. 1 ..... 011 011 ..... ..... @rd_rn_rm SUBHNB 01000101 .. 1 ..... 011 100 ..... ..... @rd_rn_rm SUBHNT 01000101 .. 1 ..... 011 101 ..... ..... @rd_rn_rm +RSUBHNB 01000101 .. 1 ..... 011 110 ..... ..... @rd_rn_rm +RSUBHNT 01000101 .. 1 ..... 011 111 ..... ..... @rd_rn_rm ### SVE2 Character Match diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ #define DO_ADDHN(N, M, SH) ((N + M) >> SH) #define DO_RADDHN(N, M, SH) ((N + M + ((__typeof(N))1 << (SH - 1))) >> SH) #define DO_SUBHN(N, M, SH) ((N - M) >> SH) +#define DO_RSUBHN(N, M, SH) ((N - M + ((__typeof(N))1 << (SH - 1))) >> SH) DO_BINOPNB(sve2_addhnb_h, uint16_t, uint8_t, 8, DO_ADDHN) DO_BINOPNB(sve2_addhnb_s, uint32_t, uint16_t, 16, DO_ADDHN) @@ -XXX,XX +XXX,XX @@ DO_BINOPNT(sve2_subhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_SUBHN) DO_BINOPNT(sve2_subhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_SUBHN) DO_BINOPNT(sve2_subhnt_d, uint64_t, uint32_t, 32, , H1_4, DO_SUBHN) +DO_BINOPNB(sve2_rsubhnb_h, uint16_t, uint8_t, 8, DO_RSUBHN) +DO_BINOPNB(sve2_rsubhnb_s, uint32_t, uint16_t, 16, DO_RSUBHN) +DO_BINOPNB(sve2_rsubhnb_d, uint64_t, uint32_t, 32, DO_RSUBHN) + +DO_BINOPNT(sve2_rsubhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_RSUBHN) +DO_BINOPNT(sve2_rsubhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_RSUBHN) +DO_BINOPNT(sve2_rsubhnt_d, uint64_t, uint32_t, 32, , H1_4, DO_RSUBHN) + +#undef DO_RSUBHN #undef DO_SUBHN #undef DO_RADDHN #undef DO_ADDHN diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_ZZZ_NARROW(RADDHNT, raddhnt) DO_SVE2_ZZZ_NARROW(SUBHNB, subhnb) DO_SVE2_ZZZ_NARROW(SUBHNT, subhnt) +DO_SVE2_ZZZ_NARROW(RSUBHNB, rsubhnb) +DO_SVE2_ZZZ_NARROW(RSUBHNT, rsubhnt) static bool do_sve2_ppzz_flags(DisasContext *s, arg_rprr_esz *a, gen_helper_gvec_flags_4 *fn) -- 2.20.1 From: Stephen Long <steplong@quicinc.com> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Stephen Long <steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-43-richard.henderson@linaro.org Message-Id: <20200416173109.8856-1-steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 7 ++ target/arm/sve.decode | 6 ++ target/arm/sve_helper.c | 131 +++++++++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 19 ++++++ 4 files changed, 163 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_nmatch_ppzz_b, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_5(sve2_nmatch_ppzz_h, TCG_CALL_NO_RWG, i32, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_histcnt_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_histcnt_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_histseg, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG, diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ &rprrr_esz rn=%reg_movprfx @rdn_pg_rm_ra ........ esz:2 . ra:5 ... pg:3 rm:5 rd:5 \ &rprrr_esz rn=%reg_movprfx +@rd_pg_rn_rm ........ esz:2 . rm:5 ... pg:3 rn:5 rd:5 &rprr_esz # One register operand, with governing predicate, vector element size @rd_pg_rn ........ esz:2 ... ... ... pg:3 rn:5 rd:5 &rpr_esz @@ -XXX,XX +XXX,XX @@ RSUBHNT 01000101 .. 1 ..... 011 111 ..... ..... @rd_rn_rm MATCH 01000101 .. 1 ..... 100 ... ..... 0 .... @pd_pg_rn_rm NMATCH 01000101 .. 1 ..... 100 ... ..... 1 .... @pd_pg_rn_rm +### SVE2 Histogram Computation + +HISTCNT 01000101 .. 1 ..... 110 ... ..... ..... @rd_pg_rn_rm +HISTSEG 01000101 .. 1 ..... 101 000 ..... ..... @rd_rn_rm + ## SVE2 floating-point pairwise operations FADDP 01100100 .. 010 00 0 100 ... ..... ..... @rdn_pg_rm diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_PPZZ_MATCH(sve2_nmatch_ppzz_b, MO_8, true) DO_PPZZ_MATCH(sve2_nmatch_ppzz_h, MO_16, true) #undef DO_PPZZ_MATCH + +void HELPER(sve2_histcnt_s)(void *vd, void *vn, void *vm, void *vg, + uint32_t desc) +{ + ARMVectorReg scratch; + intptr_t i, j; + intptr_t opr_sz = simd_oprsz(desc); + uint32_t *d = vd, *n = vn, *m = vm; + uint8_t *pg = vg; + + if (d == n) { + n = memcpy(&scratch, n, opr_sz); + if (d == m) { + m = n; + } + } else if (d == m) { + m = memcpy(&scratch, m, opr_sz); + } + + for (i = 0; i < opr_sz; i += 4) { + uint64_t count = 0; + uint8_t pred; + + pred = pg[H1(i >> 3)] >> (i & 7); + if (pred & 1) { + uint32_t nn = n[H4(i >> 2)]; + + for (j = 0; j <= i; j += 4) { + pred = pg[H1(j >> 3)] >> (j & 7); + if ((pred & 1) && nn == m[H4(j >> 2)]) { + ++count; + } + } + } + d[H4(i >> 2)] = count; + } +} + +void HELPER(sve2_histcnt_d)(void *vd, void *vn, void *vm, void *vg, + uint32_t desc) +{ + ARMVectorReg scratch; + intptr_t i, j; + intptr_t opr_sz = simd_oprsz(desc); + uint64_t *d = vd, *n = vn, *m = vm; + uint8_t *pg = vg; + + if (d == n) { + n = memcpy(&scratch, n, opr_sz); + if (d == m) { + m = n; + } + } else if (d == m) { + m = memcpy(&scratch, m, opr_sz); + } + + for (i = 0; i < opr_sz / 8; ++i) { + uint64_t count = 0; + if (pg[H1(i)] & 1) { + uint64_t nn = n[i]; + for (j = 0; j <= i; ++j) { + if ((pg[H1(j)] & 1) && nn == m[j]) { + ++count; + } + } + } + d[i] = count; + } +} + +/* + * Returns the number of bytes in m0 and m1 that match n. + * Unlike do_match2 we don't just need true/false, we need an exact count. + * This requires two extra logical operations. + */ +static inline uint64_t do_histseg_cnt(uint8_t n, uint64_t m0, uint64_t m1) +{ + const uint64_t mask = dup_const(MO_8, 0x7f); + uint64_t cmp0, cmp1; + + cmp1 = dup_const(MO_8, n); + cmp0 = cmp1 ^ m0; + cmp1 = cmp1 ^ m1; + + /* + * 1: clear msb of each byte to avoid carry to next byte (& mask) + * 2: carry in to msb if byte != 0 (+ mask) + * 3: set msb if cmp has msb set (| cmp) + * 4: set ~msb to ignore them (| mask) + * We now have 0xff for byte != 0 or 0x7f for byte == 0. + * 5: invert, resulting in 0x80 if and only if byte == 0. + */ + cmp0 = ~(((cmp0 & mask) + mask) | cmp0 | mask); + cmp1 = ~(((cmp1 & mask) + mask) | cmp1 | mask); + + /* + * Combine the two compares in a way that the bits do + * not overlap, and so preserves the count of set bits. + * If the host has an efficient instruction for ctpop, + * then ctpop(x) + ctpop(y) has the same number of + * operations as ctpop(x | (y >> 1)). If the host does + * not have an efficient ctpop, then we only want to + * use it once. + */ + return ctpop64(cmp0 | (cmp1 >> 1)); +} + +void HELPER(sve2_histseg)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, j; + intptr_t opr_sz = simd_oprsz(desc); + + for (i = 0; i < opr_sz; i += 16) { + uint64_t n0 = *(uint64_t *)(vn + i); + uint64_t m0 = *(uint64_t *)(vm + i); + uint64_t n1 = *(uint64_t *)(vn + i + 8); + uint64_t m1 = *(uint64_t *)(vm + i + 8); + uint64_t out0 = 0; + uint64_t out1 = 0; + + for (j = 0; j < 64; j += 8) { + uint64_t cnt0 = do_histseg_cnt(n0 >> j, m0, m1); + uint64_t cnt1 = do_histseg_cnt(n1 >> j, m0, m1); + out0 |= cnt0 << j; + out1 |= cnt1 << j; + } + + *(uint64_t *)(vd + i) = out0; + *(uint64_t *)(vd + i + 8) = out1; + } +} diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \ DO_SVE2_PPZZ_MATCH(MATCH, match) DO_SVE2_PPZZ_MATCH(NMATCH, nmatch) +static bool trans_HISTCNT(DisasContext *s, arg_rprr_esz *a) +{ + static gen_helper_gvec_4 * const fns[2] = { + gen_helper_sve2_histcnt_s, gen_helper_sve2_histcnt_d + }; + if (a->esz < 2) { + return false; + } + return do_sve2_zpzz_ool(s, a, fns[a->esz - 2]); +} + +static bool trans_HISTSEG(DisasContext *s, arg_rrr_esz *a) +{ + if (a->esz != 0) { + return false; + } + return do_sve2_zzz_ool(s, a, gen_helper_sve2_histseg); +} + static bool do_sve2_zpzz_fp(DisasContext *s, arg_rprr_esz *a, gen_helper_gvec_4_ptr *fn) { -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> In addition, use the same vector generator interface for AdvSIMD. This fixes a bug in which the AdvSIMD insn failed to clear the high bits of the SVE register. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-44-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 4 ++ target/arm/helper.h | 2 + target/arm/translate-a64.h | 3 ++ target/arm/sve.decode | 4 ++ target/arm/sve_helper.c | 39 ++++++++++++++ target/arm/translate-a64.c | 25 ++------- target/arm/translate-sve.c | 104 +++++++++++++++++++++++++++++++++++++ target/arm/vec_helper.c | 12 +++++ 8 files changed, 172 insertions(+), 21 deletions(-) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_histcnt_d, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_4(sve2_histseg, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_xar_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_xar_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_xar_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG, diff --git a/target/arm/helper.h b/target/arm/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.h +++ b/target/arm/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(neon_sqrdmulh_h, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_5(neon_sqrdmulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(gvec_xar_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + #ifdef TARGET_AARCH64 #include "helper-a64.h" #include "helper-sve.h" diff --git a/target/arm/translate-a64.h b/target/arm/translate-a64.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.h +++ b/target/arm/translate-a64.h @@ -XXX,XX +XXX,XX @@ bool disas_sve(DisasContext *, uint32_t); void gen_gvec_rax1(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs, uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz); +void gen_gvec_xar(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs, + uint32_t rm_ofs, int64_t shift, + uint32_t opr_sz, uint32_t max_sz); #endif /* TARGET_ARM_TRANSLATE_A64_H */ diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ &rr_dbm rd rn dbm &rrri rd rn rm imm &rri_esz rd rn imm esz +&rrri_esz rd rn rm imm esz &rrr_esz rd rn rm esz &rpr_esz rd pg rn esz &rpr_s rd pg rn s @@ -XXX,XX +XXX,XX @@ ORR_zzz 00000100 01 1 ..... 001 100 ..... ..... @rd_rn_rm_e0 EOR_zzz 00000100 10 1 ..... 001 100 ..... ..... @rd_rn_rm_e0 BIC_zzz 00000100 11 1 ..... 001 100 ..... ..... @rd_rn_rm_e0 +XAR 00000100 .. 1 ..... 001 101 rm:5 rd:5 &rrri_esz \ + rn=%reg_movprfx esz=%tszimm16_esz imm=%tszimm16_shr + # SVE2 bitwise ternary operations EOR3 00000100 00 1 ..... 001 110 ..... ..... @rdn_ra_rm_e0 BSL 00000100 00 1 ..... 001 111 ..... ..... @rdn_ra_rm_e0 diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(sve2_histseg)(void *vd, void *vn, void *vm, uint32_t desc) *(uint64_t *)(vd + i + 8) = out1; } } + +void HELPER(sve2_xar_b)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc) / 8; + int shr = simd_data(desc); + int shl = 8 - shr; + uint64_t mask = dup_const(MO_8, 0xff >> shr); + uint64_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz; ++i) { + uint64_t t = n[i] ^ m[i]; + d[i] = ((t >> shr) & mask) | ((t << shl) & ~mask); + } +} + +void HELPER(sve2_xar_h)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc) / 8; + int shr = simd_data(desc); + int shl = 16 - shr; + uint64_t mask = dup_const(MO_16, 0xffff >> shr); + uint64_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz; ++i) { + uint64_t t = n[i] ^ m[i]; + d[i] = ((t >> shr) & mask) | ((t << shl) & ~mask); + } +} + +void HELPER(sve2_xar_s)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc) / 4; + int shr = simd_data(desc); + uint32_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz; ++i) { + d[i] = ror32(n[i] ^ m[i], shr); + } +} diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void disas_crypto_xar(DisasContext *s, uint32_t insn) int imm6 = extract32(insn, 10, 6); int rn = extract32(insn, 5, 5); int rd = extract32(insn, 0, 5); - TCGv_i64 tcg_op1, tcg_op2, tcg_res[2]; - int pass; if (!dc_isar_feature(aa64_sha3, s)) { unallocated_encoding(s); @@ -XXX,XX +XXX,XX @@ static void disas_crypto_xar(DisasContext *s, uint32_t insn) return; } - tcg_op1 = tcg_temp_new_i64(); - tcg_op2 = tcg_temp_new_i64(); - tcg_res[0] = tcg_temp_new_i64(); - tcg_res[1] = tcg_temp_new_i64(); - - for (pass = 0; pass < 2; pass++) { - read_vec_element(s, tcg_op1, rn, pass, MO_64); - read_vec_element(s, tcg_op2, rm, pass, MO_64); - - tcg_gen_xor_i64(tcg_res[pass], tcg_op1, tcg_op2); - tcg_gen_rotri_i64(tcg_res[pass], tcg_res[pass], imm6); - } - write_vec_element(s, tcg_res[0], rd, 0, MO_64); - write_vec_element(s, tcg_res[1], rd, 1, MO_64); - - tcg_temp_free_i64(tcg_op1); - tcg_temp_free_i64(tcg_op2); - tcg_temp_free_i64(tcg_res[0]); - tcg_temp_free_i64(tcg_res[1]); + gen_gvec_xar(MO_64, vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + vec_full_reg_offset(s, rm), imm6, 16, + vec_full_reg_size(s)); } /* Crypto three-reg imm2 diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_BIC_zzz(DisasContext *s, arg_rrr_esz *a) return do_zzz_fn(s, a, tcg_gen_gvec_andc); } +static void gen_xar8_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, int64_t sh) +{ + TCGv_i64 t = tcg_temp_new_i64(); + uint64_t mask = dup_const(MO_8, 0xff >> sh); + + tcg_gen_xor_i64(t, n, m); + tcg_gen_shri_i64(d, t, sh); + tcg_gen_shli_i64(t, t, 8 - sh); + tcg_gen_andi_i64(d, d, mask); + tcg_gen_andi_i64(t, t, ~mask); + tcg_gen_or_i64(d, d, t); + tcg_temp_free_i64(t); +} + +static void gen_xar16_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, int64_t sh) +{ + TCGv_i64 t = tcg_temp_new_i64(); + uint64_t mask = dup_const(MO_16, 0xffff >> sh); + + tcg_gen_xor_i64(t, n, m); + tcg_gen_shri_i64(d, t, sh); + tcg_gen_shli_i64(t, t, 16 - sh); + tcg_gen_andi_i64(d, d, mask); + tcg_gen_andi_i64(t, t, ~mask); + tcg_gen_or_i64(d, d, t); + tcg_temp_free_i64(t); +} + +static void gen_xar_i32(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, int32_t sh) +{ + tcg_gen_xor_i32(d, n, m); + tcg_gen_rotri_i32(d, d, sh); +} + +static void gen_xar_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, int64_t sh) +{ + tcg_gen_xor_i64(d, n, m); + tcg_gen_rotri_i64(d, d, sh); +} + +static void gen_xar_vec(unsigned vece, TCGv_vec d, TCGv_vec n, + TCGv_vec m, int64_t sh) +{ + tcg_gen_xor_vec(vece, d, n, m); + tcg_gen_rotri_vec(vece, d, d, sh); +} + +void gen_gvec_xar(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs, + uint32_t rm_ofs, int64_t shift, + uint32_t opr_sz, uint32_t max_sz) +{ + static const TCGOpcode vecop[] = { INDEX_op_rotli_vec, 0 }; + static const GVecGen3i ops[4] = { + { .fni8 = gen_xar8_i64, + .fniv = gen_xar_vec, + .fno = gen_helper_sve2_xar_b, + .opt_opc = vecop, + .vece = MO_8 }, + { .fni8 = gen_xar16_i64, + .fniv = gen_xar_vec, + .fno = gen_helper_sve2_xar_h, + .opt_opc = vecop, + .vece = MO_16 }, + { .fni4 = gen_xar_i32, + .fniv = gen_xar_vec, + .fno = gen_helper_sve2_xar_s, + .opt_opc = vecop, + .vece = MO_32 }, + { .fni8 = gen_xar_i64, + .fniv = gen_xar_vec, + .fno = gen_helper_gvec_xar_d, + .opt_opc = vecop, + .vece = MO_64 } + }; + int esize = 8 << vece; + + /* The SVE2 range is 1 .. esize; the AdvSIMD range is 0 .. esize-1. */ + tcg_debug_assert(shift >= 0); + tcg_debug_assert(shift <= esize); + shift &= esize - 1; + + if (shift == 0) { + /* xar with no rotate devolves to xor. */ + tcg_gen_gvec_xor(vece, rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz); + } else { + tcg_gen_gvec_3i(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, + shift, &ops[vece]); + } +} + +static bool trans_XAR(DisasContext *s, arg_rrri_esz *a) +{ + if (a->esz < 0 || !dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (sve_access_check(s)) { + unsigned vsz = vec_full_reg_size(s); + gen_gvec_xar(a->esz, vec_full_reg_offset(s, a->rd), + vec_full_reg_offset(s, a->rn), + vec_full_reg_offset(s, a->rm), a->imm, vsz, vsz); + } + return true; +} + static bool do_sve2_zzzz_fn(DisasContext *s, arg_rrrr_esz *a, GVecGen4Fn *fn) { if (!dc_isar_feature(aa64_sve2, s)) { diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_umulh_d)(void *vd, void *vn, void *vm, uint32_t desc) } clear_tail(d, opr_sz, simd_maxsz(desc)); } + +void HELPER(gvec_xar_d)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc) / 8; + int shr = simd_data(desc); + uint64_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz; ++i) { + d[i] = ror64(n[i] ^ m[i], shr); + } + clear_tail(d, opr_sz * 8, simd_maxsz(desc)); +} -- 2.20.1 From: Stephen Long <steplong@quicinc.com> Add decoding logic for SVE2 64-bit/32-bit scatter non-temporal store insns. 64-bit * STNT1B (vector plus scalar) * STNT1H (vector plus scalar) * STNT1W (vector plus scalar) * STNT1D (vector plus scalar) 32-bit * STNT1B (vector plus scalar) * STNT1H (vector plus scalar) * STNT1W (vector plus scalar) Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Stephen Long <steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-45-richard.henderson@linaro.org Message-Id: <20200422141553.8037-1-steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/sve.decode | 10 ++++++++++ target/arm/translate-sve.c | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ UMLSLT_zzzw 01000100 .. 0 ..... 010 111 ..... ..... @rda_rn_rm CMLA_zzzz 01000100 esz:2 0 rm:5 0010 rot:2 rn:5 rd:5 ra=%reg_movprfx SQRDCMLAH_zzzz 01000100 esz:2 0 rm:5 0011 rot:2 rn:5 rd:5 ra=%reg_movprfx + +### SVE2 Memory Store Group + +# SVE2 64-bit scatter non-temporal store (vector plus scalar) +STNT1_zprz 1110010 .. 00 ..... 001 ... ..... ..... \ + @rprr_scatter_store xs=2 esz=3 scale=0 + +# SVE2 32-bit scatter non-temporal store (vector plus scalar) +STNT1_zprz 1110010 .. 10 ..... 001 ... ..... ..... \ + @rprr_scatter_store xs=0 esz=2 scale=0 diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_ST1_zpiz(DisasContext *s, arg_ST1_zpiz *a) return true; } +static bool trans_STNT1_zprz(DisasContext *s, arg_ST1_zprz *a) +{ + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + return trans_ST1_zprz(s, a); +} + /* * Prefetches */ -- 2.20.1 From: Stephen Long <steplong@quicinc.com> Add decoding logic for SVE2 64-bit/32-bit gather non-temporal load insns. 64-bit * LDNT1SB * LDNT1B (vector plus scalar) * LDNT1SH * LDNT1H (vector plus scalar) * LDNT1SW * LDNT1W (vector plus scalar) * LDNT1D (vector plus scalar) 32-bit * LDNT1SB * LDNT1B (vector plus scalar) * LDNT1SH * LDNT1H (vector plus scalar) * LDNT1W (vector plus scalar) Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Stephen Long <steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-46-richard.henderson@linaro.org Message-Id: <20200422152343.12493-1-steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/sve.decode | 11 +++++++++++ target/arm/translate-sve.c | 8 ++++++++ 2 files changed, 19 insertions(+) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ UMLSLT_zzzw 01000100 .. 0 ..... 010 111 ..... ..... @rda_rn_rm CMLA_zzzz 01000100 esz:2 0 rm:5 0010 rot:2 rn:5 rd:5 ra=%reg_movprfx SQRDCMLAH_zzzz 01000100 esz:2 0 rm:5 0011 rot:2 rn:5 rd:5 ra=%reg_movprfx +### SVE2 Memory Gather Load Group + +# SVE2 64-bit gather non-temporal load +# (scalar plus unpacked 32-bit unscaled offsets) +LDNT1_zprz 1100010 msz:2 00 rm:5 1 u:1 0 pg:3 rn:5 rd:5 \ + &rprr_gather_load xs=0 esz=3 scale=0 ff=0 + +# SVE2 32-bit gather non-temporal load (scalar plus 32-bit unscaled offsets) +LDNT1_zprz 1000010 msz:2 00 rm:5 10 u:1 pg:3 rn:5 rd:5 \ + &rprr_gather_load xs=0 esz=2 scale=0 ff=0 + ### SVE2 Memory Store Group # SVE2 64-bit scatter non-temporal store (vector plus scalar) diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a) return true; } +static bool trans_LDNT1_zprz(DisasContext *s, arg_LD1_zprz *a) +{ + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + return trans_LD1_zprz(s, a); +} + /* Indexed by [mte][be][xs][msz]. */ static gen_helper_gvec_mem_scatter * const scatter_store_fn32[2][2][2][3] = { { /* MTE Inactive */ -- 2.20.1 From: Stephen Long <steplong@quicinc.com> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Stephen Long <steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-47-richard.henderson@linaro.org Message-Id: <20200422165503.13511-1-steplong@quicinc.com> [rth: Fix indexing in helpers, expand macro to straight functions.] Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/cpu.h | 10 ++++++ target/arm/helper-sve.h | 3 ++ target/arm/sve.decode | 4 +++ target/arm/sve_helper.c | 74 ++++++++++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 34 ++++++++++++++++++ 5 files changed, 125 insertions(+) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -XXX,XX +XXX,XX @@ static inline bool isar_feature_aa64_sve2_bitperm(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, BITPERM) != 0; } +static inline bool isar_feature_aa64_sve_f32mm(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, F32MM) != 0; +} + +static inline bool isar_feature_aa64_sve_f64mm(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, F64MM) != 0; +} + /* * Feature tests for "does this exist in either 32-bit or 64-bit?" */ diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_6(fmmla_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(fmmla_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ UMLSLT_zzzw 01000100 .. 0 ..... 010 111 ..... ..... @rda_rn_rm CMLA_zzzz 01000100 esz:2 0 rm:5 0010 rot:2 rn:5 rd:5 ra=%reg_movprfx SQRDCMLAH_zzzz 01000100 esz:2 0 rm:5 0011 rot:2 rn:5 rd:5 ra=%reg_movprfx +### SVE2 floating point matrix multiply accumulate + +FMMLA 01100100 .. 1 ..... 111001 ..... ..... @rda_rn_rm + ### SVE2 Memory Gather Load Group # SVE2 64-bit gather non-temporal load diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(sve2_xar_s)(void *vd, void *vn, void *vm, uint32_t desc) d[i] = ror32(n[i] ^ m[i], shr); } } + +void HELPER(fmmla_s)(void *vd, void *vn, void *vm, void *va, + void *status, uint32_t desc) +{ + intptr_t s, opr_sz = simd_oprsz(desc) / (sizeof(float32) * 4); + + for (s = 0; s < opr_sz; ++s) { + float32 *n = vn + s * sizeof(float32) * 4; + float32 *m = vm + s * sizeof(float32) * 4; + float32 *a = va + s * sizeof(float32) * 4; + float32 *d = vd + s * sizeof(float32) * 4; + float32 n00 = n[H4(0)], n01 = n[H4(1)]; + float32 n10 = n[H4(2)], n11 = n[H4(3)]; + float32 m00 = m[H4(0)], m01 = m[H4(1)]; + float32 m10 = m[H4(2)], m11 = m[H4(3)]; + float32 p0, p1; + + /* i = 0, j = 0 */ + p0 = float32_mul(n00, m00, status); + p1 = float32_mul(n01, m01, status); + d[H4(0)] = float32_add(a[H4(0)], float32_add(p0, p1, status), status); + + /* i = 0, j = 1 */ + p0 = float32_mul(n00, m10, status); + p1 = float32_mul(n01, m11, status); + d[H4(1)] = float32_add(a[H4(1)], float32_add(p0, p1, status), status); + + /* i = 1, j = 0 */ + p0 = float32_mul(n10, m00, status); + p1 = float32_mul(n11, m01, status); + d[H4(2)] = float32_add(a[H4(2)], float32_add(p0, p1, status), status); + + /* i = 1, j = 1 */ + p0 = float32_mul(n10, m10, status); + p1 = float32_mul(n11, m11, status); + d[H4(3)] = float32_add(a[H4(3)], float32_add(p0, p1, status), status); + } +} + +void HELPER(fmmla_d)(void *vd, void *vn, void *vm, void *va, + void *status, uint32_t desc) +{ + intptr_t s, opr_sz = simd_oprsz(desc) / (sizeof(float64) * 4); + + for (s = 0; s < opr_sz; ++s) { + float64 *n = vn + s * sizeof(float64) * 4; + float64 *m = vm + s * sizeof(float64) * 4; + float64 *a = va + s * sizeof(float64) * 4; + float64 *d = vd + s * sizeof(float64) * 4; + float64 n00 = n[0], n01 = n[1], n10 = n[2], n11 = n[3]; + float64 m00 = m[0], m01 = m[1], m10 = m[2], m11 = m[3]; + float64 p0, p1; + + /* i = 0, j = 0 */ + p0 = float64_mul(n00, m00, status); + p1 = float64_mul(n01, m01, status); + d[0] = float64_add(a[0], float64_add(p0, p1, status), status); + + /* i = 0, j = 1 */ + p0 = float64_mul(n00, m10, status); + p1 = float64_mul(n01, m11, status); + d[1] = float64_add(a[1], float64_add(p0, p1, status), status); + + /* i = 1, j = 0 */ + p0 = float64_mul(n10, m00, status); + p1 = float64_mul(n11, m01, status); + d[2] = float64_add(a[2], float64_add(p0, p1, status), status); + + /* i = 1, j = 1 */ + p0 = float64_mul(n10, m10, status); + p1 = float64_mul(n11, m11, status); + d[3] = float64_add(a[3], float64_add(p0, p1, status), status); + } +} diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_ZPZZ_FP(FMINP, fminp) * SVE Integer Multiply-Add (unpredicated) */ +static bool trans_FMMLA(DisasContext *s, arg_rrrr_esz *a) +{ + gen_helper_gvec_4_ptr *fn; + + switch (a->esz) { + case MO_32: + if (!dc_isar_feature(aa64_sve_f32mm, s)) { + return false; + } + fn = gen_helper_fmmla_s; + break; + case MO_64: + if (!dc_isar_feature(aa64_sve_f64mm, s)) { + return false; + } + fn = gen_helper_fmmla_d; + break; + default: + return false; + } + + if (sve_access_check(s)) { + unsigned vsz = vec_full_reg_size(s); + TCGv_ptr status = fpstatus_ptr(FPST_FPCR); + tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, a->rd), + vec_full_reg_offset(s, a->rn), + vec_full_reg_offset(s, a->rm), + vec_full_reg_offset(s, a->ra), + status, vsz, vsz, 0, fn); + tcg_temp_free_ptr(status); + } + return true; +} + static bool do_sqdmlal_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel1, bool sel2) { -- 2.20.1 From: Stephen Long <steplong@quicinc.com> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Stephen Long <steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-48-richard.henderson@linaro.org Message-Id: <20200423180347.9403-1-steplong@quicinc.com> [rth: Rename the trans_* functions to *_sve2.] Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/sve.decode | 11 +++++++++-- target/arm/translate-sve.c | 35 ++++++++++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ CPY_z_i 00000101 .. 01 .... 00 . ........ ..... @rdn_pg4 imm=%sh8_i8s ### SVE Permute - Extract Group -# SVE extract vector (immediate offset) +# SVE extract vector (destructive) EXT 00000101 001 ..... 000 ... rm:5 rd:5 \ &rrri rn=%reg_movprfx imm=%imm8_16_10 +# SVE2 extract vector (constructive) +EXT_sve2 00000101 011 ..... 000 ... rn:5 rd:5 \ + &rri imm=%imm8_16_10 + ### SVE Permute - Unpredicated Group # SVE broadcast general register @@ -XXX,XX +XXX,XX @@ REVH 00000101 .. 1001 01 100 ... ..... ..... @rd_pg_rn REVW 00000101 .. 1001 10 100 ... ..... ..... @rd_pg_rn RBIT 00000101 .. 1001 11 100 ... ..... ..... @rd_pg_rn -# SVE vector splice (predicated) +# SVE vector splice (predicated, destructive) SPLICE 00000101 .. 101 100 100 ... ..... ..... @rdn_pg_rm +# SVE2 vector splice (predicated, constructive) +SPLICE_sve2 00000101 .. 101 101 100 ... ..... ..... @rd_pg_rn + ### SVE Select Vectors Group # SVE select vector elements (predicated) diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_CPY_z_i(DisasContext *s, arg_CPY_z_i *a) *** SVE Permute Extract Group */ -static bool trans_EXT(DisasContext *s, arg_EXT *a) +static bool do_EXT(DisasContext *s, int rd, int rn, int rm, int imm) { if (!sve_access_check(s)) { return true; } unsigned vsz = vec_full_reg_size(s); - unsigned n_ofs = a->imm >= vsz ? 0 : a->imm; + unsigned n_ofs = imm >= vsz ? 0 : imm; unsigned n_siz = vsz - n_ofs; - unsigned d = vec_full_reg_offset(s, a->rd); - unsigned n = vec_full_reg_offset(s, a->rn); - unsigned m = vec_full_reg_offset(s, a->rm); + unsigned d = vec_full_reg_offset(s, rd); + unsigned n = vec_full_reg_offset(s, rn); + unsigned m = vec_full_reg_offset(s, rm); /* Use host vector move insns if we have appropriate sizes * and no unfortunate overlap. @@ -XXX,XX +XXX,XX @@ static bool trans_EXT(DisasContext *s, arg_EXT *a) return true; } +static bool trans_EXT(DisasContext *s, arg_EXT *a) +{ + return do_EXT(s, a->rd, a->rn, a->rm, a->imm); +} + +static bool trans_EXT_sve2(DisasContext *s, arg_rri *a) +{ + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + return do_EXT(s, a->rd, a->rn, (a->rn + 1) % 32, a->imm); +} + /* *** SVE Permute - Unpredicated Group */ @@ -XXX,XX +XXX,XX @@ static bool trans_SPLICE(DisasContext *s, arg_rprr_esz *a) return true; } +static bool trans_SPLICE_sve2(DisasContext *s, arg_rpr_esz *a) +{ + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (sve_access_check(s)) { + gen_gvec_ool_zzzp(s, gen_helper_sve_splice, + a->rd, a->rn, (a->rn + 1) % 32, a->pg, a->esz); + } + return true; +} + /* *** SVE Integer Compare - Vectors Group */ -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> The signed dot product routines produce a signed result. Since we use -fwrapv, there is no functional change. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-49-richard.henderson@linaro.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/vec_helper.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(sve2_sqrdmlsh_d)(void *vd, void *vn, void *vm, void HELPER(gvec_sdot_b)(void *vd, void *vn, void *vm, uint32_t desc) { intptr_t i, opr_sz = simd_oprsz(desc); - uint32_t *d = vd; + int32_t *d = vd; int8_t *n = vn, *m = vm; for (i = 0; i < opr_sz / 4; ++i) { @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_udot_b)(void *vd, void *vn, void *vm, uint32_t desc) void HELPER(gvec_sdot_h)(void *vd, void *vn, void *vm, uint32_t desc) { intptr_t i, opr_sz = simd_oprsz(desc); - uint64_t *d = vd; + int64_t *d = vd; int16_t *n = vn, *m = vm; for (i = 0; i < opr_sz / 8; ++i) { @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc) { intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4; intptr_t index = simd_data(desc); - uint32_t *d = vd; + int32_t *d = vd; int8_t *n = vn; int8_t *m_indexed = (int8_t *)vm + H4(index) * 4; @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc) { intptr_t i, opr_sz = simd_oprsz(desc), opr_sz_8 = opr_sz / 8; intptr_t index = simd_data(desc); - uint64_t *d = vd; + int64_t *d = vd; int16_t *n = vn; int16_t *m_indexed = (int16_t *)vm + index * 4; -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> For SVE, we potentially have a 4th argument coming from the movprfx instruction. Currently we do not optimize movprfx, so the problem is not visible. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-50-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper.h | 20 +++--- target/arm/sve.decode | 7 ++- target/arm/translate-a64.c | 15 ++++- target/arm/translate-neon.c | 10 +-- target/arm/translate-sve.c | 13 ++-- target/arm/vec_helper.c | 120 ++++++++++++++++++++---------------- 6 files changed, 109 insertions(+), 76 deletions(-) diff --git a/target/arm/helper.h b/target/arm/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.h +++ b/target/arm/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_sqrdmlah_d, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) -DEF_HELPER_FLAGS_4(gvec_sdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) -DEF_HELPER_FLAGS_4(gvec_udot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) -DEF_HELPER_FLAGS_4(gvec_sdot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) -DEF_HELPER_FLAGS_4(gvec_udot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_sdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_udot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_sdot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_udot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) -DEF_HELPER_FLAGS_4(gvec_sdot_idx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) -DEF_HELPER_FLAGS_4(gvec_udot_idx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) -DEF_HELPER_FLAGS_4(gvec_sdot_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) -DEF_HELPER_FLAGS_4(gvec_udot_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_sdot_idx_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_udot_idx_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_sdot_idx_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_udot_idx_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(gvec_fcaddh, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ UMIN_zzi 00100101 .. 101 011 110 ........ ..... @rdn_i8u MUL_zzi 00100101 .. 110 000 110 ........ ..... @rdn_i8s # SVE integer dot product (unpredicated) -DOT_zzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 ra=%reg_movprfx +DOT_zzzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 \ + ra=%reg_movprfx # SVE integer dot product (indexed) -DOT_zzx 01000100 101 index:2 rm:3 00000 u:1 rn:5 rd:5 \ +DOT_zzxw 01000100 101 index:2 rm:3 00000 u:1 rn:5 rd:5 \ sz=0 ra=%reg_movprfx -DOT_zzx 01000100 111 index:1 rm:4 00000 u:1 rn:5 rd:5 \ +DOT_zzxw 01000100 111 index:1 rm:4 00000 u:1 rn:5 rd:5 \ sz=1 ra=%reg_movprfx # SVE floating-point complex add (predicated) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void gen_gvec_op3_qc(DisasContext *s, bool is_q, int rd, int rn, tcg_temp_free_ptr(qc_ptr); } +/* Expand a 4-operand operation using an out-of-line helper. */ +static void gen_gvec_op4_ool(DisasContext *s, bool is_q, int rd, int rn, + int rm, int ra, int data, gen_helper_gvec_4 *fn) +{ + tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + vec_full_reg_offset(s, rm), + vec_full_reg_offset(s, ra), + is_q ? 16 : 8, vec_full_reg_size(s), data, fn); +} + /* Set ZF and NF based on a 64 bit result. This is alas fiddlier * than the 32 bit equivalent. */ @@ -XXX,XX +XXX,XX @@ static void disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn) return; case 0x2: /* SDOT / UDOT */ - gen_gvec_op3_ool(s, is_q, rd, rn, rm, 0, + gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0, u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b); return; @@ -XXX,XX +XXX,XX @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn) switch (16 * u + opcode) { case 0x0e: /* SDOT */ case 0x1e: /* UDOT */ - gen_gvec_op3_ool(s, is_q, rd, rn, rm, index, + gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index, u ? gen_helper_gvec_udot_idx_b : gen_helper_gvec_sdot_idx_b); return; diff --git a/target/arm/translate-neon.c b/target/arm/translate-neon.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-neon.c +++ b/target/arm/translate-neon.c @@ -XXX,XX +XXX,XX @@ static bool trans_VCADD(DisasContext *s, arg_VCADD *a) static bool trans_VDOT(DisasContext *s, arg_VDOT *a) { int opr_sz; - gen_helper_gvec_3 *fn_gvec; + gen_helper_gvec_4 *fn_gvec; if (!dc_isar_feature(aa32_dp, s)) { return false; @@ -XXX,XX +XXX,XX @@ static bool trans_VDOT(DisasContext *s, arg_VDOT *a) opr_sz = (1 + a->q) * 8; fn_gvec = a->u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b; - tcg_gen_gvec_3_ool(vfp_reg_offset(1, a->vd), + tcg_gen_gvec_4_ool(vfp_reg_offset(1, a->vd), vfp_reg_offset(1, a->vn), vfp_reg_offset(1, a->vm), + vfp_reg_offset(1, a->vd), opr_sz, opr_sz, 0, fn_gvec); return true; } @@ -XXX,XX +XXX,XX @@ static bool trans_VCMLA_scalar(DisasContext *s, arg_VCMLA_scalar *a) static bool trans_VDOT_scalar(DisasContext *s, arg_VDOT_scalar *a) { - gen_helper_gvec_3 *fn_gvec; + gen_helper_gvec_4 *fn_gvec; int opr_sz; TCGv_ptr fpst; @@ -XXX,XX +XXX,XX @@ static bool trans_VDOT_scalar(DisasContext *s, arg_VDOT_scalar *a) fn_gvec = a->u ? gen_helper_gvec_udot_idx_b : gen_helper_gvec_sdot_idx_b; opr_sz = (1 + a->q) * 8; fpst = fpstatus_ptr(FPST_STD); - tcg_gen_gvec_3_ool(vfp_reg_offset(1, a->vd), + tcg_gen_gvec_4_ool(vfp_reg_offset(1, a->vd), vfp_reg_offset(1, a->vn), vfp_reg_offset(1, a->rm), + vfp_reg_offset(1, a->vd), opr_sz, opr_sz, a->index, fn_gvec); tcg_temp_free_ptr(fpst); return true; diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_ZZI(UMIN, umin) #undef DO_ZZI -static bool trans_DOT_zzz(DisasContext *s, arg_DOT_zzz *a) +static bool trans_DOT_zzzz(DisasContext *s, arg_DOT_zzzz *a) { - static gen_helper_gvec_3 * const fns[2][2] = { + static gen_helper_gvec_4 * const fns[2][2] = { { gen_helper_gvec_sdot_b, gen_helper_gvec_sdot_h }, { gen_helper_gvec_udot_b, gen_helper_gvec_udot_h } }; if (sve_access_check(s)) { - gen_gvec_ool_zzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm, 0); + gen_gvec_ool_zzzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm, a->ra, 0); } return true; } -static bool trans_DOT_zzx(DisasContext *s, arg_DOT_zzx *a) +static bool trans_DOT_zzxw(DisasContext *s, arg_DOT_zzxw *a) { - static gen_helper_gvec_3 * const fns[2][2] = { + static gen_helper_gvec_4 * const fns[2][2] = { { gen_helper_gvec_sdot_idx_b, gen_helper_gvec_sdot_idx_h }, { gen_helper_gvec_udot_idx_b, gen_helper_gvec_udot_idx_h } }; if (sve_access_check(s)) { - gen_gvec_ool_zzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm, a->index); + gen_gvec_ool_zzzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm, + a->ra, a->index); } return true; } diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(sve2_sqrdmlsh_d)(void *vd, void *vn, void *vm, * All elements are treated equally, no matter where they are. */ -void HELPER(gvec_sdot_b)(void *vd, void *vn, void *vm, uint32_t desc) +void HELPER(gvec_sdot_b)(void *vd, void *vn, void *vm, void *va, uint32_t desc) { intptr_t i, opr_sz = simd_oprsz(desc); - int32_t *d = vd; + int32_t *d = vd, *a = va; int8_t *n = vn, *m = vm; for (i = 0; i < opr_sz / 4; ++i) { - d[i] += n[i * 4 + 0] * m[i * 4 + 0] - + n[i * 4 + 1] * m[i * 4 + 1] - + n[i * 4 + 2] * m[i * 4 + 2] - + n[i * 4 + 3] * m[i * 4 + 3]; + d[i] = (a[i] + + n[i * 4 + 0] * m[i * 4 + 0] + + n[i * 4 + 1] * m[i * 4 + 1] + + n[i * 4 + 2] * m[i * 4 + 2] + + n[i * 4 + 3] * m[i * 4 + 3]); } clear_tail(d, opr_sz, simd_maxsz(desc)); } -void HELPER(gvec_udot_b)(void *vd, void *vn, void *vm, uint32_t desc) +void HELPER(gvec_udot_b)(void *vd, void *vn, void *vm, void *va, uint32_t desc) { intptr_t i, opr_sz = simd_oprsz(desc); - uint32_t *d = vd; + uint32_t *d = vd, *a = va; uint8_t *n = vn, *m = vm; for (i = 0; i < opr_sz / 4; ++i) { - d[i] += n[i * 4 + 0] * m[i * 4 + 0] - + n[i * 4 + 1] * m[i * 4 + 1] - + n[i * 4 + 2] * m[i * 4 + 2] - + n[i * 4 + 3] * m[i * 4 + 3]; + d[i] = (a[i] + + n[i * 4 + 0] * m[i * 4 + 0] + + n[i * 4 + 1] * m[i * 4 + 1] + + n[i * 4 + 2] * m[i * 4 + 2] + + n[i * 4 + 3] * m[i * 4 + 3]); } clear_tail(d, opr_sz, simd_maxsz(desc)); } -void HELPER(gvec_sdot_h)(void *vd, void *vn, void *vm, uint32_t desc) +void HELPER(gvec_sdot_h)(void *vd, void *vn, void *vm, void *va, uint32_t desc) { intptr_t i, opr_sz = simd_oprsz(desc); - int64_t *d = vd; + int64_t *d = vd, *a = va; int16_t *n = vn, *m = vm; for (i = 0; i < opr_sz / 8; ++i) { - d[i] += (int64_t)n[i * 4 + 0] * m[i * 4 + 0] - + (int64_t)n[i * 4 + 1] * m[i * 4 + 1] - + (int64_t)n[i * 4 + 2] * m[i * 4 + 2] - + (int64_t)n[i * 4 + 3] * m[i * 4 + 3]; + d[i] = (a[i] + + (int64_t)n[i * 4 + 0] * m[i * 4 + 0] + + (int64_t)n[i * 4 + 1] * m[i * 4 + 1] + + (int64_t)n[i * 4 + 2] * m[i * 4 + 2] + + (int64_t)n[i * 4 + 3] * m[i * 4 + 3]); } clear_tail(d, opr_sz, simd_maxsz(desc)); } -void HELPER(gvec_udot_h)(void *vd, void *vn, void *vm, uint32_t desc) +void HELPER(gvec_udot_h)(void *vd, void *vn, void *vm, void *va, uint32_t desc) { intptr_t i, opr_sz = simd_oprsz(desc); - uint64_t *d = vd; + uint64_t *d = vd, *a = va; uint16_t *n = vn, *m = vm; for (i = 0; i < opr_sz / 8; ++i) { - d[i] += (uint64_t)n[i * 4 + 0] * m[i * 4 + 0] - + (uint64_t)n[i * 4 + 1] * m[i * 4 + 1] - + (uint64_t)n[i * 4 + 2] * m[i * 4 + 2] - + (uint64_t)n[i * 4 + 3] * m[i * 4 + 3]; + d[i] = (a[i] + + (uint64_t)n[i * 4 + 0] * m[i * 4 + 0] + + (uint64_t)n[i * 4 + 1] * m[i * 4 + 1] + + (uint64_t)n[i * 4 + 2] * m[i * 4 + 2] + + (uint64_t)n[i * 4 + 3] * m[i * 4 + 3]); } clear_tail(d, opr_sz, simd_maxsz(desc)); } -void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc) +void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, + void *va, uint32_t desc) { intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4; intptr_t index = simd_data(desc); - int32_t *d = vd; + int32_t *d = vd, *a = va; int8_t *n = vn; int8_t *m_indexed = (int8_t *)vm + H4(index) * 4; @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc) int8_t m3 = m_indexed[i * 4 + 3]; do { - d[i] += n[i * 4 + 0] * m0 - + n[i * 4 + 1] * m1 - + n[i * 4 + 2] * m2 - + n[i * 4 + 3] * m3; + d[i] = (a[i] + + n[i * 4 + 0] * m0 + + n[i * 4 + 1] * m1 + + n[i * 4 + 2] * m2 + + n[i * 4 + 3] * m3); } while (++i < segend); segend = i + 4; } while (i < opr_sz_4); @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc) clear_tail(d, opr_sz, simd_maxsz(desc)); } -void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc) +void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm, + void *va, uint32_t desc) { intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4; intptr_t index = simd_data(desc); - uint32_t *d = vd; + uint32_t *d = vd, *a = va; uint8_t *n = vn; uint8_t *m_indexed = (uint8_t *)vm + H4(index) * 4; @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc) uint8_t m3 = m_indexed[i * 4 + 3]; do { - d[i] += n[i * 4 + 0] * m0 - + n[i * 4 + 1] * m1 - + n[i * 4 + 2] * m2 - + n[i * 4 + 3] * m3; + d[i] = (a[i] + + n[i * 4 + 0] * m0 + + n[i * 4 + 1] * m1 + + n[i * 4 + 2] * m2 + + n[i * 4 + 3] * m3); } while (++i < segend); segend = i + 4; } while (i < opr_sz_4); @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc) clear_tail(d, opr_sz, simd_maxsz(desc)); } -void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc) +void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm, + void *va, uint32_t desc) { intptr_t i, opr_sz = simd_oprsz(desc), opr_sz_8 = opr_sz / 8; intptr_t index = simd_data(desc); - int64_t *d = vd; + int64_t *d = vd, *a = va; int16_t *n = vn; int16_t *m_indexed = (int16_t *)vm + index * 4; @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc) * Process the entire segment all at once, writing back the results * only after we've consumed all of the inputs. */ - for (i = 0; i < opr_sz_8 ; i += 2) { - uint64_t d0, d1; + for (i = 0; i < opr_sz_8; i += 2) { + int64_t d0, d1; - d0 = n[i * 4 + 0] * (int64_t)m_indexed[i * 4 + 0]; + d0 = a[i + 0]; + d0 += n[i * 4 + 0] * (int64_t)m_indexed[i * 4 + 0]; d0 += n[i * 4 + 1] * (int64_t)m_indexed[i * 4 + 1]; d0 += n[i * 4 + 2] * (int64_t)m_indexed[i * 4 + 2]; d0 += n[i * 4 + 3] * (int64_t)m_indexed[i * 4 + 3]; - d1 = n[i * 4 + 4] * (int64_t)m_indexed[i * 4 + 0]; + + d1 = a[i + 1]; + d1 += n[i * 4 + 4] * (int64_t)m_indexed[i * 4 + 0]; d1 += n[i * 4 + 5] * (int64_t)m_indexed[i * 4 + 1]; d1 += n[i * 4 + 6] * (int64_t)m_indexed[i * 4 + 2]; d1 += n[i * 4 + 7] * (int64_t)m_indexed[i * 4 + 3]; - d[i + 0] += d0; - d[i + 1] += d1; + d[i + 0] = d0; + d[i + 1] = d1; } - clear_tail(d, opr_sz, simd_maxsz(desc)); } -void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc) +void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm, + void *va, uint32_t desc) { intptr_t i, opr_sz = simd_oprsz(desc), opr_sz_8 = opr_sz / 8; intptr_t index = simd_data(desc); - uint64_t *d = vd; + uint64_t *d = vd, *a = va; uint16_t *n = vn; uint16_t *m_indexed = (uint16_t *)vm + index * 4; @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc) * Process the entire segment all at once, writing back the results * only after we've consumed all of the inputs. */ - for (i = 0; i < opr_sz_8 ; i += 2) { + for (i = 0; i < opr_sz_8; i += 2) { uint64_t d0, d1; - d0 = n[i * 4 + 0] * (uint64_t)m_indexed[i * 4 + 0]; + d0 = a[i + 0]; + d0 += n[i * 4 + 0] * (uint64_t)m_indexed[i * 4 + 0]; d0 += n[i * 4 + 1] * (uint64_t)m_indexed[i * 4 + 1]; d0 += n[i * 4 + 2] * (uint64_t)m_indexed[i * 4 + 2]; d0 += n[i * 4 + 3] * (uint64_t)m_indexed[i * 4 + 3]; - d1 = n[i * 4 + 4] * (uint64_t)m_indexed[i * 4 + 0]; + + d1 = a[i + 1]; + d1 += n[i * 4 + 4] * (uint64_t)m_indexed[i * 4 + 0]; d1 += n[i * 4 + 5] * (uint64_t)m_indexed[i * 4 + 1]; d1 += n[i * 4 + 6] * (uint64_t)m_indexed[i * 4 + 2]; d1 += n[i * 4 + 7] * (uint64_t)m_indexed[i * 4 + 3]; - d[i + 0] += d0; - d[i + 1] += d1; + d[i + 0] = d0; + d[i + 1] = d1; } - clear_tail(d, opr_sz, simd_maxsz(desc)); } -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> For SVE, we potentially have a 4th argument coming from the movprfx instruction. Currently we do not optimize movprfx, so the problem is not visible. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-51-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper.h | 20 +++++++-------- target/arm/translate-a64.c | 28 +++++++++++++++++---- target/arm/translate-neon.c | 10 +++++--- target/arm/translate-sve.c | 5 ++-- target/arm/vec_helper.c | 50 +++++++++++++++---------------------- 5 files changed, 62 insertions(+), 51 deletions(-) diff --git a/target/arm/helper.h b/target/arm/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.h +++ b/target/arm/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(gvec_fcadds, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_5(gvec_fcaddd, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) -DEF_HELPER_FLAGS_5(gvec_fcmlah, TCG_CALL_NO_RWG, - void, ptr, ptr, ptr, ptr, i32) -DEF_HELPER_FLAGS_5(gvec_fcmlah_idx, TCG_CALL_NO_RWG, - void, ptr, ptr, ptr, ptr, i32) -DEF_HELPER_FLAGS_5(gvec_fcmlas, TCG_CALL_NO_RWG, - void, ptr, ptr, ptr, ptr, i32) -DEF_HELPER_FLAGS_5(gvec_fcmlas_idx, TCG_CALL_NO_RWG, - void, ptr, ptr, ptr, ptr, i32) -DEF_HELPER_FLAGS_5(gvec_fcmlad, TCG_CALL_NO_RWG, - void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(gvec_fcmlah, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(gvec_fcmlah_idx, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(gvec_fcmlas, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(gvec_fcmlas_idx, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(gvec_fcmlad, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(neon_paddh, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(neon_pmaxh, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void gen_gvec_op4_ool(DisasContext *s, bool is_q, int rd, int rn, is_q ? 16 : 8, vec_full_reg_size(s), data, fn); } +/* + * Expand a 4-operand + fpstatus pointer + simd data value operation using + * an out-of-line helper. + */ +static void gen_gvec_op4_fpst(DisasContext *s, bool is_q, int rd, int rn, + int rm, int ra, bool is_fp16, int data, + gen_helper_gvec_4_ptr *fn) +{ + TCGv_ptr fpst = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR); + tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + vec_full_reg_offset(s, rm), + vec_full_reg_offset(s, ra), fpst, + is_q ? 16 : 8, vec_full_reg_size(s), data, fn); + tcg_temp_free_ptr(fpst); +} + /* Set ZF and NF based on a 64 bit result. This is alas fiddlier * than the 32 bit equivalent. */ @@ -XXX,XX +XXX,XX @@ static void disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn) rot = extract32(opcode, 0, 2); switch (size) { case 1: - gen_gvec_op3_fpst(s, is_q, rd, rn, rm, true, rot, + gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, true, rot, gen_helper_gvec_fcmlah); break; case 2: - gen_gvec_op3_fpst(s, is_q, rd, rn, rm, false, rot, + gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, false, rot, gen_helper_gvec_fcmlas); break; case 3: - gen_gvec_op3_fpst(s, is_q, rd, rn, rm, false, rot, + gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, false, rot, gen_helper_gvec_fcmlad); break; default: @@ -XXX,XX +XXX,XX @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn) { int rot = extract32(insn, 13, 2); int data = (index << 2) | rot; - tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd), + tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn), - vec_full_reg_offset(s, rm), fpst, + vec_full_reg_offset(s, rm), + vec_full_reg_offset(s, rd), fpst, is_q ? 16 : 8, vec_full_reg_size(s), data, size == MO_64 ? gen_helper_gvec_fcmlas_idx diff --git a/target/arm/translate-neon.c b/target/arm/translate-neon.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-neon.c +++ b/target/arm/translate-neon.c @@ -XXX,XX +XXX,XX @@ static bool trans_VCMLA(DisasContext *s, arg_VCMLA *a) { int opr_sz; TCGv_ptr fpst; - gen_helper_gvec_3_ptr *fn_gvec_ptr; + gen_helper_gvec_4_ptr *fn_gvec_ptr; if (!dc_isar_feature(aa32_vcma, s) || (a->size == MO_16 && !dc_isar_feature(aa32_fp16_arith, s))) { @@ -XXX,XX +XXX,XX @@ static bool trans_VCMLA(DisasContext *s, arg_VCMLA *a) fpst = fpstatus_ptr(a->size == MO_16 ? FPST_STD_F16 : FPST_STD); fn_gvec_ptr = (a->size == MO_16) ? gen_helper_gvec_fcmlah : gen_helper_gvec_fcmlas; - tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd), + tcg_gen_gvec_4_ptr(vfp_reg_offset(1, a->vd), vfp_reg_offset(1, a->vn), vfp_reg_offset(1, a->vm), + vfp_reg_offset(1, a->vd), fpst, opr_sz, opr_sz, a->rot, fn_gvec_ptr); tcg_temp_free_ptr(fpst); @@ -XXX,XX +XXX,XX @@ static bool trans_VFML(DisasContext *s, arg_VFML *a) static bool trans_VCMLA_scalar(DisasContext *s, arg_VCMLA_scalar *a) { - gen_helper_gvec_3_ptr *fn_gvec_ptr; + gen_helper_gvec_4_ptr *fn_gvec_ptr; int opr_sz; TCGv_ptr fpst; @@ -XXX,XX +XXX,XX @@ static bool trans_VCMLA_scalar(DisasContext *s, arg_VCMLA_scalar *a) gen_helper_gvec_fcmlah_idx : gen_helper_gvec_fcmlas_idx; opr_sz = (1 + a->q) * 8; fpst = fpstatus_ptr(a->size == MO_16 ? FPST_STD_F16 : FPST_STD); - tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd), + tcg_gen_gvec_4_ptr(vfp_reg_offset(1, a->vd), vfp_reg_offset(1, a->vn), vfp_reg_offset(1, a->vm), + vfp_reg_offset(1, a->vd), fpst, opr_sz, opr_sz, (a->index << 2) | a->rot, fn_gvec_ptr); tcg_temp_free_ptr(fpst); diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_FCMLA_zpzzz(DisasContext *s, arg_FCMLA_zpzzz *a) static bool trans_FCMLA_zzxz(DisasContext *s, arg_FCMLA_zzxz *a) { - static gen_helper_gvec_3_ptr * const fns[2] = { + static gen_helper_gvec_4_ptr * const fns[2] = { gen_helper_gvec_fcmlah_idx, gen_helper_gvec_fcmlas_idx, }; @@ -XXX,XX +XXX,XX @@ static bool trans_FCMLA_zzxz(DisasContext *s, arg_FCMLA_zzxz *a) if (sve_access_check(s)) { unsigned vsz = vec_full_reg_size(s); TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); - tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd), + tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, a->rd), vec_full_reg_offset(s, a->rn), vec_full_reg_offset(s, a->rm), + vec_full_reg_offset(s, a->ra), status, vsz, vsz, a->index * 4 + a->rot, fns[a->esz - 1]); diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_fcaddd)(void *vd, void *vn, void *vm, clear_tail(d, opr_sz, simd_maxsz(desc)); } -void HELPER(gvec_fcmlah)(void *vd, void *vn, void *vm, +void HELPER(gvec_fcmlah)(void *vd, void *vn, void *vm, void *va, void *vfpst, uint32_t desc) { uintptr_t opr_sz = simd_oprsz(desc); - float16 *d = vd; - float16 *n = vn; - float16 *m = vm; + float16 *d = vd, *n = vn, *m = vm, *a = va; float_status *fpst = vfpst; intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1); uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1); @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_fcmlah)(void *vd, void *vn, void *vm, float16 e4 = e2; float16 e3 = m[H2(i + 1 - flip)] ^ neg_imag; - d[H2(i)] = float16_muladd(e2, e1, d[H2(i)], 0, fpst); - d[H2(i + 1)] = float16_muladd(e4, e3, d[H2(i + 1)], 0, fpst); + d[H2(i)] = float16_muladd(e2, e1, a[H2(i)], 0, fpst); + d[H2(i + 1)] = float16_muladd(e4, e3, a[H2(i + 1)], 0, fpst); } clear_tail(d, opr_sz, simd_maxsz(desc)); } -void HELPER(gvec_fcmlah_idx)(void *vd, void *vn, void *vm, +void HELPER(gvec_fcmlah_idx)(void *vd, void *vn, void *vm, void *va, void *vfpst, uint32_t desc) { uintptr_t opr_sz = simd_oprsz(desc); - float16 *d = vd; - float16 *n = vn; - float16 *m = vm; + float16 *d = vd, *n = vn, *m = vm, *a = va; float_status *fpst = vfpst; intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1); uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1); @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_fcmlah_idx)(void *vd, void *vn, void *vm, float16 e2 = n[H2(j + flip)]; float16 e4 = e2; - d[H2(j)] = float16_muladd(e2, e1, d[H2(j)], 0, fpst); - d[H2(j + 1)] = float16_muladd(e4, e3, d[H2(j + 1)], 0, fpst); + d[H2(j)] = float16_muladd(e2, e1, a[H2(j)], 0, fpst); + d[H2(j + 1)] = float16_muladd(e4, e3, a[H2(j + 1)], 0, fpst); } } clear_tail(d, opr_sz, simd_maxsz(desc)); } -void HELPER(gvec_fcmlas)(void *vd, void *vn, void *vm, +void HELPER(gvec_fcmlas)(void *vd, void *vn, void *vm, void *va, void *vfpst, uint32_t desc) { uintptr_t opr_sz = simd_oprsz(desc); - float32 *d = vd; - float32 *n = vn; - float32 *m = vm; + float32 *d = vd, *n = vn, *m = vm, *a = va; float_status *fpst = vfpst; intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1); uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1); @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_fcmlas)(void *vd, void *vn, void *vm, float32 e4 = e2; float32 e3 = m[H4(i + 1 - flip)] ^ neg_imag; - d[H4(i)] = float32_muladd(e2, e1, d[H4(i)], 0, fpst); - d[H4(i + 1)] = float32_muladd(e4, e3, d[H4(i + 1)], 0, fpst); + d[H4(i)] = float32_muladd(e2, e1, a[H4(i)], 0, fpst); + d[H4(i + 1)] = float32_muladd(e4, e3, a[H4(i + 1)], 0, fpst); } clear_tail(d, opr_sz, simd_maxsz(desc)); } -void HELPER(gvec_fcmlas_idx)(void *vd, void *vn, void *vm, +void HELPER(gvec_fcmlas_idx)(void *vd, void *vn, void *vm, void *va, void *vfpst, uint32_t desc) { uintptr_t opr_sz = simd_oprsz(desc); - float32 *d = vd; - float32 *n = vn; - float32 *m = vm; + float32 *d = vd, *n = vn, *m = vm, *a = va; float_status *fpst = vfpst; intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1); uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1); @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_fcmlas_idx)(void *vd, void *vn, void *vm, float32 e2 = n[H4(j + flip)]; float32 e4 = e2; - d[H4(j)] = float32_muladd(e2, e1, d[H4(j)], 0, fpst); - d[H4(j + 1)] = float32_muladd(e4, e3, d[H4(j + 1)], 0, fpst); + d[H4(j)] = float32_muladd(e2, e1, a[H4(j)], 0, fpst); + d[H4(j + 1)] = float32_muladd(e4, e3, a[H4(j + 1)], 0, fpst); } } clear_tail(d, opr_sz, simd_maxsz(desc)); } -void HELPER(gvec_fcmlad)(void *vd, void *vn, void *vm, +void HELPER(gvec_fcmlad)(void *vd, void *vn, void *vm, void *va, void *vfpst, uint32_t desc) { uintptr_t opr_sz = simd_oprsz(desc); - float64 *d = vd; - float64 *n = vn; - float64 *m = vm; + float64 *d = vd, *n = vn, *m = vm, *a = va; float_status *fpst = vfpst; intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1); uint64_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1); @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_fcmlad)(void *vd, void *vn, void *vm, float64 e4 = e2; float64 e3 = m[i + 1 - flip] ^ neg_imag; - d[i] = float64_muladd(e2, e1, d[i], 0, fpst); - d[i + 1] = float64_muladd(e4, e3, d[i + 1], 0, fpst); + d[i] = float64_muladd(e2, e1, a[i], 0, fpst); + d[i + 1] = float64_muladd(e4, e3, a[i + 1], 0, fpst); } clear_tail(d, opr_sz, simd_maxsz(desc)); } -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Currently only used by FMUL, but will shortly be used more. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-52-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/sve.decode | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ &rri_esz rd rn imm esz &rrri_esz rd rn rm imm esz &rrr_esz rd rn rm esz +&rrx_esz rd rn rm index esz &rpr_esz rd pg rn esz &rpr_s rd pg rn s &rprr_s rd pg rn rm s @@ -XXX,XX +XXX,XX @@ @rpri_scatter_store ....... msz:2 .. imm:5 ... pg:3 rn:5 rd:5 \ &rpri_scatter_store +# Two registers and a scalar by N-bit index +@rrx_3 ........ .. . .. rm:3 ...... rn:5 rd:5 \ + &rrx_esz index=%index3_22_19 +@rrx_2 ........ .. . index:2 rm:3 ...... rn:5 rd:5 &rrx_esz +@rrx_1 ........ .. . index:1 rm:4 ...... rn:5 rd:5 &rrx_esz + ########################################################################### # Instruction patterns. Grouped according to the SVE encodingindex.xhtml. @@ -XXX,XX +XXX,XX @@ FMLA_zzxz 01100100 111 index:1 rm:4 00000 sub:1 rn:5 rd:5 \ ### SVE FP Multiply Indexed Group # SVE floating-point multiply (indexed) -FMUL_zzx 01100100 0.1 .. rm:3 001000 rn:5 rd:5 \ - index=%index3_22_19 esz=1 -FMUL_zzx 01100100 101 index:2 rm:3 001000 rn:5 rd:5 esz=2 -FMUL_zzx 01100100 111 index:1 rm:4 001000 rn:5 rd:5 esz=3 +FMUL_zzx 01100100 0. 1 ..... 001000 ..... ..... @rrx_3 esz=1 +FMUL_zzx 01100100 10 1 ..... 001000 ..... ..... @rrx_2 esz=2 +FMUL_zzx 01100100 11 1 ..... 001000 ..... ..... @rrx_1 esz=3 ### SVE FP Fast Reduction Group -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Used by FMLA and DOT, but will shortly be used more. Split FMLA from FMLS to avoid an extra sub field; similarly for SDOT from UDOT. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-53-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/sve.decode | 29 +++++++++++++++++++---------- target/arm/translate-sve.c | 38 ++++++++++++++++++++++++++++---------- 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ &rprr_s rd pg rn rm s &rprr_esz rd pg rn rm esz &rrrr_esz rd ra rn rm esz +&rrxr_esz rd rn rm ra index esz &rprrr_esz rd pg rn rm ra esz &rpri_esz rd pg rn imm esz &ptrue rd esz pat s @@ -XXX,XX +XXX,XX @@ @rrx_2 ........ .. . index:2 rm:3 ...... rn:5 rd:5 &rrx_esz @rrx_1 ........ .. . index:1 rm:4 ...... rn:5 rd:5 &rrx_esz +# Three registers and a scalar by N-bit index +@rrxr_3 ........ .. . .. rm:3 ...... rn:5 rd:5 \ + &rrxr_esz ra=%reg_movprfx index=%index3_22_19 +@rrxr_2 ........ .. . index:2 rm:3 ...... rn:5 rd:5 \ + &rrxr_esz ra=%reg_movprfx +@rrxr_1 ........ .. . index:1 rm:4 ...... rn:5 rd:5 \ + &rrxr_esz ra=%reg_movprfx + ########################################################################### # Instruction patterns. Grouped according to the SVE encodingindex.xhtml. @@ -XXX,XX +XXX,XX @@ DOT_zzzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 \ ra=%reg_movprfx # SVE integer dot product (indexed) -DOT_zzxw 01000100 101 index:2 rm:3 00000 u:1 rn:5 rd:5 \ - sz=0 ra=%reg_movprfx -DOT_zzxw 01000100 111 index:1 rm:4 00000 u:1 rn:5 rd:5 \ - sz=1 ra=%reg_movprfx +SDOT_zzxw_s 01000100 10 1 ..... 000000 ..... ..... @rrxr_2 esz=2 +SDOT_zzxw_d 01000100 11 1 ..... 000000 ..... ..... @rrxr_1 esz=3 +UDOT_zzxw_s 01000100 10 1 ..... 000001 ..... ..... @rrxr_2 esz=2 +UDOT_zzxw_d 01000100 11 1 ..... 000001 ..... ..... @rrxr_1 esz=3 # SVE floating-point complex add (predicated) FCADD 01100100 esz:2 00000 rot:1 100 pg:3 rm:5 rd:5 \ @@ -XXX,XX +XXX,XX @@ FCMLA_zzxz 01100100 11 1 index:1 rm:4 0001 rot:2 rn:5 rd:5 \ ### SVE FP Multiply-Add Indexed Group # SVE floating-point multiply-add (indexed) -FMLA_zzxz 01100100 0.1 .. rm:3 00000 sub:1 rn:5 rd:5 \ - ra=%reg_movprfx index=%index3_22_19 esz=1 -FMLA_zzxz 01100100 101 index:2 rm:3 00000 sub:1 rn:5 rd:5 \ - ra=%reg_movprfx esz=2 -FMLA_zzxz 01100100 111 index:1 rm:4 00000 sub:1 rn:5 rd:5 \ - ra=%reg_movprfx esz=3 +FMLA_zzxz 01100100 0. 1 ..... 000000 ..... ..... @rrxr_3 esz=1 +FMLA_zzxz 01100100 10 1 ..... 000000 ..... ..... @rrxr_2 esz=2 +FMLA_zzxz 01100100 11 1 ..... 000000 ..... ..... @rrxr_1 esz=3 +FMLS_zzxz 01100100 0. 1 ..... 000001 ..... ..... @rrxr_3 esz=1 +FMLS_zzxz 01100100 10 1 ..... 000001 ..... ..... @rrxr_2 esz=2 +FMLS_zzxz 01100100 11 1 ..... 000001 ..... ..... @rrxr_1 esz=3 ### SVE FP Multiply Indexed Group diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_DOT_zzzz(DisasContext *s, arg_DOT_zzzz *a) return true; } -static bool trans_DOT_zzxw(DisasContext *s, arg_DOT_zzxw *a) +static bool do_zzxz_ool(DisasContext *s, arg_rrxr_esz *a, + gen_helper_gvec_4 *fn) { - static gen_helper_gvec_4 * const fns[2][2] = { - { gen_helper_gvec_sdot_idx_b, gen_helper_gvec_sdot_idx_h }, - { gen_helper_gvec_udot_idx_b, gen_helper_gvec_udot_idx_h } - }; - + if (fn == NULL) { + return false; + } if (sve_access_check(s)) { - gen_gvec_ool_zzzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm, - a->ra, a->index); + gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->index); } return true; } +#define DO_RRXR(NAME, FUNC) \ + static bool NAME(DisasContext *s, arg_rrxr_esz *a) \ + { return do_zzxz_ool(s, a, FUNC); } + +DO_RRXR(trans_SDOT_zzxw_s, gen_helper_gvec_sdot_idx_b) +DO_RRXR(trans_SDOT_zzxw_d, gen_helper_gvec_sdot_idx_h) +DO_RRXR(trans_UDOT_zzxw_s, gen_helper_gvec_udot_idx_b) +DO_RRXR(trans_UDOT_zzxw_d, gen_helper_gvec_udot_idx_h) + +#undef DO_RRXR /* *** SVE Floating Point Multiply-Add Indexed Group */ -static bool trans_FMLA_zzxz(DisasContext *s, arg_FMLA_zzxz *a) +static bool do_FMLA_zzxz(DisasContext *s, arg_rrxr_esz *a, bool sub) { static gen_helper_gvec_4_ptr * const fns[3] = { gen_helper_gvec_fmla_idx_h, @@ -XXX,XX +XXX,XX @@ static bool trans_FMLA_zzxz(DisasContext *s, arg_FMLA_zzxz *a) vec_full_reg_offset(s, a->rn), vec_full_reg_offset(s, a->rm), vec_full_reg_offset(s, a->ra), - status, vsz, vsz, (a->index << 1) | a->sub, + status, vsz, vsz, (a->index << 1) | sub, fns[a->esz - 1]); tcg_temp_free_ptr(status); } return true; } +static bool trans_FMLA_zzxz(DisasContext *s, arg_FMLA_zzxz *a) +{ + return do_FMLA_zzxz(s, a, false); +} + +static bool trans_FMLS_zzxz(DisasContext *s, arg_FMLA_zzxz *a) +{ + return do_FMLA_zzxz(s, a, true); +} + /* *** SVE Floating Point Multiply Indexed Group */ -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-54-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/sve.decode | 7 +++++++ target/arm/translate-sve.c | 30 ++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ MUL_zzi 00100101 .. 110 000 110 ........ ..... @rdn_i8s DOT_zzzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 \ ra=%reg_movprfx +#### SVE Multiply - Indexed + # SVE integer dot product (indexed) SDOT_zzxw_s 01000100 10 1 ..... 000000 ..... ..... @rrxr_2 esz=2 SDOT_zzxw_d 01000100 11 1 ..... 000000 ..... ..... @rrxr_1 esz=3 UDOT_zzxw_s 01000100 10 1 ..... 000001 ..... ..... @rrxr_2 esz=2 UDOT_zzxw_d 01000100 11 1 ..... 000001 ..... ..... @rrxr_1 esz=3 +# SVE2 integer multiply (indexed) +MUL_zzx_h 01000100 0. 1 ..... 111110 ..... ..... @rrx_3 esz=1 +MUL_zzx_s 01000100 10 1 ..... 111110 ..... ..... @rrx_2 esz=2 +MUL_zzx_d 01000100 11 1 ..... 111110 ..... ..... @rrx_1 esz=3 + # SVE floating-point complex add (predicated) FCADD 01100100 esz:2 00000 rot:1 100 pg:3 rm:5 rd:5 \ rn=%reg_movprfx diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_DOT_zzzz(DisasContext *s, arg_DOT_zzzz *a) return true; } +/* + * SVE Multiply - Indexed + */ + static bool do_zzxz_ool(DisasContext *s, arg_rrxr_esz *a, gen_helper_gvec_4 *fn) { @@ -XXX,XX +XXX,XX @@ DO_RRXR(trans_UDOT_zzxw_d, gen_helper_gvec_udot_idx_h) #undef DO_RRXR +static bool do_sve2_zzz_data(DisasContext *s, int rd, int rn, int rm, int data, + gen_helper_gvec_3 *fn) +{ + if (fn == NULL || !dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (sve_access_check(s)) { + unsigned vsz = vec_full_reg_size(s); + tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + vec_full_reg_offset(s, rm), + vsz, vsz, data, fn); + } + return true; +} + +#define DO_SVE2_RRX(NAME, FUNC) \ + static bool NAME(DisasContext *s, arg_rrx_esz *a) \ + { return do_sve2_zzz_data(s, a->rd, a->rn, a->rm, a->index, FUNC); } + +DO_SVE2_RRX(trans_MUL_zzx_h, gen_helper_gvec_mul_idx_h) +DO_SVE2_RRX(trans_MUL_zzx_s, gen_helper_gvec_mul_idx_s) +DO_SVE2_RRX(trans_MUL_zzx_d, gen_helper_gvec_mul_idx_d) + +#undef DO_SVE2_RRX + /* *** SVE Floating Point Multiply-Add Indexed Group */ -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-55-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/sve.decode | 8 ++++++++ target/arm/translate-sve.c | 31 +++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SDOT_zzxw_d 01000100 11 1 ..... 000000 ..... ..... @rrxr_1 esz=3 UDOT_zzxw_s 01000100 10 1 ..... 000001 ..... ..... @rrxr_2 esz=2 UDOT_zzxw_d 01000100 11 1 ..... 000001 ..... ..... @rrxr_1 esz=3 +# SVE2 integer multiply-add (indexed) +MLA_zzxz_h 01000100 0. 1 ..... 000010 ..... ..... @rrxr_3 esz=1 +MLA_zzxz_s 01000100 10 1 ..... 000010 ..... ..... @rrxr_2 esz=2 +MLA_zzxz_d 01000100 11 1 ..... 000010 ..... ..... @rrxr_1 esz=3 +MLS_zzxz_h 01000100 0. 1 ..... 000011 ..... ..... @rrxr_3 esz=1 +MLS_zzxz_s 01000100 10 1 ..... 000011 ..... ..... @rrxr_2 esz=2 +MLS_zzxz_d 01000100 11 1 ..... 000011 ..... ..... @rrxr_1 esz=3 + # SVE2 integer multiply (indexed) MUL_zzx_h 01000100 0. 1 ..... 111110 ..... ..... @rrx_3 esz=1 MUL_zzx_s 01000100 10 1 ..... 111110 ..... ..... @rrx_2 esz=2 diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_RRX(trans_MUL_zzx_d, gen_helper_gvec_mul_idx_d) #undef DO_SVE2_RRX +static bool do_sve2_zzzz_data(DisasContext *s, int rd, int rn, int rm, int ra, + int data, gen_helper_gvec_4 *fn) +{ + if (fn == NULL || !dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (sve_access_check(s)) { + unsigned vsz = vec_full_reg_size(s); + tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + vec_full_reg_offset(s, rm), + vec_full_reg_offset(s, ra), + vsz, vsz, data, fn); + } + return true; +} + +#define DO_SVE2_RRXR(NAME, FUNC) \ + static bool NAME(DisasContext *s, arg_rrxr_esz *a) \ + { return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->ra, a->index, FUNC); } + +DO_SVE2_RRXR(trans_MLA_zzxz_h, gen_helper_gvec_mla_idx_h) +DO_SVE2_RRXR(trans_MLA_zzxz_s, gen_helper_gvec_mla_idx_s) +DO_SVE2_RRXR(trans_MLA_zzxz_d, gen_helper_gvec_mla_idx_d) + +DO_SVE2_RRXR(trans_MLS_zzxz_h, gen_helper_gvec_mls_idx_h) +DO_SVE2_RRXR(trans_MLS_zzxz_s, gen_helper_gvec_mls_idx_s) +DO_SVE2_RRXR(trans_MLS_zzxz_d, gen_helper_gvec_mls_idx_d) + +#undef DO_SVE2_RRXR + /* *** SVE Floating Point Multiply-Add Indexed Group */ -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-56-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 14 ++++++++++++++ target/arm/sve.decode | 8 ++++++++ target/arm/sve_helper.c | 36 ++++++++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 8 ++++++++ 4 files changed, 66 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_d, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_6(fmmla_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_6(fmmla_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_sqrdmlah_idx_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrdmlah_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrdmlah_idx_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_idx_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_idx_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ MLS_zzxz_h 01000100 0. 1 ..... 000011 ..... ..... @rrxr_3 esz=1 MLS_zzxz_s 01000100 10 1 ..... 000011 ..... ..... @rrxr_2 esz=2 MLS_zzxz_d 01000100 11 1 ..... 000011 ..... ..... @rrxr_1 esz=3 +# SVE2 saturating multiply-add high (indexed) +SQRDMLAH_zzxz_h 01000100 0. 1 ..... 000100 ..... ..... @rrxr_3 esz=1 +SQRDMLAH_zzxz_s 01000100 10 1 ..... 000100 ..... ..... @rrxr_2 esz=2 +SQRDMLAH_zzxz_d 01000100 11 1 ..... 000100 ..... ..... @rrxr_1 esz=3 +SQRDMLSH_zzxz_h 01000100 0. 1 ..... 000101 ..... ..... @rrxr_3 esz=1 +SQRDMLSH_zzxz_s 01000100 10 1 ..... 000101 ..... ..... @rrxr_2 esz=2 +SQRDMLSH_zzxz_d 01000100 11 1 ..... 000101 ..... ..... @rrxr_1 esz=3 + # SVE2 integer multiply (indexed) MUL_zzx_h 01000100 0. 1 ..... 111110 ..... ..... @rrx_3 esz=1 MUL_zzx_s 01000100 10 1 ..... 111110 ..... ..... @rrx_2 esz=2 diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_d, int64_t, , DO_SQRDMLAH_D) #undef DO_SQRDMLAH_S #undef DO_SQRDMLAH_D +#define DO_ZZXZ(NAME, TYPE, H, OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \ +{ \ + intptr_t oprsz = simd_oprsz(desc), segment = 16 / sizeof(TYPE); \ + intptr_t i, j, idx = simd_data(desc); \ + TYPE *d = vd, *a = va, *n = vn, *m = (TYPE *)vm + H(idx); \ + for (i = 0; i < oprsz / sizeof(TYPE); i += segment) { \ + TYPE mm = m[i]; \ + for (j = 0; j < segment; j++) { \ + d[i + j] = OP(n[i + j], mm, a[i + j]); \ + } \ + } \ +} + +#define DO_SQRDMLAH_H(N, M, A) \ + ({ uint32_t discard; do_sqrdmlah_h(N, M, A, false, true, &discard); }) +#define DO_SQRDMLAH_S(N, M, A) \ + ({ uint32_t discard; do_sqrdmlah_s(N, M, A, false, true, &discard); }) +#define DO_SQRDMLAH_D(N, M, A) do_sqrdmlah_d(N, M, A, false, true) + +DO_ZZXZ(sve2_sqrdmlah_idx_h, int16_t, H2, DO_SQRDMLAH_H) +DO_ZZXZ(sve2_sqrdmlah_idx_s, int32_t, H4, DO_SQRDMLAH_S) +DO_ZZXZ(sve2_sqrdmlah_idx_d, int64_t, , DO_SQRDMLAH_D) + +#define DO_SQRDMLSH_H(N, M, A) \ + ({ uint32_t discard; do_sqrdmlah_h(N, M, A, true, true, &discard); }) +#define DO_SQRDMLSH_S(N, M, A) \ + ({ uint32_t discard; do_sqrdmlah_s(N, M, A, true, true, &discard); }) +#define DO_SQRDMLSH_D(N, M, A) do_sqrdmlah_d(N, M, A, true, true) + +DO_ZZXZ(sve2_sqrdmlsh_idx_h, int16_t, H2, DO_SQRDMLSH_H) +DO_ZZXZ(sve2_sqrdmlsh_idx_s, int32_t, H4, DO_SQRDMLSH_S) +DO_ZZXZ(sve2_sqrdmlsh_idx_d, int64_t, , DO_SQRDMLSH_D) + +#undef DO_ZZXZ + #define DO_BITPERM(NAME, TYPE, OP) \ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ { \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_RRXR(trans_MLS_zzxz_h, gen_helper_gvec_mls_idx_h) DO_SVE2_RRXR(trans_MLS_zzxz_s, gen_helper_gvec_mls_idx_s) DO_SVE2_RRXR(trans_MLS_zzxz_d, gen_helper_gvec_mls_idx_d) +DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_h, gen_helper_sve2_sqrdmlah_idx_h) +DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_s, gen_helper_sve2_sqrdmlah_idx_s) +DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_d, gen_helper_sve2_sqrdmlah_idx_d) + +DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_h, gen_helper_sve2_sqrdmlsh_idx_h) +DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_s, gen_helper_sve2_sqrdmlsh_idx_s) +DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_d, gen_helper_sve2_sqrdmlsh_idx_d) + #undef DO_SVE2_RRXR /* -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-57-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 9 +++++++++ target/arm/sve.decode | 18 ++++++++++++++++++ target/arm/sve_helper.c | 30 ++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 19 +++++++++++++++++++ 4 files changed, 76 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_sqdmlal_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqdmlal_idx_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqdmlsl_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqdmlsl_idx_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ %size_23 23:2 %dtype_23_13 23:2 13:2 %index3_22_19 22:1 19:2 +%index3_19_11 19:2 11:1 +%index2_20_11 20:1 11:1 # A combination of tsz:imm3 -- extract esize. %tszimm_esz 22:2 5:5 !function=tszimm_esz @@ -XXX,XX +XXX,XX @@ @rrxr_1 ........ .. . index:1 rm:4 ...... rn:5 rd:5 \ &rrxr_esz ra=%reg_movprfx +# Three registers and a scalar by N-bit index, alternate +@rrxr_3a ........ .. ... rm:3 ...... rn:5 rd:5 \ + &rrxr_esz ra=%reg_movprfx index=%index3_19_11 +@rrxr_2a ........ .. .. rm:4 ...... rn:5 rd:5 \ + &rrxr_esz ra=%reg_movprfx index=%index2_20_11 + ########################################################################### # Instruction patterns. Grouped according to the SVE encodingindex.xhtml. @@ -XXX,XX +XXX,XX @@ SQRDMLSH_zzxz_h 01000100 0. 1 ..... 000101 ..... ..... @rrxr_3 esz=1 SQRDMLSH_zzxz_s 01000100 10 1 ..... 000101 ..... ..... @rrxr_2 esz=2 SQRDMLSH_zzxz_d 01000100 11 1 ..... 000101 ..... ..... @rrxr_1 esz=3 +# SVE2 saturating multiply-add (indexed) +SQDMLALB_zzxw_s 01000100 10 1 ..... 0010.0 ..... ..... @rrxr_3a esz=2 +SQDMLALB_zzxw_d 01000100 11 1 ..... 0010.0 ..... ..... @rrxr_2a esz=3 +SQDMLALT_zzxw_s 01000100 10 1 ..... 0010.1 ..... ..... @rrxr_3a esz=2 +SQDMLALT_zzxw_d 01000100 11 1 ..... 0010.1 ..... ..... @rrxr_2a esz=3 +SQDMLSLB_zzxw_s 01000100 10 1 ..... 0011.0 ..... ..... @rrxr_3a esz=2 +SQDMLSLB_zzxw_d 01000100 11 1 ..... 0011.0 ..... ..... @rrxr_2a esz=3 +SQDMLSLT_zzxw_s 01000100 10 1 ..... 0011.1 ..... ..... @rrxr_3a esz=2 +SQDMLSLT_zzxw_d 01000100 11 1 ..... 0011.1 ..... ..... @rrxr_2a esz=3 + # SVE2 integer multiply (indexed) MUL_zzx_h 01000100 0. 1 ..... 111110 ..... ..... @rrx_3 esz=1 MUL_zzx_s 01000100 10 1 ..... 111110 ..... ..... @rrx_2 esz=2 diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZZXZ(sve2_sqrdmlsh_idx_d, int64_t, , DO_SQRDMLSH_D) #undef DO_ZZXZ +#define DO_ZZXW(NAME, TYPEW, TYPEN, HW, HN, OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \ +{ \ + intptr_t i, j, oprsz = simd_oprsz(desc); \ + intptr_t sel = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN); \ + intptr_t idx = extract32(desc, SIMD_DATA_SHIFT + 1, 3) * sizeof(TYPEN); \ + for (i = 0; i < oprsz; i += 16) { \ + TYPEW mm = *(TYPEN *)(vm + HN(i + idx)); \ + for (j = 0; j < 16; j += sizeof(TYPEW)) { \ + TYPEW nn = *(TYPEN *)(vn + HN(i + j + sel)); \ + TYPEW aa = *(TYPEW *)(va + HW(i + j)); \ + *(TYPEW *)(vd + HW(i + j)) = OP(nn, mm, aa); \ + } \ + } \ +} + +#define DO_SQDMLAL_S(N, M, A) DO_SQADD_S(A, do_sqdmull_s(N, M)) +#define DO_SQDMLAL_D(N, M, A) do_sqadd_d(A, do_sqdmull_d(N, M)) + +DO_ZZXW(sve2_sqdmlal_idx_s, int32_t, int16_t, H1_4, H1_2, DO_SQDMLAL_S) +DO_ZZXW(sve2_sqdmlal_idx_d, int64_t, int32_t, , H1_4, DO_SQDMLAL_D) + +#define DO_SQDMLSL_S(N, M, A) DO_SQSUB_S(A, do_sqdmull_s(N, M)) +#define DO_SQDMLSL_D(N, M, A) do_sqsub_d(A, do_sqdmull_d(N, M)) + +DO_ZZXW(sve2_sqdmlsl_idx_s, int32_t, int16_t, H1_4, H1_2, DO_SQDMLSL_S) +DO_ZZXW(sve2_sqdmlsl_idx_d, int64_t, int32_t, , H1_4, DO_SQDMLSL_D) + +#undef DO_ZZXW + #define DO_BITPERM(NAME, TYPE, OP) \ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ { \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_d, gen_helper_sve2_sqrdmlsh_idx_d) #undef DO_SVE2_RRXR +#define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \ + static bool NAME(DisasContext *s, arg_rrxr_esz *a) \ + { \ + return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->rd, \ + (a->index << 1) | TOP, FUNC); \ + } + +DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_s, gen_helper_sve2_sqdmlal_idx_s, false) +DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_d, gen_helper_sve2_sqdmlal_idx_d, false) +DO_SVE2_RRXR_TB(trans_SQDMLALT_zzxw_s, gen_helper_sve2_sqdmlal_idx_s, true) +DO_SVE2_RRXR_TB(trans_SQDMLALT_zzxw_d, gen_helper_sve2_sqdmlal_idx_d, true) + +DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, false) +DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, false) +DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, true) +DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, true) + +#undef DO_SVE2_RRXR_TB + /* *** SVE Floating Point Multiply-Add Indexed Group */ -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-58-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 5 +++++ target/arm/sve.decode | 12 ++++++++++++ target/arm/sve_helper.c | 20 ++++++++++++++++++++ target/arm/translate-sve.c | 14 ++++++++++++++ 4 files changed, 51 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_sqdmlsl_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve2_sqdmlsl_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_sqdmull_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqdmull_idx_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ @rrx_2 ........ .. . index:2 rm:3 ...... rn:5 rd:5 &rrx_esz @rrx_1 ........ .. . index:1 rm:4 ...... rn:5 rd:5 &rrx_esz +# Two registers and a scalar by N-bit index, alternate +@rrx_3a ........ .. . .. rm:3 ...... rn:5 rd:5 \ + &rrx_esz index=%index3_19_11 +@rrx_2a ........ .. . . rm:4 ...... rn:5 rd:5 \ + &rrx_esz index=%index2_20_11 + # Three registers and a scalar by N-bit index @rrxr_3 ........ .. . .. rm:3 ...... rn:5 rd:5 \ &rrxr_esz ra=%reg_movprfx index=%index3_22_19 @@ -XXX,XX +XXX,XX @@ SQDMLSLB_zzxw_d 01000100 11 1 ..... 0011.0 ..... ..... @rrxr_2a esz=3 SQDMLSLT_zzxw_s 01000100 10 1 ..... 0011.1 ..... ..... @rrxr_3a esz=2 SQDMLSLT_zzxw_d 01000100 11 1 ..... 0011.1 ..... ..... @rrxr_2a esz=3 +# SVE2 saturating multiply (indexed) +SQDMULLB_zzx_s 01000100 10 1 ..... 1110.0 ..... ..... @rrx_3a esz=2 +SQDMULLB_zzx_d 01000100 11 1 ..... 1110.0 ..... ..... @rrx_2a esz=3 +SQDMULLT_zzx_s 01000100 10 1 ..... 1110.1 ..... ..... @rrx_3a esz=2 +SQDMULLT_zzx_d 01000100 11 1 ..... 1110.1 ..... ..... @rrx_2a esz=3 + # SVE2 integer multiply (indexed) MUL_zzx_h 01000100 0. 1 ..... 111110 ..... ..... @rrx_3 esz=1 MUL_zzx_s 01000100 10 1 ..... 111110 ..... ..... @rrx_2 esz=2 diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_ZZXW(sve2_sqdmlsl_idx_d, int64_t, int32_t, , H1_4, DO_SQDMLSL_D) #undef DO_ZZXW +#define DO_ZZX(NAME, TYPEW, TYPEN, HW, HN, OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ +{ \ + intptr_t i, j, oprsz = simd_oprsz(desc); \ + intptr_t sel = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN); \ + intptr_t idx = extract32(desc, SIMD_DATA_SHIFT + 1, 3) * sizeof(TYPEN); \ + for (i = 0; i < oprsz; i += 16) { \ + TYPEW mm = *(TYPEN *)(vm + HN(i + idx)); \ + for (j = 0; j < 16; j += sizeof(TYPEW)) { \ + TYPEW nn = *(TYPEN *)(vn + HN(i + j + sel)); \ + *(TYPEW *)(vd + HW(i + j)) = OP(nn, mm); \ + } \ + } \ +} + +DO_ZZX(sve2_sqdmull_idx_s, int32_t, int16_t, H1_4, H1_2, do_sqdmull_s) +DO_ZZX(sve2_sqdmull_idx_d, int64_t, int32_t, , H1_4, do_sqdmull_d) + +#undef DO_ZZX + #define DO_BITPERM(NAME, TYPE, OP) \ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ { \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_RRX(trans_MUL_zzx_d, gen_helper_gvec_mul_idx_d) #undef DO_SVE2_RRX +#define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \ + static bool NAME(DisasContext *s, arg_rrx_esz *a) \ + { \ + return do_sve2_zzz_data(s, a->rd, a->rn, a->rm, \ + (a->index << 1) | TOP, FUNC); \ + } + +DO_SVE2_RRX_TB(trans_SQDMULLB_zzx_s, gen_helper_sve2_sqdmull_idx_s, false) +DO_SVE2_RRX_TB(trans_SQDMULLB_zzx_d, gen_helper_sve2_sqdmull_idx_d, false) +DO_SVE2_RRX_TB(trans_SQDMULLT_zzx_s, gen_helper_sve2_sqdmull_idx_s, true) +DO_SVE2_RRX_TB(trans_SQDMULLT_zzx_d, gen_helper_sve2_sqdmull_idx_d, true) + +#undef DO_SVE2_RRX_TB + static bool do_sve2_zzzz_data(DisasContext *s, int rd, int rn, int rm, int ra, int data, gen_helper_gvec_4 *fn) { -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-59-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper.h | 10 +++++ target/arm/sve.decode | 4 ++ target/arm/translate-sve.c | 18 ++++++++ target/arm/vec_helper.c | 84 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+) diff --git a/target/arm/helper.h b/target/arm/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.h +++ b/target/arm/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(neon_sqrdmulh_h, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_5(neon_sqrdmulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqdmulh_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqdmulh_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqdmulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqdmulh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_sqrdmulh_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqrdmulh_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqrdmulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqrdmulh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_4(gvec_xar_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) #ifdef TARGET_AARCH64 diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SMULH_zzz 00000100 .. 1 ..... 0110 10 ..... ..... @rd_rn_rm UMULH_zzz 00000100 .. 1 ..... 0110 11 ..... ..... @rd_rn_rm PMUL_zzz 00000100 00 1 ..... 0110 01 ..... ..... @rd_rn_rm_e0 +# SVE2 signed saturating doubling multiply high (unpredicated) +SQDMULH_zzz 00000100 .. 1 ..... 0111 00 ..... ..... @rd_rn_rm +SQRDMULH_zzz 00000100 .. 1 ..... 0111 01 ..... ..... @rd_rn_rm + ### SVE2 Integer - Predicated SADALP_zpzz 01000100 .. 000 100 101 ... ..... ..... @rdm_pg_rn diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_PMUL_zzz(DisasContext *s, arg_rrr_esz *a) return do_sve2_zzz_ool(s, a, gen_helper_gvec_pmul_b); } +static bool trans_SQDMULH_zzz(DisasContext *s, arg_rrr_esz *a) +{ + static gen_helper_gvec_3 * const fns[4] = { + gen_helper_sve2_sqdmulh_b, gen_helper_sve2_sqdmulh_h, + gen_helper_sve2_sqdmulh_s, gen_helper_sve2_sqdmulh_d, + }; + return do_sve2_zzz_ool(s, a, fns[a->esz]); +} + +static bool trans_SQRDMULH_zzz(DisasContext *s, arg_rrr_esz *a) +{ + static gen_helper_gvec_3 * const fns[4] = { + gen_helper_sve2_sqrdmulh_b, gen_helper_sve2_sqrdmulh_h, + gen_helper_sve2_sqrdmulh_s, gen_helper_sve2_sqrdmulh_d, + }; + return do_sve2_zzz_ool(s, a, fns[a->esz]); +} + /* * SVE2 Integer - Predicated */ diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(sve2_sqrdmlsh_b)(void *vd, void *vn, void *vm, } } +void HELPER(sve2_sqdmulh_b)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int8_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz; ++i) { + d[i] = do_sqrdmlah_b(n[i], m[i], 0, false, false); + } +} + +void HELPER(sve2_sqrdmulh_b)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int8_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz; ++i) { + d[i] = do_sqrdmlah_b(n[i], m[i], 0, false, true); + } +} + /* Signed saturating rounding doubling multiply-accumulate high half, 16-bit */ int16_t do_sqrdmlah_h(int16_t src1, int16_t src2, int16_t src3, bool neg, bool round, uint32_t *sat) @@ -XXX,XX +XXX,XX @@ void HELPER(sve2_sqrdmlsh_h)(void *vd, void *vn, void *vm, } } +void HELPER(sve2_sqdmulh_h)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int16_t *d = vd, *n = vn, *m = vm; + uint32_t discard; + + for (i = 0; i < opr_sz / 2; ++i) { + d[i] = do_sqrdmlah_h(n[i], m[i], 0, false, false, &discard); + } +} + +void HELPER(sve2_sqrdmulh_h)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int16_t *d = vd, *n = vn, *m = vm; + uint32_t discard; + + for (i = 0; i < opr_sz / 2; ++i) { + d[i] = do_sqrdmlah_h(n[i], m[i], 0, false, true, &discard); + } +} + /* Signed saturating rounding doubling multiply-accumulate high half, 32-bit */ int32_t do_sqrdmlah_s(int32_t src1, int32_t src2, int32_t src3, bool neg, bool round, uint32_t *sat) @@ -XXX,XX +XXX,XX @@ void HELPER(sve2_sqrdmlsh_s)(void *vd, void *vn, void *vm, } } +void HELPER(sve2_sqdmulh_s)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int32_t *d = vd, *n = vn, *m = vm; + uint32_t discard; + + for (i = 0; i < opr_sz / 4; ++i) { + d[i] = do_sqrdmlah_s(n[i], m[i], 0, false, false, &discard); + } +} + +void HELPER(sve2_sqrdmulh_s)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int32_t *d = vd, *n = vn, *m = vm; + uint32_t discard; + + for (i = 0; i < opr_sz / 4; ++i) { + d[i] = do_sqrdmlah_s(n[i], m[i], 0, false, true, &discard); + } +} + /* Signed saturating rounding doubling multiply-accumulate high half, 64-bit */ static int64_t do_sat128_d(Int128 r) { @@ -XXX,XX +XXX,XX @@ void HELPER(sve2_sqrdmlsh_d)(void *vd, void *vn, void *vm, } } +void HELPER(sve2_sqdmulh_d)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int64_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz / 8; ++i) { + d[i] = do_sqrdmlah_d(n[i], m[i], 0, false, false); + } +} + +void HELPER(sve2_sqrdmulh_d)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int64_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz / 8; ++i) { + d[i] = do_sqrdmlah_d(n[i], m[i], 0, false, true); + } +} + /* Integer 8 and 16-bit dot-product. * * Note that for the loops herein, host endianness does not matter -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-60-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper.h | 14 ++++++ target/arm/sve.decode | 8 ++++ target/arm/translate-sve.c | 8 ++++ target/arm/vec_helper.c | 88 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+) diff --git a/target/arm/helper.h b/target/arm/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.h +++ b/target/arm/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_4(sve2_sqrdmulh_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_sqrdmulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_sqrdmulh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqdmulh_idx_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqdmulh_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqdmulh_idx_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_sqrdmulh_idx_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqrdmulh_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_sqrdmulh_idx_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_4(gvec_xar_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) #ifdef TARGET_AARCH64 diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SQDMULLB_zzx_d 01000100 11 1 ..... 1110.0 ..... ..... @rrx_2a esz=3 SQDMULLT_zzx_s 01000100 10 1 ..... 1110.1 ..... ..... @rrx_3a esz=2 SQDMULLT_zzx_d 01000100 11 1 ..... 1110.1 ..... ..... @rrx_2a esz=3 +# SVE2 saturating multiply high (indexed) +SQDMULH_zzx_h 01000100 0. 1 ..... 111100 ..... ..... @rrx_3 esz=1 +SQDMULH_zzx_s 01000100 10 1 ..... 111100 ..... ..... @rrx_2 esz=2 +SQDMULH_zzx_d 01000100 11 1 ..... 111100 ..... ..... @rrx_1 esz=3 +SQRDMULH_zzx_h 01000100 0. 1 ..... 111101 ..... ..... @rrx_3 esz=1 +SQRDMULH_zzx_s 01000100 10 1 ..... 111101 ..... ..... @rrx_2 esz=2 +SQRDMULH_zzx_d 01000100 11 1 ..... 111101 ..... ..... @rrx_1 esz=3 + # SVE2 integer multiply (indexed) MUL_zzx_h 01000100 0. 1 ..... 111110 ..... ..... @rrx_3 esz=1 MUL_zzx_s 01000100 10 1 ..... 111110 ..... ..... @rrx_2 esz=2 diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_RRX(trans_MUL_zzx_h, gen_helper_gvec_mul_idx_h) DO_SVE2_RRX(trans_MUL_zzx_s, gen_helper_gvec_mul_idx_s) DO_SVE2_RRX(trans_MUL_zzx_d, gen_helper_gvec_mul_idx_d) +DO_SVE2_RRX(trans_SQDMULH_zzx_h, gen_helper_sve2_sqdmulh_idx_h) +DO_SVE2_RRX(trans_SQDMULH_zzx_s, gen_helper_sve2_sqdmulh_idx_s) +DO_SVE2_RRX(trans_SQDMULH_zzx_d, gen_helper_sve2_sqdmulh_idx_d) + +DO_SVE2_RRX(trans_SQRDMULH_zzx_h, gen_helper_sve2_sqrdmulh_idx_h) +DO_SVE2_RRX(trans_SQRDMULH_zzx_s, gen_helper_sve2_sqrdmulh_idx_s) +DO_SVE2_RRX(trans_SQRDMULH_zzx_d, gen_helper_sve2_sqrdmulh_idx_d) + #undef DO_SVE2_RRX #define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \ diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(sve2_sqrdmulh_h)(void *vd, void *vn, void *vm, uint32_t desc) } } +void HELPER(sve2_sqdmulh_idx_h)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, j, opr_sz = simd_oprsz(desc); + int idx = simd_data(desc); + int16_t *d = vd, *n = vn, *m = (int16_t *)vm + H2(idx); + uint32_t discard; + + for (i = 0; i < opr_sz / 2; i += 16 / 2) { + int16_t mm = m[i]; + for (j = 0; j < 16 / 2; ++j) { + d[i + j] = do_sqrdmlah_h(n[i + j], mm, 0, false, false, &discard); + } + } +} + +void HELPER(sve2_sqrdmulh_idx_h)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, j, opr_sz = simd_oprsz(desc); + int idx = simd_data(desc); + int16_t *d = vd, *n = vn, *m = (int16_t *)vm + H2(idx); + uint32_t discard; + + for (i = 0; i < opr_sz / 2; i += 16 / 2) { + int16_t mm = m[i]; + for (j = 0; j < 16 / 2; ++j) { + d[i + j] = do_sqrdmlah_h(n[i + j], mm, 0, false, true, &discard); + } + } +} + /* Signed saturating rounding doubling multiply-accumulate high half, 32-bit */ int32_t do_sqrdmlah_s(int32_t src1, int32_t src2, int32_t src3, bool neg, bool round, uint32_t *sat) @@ -XXX,XX +XXX,XX @@ void HELPER(sve2_sqrdmulh_s)(void *vd, void *vn, void *vm, uint32_t desc) } } +void HELPER(sve2_sqdmulh_idx_s)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, j, opr_sz = simd_oprsz(desc); + int idx = simd_data(desc); + int32_t *d = vd, *n = vn, *m = (int32_t *)vm + H4(idx); + uint32_t discard; + + for (i = 0; i < opr_sz / 4; i += 16 / 4) { + int32_t mm = m[i]; + for (j = 0; j < 16 / 4; ++j) { + d[i + j] = do_sqrdmlah_s(n[i + j], mm, 0, false, false, &discard); + } + } +} + +void HELPER(sve2_sqrdmulh_idx_s)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, j, opr_sz = simd_oprsz(desc); + int idx = simd_data(desc); + int32_t *d = vd, *n = vn, *m = (int32_t *)vm + H4(idx); + uint32_t discard; + + for (i = 0; i < opr_sz / 4; i += 16 / 4) { + int32_t mm = m[i]; + for (j = 0; j < 16 / 4; ++j) { + d[i + j] = do_sqrdmlah_s(n[i + j], mm, 0, false, true, &discard); + } + } +} + /* Signed saturating rounding doubling multiply-accumulate high half, 64-bit */ static int64_t do_sat128_d(Int128 r) { @@ -XXX,XX +XXX,XX @@ void HELPER(sve2_sqrdmulh_d)(void *vd, void *vn, void *vm, uint32_t desc) } } +void HELPER(sve2_sqdmulh_idx_d)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, j, opr_sz = simd_oprsz(desc); + int idx = simd_data(desc); + int64_t *d = vd, *n = vn, *m = (int64_t *)vm + idx; + + for (i = 0; i < opr_sz / 8; i += 16 / 8) { + int64_t mm = m[i]; + for (j = 0; j < 16 / 8; ++j) { + d[i + j] = do_sqrdmlah_d(n[i + j], mm, 0, false, false); + } + } +} + +void HELPER(sve2_sqrdmulh_idx_d)(void *vd, void *vn, void *vm, uint32_t desc) +{ + intptr_t i, j, opr_sz = simd_oprsz(desc); + int idx = simd_data(desc); + int64_t *d = vd, *n = vn, *m = (int64_t *)vm + idx; + + for (i = 0; i < opr_sz / 8; i += 16 / 8) { + int64_t mm = m[i]; + for (j = 0; j < 16 / 8; ++j) { + d[i + j] = do_sqrdmlah_d(n[i + j], mm, 0, false, true); + } + } +} + /* Integer 8 and 16-bit dot-product. * * Note that for the loops herein, host endianness does not matter -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-61-richard.henderson@linaro.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 17 +++++++++++++++++ target/arm/sve.decode | 18 ++++++++++++++++++ target/arm/sve_helper.c | 16 ++++++++++++++++ target/arm/translate-sve.c | 20 ++++++++++++++++++++ 4 files changed, 71 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_4(sve2_sqdmull_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_sqdmull_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_smlal_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_smlal_idx_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_smlsl_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_smlsl_idx_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_umlal_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_umlal_idx_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_umlsl_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_umlsl_idx_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SQDMLSLB_zzxw_d 01000100 11 1 ..... 0011.0 ..... ..... @rrxr_2a esz=3 SQDMLSLT_zzxw_s 01000100 10 1 ..... 0011.1 ..... ..... @rrxr_3a esz=2 SQDMLSLT_zzxw_d 01000100 11 1 ..... 0011.1 ..... ..... @rrxr_2a esz=3 +# SVE2 multiply-add long (indexed) +SMLALB_zzxw_s 01000100 10 1 ..... 1000.0 ..... ..... @rrxr_3a esz=2 +SMLALB_zzxw_d 01000100 11 1 ..... 1000.0 ..... ..... @rrxr_2a esz=3 +SMLALT_zzxw_s 01000100 10 1 ..... 1000.1 ..... ..... @rrxr_3a esz=2 +SMLALT_zzxw_d 01000100 11 1 ..... 1000.1 ..... ..... @rrxr_2a esz=3 +UMLALB_zzxw_s 01000100 10 1 ..... 1001.0 ..... ..... @rrxr_3a esz=2 +UMLALB_zzxw_d 01000100 11 1 ..... 1001.0 ..... ..... @rrxr_2a esz=3 +UMLALT_zzxw_s 01000100 10 1 ..... 1001.1 ..... ..... @rrxr_3a esz=2 +UMLALT_zzxw_d 01000100 11 1 ..... 1001.1 ..... ..... @rrxr_2a esz=3 +SMLSLB_zzxw_s 01000100 10 1 ..... 1010.0 ..... ..... @rrxr_3a esz=2 +SMLSLB_zzxw_d 01000100 11 1 ..... 1010.0 ..... ..... @rrxr_2a esz=3 +SMLSLT_zzxw_s 01000100 10 1 ..... 1010.1 ..... ..... @rrxr_3a esz=2 +SMLSLT_zzxw_d 01000100 11 1 ..... 1010.1 ..... ..... @rrxr_2a esz=3 +UMLSLB_zzxw_s 01000100 10 1 ..... 1011.0 ..... ..... @rrxr_3a esz=2 +UMLSLB_zzxw_d 01000100 11 1 ..... 1011.0 ..... ..... @rrxr_2a esz=3 +UMLSLT_zzxw_s 01000100 10 1 ..... 1011.1 ..... ..... @rrxr_3a esz=2 +UMLSLT_zzxw_d 01000100 11 1 ..... 1011.1 ..... ..... @rrxr_2a esz=3 + # SVE2 saturating multiply (indexed) SQDMULLB_zzx_s 01000100 10 1 ..... 1110.0 ..... ..... @rrx_3a esz=2 SQDMULLB_zzx_d 01000100 11 1 ..... 1110.0 ..... ..... @rrx_2a esz=3 diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \ } \ } +#define DO_MLA(N, M, A) (A + N * M) + +DO_ZZXW(sve2_smlal_idx_s, int32_t, int16_t, H1_4, H1_2, DO_MLA) +DO_ZZXW(sve2_smlal_idx_d, int64_t, int32_t, , H1_4, DO_MLA) +DO_ZZXW(sve2_umlal_idx_s, uint32_t, uint16_t, H1_4, H1_2, DO_MLA) +DO_ZZXW(sve2_umlal_idx_d, uint64_t, uint32_t, , H1_4, DO_MLA) + +#define DO_MLS(N, M, A) (A - N * M) + +DO_ZZXW(sve2_smlsl_idx_s, int32_t, int16_t, H1_4, H1_2, DO_MLS) +DO_ZZXW(sve2_smlsl_idx_d, int64_t, int32_t, , H1_4, DO_MLS) +DO_ZZXW(sve2_umlsl_idx_s, uint32_t, uint16_t, H1_4, H1_2, DO_MLS) +DO_ZZXW(sve2_umlsl_idx_d, uint64_t, uint32_t, , H1_4, DO_MLS) + #define DO_SQDMLAL_S(N, M, A) DO_SQADD_S(A, do_sqdmull_s(N, M)) #define DO_SQDMLAL_D(N, M, A) do_sqadd_d(A, do_sqdmull_d(N, M)) @@ -XXX,XX +XXX,XX @@ DO_ZZXW(sve2_sqdmlal_idx_d, int64_t, int32_t, , H1_4, DO_SQDMLAL_D) DO_ZZXW(sve2_sqdmlsl_idx_s, int32_t, int16_t, H1_4, H1_2, DO_SQDMLSL_S) DO_ZZXW(sve2_sqdmlsl_idx_d, int64_t, int32_t, , H1_4, DO_SQDMLSL_D) +#undef DO_MLA +#undef DO_MLS #undef DO_ZZXW #define DO_ZZX(NAME, TYPEW, TYPEN, HW, HN, OP) \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, false) DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, true) DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, true) +DO_SVE2_RRXR_TB(trans_SMLALB_zzxw_s, gen_helper_sve2_smlal_idx_s, false) +DO_SVE2_RRXR_TB(trans_SMLALB_zzxw_d, gen_helper_sve2_smlal_idx_d, false) +DO_SVE2_RRXR_TB(trans_SMLALT_zzxw_s, gen_helper_sve2_smlal_idx_s, true) +DO_SVE2_RRXR_TB(trans_SMLALT_zzxw_d, gen_helper_sve2_smlal_idx_d, true) + +DO_SVE2_RRXR_TB(trans_UMLALB_zzxw_s, gen_helper_sve2_umlal_idx_s, false) +DO_SVE2_RRXR_TB(trans_UMLALB_zzxw_d, gen_helper_sve2_umlal_idx_d, false) +DO_SVE2_RRXR_TB(trans_UMLALT_zzxw_s, gen_helper_sve2_umlal_idx_s, true) +DO_SVE2_RRXR_TB(trans_UMLALT_zzxw_d, gen_helper_sve2_umlal_idx_d, true) + +DO_SVE2_RRXR_TB(trans_SMLSLB_zzxw_s, gen_helper_sve2_smlsl_idx_s, false) +DO_SVE2_RRXR_TB(trans_SMLSLB_zzxw_d, gen_helper_sve2_smlsl_idx_d, false) +DO_SVE2_RRXR_TB(trans_SMLSLT_zzxw_s, gen_helper_sve2_smlsl_idx_s, true) +DO_SVE2_RRXR_TB(trans_SMLSLT_zzxw_d, gen_helper_sve2_smlsl_idx_d, true) + +DO_SVE2_RRXR_TB(trans_UMLSLB_zzxw_s, gen_helper_sve2_umlsl_idx_s, false) +DO_SVE2_RRXR_TB(trans_UMLSLB_zzxw_d, gen_helper_sve2_umlsl_idx_d, false) +DO_SVE2_RRXR_TB(trans_UMLSLT_zzxw_s, gen_helper_sve2_umlsl_idx_s, true) +DO_SVE2_RRXR_TB(trans_UMLSLT_zzxw_d, gen_helper_sve2_umlsl_idx_d, true) + #undef DO_SVE2_RRXR_TB /* -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-62-richard.henderson@linaro.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 5 +++++ target/arm/sve.decode | 10 ++++++++++ target/arm/sve_helper.c | 6 ++++++ target/arm/translate-sve.c | 10 ++++++++++ 4 files changed, 31 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_umlsl_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve2_umlsl_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_smull_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_smull_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_umull_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_umull_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ UMLSLB_zzxw_d 01000100 11 1 ..... 1011.0 ..... ..... @rrxr_2a esz=3 UMLSLT_zzxw_s 01000100 10 1 ..... 1011.1 ..... ..... @rrxr_3a esz=2 UMLSLT_zzxw_d 01000100 11 1 ..... 1011.1 ..... ..... @rrxr_2a esz=3 +# SVE2 integer multiply long (indexed) +SMULLB_zzx_s 01000100 10 1 ..... 1100.0 ..... ..... @rrx_3a esz=2 +SMULLB_zzx_d 01000100 11 1 ..... 1100.0 ..... ..... @rrx_2a esz=3 +SMULLT_zzx_s 01000100 10 1 ..... 1100.1 ..... ..... @rrx_3a esz=2 +SMULLT_zzx_d 01000100 11 1 ..... 1100.1 ..... ..... @rrx_2a esz=3 +UMULLB_zzx_s 01000100 10 1 ..... 1101.0 ..... ..... @rrx_3a esz=2 +UMULLB_zzx_d 01000100 11 1 ..... 1101.0 ..... ..... @rrx_2a esz=3 +UMULLT_zzx_s 01000100 10 1 ..... 1101.1 ..... ..... @rrx_3a esz=2 +UMULLT_zzx_d 01000100 11 1 ..... 1101.1 ..... ..... @rrx_2a esz=3 + # SVE2 saturating multiply (indexed) SQDMULLB_zzx_s 01000100 10 1 ..... 1110.0 ..... ..... @rrx_3a esz=2 SQDMULLB_zzx_d 01000100 11 1 ..... 1110.0 ..... ..... @rrx_2a esz=3 diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ DO_ZZX(sve2_sqdmull_idx_s, int32_t, int16_t, H1_4, H1_2, do_sqdmull_s) DO_ZZX(sve2_sqdmull_idx_d, int64_t, int32_t, , H1_4, do_sqdmull_d) +DO_ZZX(sve2_smull_idx_s, int32_t, int16_t, H1_4, H1_2, DO_MUL) +DO_ZZX(sve2_smull_idx_d, int64_t, int32_t, , H1_4, DO_MUL) + +DO_ZZX(sve2_umull_idx_s, uint32_t, uint16_t, H1_4, H1_2, DO_MUL) +DO_ZZX(sve2_umull_idx_d, uint64_t, uint32_t, , H1_4, DO_MUL) + #undef DO_ZZX #define DO_BITPERM(NAME, TYPE, OP) \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_RRX_TB(trans_SQDMULLB_zzx_d, gen_helper_sve2_sqdmull_idx_d, false) DO_SVE2_RRX_TB(trans_SQDMULLT_zzx_s, gen_helper_sve2_sqdmull_idx_s, true) DO_SVE2_RRX_TB(trans_SQDMULLT_zzx_d, gen_helper_sve2_sqdmull_idx_d, true) +DO_SVE2_RRX_TB(trans_SMULLB_zzx_s, gen_helper_sve2_smull_idx_s, false) +DO_SVE2_RRX_TB(trans_SMULLB_zzx_d, gen_helper_sve2_smull_idx_d, false) +DO_SVE2_RRX_TB(trans_SMULLT_zzx_s, gen_helper_sve2_smull_idx_s, true) +DO_SVE2_RRX_TB(trans_SMULLT_zzx_d, gen_helper_sve2_smull_idx_d, true) + +DO_SVE2_RRX_TB(trans_UMULLB_zzx_s, gen_helper_sve2_umull_idx_s, false) +DO_SVE2_RRX_TB(trans_UMULLB_zzx_d, gen_helper_sve2_umull_idx_d, false) +DO_SVE2_RRX_TB(trans_UMULLT_zzx_s, gen_helper_sve2_umull_idx_s, true) +DO_SVE2_RRX_TB(trans_UMULLT_zzx_d, gen_helper_sve2_umull_idx_d, true) + #undef DO_SVE2_RRX_TB static bool do_sve2_zzzz_data(DisasContext *s, int rd, int rn, int rm, int ra, -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-63-richard.henderson@linaro.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 9 +++++++++ target/arm/sve.decode | 12 ++++++++++++ target/arm/sve_helper.c | 28 ++++++++++++++++++++++++++++ target/arm/translate-sve.c | 15 +++++++++++++++ 4 files changed, 64 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_4(sve2_smull_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_smull_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_umull_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve2_umull_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_cmla_idx_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_cmla_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_idx_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SQDMLSLB_zzxw_d 01000100 11 1 ..... 0011.0 ..... ..... @rrxr_2a esz=3 SQDMLSLT_zzxw_s 01000100 10 1 ..... 0011.1 ..... ..... @rrxr_3a esz=2 SQDMLSLT_zzxw_d 01000100 11 1 ..... 0011.1 ..... ..... @rrxr_2a esz=3 +# SVE2 complex integer multiply-add (indexed) +CMLA_zzxz_h 01000100 10 1 index:2 rm:3 0110 rot:2 rn:5 rd:5 \ + ra=%reg_movprfx +CMLA_zzxz_s 01000100 11 1 index:1 rm:4 0110 rot:2 rn:5 rd:5 \ + ra=%reg_movprfx + +# SVE2 complex saturating integer multiply-add (indexed) +SQRDCMLAH_zzxz_h 01000100 10 1 index:2 rm:3 0111 rot:2 rn:5 rd:5 \ + ra=%reg_movprfx +SQRDCMLAH_zzxz_s 01000100 11 1 index:1 rm:4 0111 rot:2 rn:5 rd:5 \ + ra=%reg_movprfx + # SVE2 multiply-add long (indexed) SMLALB_zzxw_s 01000100 10 1 ..... 1000.0 ..... ..... @rrxr_3a esz=2 SMLALB_zzxw_d 01000100 11 1 ..... 1000.0 ..... ..... @rrxr_2a esz=3 diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_h, int16_t, H2, DO_SQRDMLAH_H) DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_s, int32_t, H4, DO_SQRDMLAH_S) DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_d, int64_t, , DO_SQRDMLAH_D) +#define DO_CMLA_IDX_FUNC(NAME, TYPE, H, OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \ +{ \ + intptr_t i, j, oprsz = simd_oprsz(desc); \ + int rot = extract32(desc, SIMD_DATA_SHIFT, 2); \ + int idx = extract32(desc, SIMD_DATA_SHIFT + 2, 2) * 2; \ + int sel_a = rot & 1, sel_b = sel_a ^ 1; \ + bool sub_r = rot == 1 || rot == 2; \ + bool sub_i = rot >= 2; \ + TYPE *d = vd, *n = vn, *m = vm, *a = va; \ + for (i = 0; i < oprsz / sizeof(TYPE); i += 16 / sizeof(TYPE)) { \ + TYPE elt2_a = m[H(i + idx + sel_a)]; \ + TYPE elt2_b = m[H(i + idx + sel_b)]; \ + for (j = 0; j < 16 / sizeof(TYPE); j += 2) { \ + TYPE elt1_a = n[H(i + j + sel_a)]; \ + d[H2(i + j)] = OP(elt1_a, elt2_a, a[H(i + j)], sub_r); \ + d[H2(i + j + 1)] = OP(elt1_a, elt2_b, a[H(i + j + 1)], sub_i); \ + } \ + } \ +} + +DO_CMLA_IDX_FUNC(sve2_cmla_idx_h, int16_t, H2, DO_CMLA) +DO_CMLA_IDX_FUNC(sve2_cmla_idx_s, int32_t, H4, DO_CMLA) + +DO_CMLA_IDX_FUNC(sve2_sqrdcmlah_idx_h, int16_t, H2, DO_SQRDMLAH_H) +DO_CMLA_IDX_FUNC(sve2_sqrdcmlah_idx_s, int32_t, H4, DO_SQRDMLAH_S) + #undef DO_CMLA #undef DO_CMLA_FUNC +#undef DO_CMLA_IDX_FUNC #undef DO_SQRDMLAH_B #undef DO_SQRDMLAH_H #undef DO_SQRDMLAH_S diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_RRXR_TB(trans_UMLSLT_zzxw_d, gen_helper_sve2_umlsl_idx_d, true) #undef DO_SVE2_RRXR_TB +#define DO_SVE2_RRXR_ROT(NAME, FUNC) \ + static bool trans_##NAME(DisasContext *s, arg_##NAME *a) \ + { \ + return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->ra, \ + (a->index << 2) | a->rot, FUNC); \ + } + +DO_SVE2_RRXR_ROT(CMLA_zzxz_h, gen_helper_sve2_cmla_idx_h) +DO_SVE2_RRXR_ROT(CMLA_zzxz_s, gen_helper_sve2_cmla_idx_s) + +DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h, gen_helper_sve2_sqrdcmlah_idx_h) +DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s, gen_helper_sve2_sqrdcmlah_idx_s) + +#undef DO_SVE2_RRXR_ROT + /* *** SVE Floating Point Multiply-Add Indexed Group */ -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-64-richard.henderson@linaro.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 10 ++++ target/arm/sve.decode | 9 ++++ target/arm/sve_helper.c | 99 ++++++++++++++++++++++++++++++++++++++ target/arm/translate-sve.c | 17 +++++++ 4 files changed, 135 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_cdot_zzzz_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_cdot_zzzz_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_cdot_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_cdot_idx_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ MUL_zzi 00100101 .. 110 000 110 ........ ..... @rdn_i8s DOT_zzzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 \ ra=%reg_movprfx +# SVE2 complex dot product (vectors) +CDOT_zzzz 01000100 esz:2 0 rm:5 0001 rot:2 rn:5 rd:5 ra=%reg_movprfx + #### SVE Multiply - Indexed # SVE integer dot product (indexed) @@ -XXX,XX +XXX,XX @@ SQDMLSLB_zzxw_d 01000100 11 1 ..... 0011.0 ..... ..... @rrxr_2a esz=3 SQDMLSLT_zzxw_s 01000100 10 1 ..... 0011.1 ..... ..... @rrxr_3a esz=2 SQDMLSLT_zzxw_d 01000100 11 1 ..... 0011.1 ..... ..... @rrxr_2a esz=3 +# SVE2 complex integer dot product (indexed) +CDOT_zzxw_s 01000100 10 1 index:2 rm:3 0100 rot:2 rn:5 rd:5 \ + ra=%reg_movprfx +CDOT_zzxw_d 01000100 11 1 index:1 rm:4 0100 rot:2 rn:5 rd:5 \ + ra=%reg_movprfx + # SVE2 complex integer multiply-add (indexed) CMLA_zzxz_h 01000100 10 1 index:2 rm:3 0110 rot:2 rn:5 rd:5 \ ra=%reg_movprfx diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ DO_CMLA_IDX_FUNC(sve2_sqrdcmlah_idx_s, int32_t, H4, DO_SQRDMLAH_S) #undef DO_SQRDMLAH_S #undef DO_SQRDMLAH_D +/* Note N and M are 4 elements bundled into one unit. */ +static int32_t do_cdot_s(uint32_t n, uint32_t m, int32_t a, + int sel_a, int sel_b, int sub_i) +{ + for (int i = 0; i <= 1; i++) { + int32_t elt1_r = (int8_t)(n >> (16 * i)); + int32_t elt1_i = (int8_t)(n >> (16 * i + 8)); + int32_t elt2_a = (int8_t)(m >> (16 * i + 8 * sel_a)); + int32_t elt2_b = (int8_t)(m >> (16 * i + 8 * sel_b)); + + a += elt1_r * elt2_a + elt1_i * elt2_b * sub_i; + } + return a; +} + +static int64_t do_cdot_d(uint64_t n, uint64_t m, int64_t a, + int sel_a, int sel_b, int sub_i) +{ + for (int i = 0; i <= 1; i++) { + int64_t elt1_r = (int16_t)(n >> (32 * i + 0)); + int64_t elt1_i = (int16_t)(n >> (32 * i + 16)); + int64_t elt2_a = (int16_t)(m >> (32 * i + 16 * sel_a)); + int64_t elt2_b = (int16_t)(m >> (32 * i + 16 * sel_b)); + + a += elt1_r * elt2_a + elt1_i * elt2_b * sub_i; + } + return a; +} + +void HELPER(sve2_cdot_zzzz_s)(void *vd, void *vn, void *vm, + void *va, uint32_t desc) +{ + int opr_sz = simd_oprsz(desc); + int rot = simd_data(desc); + int sel_a = rot & 1; + int sel_b = sel_a ^ 1; + int sub_i = (rot == 0 || rot == 3 ? -1 : 1); + uint32_t *d = vd, *n = vn, *m = vm, *a = va; + + for (int e = 0; e < opr_sz / 4; e++) { + d[e] = do_cdot_s(n[e], m[e], a[e], sel_a, sel_b, sub_i); + } +} + +void HELPER(sve2_cdot_zzzz_d)(void *vd, void *vn, void *vm, + void *va, uint32_t desc) +{ + int opr_sz = simd_oprsz(desc); + int rot = simd_data(desc); + int sel_a = rot & 1; + int sel_b = sel_a ^ 1; + int sub_i = (rot == 0 || rot == 3 ? -1 : 1); + uint64_t *d = vd, *n = vn, *m = vm, *a = va; + + for (int e = 0; e < opr_sz / 8; e++) { + d[e] = do_cdot_d(n[e], m[e], a[e], sel_a, sel_b, sub_i); + } +} + +void HELPER(sve2_cdot_idx_s)(void *vd, void *vn, void *vm, + void *va, uint32_t desc) +{ + int opr_sz = simd_oprsz(desc); + int rot = extract32(desc, SIMD_DATA_SHIFT, 2); + int idx = H4(extract32(desc, SIMD_DATA_SHIFT + 2, 2)); + int sel_a = rot & 1; + int sel_b = sel_a ^ 1; + int sub_i = (rot == 0 || rot == 3 ? -1 : 1); + uint32_t *d = vd, *n = vn, *m = vm, *a = va; + + for (int seg = 0; seg < opr_sz / 4; seg += 4) { + uint32_t seg_m = m[seg + idx]; + for (int e = 0; e < 4; e++) { + d[seg + e] = do_cdot_s(n[seg + e], seg_m, a[seg + e], + sel_a, sel_b, sub_i); + } + } +} + +void HELPER(sve2_cdot_idx_d)(void *vd, void *vn, void *vm, + void *va, uint32_t desc) +{ + int seg, opr_sz = simd_oprsz(desc); + int rot = extract32(desc, SIMD_DATA_SHIFT, 2); + int idx = extract32(desc, SIMD_DATA_SHIFT + 2, 2); + int sel_a = rot & 1; + int sel_b = sel_a ^ 1; + int sub_i = (rot == 0 || rot == 3 ? -1 : 1); + uint64_t *d = vd, *n = vn, *m = vm, *a = va; + + for (seg = 0; seg < opr_sz / 8; seg += 2) { + uint64_t seg_m = m[seg + idx]; + for (int e = 0; e < 2; e++) { + d[seg + e] = do_cdot_d(n[seg + e], seg_m, a[seg + e], + sel_a, sel_b, sub_i); + } + } +} + #define DO_ZZXZ(NAME, TYPE, H, OP) \ void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \ { \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_SVE2_RRXR_ROT(CMLA_zzxz_s, gen_helper_sve2_cmla_idx_s) DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h, gen_helper_sve2_sqrdcmlah_idx_h) DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s, gen_helper_sve2_sqrdcmlah_idx_s) +DO_SVE2_RRXR_ROT(CDOT_zzxw_s, gen_helper_sve2_cdot_idx_s) +DO_SVE2_RRXR_ROT(CDOT_zzxw_d, gen_helper_sve2_cdot_idx_d) + #undef DO_SVE2_RRXR_ROT /* @@ -XXX,XX +XXX,XX @@ static bool trans_CMLA_zzzz(DisasContext *s, arg_CMLA_zzzz *a) return true; } +static bool trans_CDOT_zzzz(DisasContext *s, arg_CMLA_zzzz *a) +{ + if (!dc_isar_feature(aa64_sve2, s) || a->esz < MO_32) { + return false; + } + if (sve_access_check(s)) { + gen_helper_gvec_4 *fn = (a->esz == MO_32 + ? gen_helper_sve2_cdot_zzzz_s + : gen_helper_sve2_cdot_zzzz_d); + gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->rot); + } + return true; +} + static bool trans_SQRDCMLAH_zzzz(DisasContext *s, arg_SQRDCMLAH_zzzz *a) { static gen_helper_gvec_4 * const fns[] = { -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> We're about to add more variations on this theme. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-65-richard.henderson@linaro.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/vec_helper.c | 82 ++++++++++------------------------------- 1 file changed, 20 insertions(+), 62 deletions(-) diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(sve2_sqrdmulh_idx_d)(void *vd, void *vn, void *vm, uint32_t desc) /* Integer 8 and 16-bit dot-product. * * Note that for the loops herein, host endianness does not matter - * with respect to the ordering of data within the 64-bit lanes. + * with respect to the ordering of data within the quad-width lanes. * All elements are treated equally, no matter where they are. */ -void HELPER(gvec_sdot_b)(void *vd, void *vn, void *vm, void *va, uint32_t desc) -{ - intptr_t i, opr_sz = simd_oprsz(desc); - int32_t *d = vd, *a = va; - int8_t *n = vn, *m = vm; - - for (i = 0; i < opr_sz / 4; ++i) { - d[i] = (a[i] + - n[i * 4 + 0] * m[i * 4 + 0] + - n[i * 4 + 1] * m[i * 4 + 1] + - n[i * 4 + 2] * m[i * 4 + 2] + - n[i * 4 + 3] * m[i * 4 + 3]); - } - clear_tail(d, opr_sz, simd_maxsz(desc)); +#define DO_DOT(NAME, TYPED, TYPEN, TYPEM) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc); \ + TYPED *d = vd, *a = va; \ + TYPEN *n = vn; \ + TYPEM *m = vm; \ + for (i = 0; i < opr_sz / sizeof(TYPED); ++i) { \ + d[i] = (a[i] + \ + (TYPED)n[i * 4 + 0] * m[i * 4 + 0] + \ + (TYPED)n[i * 4 + 1] * m[i * 4 + 1] + \ + (TYPED)n[i * 4 + 2] * m[i * 4 + 2] + \ + (TYPED)n[i * 4 + 3] * m[i * 4 + 3]); \ + } \ + clear_tail(d, opr_sz, simd_maxsz(desc)); \ } -void HELPER(gvec_udot_b)(void *vd, void *vn, void *vm, void *va, uint32_t desc) -{ - intptr_t i, opr_sz = simd_oprsz(desc); - uint32_t *d = vd, *a = va; - uint8_t *n = vn, *m = vm; - - for (i = 0; i < opr_sz / 4; ++i) { - d[i] = (a[i] + - n[i * 4 + 0] * m[i * 4 + 0] + - n[i * 4 + 1] * m[i * 4 + 1] + - n[i * 4 + 2] * m[i * 4 + 2] + - n[i * 4 + 3] * m[i * 4 + 3]); - } - clear_tail(d, opr_sz, simd_maxsz(desc)); -} - -void HELPER(gvec_sdot_h)(void *vd, void *vn, void *vm, void *va, uint32_t desc) -{ - intptr_t i, opr_sz = simd_oprsz(desc); - int64_t *d = vd, *a = va; - int16_t *n = vn, *m = vm; - - for (i = 0; i < opr_sz / 8; ++i) { - d[i] = (a[i] + - (int64_t)n[i * 4 + 0] * m[i * 4 + 0] + - (int64_t)n[i * 4 + 1] * m[i * 4 + 1] + - (int64_t)n[i * 4 + 2] * m[i * 4 + 2] + - (int64_t)n[i * 4 + 3] * m[i * 4 + 3]); - } - clear_tail(d, opr_sz, simd_maxsz(desc)); -} - -void HELPER(gvec_udot_h)(void *vd, void *vn, void *vm, void *va, uint32_t desc) -{ - intptr_t i, opr_sz = simd_oprsz(desc); - uint64_t *d = vd, *a = va; - uint16_t *n = vn, *m = vm; - - for (i = 0; i < opr_sz / 8; ++i) { - d[i] = (a[i] + - (uint64_t)n[i * 4 + 0] * m[i * 4 + 0] + - (uint64_t)n[i * 4 + 1] * m[i * 4 + 1] + - (uint64_t)n[i * 4 + 2] * m[i * 4 + 2] + - (uint64_t)n[i * 4 + 3] * m[i * 4 + 3]); - } - clear_tail(d, opr_sz, simd_maxsz(desc)); -} +DO_DOT(gvec_sdot_b, int32_t, int8_t, int8_t) +DO_DOT(gvec_udot_b, uint32_t, uint8_t, uint8_t) +DO_DOT(gvec_sdot_h, int64_t, int16_t, int16_t) +DO_DOT(gvec_udot_h, uint64_t, uint16_t, uint16_t) void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, void *va, uint32_t desc) -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> We're about to add more variations on this theme. Accept the inner loop for the _h variants, rather than keep it unrolled. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-66-richard.henderson@linaro.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/vec_helper.c | 160 ++++++++-------------------------------- 1 file changed, 29 insertions(+), 131 deletions(-) diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ DO_DOT(gvec_udot_b, uint32_t, uint8_t, uint8_t) DO_DOT(gvec_sdot_h, int64_t, int16_t, int16_t) DO_DOT(gvec_udot_h, uint64_t, uint16_t, uint16_t) -void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, - void *va, uint32_t desc) -{ - intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4; - intptr_t index = simd_data(desc); - int32_t *d = vd, *a = va; - int8_t *n = vn; - int8_t *m_indexed = (int8_t *)vm + H4(index) * 4; - - /* Notice the special case of opr_sz == 8, from aa64/aa32 advsimd. - * Otherwise opr_sz is a multiple of 16. - */ - segend = MIN(4, opr_sz_4); - i = 0; - do { - int8_t m0 = m_indexed[i * 4 + 0]; - int8_t m1 = m_indexed[i * 4 + 1]; - int8_t m2 = m_indexed[i * 4 + 2]; - int8_t m3 = m_indexed[i * 4 + 3]; - - do { - d[i] = (a[i] + - n[i * 4 + 0] * m0 + - n[i * 4 + 1] * m1 + - n[i * 4 + 2] * m2 + - n[i * 4 + 3] * m3); - } while (++i < segend); - segend = i + 4; - } while (i < opr_sz_4); - - clear_tail(d, opr_sz, simd_maxsz(desc)); +#define DO_DOT_IDX(NAME, TYPED, TYPEN, TYPEM, HD) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \ +{ \ + intptr_t i = 0, opr_sz = simd_oprsz(desc); \ + intptr_t opr_sz_n = opr_sz / sizeof(TYPED); \ + intptr_t segend = MIN(16 / sizeof(TYPED), opr_sz_n); \ + intptr_t index = simd_data(desc); \ + TYPED *d = vd, *a = va; \ + TYPEN *n = vn; \ + TYPEM *m_indexed = (TYPEM *)vm + HD(index) * 4; \ + do { \ + TYPED m0 = m_indexed[i * 4 + 0]; \ + TYPED m1 = m_indexed[i * 4 + 1]; \ + TYPED m2 = m_indexed[i * 4 + 2]; \ + TYPED m3 = m_indexed[i * 4 + 3]; \ + do { \ + d[i] = (a[i] + \ + n[i * 4 + 0] * m0 + \ + n[i * 4 + 1] * m1 + \ + n[i * 4 + 2] * m2 + \ + n[i * 4 + 3] * m3); \ + } while (++i < segend); \ + segend = i + 4; \ + } while (i < opr_sz_n); \ + clear_tail(d, opr_sz, simd_maxsz(desc)); \ } -void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm, - void *va, uint32_t desc) -{ - intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4; - intptr_t index = simd_data(desc); - uint32_t *d = vd, *a = va; - uint8_t *n = vn; - uint8_t *m_indexed = (uint8_t *)vm + H4(index) * 4; - - /* Notice the special case of opr_sz == 8, from aa64/aa32 advsimd. - * Otherwise opr_sz is a multiple of 16. - */ - segend = MIN(4, opr_sz_4); - i = 0; - do { - uint8_t m0 = m_indexed[i * 4 + 0]; - uint8_t m1 = m_indexed[i * 4 + 1]; - uint8_t m2 = m_indexed[i * 4 + 2]; - uint8_t m3 = m_indexed[i * 4 + 3]; - - do { - d[i] = (a[i] + - n[i * 4 + 0] * m0 + - n[i * 4 + 1] * m1 + - n[i * 4 + 2] * m2 + - n[i * 4 + 3] * m3); - } while (++i < segend); - segend = i + 4; - } while (i < opr_sz_4); - - clear_tail(d, opr_sz, simd_maxsz(desc)); -} - -void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm, - void *va, uint32_t desc) -{ - intptr_t i, opr_sz = simd_oprsz(desc), opr_sz_8 = opr_sz / 8; - intptr_t index = simd_data(desc); - int64_t *d = vd, *a = va; - int16_t *n = vn; - int16_t *m_indexed = (int16_t *)vm + index * 4; - - /* This is supported by SVE only, so opr_sz is always a multiple of 16. - * Process the entire segment all at once, writing back the results - * only after we've consumed all of the inputs. - */ - for (i = 0; i < opr_sz_8; i += 2) { - int64_t d0, d1; - - d0 = a[i + 0]; - d0 += n[i * 4 + 0] * (int64_t)m_indexed[i * 4 + 0]; - d0 += n[i * 4 + 1] * (int64_t)m_indexed[i * 4 + 1]; - d0 += n[i * 4 + 2] * (int64_t)m_indexed[i * 4 + 2]; - d0 += n[i * 4 + 3] * (int64_t)m_indexed[i * 4 + 3]; - - d1 = a[i + 1]; - d1 += n[i * 4 + 4] * (int64_t)m_indexed[i * 4 + 0]; - d1 += n[i * 4 + 5] * (int64_t)m_indexed[i * 4 + 1]; - d1 += n[i * 4 + 6] * (int64_t)m_indexed[i * 4 + 2]; - d1 += n[i * 4 + 7] * (int64_t)m_indexed[i * 4 + 3]; - - d[i + 0] = d0; - d[i + 1] = d1; - } - clear_tail(d, opr_sz, simd_maxsz(desc)); -} - -void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm, - void *va, uint32_t desc) -{ - intptr_t i, opr_sz = simd_oprsz(desc), opr_sz_8 = opr_sz / 8; - intptr_t index = simd_data(desc); - uint64_t *d = vd, *a = va; - uint16_t *n = vn; - uint16_t *m_indexed = (uint16_t *)vm + index * 4; - - /* This is supported by SVE only, so opr_sz is always a multiple of 16. - * Process the entire segment all at once, writing back the results - * only after we've consumed all of the inputs. - */ - for (i = 0; i < opr_sz_8; i += 2) { - uint64_t d0, d1; - - d0 = a[i + 0]; - d0 += n[i * 4 + 0] * (uint64_t)m_indexed[i * 4 + 0]; - d0 += n[i * 4 + 1] * (uint64_t)m_indexed[i * 4 + 1]; - d0 += n[i * 4 + 2] * (uint64_t)m_indexed[i * 4 + 2]; - d0 += n[i * 4 + 3] * (uint64_t)m_indexed[i * 4 + 3]; - - d1 = a[i + 1]; - d1 += n[i * 4 + 4] * (uint64_t)m_indexed[i * 4 + 0]; - d1 += n[i * 4 + 5] * (uint64_t)m_indexed[i * 4 + 1]; - d1 += n[i * 4 + 6] * (uint64_t)m_indexed[i * 4 + 2]; - d1 += n[i * 4 + 7] * (uint64_t)m_indexed[i * 4 + 3]; - - d[i + 0] = d0; - d[i + 1] = d1; - } - clear_tail(d, opr_sz, simd_maxsz(desc)); -} +DO_DOT_IDX(gvec_sdot_idx_b, int32_t, int8_t, int8_t, H4) +DO_DOT_IDX(gvec_udot_idx_b, uint32_t, uint8_t, uint8_t, H4) +DO_DOT_IDX(gvec_sdot_idx_h, int64_t, int16_t, int16_t, ) +DO_DOT_IDX(gvec_udot_idx_h, uint64_t, uint16_t, uint16_t, ) void HELPER(gvec_fcaddh)(void *vd, void *vn, void *vm, void *vfpst, uint32_t desc) -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-67-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/cpu.h | 5 +++++ target/arm/helper.h | 4 ++++ target/arm/sve.decode | 4 ++++ target/arm/translate-sve.c | 16 ++++++++++++++++ target/arm/vec_helper.c | 2 ++ 5 files changed, 31 insertions(+) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -XXX,XX +XXX,XX @@ static inline bool isar_feature_aa64_sve2_bitperm(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, BITPERM) != 0; } +static inline bool isar_feature_aa64_sve_i8mm(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, I8MM) != 0; +} + static inline bool isar_feature_aa64_sve_f32mm(const ARMISARegisters *id) { return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, F32MM) != 0; diff --git a/target/arm/helper.h b/target/arm/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.h +++ b/target/arm/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(gvec_sdot_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(gvec_udot_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_sudot_idx_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_usdot_idx_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(gvec_fcaddh, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SQRDMLSH_zzxz_h 01000100 0. 1 ..... 000101 ..... ..... @rrxr_3 esz=1 SQRDMLSH_zzxz_s 01000100 10 1 ..... 000101 ..... ..... @rrxr_2 esz=2 SQRDMLSH_zzxz_d 01000100 11 1 ..... 000101 ..... ..... @rrxr_1 esz=3 +# SVE mixed sign dot product (indexed) +USDOT_zzxw_s 01000100 10 1 ..... 000110 ..... ..... @rrxr_2 esz=2 +SUDOT_zzxw_s 01000100 10 1 ..... 000111 ..... ..... @rrxr_2 esz=2 + # SVE2 saturating multiply-add (indexed) SQDMLALB_zzxw_s 01000100 10 1 ..... 0010.0 ..... ..... @rrxr_3a esz=2 SQDMLALB_zzxw_d 01000100 11 1 ..... 0010.0 ..... ..... @rrxr_2a esz=3 diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ DO_RRXR(trans_SDOT_zzxw_d, gen_helper_gvec_sdot_idx_h) DO_RRXR(trans_UDOT_zzxw_s, gen_helper_gvec_udot_idx_b) DO_RRXR(trans_UDOT_zzxw_d, gen_helper_gvec_udot_idx_h) +static bool trans_SUDOT_zzxw_s(DisasContext *s, arg_rrxr_esz *a) +{ + if (!dc_isar_feature(aa64_sve_i8mm, s)) { + return false; + } + return do_zzxz_ool(s, a, gen_helper_gvec_sudot_idx_b); +} + +static bool trans_USDOT_zzxw_s(DisasContext *s, arg_rrxr_esz *a) +{ + if (!dc_isar_feature(aa64_sve_i8mm, s)) { + return false; + } + return do_zzxz_ool(s, a, gen_helper_gvec_usdot_idx_b); +} + #undef DO_RRXR static bool do_sve2_zzz_data(DisasContext *s, int rd, int rn, int rm, int data, diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \ DO_DOT_IDX(gvec_sdot_idx_b, int32_t, int8_t, int8_t, H4) DO_DOT_IDX(gvec_udot_idx_b, uint32_t, uint8_t, uint8_t, H4) +DO_DOT_IDX(gvec_sudot_idx_b, int32_t, int8_t, uint8_t, H4) +DO_DOT_IDX(gvec_usdot_idx_b, int32_t, uint8_t, int8_t, H4) DO_DOT_IDX(gvec_sdot_idx_h, int64_t, int16_t, int16_t, ) DO_DOT_IDX(gvec_udot_idx_h, uint64_t, uint16_t, uint16_t, ) -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-68-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper.h | 1 + target/arm/sve.decode | 4 ++++ target/arm/translate-sve.c | 16 ++++++++++++++++ target/arm/vec_helper.c | 1 + 4 files changed, 22 insertions(+) diff --git a/target/arm/helper.h b/target/arm/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.h +++ b/target/arm/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(gvec_sdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(gvec_udot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(gvec_sdot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(gvec_udot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_usdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(gvec_sdot_idx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ UMLSLT_zzzw 01000100 .. 0 ..... 010 111 ..... ..... @rda_rn_rm CMLA_zzzz 01000100 esz:2 0 rm:5 0010 rot:2 rn:5 rd:5 ra=%reg_movprfx SQRDCMLAH_zzzz 01000100 esz:2 0 rm:5 0011 rot:2 rn:5 rd:5 ra=%reg_movprfx +## SVE mixed sign dot product + +USDOT_zzzz 01000100 .. 0 ..... 011 110 ..... ..... @rda_rn_rm + ### SVE2 floating point matrix multiply accumulate FMMLA 01100100 .. 1 ..... 111001 ..... ..... @rda_rn_rm diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_SQRDCMLAH_zzzz(DisasContext *s, arg_SQRDCMLAH_zzzz *a) } return true; } + +static bool trans_USDOT_zzzz(DisasContext *s, arg_USDOT_zzzz *a) +{ + if (a->esz != 2 || !dc_isar_feature(aa64_sve_i8mm, s)) { + return false; + } + if (sve_access_check(s)) { + unsigned vsz = vec_full_reg_size(s); + tcg_gen_gvec_4_ool(vec_full_reg_offset(s, a->rd), + vec_full_reg_offset(s, a->rn), + vec_full_reg_offset(s, a->rm), + vec_full_reg_offset(s, a->ra), + vsz, vsz, 0, gen_helper_gvec_usdot_b); + } + return true; +} diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \ DO_DOT(gvec_sdot_b, int32_t, int8_t, int8_t) DO_DOT(gvec_udot_b, uint32_t, uint8_t, uint8_t) +DO_DOT(gvec_usdot_b, uint32_t, uint8_t, int8_t) DO_DOT(gvec_sdot_h, int64_t, int16_t, int16_t) DO_DOT(gvec_udot_h, uint64_t, uint16_t, uint16_t) -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-69-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/sve.decode | 6 ++++++ target/arm/translate-sve.c | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ STNT1_zprz 1110010 .. 00 ..... 001 ... ..... ..... \ # SVE2 32-bit scatter non-temporal store (vector plus scalar) STNT1_zprz 1110010 .. 10 ..... 001 ... ..... ..... \ @rprr_scatter_store xs=0 esz=2 scale=0 + +### SVE2 Crypto Extensions + +# SVE2 crypto unary operations +# AESMC and AESIMC +AESMC 01000101 00 10000011100 decrypt:1 00000 rd:5 diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_USDOT_zzzz(DisasContext *s, arg_USDOT_zzzz *a) } return true; } + +static bool trans_AESMC(DisasContext *s, arg_AESMC *a) +{ + if (!dc_isar_feature(aa64_sve2_aes, s)) { + return false; + } + if (sve_access_check(s)) { + gen_gvec_ool_zz(s, gen_helper_crypto_aesmc, a->rd, a->rd, a->decrypt); + } + return true; +} -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-70-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/cpu.h | 5 +++++ target/arm/sve.decode | 7 +++++++ target/arm/translate-sve.c | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -XXX,XX +XXX,XX @@ static inline bool isar_feature_aa64_sve2_bitperm(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, BITPERM) != 0; } +static inline bool isar_feature_aa64_sve2_sm4(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, SM4) != 0; +} + static inline bool isar_feature_aa64_sve_i8mm(const ARMISARegisters *id) { return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, I8MM) != 0; diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ @pd_pn_pm ........ esz:2 .. rm:4 ....... rn:4 . rd:4 &rrr_esz @rdn_rm ........ esz:2 ...... ...... rm:5 rd:5 \ &rrr_esz rn=%reg_movprfx +@rdn_rm_e0 ........ .. ...... ...... rm:5 rd:5 \ + &rrr_esz rn=%reg_movprfx esz=0 @rdn_sh_i8u ........ esz:2 ...... ...... ..... rd:5 \ &rri_esz rn=%reg_movprfx imm=%sh8_i8u @rdn_i8u ........ esz:2 ...... ... imm:8 rd:5 \ @@ -XXX,XX +XXX,XX @@ STNT1_zprz 1110010 .. 10 ..... 001 ... ..... ..... \ # SVE2 crypto unary operations # AESMC and AESIMC AESMC 01000101 00 10000011100 decrypt:1 00000 rd:5 + +# SVE2 crypto destructive binary operations +AESE 01000101 00 10001 0 11100 0 ..... ..... @rdn_rm_e0 +AESD 01000101 00 10001 0 11100 1 ..... ..... @rdn_rm_e0 +SM4E 01000101 00 10001 1 11100 0 ..... ..... @rdn_rm_e0 diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_AESMC(DisasContext *s, arg_AESMC *a) } return true; } + +static bool do_aese(DisasContext *s, arg_rrr_esz *a, bool decrypt) +{ + if (!dc_isar_feature(aa64_sve2_aes, s)) { + return false; + } + if (sve_access_check(s)) { + gen_gvec_ool_zzz(s, gen_helper_crypto_aese, + a->rd, a->rn, a->rm, decrypt); + } + return true; +} + +static bool trans_AESE(DisasContext *s, arg_rrr_esz *a) +{ + return do_aese(s, a, false); +} + +static bool trans_AESD(DisasContext *s, arg_rrr_esz *a) +{ + return do_aese(s, a, true); +} + +static bool do_sm4(DisasContext *s, arg_rrr_esz *a, gen_helper_gvec_3 *fn) +{ + if (!dc_isar_feature(aa64_sve2_sm4, s)) { + return false; + } + if (sve_access_check(s)) { + gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, 0); + } + return true; +} + +static bool trans_SM4E(DisasContext *s, arg_rrr_esz *a) +{ + return do_sm4(s, a, gen_helper_crypto_sm4e); +} -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-71-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/cpu.h | 5 +++++ target/arm/sve.decode | 4 ++++ target/arm/translate-sve.c | 16 ++++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -XXX,XX +XXX,XX @@ static inline bool isar_feature_aa64_sve2_bitperm(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, BITPERM) != 0; } +static inline bool isar_feature_aa64_sve2_sha3(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, SHA3) != 0; +} + static inline bool isar_feature_aa64_sve2_sm4(const ARMISARegisters *id) { return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, SM4) != 0; diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ AESMC 01000101 00 10000011100 decrypt:1 00000 rd:5 AESE 01000101 00 10001 0 11100 0 ..... ..... @rdn_rm_e0 AESD 01000101 00 10001 0 11100 1 ..... ..... @rdn_rm_e0 SM4E 01000101 00 10001 1 11100 0 ..... ..... @rdn_rm_e0 + +# SVE2 crypto constructive binary operations +SM4EKEY 01000101 00 1 ..... 11110 0 ..... ..... @rd_rn_rm_e0 +RAX1 01000101 00 1 ..... 11110 1 ..... ..... @rd_rn_rm_e0 diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_SM4E(DisasContext *s, arg_rrr_esz *a) { return do_sm4(s, a, gen_helper_crypto_sm4e); } + +static bool trans_SM4EKEY(DisasContext *s, arg_rrr_esz *a) +{ + return do_sm4(s, a, gen_helper_crypto_sm4ekey); +} + +static bool trans_RAX1(DisasContext *s, arg_rrr_esz *a) +{ + if (!dc_isar_feature(aa64_sve2_sha3, s)) { + return false; + } + if (sve_access_check(s)) { + gen_gvec_fn_zzz(s, gen_gvec_rax1, MO_64, a->rd, a->rn, a->rm); + } + return true; +} -- 2.20.1 From: Stephen Long <steplong@quicinc.com> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Stephen Long <steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-72-richard.henderson@linaro.org Message-Id: <20200428144352.9275-1-steplong@quicinc.com> [rth: rearrange the macros a little and rebase] Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 10 +++++ target/arm/sve.decode | 5 +++ target/arm/sve_helper.c | 90 ++++++++++++++++++++++++++++++-------- target/arm/translate-sve.c | 33 ++++++++++++++ 4 files changed, 119 insertions(+), 19 deletions(-) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_4(sve_tbl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve_tbl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve_tbl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_tbl_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_tbl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_tbl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_tbl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve2_tbx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_tbx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_tbx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve2_tbx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_3(sve_sunpk_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve_sunpk_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve_sunpk_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ TBL 00000101 .. 1 ..... 001100 ..... ..... @rd_rn_rm # SVE unpack vector elements UNPK 00000101 esz:2 1100 u:1 h:1 001110 rn:5 rd:5 +# SVE2 Table Lookup (three sources) + +TBL_sve2 00000101 .. 1 ..... 001010 ..... ..... @rd_rn_rm +TBX 00000101 .. 1 ..... 001011 ..... ..... @rd_rn_rm + ### SVE Permute - Predicates Group # SVE permute predicate elements diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(sve_rev_d)(void *vd, void *vn, uint32_t desc) } } -#define DO_TBL(NAME, TYPE, H) \ -void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ -{ \ - intptr_t i, opr_sz = simd_oprsz(desc); \ - uintptr_t elem = opr_sz / sizeof(TYPE); \ - TYPE *d = vd, *n = vn, *m = vm; \ - ARMVectorReg tmp; \ - if (unlikely(vd == vn)) { \ - n = memcpy(&tmp, vn, opr_sz); \ - } \ - for (i = 0; i < elem; i++) { \ - TYPE j = m[H(i)]; \ - d[H(i)] = j < elem ? n[H(j)] : 0; \ - } \ +typedef void tb_impl_fn(void *, void *, void *, void *, uintptr_t, bool); + +static inline void do_tbl1(void *vd, void *vn, void *vm, uint32_t desc, + bool is_tbx, tb_impl_fn *fn) +{ + ARMVectorReg scratch; + uintptr_t oprsz = simd_oprsz(desc); + + if (unlikely(vd == vn)) { + vn = memcpy(&scratch, vn, oprsz); + } + + fn(vd, vn, NULL, vm, oprsz, is_tbx); } -DO_TBL(sve_tbl_b, uint8_t, H1) -DO_TBL(sve_tbl_h, uint16_t, H2) -DO_TBL(sve_tbl_s, uint32_t, H4) -DO_TBL(sve_tbl_d, uint64_t, ) +static inline void do_tbl2(void *vd, void *vn0, void *vn1, void *vm, + uint32_t desc, bool is_tbx, tb_impl_fn *fn) +{ + ARMVectorReg scratch; + uintptr_t oprsz = simd_oprsz(desc); -#undef TBL + if (unlikely(vd == vn0)) { + vn0 = memcpy(&scratch, vn0, oprsz); + if (vd == vn1) { + vn1 = vn0; + } + } else if (unlikely(vd == vn1)) { + vn1 = memcpy(&scratch, vn1, oprsz); + } + + fn(vd, vn0, vn1, vm, oprsz, is_tbx); +} + +#define DO_TB(SUFF, TYPE, H) \ +static inline void do_tb_##SUFF(void *vd, void *vt0, void *vt1, \ + void *vm, uintptr_t oprsz, bool is_tbx) \ +{ \ + TYPE *d = vd, *tbl0 = vt0, *tbl1 = vt1, *indexes = vm; \ + uintptr_t i, nelem = oprsz / sizeof(TYPE); \ + for (i = 0; i < nelem; ++i) { \ + TYPE index = indexes[H1(i)], val = 0; \ + if (index < nelem) { \ + val = tbl0[H(index)]; \ + } else { \ + index -= nelem; \ + if (tbl1 && index < nelem) { \ + val = tbl1[H(index)]; \ + } else if (is_tbx) { \ + continue; \ + } \ + } \ + d[H(i)] = val; \ + } \ +} \ +void HELPER(sve_tbl_##SUFF)(void *vd, void *vn, void *vm, uint32_t desc) \ +{ \ + do_tbl1(vd, vn, vm, desc, false, do_tb_##SUFF); \ +} \ +void HELPER(sve2_tbl_##SUFF)(void *vd, void *vn0, void *vn1, \ + void *vm, uint32_t desc) \ +{ \ + do_tbl2(vd, vn0, vn1, vm, desc, false, do_tb_##SUFF); \ +} \ +void HELPER(sve2_tbx_##SUFF)(void *vd, void *vn, void *vm, uint32_t desc) \ +{ \ + do_tbl1(vd, vn, vm, desc, true, do_tb_##SUFF); \ +} + +DO_TB(b, uint8_t, H1) +DO_TB(h, uint16_t, H2) +DO_TB(s, uint32_t, H4) +DO_TB(d, uint64_t, ) + +#undef DO_TB #define DO_UNPK(NAME, TYPED, TYPES, HD, HS) \ void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \ diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_TBL(DisasContext *s, arg_rrr_esz *a) return true; } +static bool trans_TBL_sve2(DisasContext *s, arg_rrr_esz *a) +{ + static gen_helper_gvec_4 * const fns[4] = { + gen_helper_sve2_tbl_b, gen_helper_sve2_tbl_h, + gen_helper_sve2_tbl_s, gen_helper_sve2_tbl_d + }; + + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (sve_access_check(s)) { + gen_gvec_ool_zzzz(s, fns[a->esz], a->rd, a->rn, + (a->rn + 1) % 32, a->rm, 0); + } + return true; +} + +static bool trans_TBX(DisasContext *s, arg_rrr_esz *a) +{ + static gen_helper_gvec_3 * const fns[4] = { + gen_helper_sve2_tbx_b, gen_helper_sve2_tbx_h, + gen_helper_sve2_tbx_s, gen_helper_sve2_tbx_d + }; + + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + if (sve_access_check(s)) { + gen_gvec_ool_zzz(s, fns[a->esz], a->rd, a->rn, a->rm, 0); + } + return true; +} + static bool trans_UNPK(DisasContext *s, arg_UNPK *a) { static gen_helper_gvec_2 * const fns[4][2] = { -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Stephen Long <steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20210525010358.152808-73-richard.henderson@linaro.org Message-Id: <20200428174332.17162-2-steplong@quicinc.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 5 +++++ target/arm/sve.decode | 4 ++++ target/arm/sve_helper.c | 20 ++++++++++++++++++++ target/arm/translate-sve.c | 16 ++++++++++++++++ 4 files changed, 45 insertions(+) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(sve2_cdot_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve2_cdot_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(sve2_fcvtnt_sh, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(sve2_fcvtnt_ds, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ SM4E 01000101 00 10001 1 11100 0 ..... ..... @rdn_rm_e0 # SVE2 crypto constructive binary operations SM4EKEY 01000101 00 1 ..... 11110 0 ..... ..... @rd_rn_rm_e0 RAX1 01000101 00 1 ..... 11110 1 ..... ..... @rd_rn_rm_e0 + +### SVE2 floating-point convert precision odd elements +FCVTNT_sh 01100100 10 0010 00 101 ... ..... ..... @rd_pg_rn_e0 +FCVTNT_ds 01100100 11 0010 10 101 ... ..... ..... @rd_pg_rn_e0 diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(fmmla_d)(void *vd, void *vn, void *vm, void *va, d[3] = float64_add(a[3], float64_add(p0, p1, status), status); } } + +#define DO_FCVTNT(NAME, TYPEW, TYPEN, HW, HN, OP) \ +void HELPER(NAME)(void *vd, void *vn, void *vg, void *status, uint32_t desc) \ +{ \ + intptr_t i = simd_oprsz(desc); \ + uint64_t *g = vg; \ + do { \ + uint64_t pg = g[(i - 1) >> 6]; \ + do { \ + i -= sizeof(TYPEW); \ + if (likely((pg >> (i & 63)) & 1)) { \ + TYPEW nn = *(TYPEW *)(vn + HW(i)); \ + *(TYPEN *)(vd + HN(i + sizeof(TYPEN))) = OP(nn, status); \ + } \ + } while (i & 63); \ + } while (i != 0); \ +} + +DO_FCVTNT(sve2_fcvtnt_sh, uint32_t, uint16_t, H1_4, H1_2, sve_f32_to_f16) +DO_FCVTNT(sve2_fcvtnt_ds, uint64_t, uint32_t, , H1_4, float64_to_float32) diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_RAX1(DisasContext *s, arg_rrr_esz *a) } return true; } + +static bool trans_FCVTNT_sh(DisasContext *s, arg_rpr_esz *a) +{ + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve2_fcvtnt_sh); +} + +static bool trans_FCVTNT_ds(DisasContext *s, arg_rpr_esz *a) +{ + if (!dc_isar_feature(aa64_sve2, s)) { + return false; + } + return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve2_fcvtnt_ds); +} -- 2.20.1