Hook up the generic vDSO implementation to the aarch64 vDSO data page.
The _vdso_rng_data required data is placed within the _vdso_data vvar
page, by using a offset larger than the vdso_data.
The vDSO function requires a ChaCha20 implementation that does not write
to the stack, and that can do an entire ChaCha20 permutation. The one
provided uses NEON on the permute operation, with a fallback to the
syscall for chips that do not support AdvSIMD.
This also passes the vdso_test_chacha test along with
vdso_test_getrandom. The vdso_test_getrandom bench-single result on
Neoverse-N1 shows:
vdso: 25000000 times in 0.783884250 seconds
libc: 25000000 times in 8.780275399 seconds
syscall: 25000000 times in 8.786581518 seconds
A small fixup to arch/arm64/include/asm/mman.h was required to avoid
pulling kernel code into the vDSO, similar to what's already done in
arch/arm64/include/asm/rwonce.h.
Signed-off-by: Adhemerval Zanella <adhemerval.zanella@linaro.org>
---
arch/arm64/Kconfig | 1 +
arch/arm64/include/asm/mman.h | 6 +-
arch/arm64/include/asm/vdso.h | 6 +
arch/arm64/include/asm/vdso/getrandom.h | 50 ++++++
arch/arm64/include/asm/vdso/vsyscall.h | 10 ++
arch/arm64/kernel/vdso.c | 6 -
arch/arm64/kernel/vdso/Makefile | 25 ++-
arch/arm64/kernel/vdso/vdso | 1 +
arch/arm64/kernel/vdso/vdso.lds.S | 4 +
arch/arm64/kernel/vdso/vgetrandom-chacha.S | 178 +++++++++++++++++++++
arch/arm64/kernel/vdso/vgetrandom.c | 15 ++
tools/arch/arm64/vdso | 1 +
tools/include/linux/compiler.h | 4 +
tools/testing/selftests/vDSO/Makefile | 3 +-
14 files changed, 294 insertions(+), 16 deletions(-)
create mode 100644 arch/arm64/include/asm/vdso/getrandom.h
create mode 120000 arch/arm64/kernel/vdso/vdso
create mode 100644 arch/arm64/kernel/vdso/vgetrandom-chacha.S
create mode 100644 arch/arm64/kernel/vdso/vgetrandom.c
create mode 120000 tools/arch/arm64/vdso
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index a2f8ff354ca6..7f7424d1b3b8 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -262,6 +262,7 @@ config ARM64
select TRACE_IRQFLAGS_NMI_SUPPORT
select HAVE_SOFTIRQ_ON_OWN_STACK
select USER_STACKTRACE_SUPPORT
+ select VDSO_GETRANDOM
help
ARM 64-bit (AArch64) Linux support.
diff --git a/arch/arm64/include/asm/mman.h b/arch/arm64/include/asm/mman.h
index 5966ee4a6154..ceb4370a69c5 100644
--- a/arch/arm64/include/asm/mman.h
+++ b/arch/arm64/include/asm/mman.h
@@ -2,9 +2,11 @@
#ifndef __ASM_MMAN_H__
#define __ASM_MMAN_H__
+#include <uapi/asm/mman.h>
+
+#ifndef BUILD_VDSO
#include <linux/compiler.h>
#include <linux/types.h>
-#include <uapi/asm/mman.h>
static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot,
unsigned long pkey __always_unused)
@@ -60,4 +62,6 @@ static inline bool arch_validate_flags(unsigned long vm_flags)
}
#define arch_validate_flags(vm_flags) arch_validate_flags(vm_flags)
+#endif /* !BUILD_VDSO */
+
#endif /* ! __ASM_MMAN_H__ */
diff --git a/arch/arm64/include/asm/vdso.h b/arch/arm64/include/asm/vdso.h
index 4305995c8f82..18407b757c95 100644
--- a/arch/arm64/include/asm/vdso.h
+++ b/arch/arm64/include/asm/vdso.h
@@ -16,6 +16,12 @@
#ifndef __ASSEMBLY__
+enum vvar_pages {
+ VVAR_DATA_PAGE_OFFSET,
+ VVAR_TIMENS_PAGE_OFFSET,
+ VVAR_NR_PAGES,
+};
+
#include <generated/vdso-offsets.h>
#define VDSO_SYMBOL(base, name) \
diff --git a/arch/arm64/include/asm/vdso/getrandom.h b/arch/arm64/include/asm/vdso/getrandom.h
new file mode 100644
index 000000000000..6cda64e6da83
--- /dev/null
+++ b/arch/arm64/include/asm/vdso/getrandom.h
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __ASM_VDSO_GETRANDOM_H
+#define __ASM_VDSO_GETRANDOM_H
+
+#ifndef __ASSEMBLY__
+
+#include <asm/vdso.h>
+#include <asm/unistd.h>
+#include <vdso/datapage.h>
+
+/**
+ * getrandom_syscall - Invoke the getrandom() syscall.
+ * @buffer: Destination buffer to fill with random bytes.
+ * @len: Size of @buffer in bytes.
+ * @flags: Zero or more GRND_* flags.
+ * Returns: The number of random bytes written to @buffer, or a negative value indicating an error.
+ */
+static __always_inline ssize_t getrandom_syscall(void *_buffer, size_t _len, unsigned int _flags)
+{
+ register void *buffer asm ("x0") = _buffer;
+ register size_t len asm ("x1") = _len;
+ register unsigned int flags asm ("x2") = _flags;
+ register long ret asm ("x0");
+ register long nr asm ("x8") = __NR_getrandom;
+
+ asm volatile(
+ " svc #0\n"
+ : "=r" (ret)
+ : "r" (buffer), "r" (len), "r" (flags), "r" (nr)
+ : "memory");
+
+ return ret;
+}
+
+static __always_inline const struct vdso_rng_data *__arch_get_vdso_rng_data(void)
+{
+ /*
+ * The RNG data is in the real VVAR data page, but if a task belongs to a time namespace
+ * then VVAR_DATA_PAGE_OFFSET points to the namespace-specific VVAR page and VVAR_TIMENS_
+ * PAGE_OFFSET points to the real VVAR page.
+ */
+ if (IS_ENABLED(CONFIG_TIME_NS) && _vdso_data->clock_mode == VDSO_CLOCKMODE_TIMENS)
+ return (void *)&_vdso_rng_data + VVAR_TIMENS_PAGE_OFFSET * (1UL << CONFIG_PAGE_SHIFT);
+ return &_vdso_rng_data;
+}
+
+#endif /* !__ASSEMBLY__ */
+
+#endif /* __ASM_VDSO_GETRANDOM_H */
diff --git a/arch/arm64/include/asm/vdso/vsyscall.h b/arch/arm64/include/asm/vdso/vsyscall.h
index f94b1457c117..b4241510dc36 100644
--- a/arch/arm64/include/asm/vdso/vsyscall.h
+++ b/arch/arm64/include/asm/vdso/vsyscall.h
@@ -2,8 +2,11 @@
#ifndef __ASM_VDSO_VSYSCALL_H
#define __ASM_VDSO_VSYSCALL_H
+#define __VDSO_RND_DATA_OFFSET 480
+
#ifndef __ASSEMBLY__
+#include <asm/vdso.h>
#include <linux/timekeeper_internal.h>
#include <vdso/datapage.h>
@@ -21,6 +24,13 @@ struct vdso_data *__arm64_get_k_vdso_data(void)
}
#define __arch_get_k_vdso_data __arm64_get_k_vdso_data
+static __always_inline
+struct vdso_rng_data *__arm64_get_k_vdso_rnd_data(void)
+{
+ return (void *)vdso_data + __VDSO_RND_DATA_OFFSET;
+}
+#define __arch_get_k_vdso_rng_data __arm64_get_k_vdso_rnd_data
+
static __always_inline
void __arm64_update_vsyscall(struct vdso_data *vdata, struct timekeeper *tk)
{
diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c
index 89b6e7840002..706c9c3a7a50 100644
--- a/arch/arm64/kernel/vdso.c
+++ b/arch/arm64/kernel/vdso.c
@@ -34,12 +34,6 @@ enum vdso_abi {
VDSO_ABI_AA32,
};
-enum vvar_pages {
- VVAR_DATA_PAGE_OFFSET,
- VVAR_TIMENS_PAGE_OFFSET,
- VVAR_NR_PAGES,
-};
-
struct vdso_abi_info {
const char *name;
const char *vdso_code_start;
diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile
index d11da6461278..35685c036044 100644
--- a/arch/arm64/kernel/vdso/Makefile
+++ b/arch/arm64/kernel/vdso/Makefile
@@ -9,7 +9,7 @@
# Include the generic Makefile to check the built vdso.
include $(srctree)/lib/vdso/Makefile
-obj-vdso := vgettimeofday.o note.o sigreturn.o
+obj-vdso := vgettimeofday.o note.o sigreturn.o vgetrandom.o vgetrandom-chacha.o
# Build rules
targets := $(obj-vdso) vdso.so vdso.so.dbg
@@ -34,19 +34,28 @@ ccflags-y := -fno-common -fno-builtin -fno-stack-protector -ffixed-x18
ccflags-y += -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO
# -Wmissing-prototypes and -Wmissing-declarations are removed from
-# the CFLAGS of vgettimeofday.c to make possible to build the
-# kernel with CONFIG_WERROR enabled.
-CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) \
- $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) \
- $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) \
- -Wmissing-prototypes -Wmissing-declarations
+# the CFLAGS to make possible to build the kernel with CONFIG_WERROR enabled.
+CC_FLAGS_REMOVE_VDSO := $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) \
+ $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) \
+ $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) \
+ -Wmissing-prototypes -Wmissing-declarations
-CFLAGS_vgettimeofday.o = -O2 -mcmodel=tiny -fasynchronous-unwind-tables
+CC_FLAGS_ADD_VDSO := -O2 -mcmodel=tiny -fasynchronous-unwind-tables
+
+CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_REMOVE_VDSO)
+CFLAGS_REMOVE_vgetrandom.o = $(CC_FLAGS_REMOVE_VDSO)
+
+CFLAGS_vgettimeofday.o = $(CC_FLAGS_ADD_VDSO)
+CFLAGS_vgetrandom.o = $(CC_FLAGS_ADD_VDSO)
ifneq ($(c-gettimeofday-y),)
CFLAGS_vgettimeofday.o += -include $(c-gettimeofday-y)
endif
+ifneq ($(c-getrandom-y),)
+ CFLAGS_vgetrandom.o += -include $(c-getrandom-y)
+endif
+
targets += vdso.lds
CPPFLAGS_vdso.lds += -P -C -U$(ARCH)
diff --git a/arch/arm64/kernel/vdso/vdso b/arch/arm64/kernel/vdso/vdso
new file mode 120000
index 000000000000..233c7a26f6e5
--- /dev/null
+++ b/arch/arm64/kernel/vdso/vdso
@@ -0,0 +1 @@
+../../../arch/arm64/kernel/vdso
\ No newline at end of file
diff --git a/arch/arm64/kernel/vdso/vdso.lds.S b/arch/arm64/kernel/vdso/vdso.lds.S
index 45354f2ddf70..f204a9ddc833 100644
--- a/arch/arm64/kernel/vdso/vdso.lds.S
+++ b/arch/arm64/kernel/vdso/vdso.lds.S
@@ -11,7 +11,9 @@
#include <linux/const.h>
#include <asm/page.h>
#include <asm/vdso.h>
+#include <asm/vdso/vsyscall.h>
#include <asm-generic/vmlinux.lds.h>
+#include <vdso/datapage.h>
OUTPUT_FORMAT("elf64-littleaarch64", "elf64-bigaarch64", "elf64-littleaarch64")
OUTPUT_ARCH(aarch64)
@@ -19,6 +21,7 @@ OUTPUT_ARCH(aarch64)
SECTIONS
{
PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE);
+ PROVIDE(_vdso_rng_data = _vdso_data + __VDSO_RND_DATA_OFFSET);
#ifdef CONFIG_TIME_NS
PROVIDE(_timens_data = _vdso_data + PAGE_SIZE);
#endif
@@ -102,6 +105,7 @@ VERSION
__kernel_gettimeofday;
__kernel_clock_gettime;
__kernel_clock_getres;
+ __kernel_getrandom;
local: *;
};
}
diff --git a/arch/arm64/kernel/vdso/vgetrandom-chacha.S b/arch/arm64/kernel/vdso/vgetrandom-chacha.S
new file mode 100644
index 000000000000..4e5f9c349522
--- /dev/null
+++ b/arch/arm64/kernel/vdso/vgetrandom-chacha.S
@@ -0,0 +1,178 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/linkage.h>
+#include <asm/cache.h>
+#include <asm/assembler.h>
+
+ .text
+
+#define state0 v0
+#define state1 v1
+#define state2 v2
+#define state3 v3
+#define copy0 v4
+#define copy0_q q4
+#define copy1 v5
+#define copy2 v6
+#define copy3 v7
+#define copy3_d d7
+#define one_d d16
+#define one_q q16
+#define one_v v16
+#define tmp v17
+#define rot8 v18
+
+/*
+ * ARM64 ChaCha20 implementation meant for vDSO. Produces a given positive
+ * number of blocks of output with nonce 0, taking an input key and 8-bytes
+ * counter. Importantly does not spill to the stack.
+ *
+ * This implementation avoids d8-d15 because they are callee-save in user
+ * space.
+ *
+ * void __arch_chacha20_blocks_nostack(uint8_t *dst_bytes,
+ * const uint8_t *key,
+ * uint32_t *counter,
+ * size_t nblocks)
+ *
+ * x0: output bytes
+ * x1: 32-byte key input
+ * x2: 8-byte counter input/output
+ * x3: number of 64-byte block to write to output
+ */
+SYM_FUNC_START(__arch_chacha20_blocks_nostack)
+
+ /* copy0 = "expand 32-byte k" */
+ mov_q x8, 0x3320646e61707865
+ mov_q x9, 0x6b20657479622d32
+ mov copy0.d[0], x8
+ mov copy0.d[1], x9
+
+ /* copy1,copy2 = key */
+ ld1 { copy1.4s, copy2.4s }, [x1]
+ /* copy3 = counter || zero nonce */
+ ldr copy3_d, [x2]
+CPU_BE( rev64 copy3.4s, copy3.4s)
+
+ movi one_v.2s, #1
+ uzp1 one_v.4s, one_v.4s, one_v.4s
+
+.Lblock:
+ /* copy state to auxiliary vectors for the final add after the permute. */
+ mov state0.16b, copy0.16b
+ mov state1.16b, copy1.16b
+ mov state2.16b, copy2.16b
+ mov state3.16b, copy3.16b
+
+ mov w4, 20
+.Lpermute:
+ /*
+ * Permute one 64-byte block where the state matrix is stored in the four NEON
+ * registers state0-state3. It performs matrix operations on four words in parallel,
+ * but requires shuffling to rearrange the words after each round.
+ */
+
+.Ldoubleround:
+ /* state0 += state1, state3 = rotl32(state3 ^ state0, 16) */
+ add state0.4s, state0.4s, state1.4s
+ eor state3.16b, state3.16b, state0.16b
+ rev32 state3.8h, state3.8h
+
+ /* state2 += state3, state1 = rotl32(state1 ^ state2, 12) */
+ add state2.4s, state2.4s, state3.4s
+ eor tmp.16b, state1.16b, state2.16b
+ shl state1.4s, tmp.4s, #12
+ sri state1.4s, tmp.4s, #20
+
+ /* state0 += state1, state3 = rotl32(state3 ^ state0, 8) */
+ add state0.4s, state0.4s, state1.4s
+ eor tmp.16b, state3.16b, state0.16b
+ shl state3.4s, tmp.4s, #8
+ sri state3.4s, tmp.4s, #24
+
+ /* state2 += state3, state1 = rotl32(state1 ^ state2, 7) */
+ add state2.4s, state2.4s, state3.4s
+ eor tmp.16b, state1.16b, state2.16b
+ shl state1.4s, tmp.4s, #7
+ sri state1.4s, tmp.4s, #25
+
+ /* state1[0,1,2,3] = state1[1,2,3,0] */
+ ext state1.16b, state1.16b, state1.16b, #4
+ /* state2[0,1,2,3] = state2[2,3,0,1] */
+ ext state2.16b, state2.16b, state2.16b, #8
+ /* state3[0,1,2,3] = state3[1,2,3,0] */
+ ext state3.16b, state3.16b, state3.16b, #12
+
+ /* state0 += state1, state3 = rotl32(state3 ^ state0, 16) */
+ add state0.4s, state0.4s, state1.4s
+ eor state3.16b, state3.16b, state0.16b
+ rev32 state3.8h, state3.8h
+
+ /* state2 += state3, state1 = rotl32(state1 ^ state2, 12) */
+ add state2.4s, state2.4s, state3.4s
+ eor tmp.16b, state1.16b, state2.16b
+ shl state1.4s, tmp.4s, #12
+ sri state1.4s, tmp.4s, #20
+
+ /* state0 += state1, state3 = rotl32(state3 ^ state0, 8) */
+ add state0.4s, state0.4s, state1.4s
+ eor tmp.16b, state3.16b, state0.16b
+ shl state3.4s, tmp.4s, #8
+ sri state3.4s, tmp.4s, #24
+
+ /* state2 += state3, state1 = rotl32(state1 ^ state2, 7) */
+ add state2.4s, state2.4s, state3.4s
+ eor tmp.16b, state1.16b, state2.16b
+ shl state1.4s, tmp.4s, #7
+ sri state1.4s, tmp.4s, #25
+
+ /* state1[0,1,2,3] = state1[3,0,1,2] */
+ ext state1.16b, state1.16b, state1.16b, #12
+ /* state2[0,1,2,3] = state2[2,3,0,1] */
+ ext state2.16b, state2.16b, state2.16b, #8
+ /* state3[0,1,2,3] = state3[1,2,3,0] */
+ ext state3.16b, state3.16b, state3.16b, #4
+
+ subs w4, w4, #2
+ b.ne .Ldoubleround
+
+ /* output0 = state0 + state0 */
+ add state0.4s, state0.4s, copy0.4s
+CPU_BE( rev32 state0.16b, state0.16b)
+ /* output1 = state1 + state1 */
+ add state1.4s, state1.4s, copy1.4s
+CPU_BE( rev32 state1.16b, state1.16b)
+ /* output2 = state2 + state2 */
+ add state2.4s, state2.4s, copy2.4s
+CPU_BE( rev32 state2.16b, state2.16b)
+ /* output2 = state3 + state3 */
+ add state3.4s, state3.4s, copy3.4s
+CPU_BE( rev32 state3.16b, state3.16b)
+ st1 { state0.4s - state3.4s }, [x0]
+
+ /*
+ * ++copy3.counter, the 'add' clears the upper half of the SIMD register
+ * which is the expected behaviour here.
+ */
+ add copy3_d, copy3_d, one_d
+
+ /* output += 64, --nblocks */
+ add x0, x0, 64
+ subs x3, x3, #1
+ b.ne .Lblock
+
+ /* counter = copy3.counter */
+CPU_BE( rev64 copy3.4s, copy3.4s)
+ str copy3_d, [x2]
+
+ /* Zero out the potentially sensitive regs, in case nothing uses these again. */
+ movi state0.16b, #0
+ movi state1.16b, #0
+ movi state2.16b, #0
+ movi state3.16b, #0
+ movi copy1.16b, #0
+ movi copy2.16b, #0
+ ret
+SYM_FUNC_END(__arch_chacha20_blocks_nostack)
+
+emit_aarch64_feature_1_and
diff --git a/arch/arm64/kernel/vdso/vgetrandom.c b/arch/arm64/kernel/vdso/vgetrandom.c
new file mode 100644
index 000000000000..832fe195292b
--- /dev/null
+++ b/arch/arm64/kernel/vdso/vgetrandom.c
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <uapi/asm-generic/errno.h>
+
+typeof(__cvdso_getrandom) __kernel_getrandom;
+
+ssize_t __kernel_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len)
+{
+ if (alternative_has_cap_likely(ARM64_HAS_FPSIMD))
+ return __cvdso_getrandom(buffer, len, flags, opaque_state, opaque_len);
+
+ if (unlikely(opaque_len == ~0UL && !buffer && !len && !flags))
+ return -ENOSYS;
+ return getrandom_syscall(buffer, len, flags);
+}
diff --git a/tools/arch/arm64/vdso b/tools/arch/arm64/vdso
new file mode 120000
index 000000000000..233c7a26f6e5
--- /dev/null
+++ b/tools/arch/arm64/vdso
@@ -0,0 +1 @@
+../../../arch/arm64/kernel/vdso
\ No newline at end of file
diff --git a/tools/include/linux/compiler.h b/tools/include/linux/compiler.h
index 6f7f22ac9da5..4366da278033 100644
--- a/tools/include/linux/compiler.h
+++ b/tools/include/linux/compiler.h
@@ -2,6 +2,8 @@
#ifndef _TOOLS_LINUX_COMPILER_H_
#define _TOOLS_LINUX_COMPILER_H_
+#ifndef __ASSEMBLY__
+
#include <linux/compiler_types.h>
#ifndef __compiletime_error
@@ -224,4 +226,6 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s
__asm__ ("" : "=r" (var) : "0" (var))
#endif
+#endif /* __ASSEMBLY__ */
+
#endif /* _TOOLS_LINUX_COMPILER_H */
diff --git a/tools/testing/selftests/vDSO/Makefile b/tools/testing/selftests/vDSO/Makefile
index 04930125035e..3c6fafbd83a6 100644
--- a/tools/testing/selftests/vDSO/Makefile
+++ b/tools/testing/selftests/vDSO/Makefile
@@ -9,7 +9,7 @@ ifeq ($(ARCH),$(filter $(ARCH),x86 x86_64))
TEST_GEN_PROGS += vdso_standalone_test_x86
endif
TEST_GEN_PROGS += vdso_test_correctness
-ifeq ($(ARCH)$(CONFIG_X86_32),$(filter $(ARCH)$(CONFIG_X86_32),x86 x86_64 loongarch))
+ifeq ($(ARCH)$(CONFIG_X86_32),$(filter $(ARCH)$(CONFIG_X86_32),x86 x86_64 loongarch arm64))
TEST_GEN_PROGS += vdso_test_getrandom
TEST_GEN_PROGS += vdso_test_chacha
endif
@@ -40,5 +40,6 @@ $(OUTPUT)/vdso_test_getrandom: CFLAGS += -isystem $(top_srcdir)/tools/include \
$(OUTPUT)/vdso_test_chacha: $(top_srcdir)/tools/arch/$(SRCARCH)/vdso/vgetrandom-chacha.S
$(OUTPUT)/vdso_test_chacha: CFLAGS += -idirafter $(top_srcdir)/tools/include \
-idirafter $(top_srcdir)/arch/$(SRCARCH)/include \
+ -idirafter $(top_srcdir)/arch/$(SRCARCH)/include/generated \
-idirafter $(top_srcdir)/include \
-D__ASSEMBLY__ -Wa,--noexecstack
--
2.43.0Hi Adhemerval,
I have just a couple of more points below, on the BE handling in the asm.
On Mon, 2 Sept 2024 at 18:19, Adhemerval Zanella
<adhemerval.zanella@linaro.org> wrote:
>
> Hook up the generic vDSO implementation to the aarch64 vDSO data page.
> The _vdso_rng_data required data is placed within the _vdso_data vvar
> page, by using a offset larger than the vdso_data.
>
> The vDSO function requires a ChaCha20 implementation that does not write
> to the stack, and that can do an entire ChaCha20 permutation. The one
> provided uses NEON on the permute operation, with a fallback to the
> syscall for chips that do not support AdvSIMD.
>
> This also passes the vdso_test_chacha test along with
> vdso_test_getrandom. The vdso_test_getrandom bench-single result on
> Neoverse-N1 shows:
>
> vdso: 25000000 times in 0.783884250 seconds
> libc: 25000000 times in 8.780275399 seconds
> syscall: 25000000 times in 8.786581518 seconds
>
> A small fixup to arch/arm64/include/asm/mman.h was required to avoid
> pulling kernel code into the vDSO, similar to what's already done in
> arch/arm64/include/asm/rwonce.h.
>
> Signed-off-by: Adhemerval Zanella <adhemerval.zanella@linaro.org>
> ---
> arch/arm64/Kconfig | 1 +
> arch/arm64/include/asm/mman.h | 6 +-
> arch/arm64/include/asm/vdso.h | 6 +
> arch/arm64/include/asm/vdso/getrandom.h | 50 ++++++
> arch/arm64/include/asm/vdso/vsyscall.h | 10 ++
> arch/arm64/kernel/vdso.c | 6 -
> arch/arm64/kernel/vdso/Makefile | 25 ++-
> arch/arm64/kernel/vdso/vdso | 1 +
> arch/arm64/kernel/vdso/vdso.lds.S | 4 +
> arch/arm64/kernel/vdso/vgetrandom-chacha.S | 178 +++++++++++++++++++++
> arch/arm64/kernel/vdso/vgetrandom.c | 15 ++
> tools/arch/arm64/vdso | 1 +
> tools/include/linux/compiler.h | 4 +
> tools/testing/selftests/vDSO/Makefile | 3 +-
> 14 files changed, 294 insertions(+), 16 deletions(-)
> create mode 100644 arch/arm64/include/asm/vdso/getrandom.h
> create mode 120000 arch/arm64/kernel/vdso/vdso
> create mode 100644 arch/arm64/kernel/vdso/vgetrandom-chacha.S
> create mode 100644 arch/arm64/kernel/vdso/vgetrandom.c
> create mode 120000 tools/arch/arm64/vdso
>
...
> diff --git a/arch/arm64/kernel/vdso/vgetrandom-chacha.S b/arch/arm64/kernel/vdso/vgetrandom-chacha.S
> new file mode 100644
> index 000000000000..4e5f9c349522
> --- /dev/null
> +++ b/arch/arm64/kernel/vdso/vgetrandom-chacha.S
> @@ -0,0 +1,178 @@
> +// SPDX-License-Identifier: GPL-2.0
> +
> +#include <linux/linkage.h>
> +#include <asm/cache.h>
> +#include <asm/assembler.h>
> +
> + .text
> +
> +#define state0 v0
> +#define state1 v1
> +#define state2 v2
> +#define state3 v3
> +#define copy0 v4
> +#define copy0_q q4
> +#define copy1 v5
> +#define copy2 v6
> +#define copy3 v7
> +#define copy3_d d7
> +#define one_d d16
> +#define one_q q16
> +#define one_v v16
> +#define tmp v17
> +#define rot8 v18
> +
> +/*
> + * ARM64 ChaCha20 implementation meant for vDSO. Produces a given positive
> + * number of blocks of output with nonce 0, taking an input key and 8-bytes
> + * counter. Importantly does not spill to the stack.
> + *
> + * This implementation avoids d8-d15 because they are callee-save in user
> + * space.
> + *
> + * void __arch_chacha20_blocks_nostack(uint8_t *dst_bytes,
> + * const uint8_t *key,
> + * uint32_t *counter,
> + * size_t nblocks)
> + *
> + * x0: output bytes
> + * x1: 32-byte key input
> + * x2: 8-byte counter input/output
> + * x3: number of 64-byte block to write to output
> + */
> +SYM_FUNC_START(__arch_chacha20_blocks_nostack)
> +
> + /* copy0 = "expand 32-byte k" */
> + mov_q x8, 0x3320646e61707865
> + mov_q x9, 0x6b20657479622d32
> + mov copy0.d[0], x8
> + mov copy0.d[1], x9
> +
> + /* copy1,copy2 = key */
> + ld1 { copy1.4s, copy2.4s }, [x1]
> + /* copy3 = counter || zero nonce */
> + ldr copy3_d, [x2]
> +CPU_BE( rev64 copy3.4s, copy3.4s)
> +
This loads 2 u32s as a single u64, and then swaps them if we are running on BE.
So better to just use
ld1 {copy3.2s}, [x2]
here, and drop the CPU_BE() special case.
> + movi one_v.2s, #1
> + uzp1 one_v.4s, one_v.4s, one_v.4s
> +
> +.Lblock:
> + /* copy state to auxiliary vectors for the final add after the permute. */
> + mov state0.16b, copy0.16b
> + mov state1.16b, copy1.16b
> + mov state2.16b, copy2.16b
> + mov state3.16b, copy3.16b
> +
> + mov w4, 20
> +.Lpermute:
> + /*
> + * Permute one 64-byte block where the state matrix is stored in the four NEON
> + * registers state0-state3. It performs matrix operations on four words in parallel,
> + * but requires shuffling to rearrange the words after each round.
> + */
> +
> +.Ldoubleround:
> + /* state0 += state1, state3 = rotl32(state3 ^ state0, 16) */
> + add state0.4s, state0.4s, state1.4s
> + eor state3.16b, state3.16b, state0.16b
> + rev32 state3.8h, state3.8h
> +
> + /* state2 += state3, state1 = rotl32(state1 ^ state2, 12) */
> + add state2.4s, state2.4s, state3.4s
> + eor tmp.16b, state1.16b, state2.16b
> + shl state1.4s, tmp.4s, #12
> + sri state1.4s, tmp.4s, #20
> +
> + /* state0 += state1, state3 = rotl32(state3 ^ state0, 8) */
> + add state0.4s, state0.4s, state1.4s
> + eor tmp.16b, state3.16b, state0.16b
> + shl state3.4s, tmp.4s, #8
> + sri state3.4s, tmp.4s, #24
> +
> + /* state2 += state3, state1 = rotl32(state1 ^ state2, 7) */
> + add state2.4s, state2.4s, state3.4s
> + eor tmp.16b, state1.16b, state2.16b
> + shl state1.4s, tmp.4s, #7
> + sri state1.4s, tmp.4s, #25
> +
> + /* state1[0,1,2,3] = state1[1,2,3,0] */
> + ext state1.16b, state1.16b, state1.16b, #4
> + /* state2[0,1,2,3] = state2[2,3,0,1] */
> + ext state2.16b, state2.16b, state2.16b, #8
> + /* state3[0,1,2,3] = state3[1,2,3,0] */
> + ext state3.16b, state3.16b, state3.16b, #12
> +
> + /* state0 += state1, state3 = rotl32(state3 ^ state0, 16) */
> + add state0.4s, state0.4s, state1.4s
> + eor state3.16b, state3.16b, state0.16b
> + rev32 state3.8h, state3.8h
> +
> + /* state2 += state3, state1 = rotl32(state1 ^ state2, 12) */
> + add state2.4s, state2.4s, state3.4s
> + eor tmp.16b, state1.16b, state2.16b
> + shl state1.4s, tmp.4s, #12
> + sri state1.4s, tmp.4s, #20
> +
> + /* state0 += state1, state3 = rotl32(state3 ^ state0, 8) */
> + add state0.4s, state0.4s, state1.4s
> + eor tmp.16b, state3.16b, state0.16b
> + shl state3.4s, tmp.4s, #8
> + sri state3.4s, tmp.4s, #24
> +
> + /* state2 += state3, state1 = rotl32(state1 ^ state2, 7) */
> + add state2.4s, state2.4s, state3.4s
> + eor tmp.16b, state1.16b, state2.16b
> + shl state1.4s, tmp.4s, #7
> + sri state1.4s, tmp.4s, #25
> +
> + /* state1[0,1,2,3] = state1[3,0,1,2] */
> + ext state1.16b, state1.16b, state1.16b, #12
> + /* state2[0,1,2,3] = state2[2,3,0,1] */
> + ext state2.16b, state2.16b, state2.16b, #8
> + /* state3[0,1,2,3] = state3[1,2,3,0] */
> + ext state3.16b, state3.16b, state3.16b, #4
> +
> + subs w4, w4, #2
> + b.ne .Ldoubleround
> +
> + /* output0 = state0 + state0 */
> + add state0.4s, state0.4s, copy0.4s
> +CPU_BE( rev32 state0.16b, state0.16b)
> + /* output1 = state1 + state1 */
> + add state1.4s, state1.4s, copy1.4s
> +CPU_BE( rev32 state1.16b, state1.16b)
> + /* output2 = state2 + state2 */
> + add state2.4s, state2.4s, copy2.4s
> +CPU_BE( rev32 state2.16b, state2.16b)
> + /* output2 = state3 + state3 */
> + add state3.4s, state3.4s, copy3.4s
> +CPU_BE( rev32 state3.16b, state3.16b)
> + st1 { state0.4s - state3.4s }, [x0]
> +
If the u32s shouldn't be swabbed for BE, you should simply be able to do
st1 {state0.16b - state3.16b}, [x0]
here, and drop the CPU_BE(*).
> + /*
> + * ++copy3.counter, the 'add' clears the upper half of the SIMD register
> + * which is the expected behaviour here.
> + */
> + add copy3_d, copy3_d, one_d
> +
> + /* output += 64, --nblocks */
> + add x0, x0, 64
> + subs x3, x3, #1
> + b.ne .Lblock
> +
> + /* counter = copy3.counter */
> +CPU_BE( rev64 copy3.4s, copy3.4s)
> + str copy3_d, [x2]
> +
... and this could be
st1 {copy3.2s}, [x2]
> + /* Zero out the potentially sensitive regs, in case nothing uses these again. */
> + movi state0.16b, #0
> + movi state1.16b, #0
> + movi state2.16b, #0
> + movi state3.16b, #0
> + movi copy1.16b, #0
> + movi copy2.16b, #0
> + ret
> +SYM_FUNC_END(__arch_chacha20_blocks_nostack)
> +
> +emit_aarch64_feature_1_and
On 02/09/24 17:57, Ard Biesheuvel wrote:
> Hi Adhemerval,
>
> I have just a couple of more points below, on the BE handling in the asm.
>
> On Mon, 2 Sept 2024 at 18:19, Adhemerval Zanella
> <adhemerval.zanella@linaro.org> wrote:
>>
>> Hook up the generic vDSO implementation to the aarch64 vDSO data page.
>> The _vdso_rng_data required data is placed within the _vdso_data vvar
>> page, by using a offset larger than the vdso_data.
>>
>> The vDSO function requires a ChaCha20 implementation that does not write
>> to the stack, and that can do an entire ChaCha20 permutation. The one
>> provided uses NEON on the permute operation, with a fallback to the
>> syscall for chips that do not support AdvSIMD.
>>
>> This also passes the vdso_test_chacha test along with
>> vdso_test_getrandom. The vdso_test_getrandom bench-single result on
>> Neoverse-N1 shows:
>>
>> vdso: 25000000 times in 0.783884250 seconds
>> libc: 25000000 times in 8.780275399 seconds
>> syscall: 25000000 times in 8.786581518 seconds
>>
>> A small fixup to arch/arm64/include/asm/mman.h was required to avoid
>> pulling kernel code into the vDSO, similar to what's already done in
>> arch/arm64/include/asm/rwonce.h.
>>
>> Signed-off-by: Adhemerval Zanella <adhemerval.zanella@linaro.org>
>> ---
>> arch/arm64/Kconfig | 1 +
>> arch/arm64/include/asm/mman.h | 6 +-
>> arch/arm64/include/asm/vdso.h | 6 +
>> arch/arm64/include/asm/vdso/getrandom.h | 50 ++++++
>> arch/arm64/include/asm/vdso/vsyscall.h | 10 ++
>> arch/arm64/kernel/vdso.c | 6 -
>> arch/arm64/kernel/vdso/Makefile | 25 ++-
>> arch/arm64/kernel/vdso/vdso | 1 +
>> arch/arm64/kernel/vdso/vdso.lds.S | 4 +
>> arch/arm64/kernel/vdso/vgetrandom-chacha.S | 178 +++++++++++++++++++++
>> arch/arm64/kernel/vdso/vgetrandom.c | 15 ++
>> tools/arch/arm64/vdso | 1 +
>> tools/include/linux/compiler.h | 4 +
>> tools/testing/selftests/vDSO/Makefile | 3 +-
>> 14 files changed, 294 insertions(+), 16 deletions(-)
>> create mode 100644 arch/arm64/include/asm/vdso/getrandom.h
>> create mode 120000 arch/arm64/kernel/vdso/vdso
>> create mode 100644 arch/arm64/kernel/vdso/vgetrandom-chacha.S
>> create mode 100644 arch/arm64/kernel/vdso/vgetrandom.c
>> create mode 120000 tools/arch/arm64/vdso
>>
> ...
>> diff --git a/arch/arm64/kernel/vdso/vgetrandom-chacha.S b/arch/arm64/kernel/vdso/vgetrandom-chacha.S
>> new file mode 100644
>> index 000000000000..4e5f9c349522
>> --- /dev/null
>> +++ b/arch/arm64/kernel/vdso/vgetrandom-chacha.S
>> @@ -0,0 +1,178 @@
>> +// SPDX-License-Identifier: GPL-2.0
>> +
>> +#include <linux/linkage.h>
>> +#include <asm/cache.h>
>> +#include <asm/assembler.h>
>> +
>> + .text
>> +
>> +#define state0 v0
>> +#define state1 v1
>> +#define state2 v2
>> +#define state3 v3
>> +#define copy0 v4
>> +#define copy0_q q4
>> +#define copy1 v5
>> +#define copy2 v6
>> +#define copy3 v7
>> +#define copy3_d d7
>> +#define one_d d16
>> +#define one_q q16
>> +#define one_v v16
>> +#define tmp v17
>> +#define rot8 v18
>> +
>> +/*
>> + * ARM64 ChaCha20 implementation meant for vDSO. Produces a given positive
>> + * number of blocks of output with nonce 0, taking an input key and 8-bytes
>> + * counter. Importantly does not spill to the stack.
>> + *
>> + * This implementation avoids d8-d15 because they are callee-save in user
>> + * space.
>> + *
>> + * void __arch_chacha20_blocks_nostack(uint8_t *dst_bytes,
>> + * const uint8_t *key,
>> + * uint32_t *counter,
>> + * size_t nblocks)
>> + *
>> + * x0: output bytes
>> + * x1: 32-byte key input
>> + * x2: 8-byte counter input/output
>> + * x3: number of 64-byte block to write to output
>> + */
>> +SYM_FUNC_START(__arch_chacha20_blocks_nostack)
>> +
>> + /* copy0 = "expand 32-byte k" */
>> + mov_q x8, 0x3320646e61707865
>> + mov_q x9, 0x6b20657479622d32
>> + mov copy0.d[0], x8
>> + mov copy0.d[1], x9
>> +
>> + /* copy1,copy2 = key */
>> + ld1 { copy1.4s, copy2.4s }, [x1]
>> + /* copy3 = counter || zero nonce */
>> + ldr copy3_d, [x2]
>> +CPU_BE( rev64 copy3.4s, copy3.4s)
>> +
>
> This loads 2 u32s as a single u64, and then swaps them if we are running on BE.
> So better to just use
>
> ld1 {copy3.2s}, [x2]
>
> here, and drop the CPU_BE() special case.
Ack.
>
>> + movi one_v.2s, #1
>> + uzp1 one_v.4s, one_v.4s, one_v.4s
>> +
>> +.Lblock:
>> + /* copy state to auxiliary vectors for the final add after the permute. */
>> + mov state0.16b, copy0.16b
>> + mov state1.16b, copy1.16b
>> + mov state2.16b, copy2.16b
>> + mov state3.16b, copy3.16b
>> +
>> + mov w4, 20
>> +.Lpermute:
>> + /*
>> + * Permute one 64-byte block where the state matrix is stored in the four NEON
>> + * registers state0-state3. It performs matrix operations on four words in parallel,
>> + * but requires shuffling to rearrange the words after each round.
>> + */
>> +
>> +.Ldoubleround:
>> + /* state0 += state1, state3 = rotl32(state3 ^ state0, 16) */
>> + add state0.4s, state0.4s, state1.4s
>> + eor state3.16b, state3.16b, state0.16b
>> + rev32 state3.8h, state3.8h
>> +
>> + /* state2 += state3, state1 = rotl32(state1 ^ state2, 12) */
>> + add state2.4s, state2.4s, state3.4s
>> + eor tmp.16b, state1.16b, state2.16b
>> + shl state1.4s, tmp.4s, #12
>> + sri state1.4s, tmp.4s, #20
>> +
>> + /* state0 += state1, state3 = rotl32(state3 ^ state0, 8) */
>> + add state0.4s, state0.4s, state1.4s
>> + eor tmp.16b, state3.16b, state0.16b
>> + shl state3.4s, tmp.4s, #8
>> + sri state3.4s, tmp.4s, #24
>> +
>> + /* state2 += state3, state1 = rotl32(state1 ^ state2, 7) */
>> + add state2.4s, state2.4s, state3.4s
>> + eor tmp.16b, state1.16b, state2.16b
>> + shl state1.4s, tmp.4s, #7
>> + sri state1.4s, tmp.4s, #25
>> +
>> + /* state1[0,1,2,3] = state1[1,2,3,0] */
>> + ext state1.16b, state1.16b, state1.16b, #4
>> + /* state2[0,1,2,3] = state2[2,3,0,1] */
>> + ext state2.16b, state2.16b, state2.16b, #8
>> + /* state3[0,1,2,3] = state3[1,2,3,0] */
>> + ext state3.16b, state3.16b, state3.16b, #12
>> +
>> + /* state0 += state1, state3 = rotl32(state3 ^ state0, 16) */
>> + add state0.4s, state0.4s, state1.4s
>> + eor state3.16b, state3.16b, state0.16b
>> + rev32 state3.8h, state3.8h
>> +
>> + /* state2 += state3, state1 = rotl32(state1 ^ state2, 12) */
>> + add state2.4s, state2.4s, state3.4s
>> + eor tmp.16b, state1.16b, state2.16b
>> + shl state1.4s, tmp.4s, #12
>> + sri state1.4s, tmp.4s, #20
>> +
>> + /* state0 += state1, state3 = rotl32(state3 ^ state0, 8) */
>> + add state0.4s, state0.4s, state1.4s
>> + eor tmp.16b, state3.16b, state0.16b
>> + shl state3.4s, tmp.4s, #8
>> + sri state3.4s, tmp.4s, #24
>> +
>> + /* state2 += state3, state1 = rotl32(state1 ^ state2, 7) */
>> + add state2.4s, state2.4s, state3.4s
>> + eor tmp.16b, state1.16b, state2.16b
>> + shl state1.4s, tmp.4s, #7
>> + sri state1.4s, tmp.4s, #25
>> +
>> + /* state1[0,1,2,3] = state1[3,0,1,2] */
>> + ext state1.16b, state1.16b, state1.16b, #12
>> + /* state2[0,1,2,3] = state2[2,3,0,1] */
>> + ext state2.16b, state2.16b, state2.16b, #8
>> + /* state3[0,1,2,3] = state3[1,2,3,0] */
>> + ext state3.16b, state3.16b, state3.16b, #4
>> +
>> + subs w4, w4, #2
>> + b.ne .Ldoubleround
>> +
>> + /* output0 = state0 + state0 */
>> + add state0.4s, state0.4s, copy0.4s
>> +CPU_BE( rev32 state0.16b, state0.16b)
>> + /* output1 = state1 + state1 */
>> + add state1.4s, state1.4s, copy1.4s
>> +CPU_BE( rev32 state1.16b, state1.16b)
>> + /* output2 = state2 + state2 */
>> + add state2.4s, state2.4s, copy2.4s
>> +CPU_BE( rev32 state2.16b, state2.16b)
>> + /* output2 = state3 + state3 */
>> + add state3.4s, state3.4s, copy3.4s
>> +CPU_BE( rev32 state3.16b, state3.16b)
>> + st1 { state0.4s - state3.4s }, [x0]
>> +
>
> If the u32s shouldn't be swabbed for BE, you should simply be able to do
>
> st1 {state0.16b - state3.16b}, [x0]
>
> here, and drop the CPU_BE(*).
Ack.
>
>> + /*
>> + * ++copy3.counter, the 'add' clears the upper half of the SIMD register
>> + * which is the expected behaviour here.
>> + */
>> + add copy3_d, copy3_d, one_d
>> +
>> + /* output += 64, --nblocks */
>> + add x0, x0, 64
>> + subs x3, x3, #1
>> + b.ne .Lblock
>> +
>> + /* counter = copy3.counter */
>> +CPU_BE( rev64 copy3.4s, copy3.4s)
>> + str copy3_d, [x2]
>> +
>
> ... and this could be
>
> st1 {copy3.2s}, [x2]
Ack.
I just sent a v5 with your proposed suggestions, thanks!
>
>> + /* Zero out the potentially sensitive regs, in case nothing uses these again. */
>> + movi state0.16b, #0
>> + movi state1.16b, #0
>> + movi state2.16b, #0
>> + movi state3.16b, #0
>> + movi copy1.16b, #0
>> + movi copy2.16b, #0
>> + ret
>> +SYM_FUNC_END(__arch_chacha20_blocks_nostack)
>> +
>> +emit_aarch64_feature_1_and
© 2016 - 2025 Red Hat, Inc.