From: Eric Biggers <ebiggers@google.com>
Update a caller of aegis128_aesni_ad() to round down the length to a
block boundary. After that, aegis128_aesni_ad(), aegis128_aesni_enc(),
and aegis128_aesni_dec() are only passed whole blocks. Update the
assembly code to take advantage of that, which eliminates some unneeded
instructions. For aegis128_aesni_enc() and aegis128_aesni_dec(), the
length is also always nonzero, so stop checking for zero length.
Reviewed-by: Ondrej Mosnacek <omosnace@redhat.com>
Signed-off-by: Eric Biggers <ebiggers@google.com>
---
arch/x86/crypto/aegis128-aesni-asm.S | 37 +++++++++++----------------
arch/x86/crypto/aegis128-aesni-glue.c | 4 +--
2 files changed, 17 insertions(+), 24 deletions(-)
diff --git a/arch/x86/crypto/aegis128-aesni-asm.S b/arch/x86/crypto/aegis128-aesni-asm.S
index e650330ef6951..345b1eafe45af 100644
--- a/arch/x86/crypto/aegis128-aesni-asm.S
+++ b/arch/x86/crypto/aegis128-aesni-asm.S
@@ -188,19 +188,21 @@ SYM_FUNC_START(aegis128_aesni_init)
SYM_FUNC_END(aegis128_aesni_init)
/*
* void aegis128_aesni_ad(struct aegis_state *state, const u8 *data,
* unsigned int len);
+ *
+ * len must be a multiple of 16.
*/
SYM_FUNC_START(aegis128_aesni_ad)
.set STATEP, %rdi
.set SRC, %rsi
.set LEN, %edx
FRAME_BEGIN
- cmp $0x10, LEN
- jb .Lad_out
+ test LEN, LEN
+ jz .Lad_out
/* load the state: */
movdqu 0x00(STATEP), STATE0
movdqu 0x10(STATEP), STATE1
movdqu 0x20(STATEP), STATE2
@@ -211,40 +213,35 @@ SYM_FUNC_START(aegis128_aesni_ad)
.Lad_loop:
movdqu 0x00(SRC), MSG
aegis128_update
pxor MSG, STATE4
sub $0x10, LEN
- cmp $0x10, LEN
- jl .Lad_out_1
+ jz .Lad_out_1
movdqu 0x10(SRC), MSG
aegis128_update
pxor MSG, STATE3
sub $0x10, LEN
- cmp $0x10, LEN
- jl .Lad_out_2
+ jz .Lad_out_2
movdqu 0x20(SRC), MSG
aegis128_update
pxor MSG, STATE2
sub $0x10, LEN
- cmp $0x10, LEN
- jl .Lad_out_3
+ jz .Lad_out_3
movdqu 0x30(SRC), MSG
aegis128_update
pxor MSG, STATE1
sub $0x10, LEN
- cmp $0x10, LEN
- jl .Lad_out_4
+ jz .Lad_out_4
movdqu 0x40(SRC), MSG
aegis128_update
pxor MSG, STATE0
sub $0x10, LEN
- cmp $0x10, LEN
- jl .Lad_out_0
+ jz .Lad_out_0
add $0x50, SRC
jmp .Lad_loop
/* store the state: */
@@ -310,28 +307,26 @@ SYM_FUNC_END(aegis128_aesni_ad)
aegis128_update
pxor MSG, \s4
sub $0x10, LEN
- cmp $0x10, LEN
- jl .Lenc_out_\i
+ jz .Lenc_out_\i
.endm
/*
* void aegis128_aesni_enc(struct aegis_state *state, const u8 *src, u8 *dst,
* unsigned int len);
+ *
+ * len must be nonzero and a multiple of 16.
*/
SYM_FUNC_START(aegis128_aesni_enc)
.set STATEP, %rdi
.set SRC, %rsi
.set DST, %rdx
.set LEN, %ecx
FRAME_BEGIN
- cmp $0x10, LEN
- jb .Lenc_out
-
/* load the state: */
movdqu 0x00(STATEP), STATE0
movdqu 0x10(STATEP), STATE1
movdqu 0x20(STATEP), STATE2
movdqu 0x30(STATEP), STATE3
@@ -457,28 +452,26 @@ SYM_FUNC_END(aegis128_aesni_enc_tail)
aegis128_update
pxor MSG, \s4
sub $0x10, LEN
- cmp $0x10, LEN
- jl .Ldec_out_\i
+ jz .Ldec_out_\i
.endm
/*
* void aegis128_aesni_dec(struct aegis_state *state, const u8 *src, u8 *dst,
* unsigned int len);
+ *
+ * len must be nonzero and a multiple of 16.
*/
SYM_FUNC_START(aegis128_aesni_dec)
.set STATEP, %rdi
.set SRC, %rsi
.set DST, %rdx
.set LEN, %ecx
FRAME_BEGIN
- cmp $0x10, LEN
- jb .Ldec_out
-
/* load the state: */
movdqu 0x00(STATEP), STATE0
movdqu 0x10(STATEP), STATE1
movdqu 0x20(STATEP), STATE2
movdqu 0x30(STATEP), STATE3
diff --git a/arch/x86/crypto/aegis128-aesni-glue.c b/arch/x86/crypto/aegis128-aesni-glue.c
index 9555958e4089d..c19d8e3d96a35 100644
--- a/arch/x86/crypto/aegis128-aesni-glue.c
+++ b/arch/x86/crypto/aegis128-aesni-glue.c
@@ -85,12 +85,12 @@ static void crypto_aegis128_aesni_process_ad(
pos = 0;
left -= fill;
src += fill;
}
- aegis128_aesni_ad(state, src, left);
-
+ aegis128_aesni_ad(state, src,
+ left & ~(AEGIS128_BLOCK_SIZE - 1));
src += left & ~(AEGIS128_BLOCK_SIZE - 1);
left &= AEGIS128_BLOCK_SIZE - 1;
}
memcpy(buf.bytes + pos, src, left);
--
2.47.0