From nobody Wed Sep 10 02:04:10 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 89EEEC64ED6 for ; Tue, 28 Feb 2023 00:12:26 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229698AbjB1AMZ (ORCPT ); Mon, 27 Feb 2023 19:12:25 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45128 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229677AbjB1AMV (ORCPT ); Mon, 27 Feb 2023 19:12:21 -0500 Received: from gloria.sntech.de (gloria.sntech.de [185.11.138.130]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id C6282298D0 for ; Mon, 27 Feb 2023 16:12:02 -0800 (PST) Received: from ip5b412258.dynamic.kabel-deutschland.de ([91.65.34.88] helo=phil.lan) by gloria.sntech.de with esmtpsa (TLS1.3) tls TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 (Exim 4.94.2) (envelope-from ) id 1pWnVA-000552-QF; Tue, 28 Feb 2023 01:05:52 +0100 From: Heiko Stuebner To: palmer@rivosinc.com Cc: greentime.hu@sifive.com, conor@kernel.org, linux-kernel@vger.kernel.org, linux-riscv@lists.infradead.org, christoph.muellner@vrull.eu, heiko@sntech.de, Heiko Stuebner Subject: [PATCH RFC v2 11/16] RISC-V: crypto: add Zvkg accelerated GCM GHASH implementation Date: Tue, 28 Feb 2023 01:05:39 +0100 Message-Id: <20230228000544.2234136-12-heiko@sntech.de> X-Mailer: git-send-email 2.39.0 In-Reply-To: <20230228000544.2234136-1-heiko@sntech.de> References: <20230228000544.2234136-1-heiko@sntech.de> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Heiko Stuebner When the Zvkg vector crypto extension is available another optimized gcm ghash variant is possible, so add it as another implmentation. Signed-off-by: Heiko Stuebner --- arch/riscv/crypto/Kconfig | 1 + arch/riscv/crypto/Makefile | 7 +- arch/riscv/crypto/ghash-riscv64-glue.c | 80 ++++++++++++ arch/riscv/crypto/ghash-riscv64-zvkg.pl | 161 ++++++++++++++++++++++++ 4 files changed, 247 insertions(+), 2 deletions(-) create mode 100644 arch/riscv/crypto/ghash-riscv64-zvkg.pl diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index 404fd9b3cb7c..84da19bdde8b 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -13,5 +13,6 @@ config CRYPTO_GHASH_RISCV64 Architecture: riscv64 using one of: - ZBC extension - ZVKB vector crypto extension + - ZVKG vector crypto extension =20 endmenu diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile index 8ab9a0ae8f2d..1ee0ce7d3264 100644 --- a/arch/riscv/crypto/Makefile +++ b/arch/riscv/crypto/Makefile @@ -9,7 +9,7 @@ ifdef CONFIG_RISCV_ISA_ZBC ghash-riscv64-y +=3D ghash-riscv64-zbc.o endif ifdef CONFIG_RISCV_ISA_V -ghash-riscv64-y +=3D ghash-riscv64-zvkb.o +ghash-riscv64-y +=3D ghash-riscv64-zvkb.o ghash-riscv64-zvkg.o endif =20 quiet_cmd_perlasm =3D PERLASM $@ @@ -21,4 +21,7 @@ $(obj)/ghash-riscv64-zbc.S: $(src)/ghash-riscv64-zbc.pl $(obj)/ghash-riscv64-zvkb.S: $(src)/ghash-riscv64-zvkb.pl $(call cmd,perlasm) =20 -clean-files +=3D ghash-riscv64-zbc.S ghash-riscv64-zvkb.S +$(obj)/ghash-riscv64-zvkg.S: $(src)/ghash-riscv64-zvkg.pl + $(call cmd,perlasm) + +clean-files +=3D ghash-riscv64-zbc.S ghash-riscv64-zvkb.S ghash-riscv64-zv= kg.S diff --git a/arch/riscv/crypto/ghash-riscv64-glue.c b/arch/riscv/crypto/gha= sh-riscv64-glue.c index 004a1a11d7d8..9c7a8616049e 100644 --- a/arch/riscv/crypto/ghash-riscv64-glue.c +++ b/arch/riscv/crypto/ghash-riscv64-glue.c @@ -26,6 +26,10 @@ void gcm_ghash_rv64i_zbc__zbkb(u64 Xi[2], const u128 Hta= ble[16], void gcm_ghash_rv64i_zvkb(u64 Xi[2], const u128 Htable[16], const u8 *inp, size_t len); =20 +/* Zvkg (vector crypto with vghmac.vv). */ +void gcm_ghash_rv64i_zvkg(u64 Xi[2], const u128 Htable[16], + const u8 *inp, size_t len); + struct riscv64_ghash_ctx { void (*ghash_func)(u64 Xi[2], const u128 Htable[16], const u8 *inp, size_t len); @@ -183,6 +187,63 @@ struct shash_alg riscv64_zvkb_ghash_alg =3D { }, }; =20 +RISCV64_ZVK_SETKEY(zvkg, zvkg); +struct shash_alg riscv64_zvkg_ghash_alg =3D { + .digestsize =3D GHASH_DIGEST_SIZE, + .init =3D riscv64_ghash_init, + .update =3D riscv64_zvk_ghash_update, + .final =3D riscv64_zvk_ghash_final, + .setkey =3D riscv64_zvk_ghash_setkey_zvkg, + .descsize =3D sizeof(struct riscv64_ghash_desc_ctx) + + sizeof(struct ghash_desc_ctx), + .base =3D { + .cra_name =3D "ghash", + .cra_driver_name =3D "riscv64_zvkg_ghash", + .cra_priority =3D 301, + .cra_blocksize =3D GHASH_BLOCK_SIZE, + .cra_ctxsize =3D sizeof(struct riscv64_ghash_ctx), + .cra_module =3D THIS_MODULE, + }, +}; + +RISCV64_ZVK_SETKEY(zvkg__zbb_or_zbkb, zvkg); +struct shash_alg riscv64_zvkg_zbb_or_zbkb_ghash_alg =3D { + .digestsize =3D GHASH_DIGEST_SIZE, + .init =3D riscv64_ghash_init, + .update =3D riscv64_zvk_ghash_update, + .final =3D riscv64_zvk_ghash_final, + .setkey =3D riscv64_zvk_ghash_setkey_zvkg__zbb_or_zbkb, + .descsize =3D sizeof(struct riscv64_ghash_desc_ctx) + + sizeof(struct ghash_desc_ctx), + .base =3D { + .cra_name =3D "ghash", + .cra_driver_name =3D "riscv64_zvkg_zbb_or_zbkb_ghash", + .cra_priority =3D 302, + .cra_blocksize =3D GHASH_BLOCK_SIZE, + .cra_ctxsize =3D sizeof(struct riscv64_ghash_ctx), + .cra_module =3D THIS_MODULE, + }, +}; + +RISCV64_ZVK_SETKEY(zvkg__zvkb, zvkg); +struct shash_alg riscv64_zvkg_zvkb_ghash_alg =3D { + .digestsize =3D GHASH_DIGEST_SIZE, + .init =3D riscv64_ghash_init, + .update =3D riscv64_zvk_ghash_update, + .final =3D riscv64_zvk_ghash_final, + .setkey =3D riscv64_zvk_ghash_setkey_zvkg__zvkb, + .descsize =3D sizeof(struct riscv64_ghash_desc_ctx) + + sizeof(struct ghash_desc_ctx), + .base =3D { + .cra_name =3D "ghash", + .cra_driver_name =3D "riscv64_zvkg_zvkb_ghash", + .cra_priority =3D 303, + .cra_blocksize =3D GHASH_BLOCK_SIZE, + .cra_ctxsize =3D sizeof(struct riscv64_ghash_ctx), + .cra_module =3D THIS_MODULE, + }, +}; + #endif /* CONFIG_RISCV_ISA_V */ =20 #ifdef CONFIG_RISCV_ISA_ZBC @@ -381,6 +442,25 @@ static int __init riscv64_ghash_mod_init(void) if (ret < 0) return ret; } + + if (riscv_isa_extension_available(NULL, ZVKG)) { + ret =3D riscv64_ghash_register(&riscv64_zvkg_ghash_alg); + if (ret < 0) + return ret; + + if (riscv_isa_extension_available(NULL, ZVKB)) { + ret =3D riscv64_ghash_register(&riscv64_zvkg_zvkb_ghash_alg); + if (ret < 0) + return ret; + } + + if (riscv_isa_extension_available(NULL, ZBB) || + riscv_isa_extension_available(NULL, ZBKB)) { + ret =3D riscv64_ghash_register(&riscv64_zvkg_zbb_or_zbkb_ghash_alg); + if (ret < 0) + return ret; + } + } #endif =20 return 0; diff --git a/arch/riscv/crypto/ghash-riscv64-zvkg.pl b/arch/riscv/crypto/gh= ash-riscv64-zvkg.pl new file mode 100644 index 000000000000..c13dd9c4ee31 --- /dev/null +++ b/arch/riscv/crypto/ghash-riscv64-zvkg.pl @@ -0,0 +1,161 @@ +#! /usr/bin/env perl +# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +# - RV64I +# - RISC-V vector ('V') with VLEN >=3D 128 +# - RISC-V vector crypto GHASH extension ('Zvkg') + +use strict; +use warnings; + +use FindBin qw($Bin); +use lib "$Bin"; +use lib "$Bin/../../perlasm"; +use riscv; + +# $output is the last argument if it looks like a file (it has an extensio= n) +# $flavour is the first argument if it doesn't look like a file +my $output =3D $#ARGV >=3D 0 && $ARGV[$#ARGV] =3D~ m|\.\w+$| ? pop : undef; +my $flavour =3D $#ARGV >=3D 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$output and open STDOUT,">$output"; + +my $code=3D<<___; +.text +___ + +##########################################################################= ###### +# void gcm_init_rv64i_zvkg(u128 Htable[16], const u64 H[2]); +# void gcm_init_rv64i_zvkg__zbb_or_zbkb(u128 Htable[16], const u64 H[2]); +# void gcm_init_rv64i_zvkg__zvkb(u128 Htable[16], const u64 H[2]); +# +# input: H: 128-bit H - secret parameter E(K, 0^128) +# output: Htable: Copy of secret parameter (in normalized byte order) +# +# All callers of this function revert the byte-order unconditionally +# on little-endian machines. So we need to revert the byte-order back. +{ +my ($Htable,$H,$VAL0,$VAL1,$TMP0) =3D ("a0","a1","a2","a3","t0"); + +$code .=3D <<___; +.p2align 3 +.globl gcm_init_rv64i_zvkg +.type gcm_init_rv64i_zvkg,\@function +gcm_init_rv64i_zvkg: + # First word + ld $VAL0, 0($H) + ld $VAL1, 8($H) + @{[sd_rev8_rv64i $VAL0, $Htable, 0, $TMP0]} + @{[sd_rev8_rv64i $VAL1, $Htable, 8, $TMP0]} + ret +.size gcm_init_rv64i_zvkg,.-gcm_init_rv64i_zvkg +___ +} + +{ +my ($Htable,$H,$TMP0,$TMP1) =3D ("a0","a1","t0","t1"); + +$code .=3D <<___; +.p2align 3 +.globl gcm_init_rv64i_zvkg__zbb_or_zbkb +.type gcm_init_rv64i_zvkg__zbb_or_zbkb,\@function +gcm_init_rv64i_zvkg__zbb_or_zbkb: + ld $TMP0,0($H) + ld $TMP1,8($H) + @{[rev8 $TMP0, $TMP0]} #rev8 $TMP0, $TMP0 + @{[rev8 $TMP1, $TMP1]} #rev8 $TMP1, $TMP1 + sd $TMP0,0($Htable) + sd $TMP1,8($Htable) + ret +.size gcm_init_rv64i_zvkg__zbb_or_zbkb,.-gcm_init_rv64i_zvkg__zbb_or_zbkb +___ +} + +{ +my ($Htable,$H,$V0) =3D ("a0","a1","v0"); + +$code .=3D <<___; +.p2align 3 +.globl gcm_init_rv64i_zvkg__zvkb +.type gcm_init_rv64i_zvkg__zvkb,\@function +gcm_init_rv64i_zvkg__zvkb: + # All callers of this function revert the byte-order unconditionally + # on little-endian machines. So we need to revert the byte-order back. + @{[vsetivli__x0_2_e64_m1_ta_ma]} # vsetivli x0, 2, e64, m1, ta, ma + @{[vle64_v $V0, $H]} # vle64.v v0, (a1) + @{[vrev8_v $V0, $V0]} # vrev8.v v0, v0 + @{[vse64_v $V0, $Htable]} # vse64.v v0, (a0) + ret +.size gcm_init_rv64i_zvkg__zvkb,.-gcm_init_rv64i_zvkg__zvkb +___ +} + +##########################################################################= ###### +# void gcm_gmult_rv64i_zvkg(u64 Xi[2], const u128 Htable[16]); +# +# input: Xi: current hash value +# Htable: copy of H +# output: Xi: next hash value Xi +{ +my ($Xi,$Htable) =3D ("a0","a1"); +my ($VD,$VS2) =3D ("v1","v2"); + +$code .=3D <<___; +.p2align 3 +.globl gcm_gmult_rv64i_zvkg +.type gcm_gmult_rv64i_zvkg,\@function +gcm_gmult_rv64i_zvkg: + @{[vsetivli__x0_4_e32_m1_ta_ma]} + @{[vle32_v $VS2, $Htable]} + @{[vle32_v $VD, $Xi]} + @{[vgmul_vv $VD, $VS2]} + @{[vse32_v $VD, $Xi]} + ret +.size gcm_gmult_rv64i_zvkg,.-gcm_gmult_rv64i_zvkg +___ +} + +##########################################################################= ###### +# void gcm_ghash_rv64i_zvkg(u64 Xi[2], const u128 Htable[16], +# const u8 *inp, size_t len); +# +# input: Xi: current hash value +# Htable: copy of H +# inp: pointer to input data +# len: length of input data in bytes (mutiple of block size) +# output: Xi: Xi+1 (next hash value Xi) +{ +my ($Xi,$Htable,$inp,$len) =3D ("a0","a1","a2","a3"); +my ($vXi,$vH,$vinp,$Vzero) =3D ("v1","v2","v3","v4"); + +$code .=3D <<___; +.p2align 3 +.globl gcm_ghash_rv64i_zvkg +.type gcm_ghash_rv64i_zvkg,\@function +gcm_ghash_rv64i_zvkg: + @{[vsetivli__x0_4_e32_m1_ta_ma]} + @{[vle32_v $vH, $Htable]} + @{[vle32_v $vXi, $Xi]} + +Lstep: + @{[vle32_v $vinp, $inp]} + add $inp, $inp, 16 + add $len, $len, -16 + @{[vghsh_vv $vXi, $vinp, $vH]} + bnez $len, Lstep + + @{[vse32_v $vXi, $Xi]} + ret + +.size gcm_ghash_rv64i_zvkg,.-gcm_ghash_rv64i_zvkg +___ +} + +print $code; + +close STDOUT or die "error closing STDOUT: $!"; --=20 2.39.0