Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib/crypto: riscv/ghash: Migrate optimized code into library
Remove the "ghash-riscv64-zvkg" crypto_shash algorithm. Move the
corresponding assembly code into lib/crypto/, modify it to take the
length in blocks instead of bytes, and wire it up to the GHASH library.
This makes the GHASH library be optimized with the RISC-V Vector
Cryptography Extension. It also greatly reduces the amount of
riscv-specific glue code that is needed, and it fixes the issue where
this optimized GHASH code was disabled by default.
Note that this RISC-V code has multiple opportunities for improvement,
such as adding more parallelism, providing an optimized multiplication
function, and directly supporting POLYVAL. But for now, this commit
simply tweaks ghash_zvkg() slightly to make it compatible with the
library, then wires it up to ghash_blocks_arch().
ghash_preparekey_arch() is also implemented to store the copy of the raw
key needed by the vghsh.vv instruction.
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260319061723.1140720-13-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
+70
-166
-11
arch/riscv/crypto/Kconfig
-11
arch/riscv/crypto/Kconfig
···
17
17
- Zvkb vector crypto extension (CTR)
18
18
- Zvkg vector crypto extension (XTS)
19
19
20
-
config CRYPTO_GHASH_RISCV64
21
-
tristate "Hash functions: GHASH"
22
-
depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
23
-
RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
24
-
select CRYPTO_GCM
25
-
help
26
-
GCM GHASH function (NIST SP 800-38D)
27
-
28
-
Architecture: riscv64 using:
29
-
- Zvkg vector crypto extension
30
-
31
20
config CRYPTO_SM3_RISCV64
32
21
tristate "Hash functions: SM3 (ShangMi 3)"
33
22
depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
-3
arch/riscv/crypto/Makefile
-3
arch/riscv/crypto/Makefile
···
4
4
aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \
5
5
aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o
6
6
7
-
obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o
8
-
ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o
9
-
10
7
obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o
11
8
sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o
12
9
-146
arch/riscv/crypto/ghash-riscv64-glue.c
-146
arch/riscv/crypto/ghash-riscv64-glue.c
···
1
-
// SPDX-License-Identifier: GPL-2.0-only
2
-
/*
3
-
* GHASH using the RISC-V vector crypto extensions
4
-
*
5
-
* Copyright (C) 2023 VRULL GmbH
6
-
* Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
7
-
*
8
-
* Copyright (C) 2023 SiFive, Inc.
9
-
* Author: Jerry Shih <jerry.shih@sifive.com>
10
-
*/
11
-
12
-
#include <asm/simd.h>
13
-
#include <asm/vector.h>
14
-
#include <crypto/b128ops.h>
15
-
#include <crypto/gf128mul.h>
16
-
#include <crypto/ghash.h>
17
-
#include <crypto/internal/hash.h>
18
-
#include <crypto/internal/simd.h>
19
-
#include <crypto/utils.h>
20
-
#include <linux/errno.h>
21
-
#include <linux/kernel.h>
22
-
#include <linux/module.h>
23
-
#include <linux/string.h>
24
-
25
-
asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data,
26
-
size_t len);
27
-
28
-
struct riscv64_ghash_tfm_ctx {
29
-
be128 key;
30
-
};
31
-
32
-
struct riscv64_ghash_desc_ctx {
33
-
be128 accumulator;
34
-
};
35
-
36
-
static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
37
-
unsigned int keylen)
38
-
{
39
-
struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(tfm);
40
-
41
-
if (keylen != GHASH_BLOCK_SIZE)
42
-
return -EINVAL;
43
-
44
-
memcpy(&tctx->key, key, GHASH_BLOCK_SIZE);
45
-
46
-
return 0;
47
-
}
48
-
49
-
static int riscv64_ghash_init(struct shash_desc *desc)
50
-
{
51
-
struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
52
-
53
-
*dctx = (struct riscv64_ghash_desc_ctx){};
54
-
55
-
return 0;
56
-
}
57
-
58
-
static inline void
59
-
riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx *tctx,
60
-
struct riscv64_ghash_desc_ctx *dctx,
61
-
const u8 *src, size_t srclen)
62
-
{
63
-
/* The srclen is nonzero and a multiple of 16. */
64
-
if (crypto_simd_usable()) {
65
-
kernel_vector_begin();
66
-
ghash_zvkg(&dctx->accumulator, &tctx->key, src, srclen);
67
-
kernel_vector_end();
68
-
} else {
69
-
do {
70
-
crypto_xor((u8 *)&dctx->accumulator, src,
71
-
GHASH_BLOCK_SIZE);
72
-
gf128mul_lle(&dctx->accumulator, &tctx->key);
73
-
src += GHASH_BLOCK_SIZE;
74
-
srclen -= GHASH_BLOCK_SIZE;
75
-
} while (srclen);
76
-
}
77
-
}
78
-
79
-
static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src,
80
-
unsigned int srclen)
81
-
{
82
-
const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
83
-
struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
84
-
85
-
riscv64_ghash_blocks(tctx, dctx, src,
86
-
round_down(srclen, GHASH_BLOCK_SIZE));
87
-
return srclen - round_down(srclen, GHASH_BLOCK_SIZE);
88
-
}
89
-
90
-
static int riscv64_ghash_finup(struct shash_desc *desc, const u8 *src,
91
-
unsigned int len, u8 *out)
92
-
{
93
-
const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
94
-
struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
95
-
96
-
if (len) {
97
-
u8 buf[GHASH_BLOCK_SIZE] = {};
98
-
99
-
memcpy(buf, src, len);
100
-
riscv64_ghash_blocks(tctx, dctx, buf, GHASH_BLOCK_SIZE);
101
-
memzero_explicit(buf, sizeof(buf));
102
-
}
103
-
104
-
memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE);
105
-
return 0;
106
-
}
107
-
108
-
static struct shash_alg riscv64_ghash_alg = {
109
-
.init = riscv64_ghash_init,
110
-
.update = riscv64_ghash_update,
111
-
.finup = riscv64_ghash_finup,
112
-
.setkey = riscv64_ghash_setkey,
113
-
.descsize = sizeof(struct riscv64_ghash_desc_ctx),
114
-
.digestsize = GHASH_DIGEST_SIZE,
115
-
.base = {
116
-
.cra_blocksize = GHASH_BLOCK_SIZE,
117
-
.cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx),
118
-
.cra_priority = 300,
119
-
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
120
-
.cra_name = "ghash",
121
-
.cra_driver_name = "ghash-riscv64-zvkg",
122
-
.cra_module = THIS_MODULE,
123
-
},
124
-
};
125
-
126
-
static int __init riscv64_ghash_mod_init(void)
127
-
{
128
-
if (riscv_isa_extension_available(NULL, ZVKG) &&
129
-
riscv_vector_vlen() >= 128)
130
-
return crypto_register_shash(&riscv64_ghash_alg);
131
-
132
-
return -ENODEV;
133
-
}
134
-
135
-
static void __exit riscv64_ghash_mod_exit(void)
136
-
{
137
-
crypto_unregister_shash(&riscv64_ghash_alg);
138
-
}
139
-
140
-
module_init(riscv64_ghash_mod_init);
141
-
module_exit(riscv64_ghash_mod_exit);
142
-
143
-
MODULE_DESCRIPTION("GHASH (RISC-V accelerated)");
144
-
MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>");
145
-
MODULE_LICENSE("GPL");
146
-
MODULE_ALIAS_CRYPTO("ghash");
+7
-6
arch/riscv/crypto/ghash-riscv64-zvkg.S
lib/crypto/riscv/ghash-riscv64-zvkg.S
+7
-6
arch/riscv/crypto/ghash-riscv64-zvkg.S
lib/crypto/riscv/ghash-riscv64-zvkg.S
···
50
50
#define ACCUMULATOR a0
51
51
#define KEY a1
52
52
#define DATA a2
53
-
#define LEN a3
53
+
#define NBLOCKS a3
54
54
55
-
// void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data,
56
-
// size_t len);
55
+
// void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE],
56
+
// const u8 key[GHASH_BLOCK_SIZE],
57
+
// const u8 *data, size_t nblocks);
57
58
//
58
-
// |len| must be nonzero and a multiple of 16 (GHASH_BLOCK_SIZE).
59
+
// |nblocks| must be nonzero.
59
60
SYM_FUNC_START(ghash_zvkg)
60
61
vsetivli zero, 4, e32, m1, ta, ma
61
62
vle32.v v1, (ACCUMULATOR)
···
65
64
vle32.v v3, (DATA)
66
65
vghsh.vv v1, v2, v3
67
66
addi DATA, DATA, 16
68
-
addi LEN, LEN, -16
69
-
bnez LEN, .Lnext_block
67
+
addi NBLOCKS, NBLOCKS, -1
68
+
bnez NBLOCKS, .Lnext_block
70
69
71
70
vse32.v v1, (ACCUMULATOR)
72
71
ret
+3
include/crypto/gf128hash.h
+3
include/crypto/gf128hash.h
···
44
44
#if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_PPC64)
45
45
/** @htable: GHASH key format used by the POWER8 assembly code */
46
46
u64 htable[4][2];
47
+
#elif defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_RISCV)
48
+
/** @h_raw: The hash key H, in GHASH format */
49
+
u8 h_raw[GHASH_BLOCK_SIZE];
47
50
#endif
48
51
/** @h: The hash key H, in POLYVAL format */
49
52
struct polyval_elem h;
+2
lib/crypto/Kconfig
+2
lib/crypto/Kconfig
+1
lib/crypto/Makefile
+1
lib/crypto/Makefile
+57
lib/crypto/riscv/gf128hash.h
+57
lib/crypto/riscv/gf128hash.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
/*
3
+
* GHASH, RISC-V optimized
4
+
*
5
+
* Copyright (C) 2023 VRULL GmbH
6
+
* Copyright (C) 2023 SiFive, Inc.
7
+
* Copyright 2026 Google LLC
8
+
*/
9
+
10
+
#include <asm/simd.h>
11
+
#include <asm/vector.h>
12
+
13
+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkg);
14
+
15
+
asmlinkage void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE],
16
+
const u8 key[GHASH_BLOCK_SIZE],
17
+
const u8 *data, size_t nblocks);
18
+
19
+
#define ghash_preparekey_arch ghash_preparekey_arch
20
+
static void ghash_preparekey_arch(struct ghash_key *key,
21
+
const u8 raw_key[GHASH_BLOCK_SIZE])
22
+
{
23
+
/* Save key in POLYVAL format for fallback */
24
+
ghash_key_to_polyval(raw_key, &key->h);
25
+
26
+
/* Save key in GHASH format for zvkg */
27
+
memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE);
28
+
}
29
+
30
+
#define ghash_blocks_arch ghash_blocks_arch
31
+
static void ghash_blocks_arch(struct polyval_elem *acc,
32
+
const struct ghash_key *key,
33
+
const u8 *data, size_t nblocks)
34
+
{
35
+
if (static_branch_likely(&have_zvkg) && likely(may_use_simd())) {
36
+
u8 ghash_acc[GHASH_BLOCK_SIZE];
37
+
38
+
polyval_acc_to_ghash(acc, ghash_acc);
39
+
40
+
kernel_vector_begin();
41
+
ghash_zvkg(ghash_acc, key->h_raw, data, nblocks);
42
+
kernel_vector_end();
43
+
44
+
ghash_acc_to_polyval(ghash_acc, acc);
45
+
memzero_explicit(ghash_acc, sizeof(ghash_acc));
46
+
} else {
47
+
ghash_blocks_generic(acc, &key->h, data, nblocks);
48
+
}
49
+
}
50
+
51
+
#define gf128hash_mod_init_arch gf128hash_mod_init_arch
52
+
static void gf128hash_mod_init_arch(void)
53
+
{
54
+
if (riscv_isa_extension_available(NULL, ZVKG) &&
55
+
riscv_vector_vlen() >= 128)
56
+
static_branch_enable(&have_zvkg);
57
+
}