Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib/crypto: arm64/sha3: Migrate optimized code into library
Instead of exposing the arm64-optimized SHA-3 code via arm64-specific
crypto_shash algorithms, instead just implement the sha3_absorb_blocks()
and sha3_keccakf() library functions. This is much simpler, it makes
the SHA-3 library functions be arm64-optimized, and it fixes the
longstanding issue where the arm64-optimized SHA-3 code was disabled by
default. SHA-3 still remains available through crypto_shash, but
individual architectures no longer need to handle it.
Note: to see the diff from arch/arm64/crypto/sha3-ce-glue.c to
lib/crypto/arm64/sha3.h, view this commit with 'git show -M10'.
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20251026055032.1413733-10-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
+73
-165
+1
-1
arch/arm64/configs/defconfig
+1
-1
arch/arm64/configs/defconfig
···
1783
1783
CONFIG_CRYPTO_BENCHMARK=m
1784
1784
CONFIG_CRYPTO_ECHAINIV=y
1785
1785
CONFIG_CRYPTO_MICHAEL_MIC=m
1786
+
CONFIG_CRYPTO_SHA3=m
1786
1787
CONFIG_CRYPTO_ANSI_CPRNG=y
1787
1788
CONFIG_CRYPTO_USER_API_RNG=m
1788
1789
CONFIG_CRYPTO_GHASH_ARM64_CE=y
1789
-
CONFIG_CRYPTO_SHA3_ARM64=m
1790
1790
CONFIG_CRYPTO_SM3_ARM64_CE=m
1791
1791
CONFIG_CRYPTO_AES_ARM64_CE_BLK=y
1792
1792
CONFIG_CRYPTO_AES_ARM64_BS=m
-11
arch/arm64/crypto/Kconfig
-11
arch/arm64/crypto/Kconfig
···
25
25
Architecture: arm64 using:
26
26
- NEON (Advanced SIMD) extensions
27
27
28
-
config CRYPTO_SHA3_ARM64
29
-
tristate "Hash functions: SHA-3 (ARMv8.2 Crypto Extensions)"
30
-
depends on KERNEL_MODE_NEON
31
-
select CRYPTO_HASH
32
-
select CRYPTO_SHA3
33
-
help
34
-
SHA-3 secure hash algorithms (FIPS 202)
35
-
36
-
Architecture: arm64 using:
37
-
- ARMv8.2 Crypto Extensions
38
-
39
28
config CRYPTO_SM3_NEON
40
29
tristate "Hash functions: SM3 (NEON)"
41
30
depends on KERNEL_MODE_NEON
-3
arch/arm64/crypto/Makefile
-3
arch/arm64/crypto/Makefile
arch/arm64/crypto/sha3-ce-core.S
lib/crypto/arm64/sha3-ce-core.S
arch/arm64/crypto/sha3-ce-core.S
lib/crypto/arm64/sha3-ce-core.S
-150
arch/arm64/crypto/sha3-ce-glue.c
-150
arch/arm64/crypto/sha3-ce-glue.c
···
1
-
// SPDX-License-Identifier: GPL-2.0
2
-
/*
3
-
* sha3-ce-glue.c - core SHA-3 transform using v8.2 Crypto Extensions
4
-
*
5
-
* Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
6
-
*
7
-
* This program is free software; you can redistribute it and/or modify
8
-
* it under the terms of the GNU General Public License version 2 as
9
-
* published by the Free Software Foundation.
10
-
*/
11
-
12
-
#include <asm/hwcap.h>
13
-
#include <asm/neon.h>
14
-
#include <asm/simd.h>
15
-
#include <crypto/internal/hash.h>
16
-
#include <crypto/sha3.h>
17
-
#include <linux/cpufeature.h>
18
-
#include <linux/kernel.h>
19
-
#include <linux/module.h>
20
-
#include <linux/string.h>
21
-
#include <linux/unaligned.h>
22
-
23
-
MODULE_DESCRIPTION("SHA3 secure hash using ARMv8 Crypto Extensions");
24
-
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
25
-
MODULE_LICENSE("GPL v2");
26
-
MODULE_ALIAS_CRYPTO("sha3-224");
27
-
MODULE_ALIAS_CRYPTO("sha3-256");
28
-
MODULE_ALIAS_CRYPTO("sha3-384");
29
-
MODULE_ALIAS_CRYPTO("sha3-512");
30
-
31
-
asmlinkage size_t sha3_ce_transform(struct sha3_state *state, const u8 *data,
32
-
size_t nblocks, size_t block_size);
33
-
34
-
static int arm64_sha3_update(struct shash_desc *desc, const u8 *data,
35
-
unsigned int len)
36
-
{
37
-
struct sha3_state *sctx = shash_desc_ctx(desc);
38
-
struct crypto_shash *tfm = desc->tfm;
39
-
unsigned int bs;
40
-
int blocks;
41
-
42
-
bs = crypto_shash_blocksize(tfm);
43
-
blocks = len / bs;
44
-
len -= blocks * bs;
45
-
do {
46
-
int rem;
47
-
48
-
kernel_neon_begin();
49
-
rem = sha3_ce_transform(sctx, data, blocks, bs);
50
-
kernel_neon_end();
51
-
data += (blocks - rem) * bs;
52
-
blocks = rem;
53
-
} while (blocks);
54
-
return len;
55
-
}
56
-
57
-
static int sha3_finup(struct shash_desc *desc, const u8 *src, unsigned int len,
58
-
u8 *out)
59
-
{
60
-
struct sha3_state *sctx = shash_desc_ctx(desc);
61
-
struct crypto_shash *tfm = desc->tfm;
62
-
__le64 *digest = (__le64 *)out;
63
-
u8 block[SHA3_224_BLOCK_SIZE];
64
-
unsigned int bs, ds;
65
-
int i;
66
-
67
-
ds = crypto_shash_digestsize(tfm);
68
-
bs = crypto_shash_blocksize(tfm);
69
-
memcpy(block, src, len);
70
-
71
-
block[len++] = 0x06;
72
-
memset(block + len, 0, bs - len);
73
-
block[bs - 1] |= 0x80;
74
-
75
-
kernel_neon_begin();
76
-
sha3_ce_transform(sctx, block, 1, bs);
77
-
kernel_neon_end();
78
-
memzero_explicit(block , sizeof(block));
79
-
80
-
for (i = 0; i < ds / 8; i++)
81
-
put_unaligned_le64(sctx->st[i], digest++);
82
-
83
-
if (ds & 4)
84
-
put_unaligned_le32(sctx->st[i], (__le32 *)digest);
85
-
86
-
return 0;
87
-
}
88
-
89
-
static struct shash_alg algs[] = { {
90
-
.digestsize = SHA3_224_DIGEST_SIZE,
91
-
.init = crypto_sha3_init,
92
-
.update = arm64_sha3_update,
93
-
.finup = sha3_finup,
94
-
.descsize = SHA3_STATE_SIZE,
95
-
.base.cra_name = "sha3-224",
96
-
.base.cra_driver_name = "sha3-224-ce",
97
-
.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
98
-
.base.cra_blocksize = SHA3_224_BLOCK_SIZE,
99
-
.base.cra_module = THIS_MODULE,
100
-
.base.cra_priority = 200,
101
-
}, {
102
-
.digestsize = SHA3_256_DIGEST_SIZE,
103
-
.init = crypto_sha3_init,
104
-
.update = arm64_sha3_update,
105
-
.finup = sha3_finup,
106
-
.descsize = SHA3_STATE_SIZE,
107
-
.base.cra_name = "sha3-256",
108
-
.base.cra_driver_name = "sha3-256-ce",
109
-
.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
110
-
.base.cra_blocksize = SHA3_256_BLOCK_SIZE,
111
-
.base.cra_module = THIS_MODULE,
112
-
.base.cra_priority = 200,
113
-
}, {
114
-
.digestsize = SHA3_384_DIGEST_SIZE,
115
-
.init = crypto_sha3_init,
116
-
.update = arm64_sha3_update,
117
-
.finup = sha3_finup,
118
-
.descsize = SHA3_STATE_SIZE,
119
-
.base.cra_name = "sha3-384",
120
-
.base.cra_driver_name = "sha3-384-ce",
121
-
.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
122
-
.base.cra_blocksize = SHA3_384_BLOCK_SIZE,
123
-
.base.cra_module = THIS_MODULE,
124
-
.base.cra_priority = 200,
125
-
}, {
126
-
.digestsize = SHA3_512_DIGEST_SIZE,
127
-
.init = crypto_sha3_init,
128
-
.update = arm64_sha3_update,
129
-
.finup = sha3_finup,
130
-
.descsize = SHA3_STATE_SIZE,
131
-
.base.cra_name = "sha3-512",
132
-
.base.cra_driver_name = "sha3-512-ce",
133
-
.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
134
-
.base.cra_blocksize = SHA3_512_BLOCK_SIZE,
135
-
.base.cra_module = THIS_MODULE,
136
-
.base.cra_priority = 200,
137
-
} };
138
-
139
-
static int __init sha3_neon_mod_init(void)
140
-
{
141
-
return crypto_register_shashes(algs, ARRAY_SIZE(algs));
142
-
}
143
-
144
-
static void __exit sha3_neon_mod_fini(void)
145
-
{
146
-
crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
147
-
}
148
-
149
-
module_cpu_feature_match(SHA3, sha3_neon_mod_init);
150
-
module_exit(sha3_neon_mod_fini);
+5
lib/crypto/Kconfig
+5
lib/crypto/Kconfig
···
202
202
The SHA3 library functions. Select this if your module uses any of
203
203
the functions from <crypto/sha3.h>.
204
204
205
+
config CRYPTO_LIB_SHA3_ARCH
206
+
bool
207
+
depends on CRYPTO_LIB_SHA3 && !UML
208
+
default y if ARM64 && KERNEL_MODE_NEON
209
+
205
210
config CRYPTO_LIB_SM3
206
211
tristate
207
212
+5
lib/crypto/Makefile
+5
lib/crypto/Makefile
···
281
281
obj-$(CONFIG_CRYPTO_LIB_SHA3) += libsha3.o
282
282
libsha3-y := sha3.o
283
283
284
+
ifeq ($(CONFIG_CRYPTO_LIB_SHA3_ARCH),y)
285
+
CFLAGS_sha3.o += -I$(src)/$(SRCARCH)
286
+
libsha3-$(CONFIG_ARM64) += arm64/sha3-ce-core.o
287
+
endif # CONFIG_CRYPTO_LIB_SHA3_ARCH
288
+
284
289
################################################################################
285
290
286
291
obj-$(CONFIG_MPILIB) += mpi/
+62
lib/crypto/arm64/sha3.h
+62
lib/crypto/arm64/sha3.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/*
3
+
* Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
4
+
*
5
+
* This program is free software; you can redistribute it and/or modify
6
+
* it under the terms of the GNU General Public License version 2 as
7
+
* published by the Free Software Foundation.
8
+
*/
9
+
10
+
#include <asm/neon.h>
11
+
#include <asm/simd.h>
12
+
#include <linux/cpufeature.h>
13
+
14
+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3);
15
+
16
+
asmlinkage size_t sha3_ce_transform(struct sha3_state *state, const u8 *data,
17
+
size_t nblocks, size_t block_size);
18
+
19
+
static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data,
20
+
size_t nblocks, size_t block_size)
21
+
{
22
+
if (static_branch_likely(&have_sha3) && likely(may_use_simd())) {
23
+
do {
24
+
size_t rem;
25
+
26
+
kernel_neon_begin();
27
+
rem = sha3_ce_transform(state, data, nblocks,
28
+
block_size);
29
+
kernel_neon_end();
30
+
data += (nblocks - rem) * block_size;
31
+
nblocks = rem;
32
+
} while (nblocks);
33
+
} else {
34
+
sha3_absorb_blocks_generic(state, data, nblocks, block_size);
35
+
}
36
+
}
37
+
38
+
static void sha3_keccakf(struct sha3_state *state)
39
+
{
40
+
if (static_branch_likely(&have_sha3) && likely(may_use_simd())) {
41
+
/*
42
+
* Passing zeroes into sha3_ce_transform() gives the plain
43
+
* Keccak-f permutation, which is what we want here. Any
44
+
* supported block size may be used. Use SHA3_512_BLOCK_SIZE
45
+
* since it's the shortest.
46
+
*/
47
+
static const u8 zeroes[SHA3_512_BLOCK_SIZE];
48
+
49
+
kernel_neon_begin();
50
+
sha3_ce_transform(state, zeroes, 1, sizeof(zeroes));
51
+
kernel_neon_end();
52
+
} else {
53
+
sha3_keccakf_generic(state);
54
+
}
55
+
}
56
+
57
+
#define sha3_mod_init_arch sha3_mod_init_arch
58
+
static void sha3_mod_init_arch(void)
59
+
{
60
+
if (cpu_have_named_feature(SHA3))
61
+
static_branch_enable(&have_sha3);
62
+
}