Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib/crypto: sha256: Remove sha256_blocks_simd()
Instead of having both sha256_blocks_arch() and sha256_blocks_simd(),
instead have just sha256_blocks_arch() which uses the most efficient
implementation that is available in the calling context.
This is simpler, as it reduces the API surface. It's also safer, since
sha256_blocks_arch() just works in all contexts, including contexts
where the FPU/SIMD/vector registers cannot be used. This doesn't mean
that SHA-256 computations *should* be done in such contexts, but rather
we should just do the right thing instead of corrupting a random task's
registers. Eliminating this footgun and simplifying the code is well
worth the very small performance cost of doing the check.
Note: in the case of arm and arm64, what used to be sha256_blocks_arch()
is renamed back to its original name of sha256_block_data_order().
sha256_blocks_arch() is now used for the higher-level dispatch function.
This renaming also required an update to lib/crypto/arm64/sha512.h,
since sha2-armv8.pl is shared by both SHA-256 and SHA-512.
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20250630160645.3198-5-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
+34
-64
-6
include/crypto/internal/sha2.h
-6
include/crypto/internal/sha2.h
···
3
3
#ifndef _CRYPTO_INTERNAL_SHA2_H
4
4
#define _CRYPTO_INTERNAL_SHA2_H
5
5
6
-
#include <crypto/internal/simd.h>
7
6
#include <crypto/sha2.h>
8
7
#include <linux/compiler_attributes.h>
9
8
#include <linux/string.h>
···
21
22
const u8 *data, size_t nblocks);
22
23
void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
23
24
const u8 *data, size_t nblocks);
24
-
void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS],
25
-
const u8 *data, size_t nblocks);
26
25
27
26
static __always_inline void sha256_choose_blocks(
28
27
u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks,
···
28
31
{
29
32
if (!IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) || force_generic)
30
33
sha256_blocks_generic(state, data, nblocks);
31
-
else if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD) &&
32
-
(force_simd || crypto_simd_usable()))
33
-
sha256_blocks_simd(state, data, nblocks);
34
34
else
35
35
sha256_blocks_arch(state, data, nblocks);
36
36
}
-8
lib/crypto/Kconfig
-8
lib/crypto/Kconfig
···
150
150
Declares whether the architecture provides an arch-specific
151
151
accelerated implementation of the SHA-256 library interface.
152
152
153
-
config CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD
154
-
bool
155
-
help
156
-
Declares whether the architecture provides an arch-specific
157
-
accelerated implementation of the SHA-256 library interface
158
-
that is SIMD-based and therefore not usable in hardirq
159
-
context.
160
-
161
153
config CRYPTO_LIB_SHA256_GENERIC
162
154
tristate
163
155
default CRYPTO_LIB_SHA256 if !CRYPTO_ARCH_HAVE_LIB_SHA256
-1
lib/crypto/arm/Kconfig
-1
lib/crypto/arm/Kconfig
+10
-10
lib/crypto/arm/sha256-armv4.pl
+10
-10
lib/crypto/arm/sha256-armv4.pl
···
204
204
.word 0 @ terminator
205
205
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
206
206
.LOPENSSL_armcap:
207
-
.word OPENSSL_armcap_P-sha256_blocks_arch
207
+
.word OPENSSL_armcap_P-sha256_block_data_order
208
208
#endif
209
209
.align 5
210
210
211
-
.global sha256_blocks_arch
212
-
.type sha256_blocks_arch,%function
213
-
sha256_blocks_arch:
214
-
.Lsha256_blocks_arch:
211
+
.global sha256_block_data_order
212
+
.type sha256_block_data_order,%function
213
+
sha256_block_data_order:
214
+
.Lsha256_block_data_order:
215
215
#if __ARM_ARCH__<7
216
-
sub r3,pc,#8 @ sha256_blocks_arch
216
+
sub r3,pc,#8 @ sha256_block_data_order
217
217
#else
218
-
adr r3,.Lsha256_blocks_arch
218
+
adr r3,.Lsha256_block_data_order
219
219
#endif
220
220
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
221
221
ldr r12,.LOPENSSL_armcap
···
282
282
moveq pc,lr @ be binary compatible with V4, yet
283
283
bx lr @ interoperable with Thumb ISA:-)
284
284
#endif
285
-
.size sha256_blocks_arch,.-sha256_blocks_arch
285
+
.size sha256_block_data_order,.-sha256_block_data_order
286
286
___
287
287
######################################################################
288
288
# NEON stuff
···
470
470
stmdb sp!,{r4-r12,lr}
471
471
472
472
sub $H,sp,#16*4+16
473
-
adr $Ktbl,.Lsha256_blocks_arch
474
-
sub $Ktbl,$Ktbl,#.Lsha256_blocks_arch-K256
473
+
adr $Ktbl,.Lsha256_block_data_order
474
+
sub $Ktbl,$Ktbl,#.Lsha256_block_data_order-K256
475
475
bic $H,$H,#15 @ align for 128-bit stores
476
476
mov $t2,sp
477
477
mov sp,$H @ alloca
+7
-7
lib/crypto/arm/sha256.c
+7
-7
lib/crypto/arm/sha256.c
···
6
6
*/
7
7
#include <asm/neon.h>
8
8
#include <crypto/internal/sha2.h>
9
+
#include <crypto/internal/simd.h>
9
10
#include <linux/kernel.h>
10
11
#include <linux/module.h>
11
12
12
-
asmlinkage void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
13
-
const u8 *data, size_t nblocks);
14
-
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
13
+
asmlinkage void sha256_block_data_order(u32 state[SHA256_STATE_WORDS],
14
+
const u8 *data, size_t nblocks);
15
15
asmlinkage void sha256_block_data_order_neon(u32 state[SHA256_STATE_WORDS],
16
16
const u8 *data, size_t nblocks);
17
17
asmlinkage void sha256_ce_transform(u32 state[SHA256_STATE_WORDS],
···
20
20
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
21
21
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
22
22
23
-
void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS],
23
+
void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
24
24
const u8 *data, size_t nblocks)
25
25
{
26
26
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
27
-
static_branch_likely(&have_neon)) {
27
+
static_branch_likely(&have_neon) && crypto_simd_usable()) {
28
28
kernel_neon_begin();
29
29
if (static_branch_likely(&have_ce))
30
30
sha256_ce_transform(state, data, nblocks);
···
32
32
sha256_block_data_order_neon(state, data, nblocks);
33
33
kernel_neon_end();
34
34
} else {
35
-
sha256_blocks_arch(state, data, nblocks);
35
+
sha256_block_data_order(state, data, nblocks);
36
36
}
37
37
}
38
-
EXPORT_SYMBOL_GPL(sha256_blocks_simd);
38
+
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
39
39
40
40
bool sha256_is_arch_optimized(void)
41
41
{
-1
lib/crypto/arm64/Kconfig
-1
lib/crypto/arm64/Kconfig
+1
-1
lib/crypto/arm64/sha2-armv8.pl
+1
-1
lib/crypto/arm64/sha2-armv8.pl
+7
-7
lib/crypto/arm64/sha256.c
+7
-7
lib/crypto/arm64/sha256.c
···
6
6
*/
7
7
#include <asm/neon.h>
8
8
#include <crypto/internal/sha2.h>
9
+
#include <crypto/internal/simd.h>
9
10
#include <linux/kernel.h>
10
11
#include <linux/module.h>
11
12
12
-
asmlinkage void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
13
-
const u8 *data, size_t nblocks);
14
-
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
13
+
asmlinkage void sha256_block_data_order(u32 state[SHA256_STATE_WORDS],
14
+
const u8 *data, size_t nblocks);
15
15
asmlinkage void sha256_block_neon(u32 state[SHA256_STATE_WORDS],
16
16
const u8 *data, size_t nblocks);
17
17
asmlinkage size_t __sha256_ce_transform(u32 state[SHA256_STATE_WORDS],
···
20
20
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
21
21
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
22
22
23
-
void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS],
23
+
void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
24
24
const u8 *data, size_t nblocks)
25
25
{
26
26
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
27
-
static_branch_likely(&have_neon)) {
27
+
static_branch_likely(&have_neon) && crypto_simd_usable()) {
28
28
if (static_branch_likely(&have_ce)) {
29
29
do {
30
30
size_t rem;
···
42
42
kernel_neon_end();
43
43
}
44
44
} else {
45
-
sha256_blocks_arch(state, data, nblocks);
45
+
sha256_block_data_order(state, data, nblocks);
46
46
}
47
47
}
48
-
EXPORT_SYMBOL_GPL(sha256_blocks_simd);
48
+
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
49
49
50
50
bool sha256_is_arch_optimized(void)
51
51
{
+3
-3
lib/crypto/arm64/sha512.h
+3
-3
lib/crypto/arm64/sha512.h
···
11
11
12
12
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_insns);
13
13
14
-
asmlinkage void sha512_blocks_arch(struct sha512_block_state *state,
15
-
const u8 *data, size_t nblocks);
14
+
asmlinkage void sha512_block_data_order(struct sha512_block_state *state,
15
+
const u8 *data, size_t nblocks);
16
16
asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state,
17
17
const u8 *data, size_t nblocks);
18
18
···
32
32
nblocks = rem;
33
33
} while (nblocks);
34
34
} else {
35
-
sha512_blocks_arch(state, data, nblocks);
35
+
sha512_block_data_order(state, data, nblocks);
36
36
}
37
37
}
38
38
-1
lib/crypto/riscv/Kconfig
-1
lib/crypto/riscv/Kconfig
+3
-9
lib/crypto/riscv/sha256.c
+3
-9
lib/crypto/riscv/sha256.c
···
11
11
12
12
#include <asm/vector.h>
13
13
#include <crypto/internal/sha2.h>
14
+
#include <crypto/internal/simd.h>
14
15
#include <linux/kernel.h>
15
16
#include <linux/module.h>
16
17
···
20
19
21
20
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions);
22
21
23
-
void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS],
22
+
void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
24
23
const u8 *data, size_t nblocks)
25
24
{
26
-
if (static_branch_likely(&have_extensions)) {
25
+
if (static_branch_likely(&have_extensions) && crypto_simd_usable()) {
27
26
kernel_vector_begin();
28
27
sha256_transform_zvknha_or_zvknhb_zvkb(state, data, nblocks);
29
28
kernel_vector_end();
30
29
} else {
31
30
sha256_blocks_generic(state, data, nblocks);
32
31
}
33
-
}
34
-
EXPORT_SYMBOL_GPL(sha256_blocks_simd);
35
-
36
-
void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
37
-
const u8 *data, size_t nblocks)
38
-
{
39
-
sha256_blocks_generic(state, data, nblocks);
40
32
}
41
33
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
42
34
-1
lib/crypto/x86/Kconfig
-1
lib/crypto/x86/Kconfig
+3
-9
lib/crypto/x86/sha256.c
+3
-9
lib/crypto/x86/sha256.c
···
6
6
*/
7
7
#include <asm/fpu/api.h>
8
8
#include <crypto/internal/sha2.h>
9
+
#include <crypto/internal/simd.h>
9
10
#include <linux/kernel.h>
10
11
#include <linux/module.h>
11
12
#include <linux/static_call.h>
···
24
23
25
24
DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_transform_ssse3);
26
25
27
-
void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS],
26
+
void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
28
27
const u8 *data, size_t nblocks)
29
28
{
30
-
if (static_branch_likely(&have_sha256_x86)) {
29
+
if (static_branch_likely(&have_sha256_x86) && crypto_simd_usable()) {
31
30
kernel_fpu_begin();
32
31
static_call(sha256_blocks_x86)(state, data, nblocks);
33
32
kernel_fpu_end();
34
33
} else {
35
34
sha256_blocks_generic(state, data, nblocks);
36
35
}
37
-
}
38
-
EXPORT_SYMBOL_GPL(sha256_blocks_simd);
39
-
40
-
void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
41
-
const u8 *data, size_t nblocks)
42
-
{
43
-
sha256_blocks_generic(state, data, nblocks);
44
36
}
45
37
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
46
38