Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
crypto: x86/chacha - Restore SSSE3 fallback path
The chacha_use_simd static branch is required for x86 machines that
lack SSSE3 support. Restore it and the generic fallback code.
Reported-by: Eric Biggers <ebiggers@kernel.org>
Fixes: 9b4400215e0e ("crypto: x86/chacha - Remove SIMD fallback path")
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
+17
-5
+17
-5
arch/x86/crypto/chacha_glue.c
+17
-5
arch/x86/crypto/chacha_glue.c
···
5
5
* Copyright (C) 2015 Martin Willi
6
6
*/
7
7
8
+
#include <asm/simd.h>
8
9
#include <crypto/chacha.h>
10
+
#include <linux/jump_label.h>
9
11
#include <linux/kernel.h>
10
12
#include <linux/module.h>
11
13
#include <linux/sizes.h>
12
-
#include <asm/simd.h>
13
14
14
15
asmlinkage void chacha_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src,
15
16
unsigned int len, int nrounds);
···
32
31
asmlinkage void chacha_8block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
33
32
unsigned int len, int nrounds);
34
33
34
+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_simd);
35
35
static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_avx2);
36
36
static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_avx512vl);
37
37
···
119
117
120
118
void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
121
119
{
122
-
kernel_fpu_begin();
123
-
hchacha_block_ssse3(state, stream, nrounds);
124
-
kernel_fpu_end();
120
+
if (!static_branch_likely(&chacha_use_simd)) {
121
+
hchacha_block_generic(state, stream, nrounds);
122
+
} else {
123
+
kernel_fpu_begin();
124
+
hchacha_block_ssse3(state, stream, nrounds);
125
+
kernel_fpu_end();
126
+
}
125
127
}
126
128
EXPORT_SYMBOL(hchacha_block_arch);
127
129
128
130
void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
129
131
int nrounds)
130
132
{
133
+
if (!static_branch_likely(&chacha_use_simd) ||
134
+
bytes <= CHACHA_BLOCK_SIZE)
135
+
return chacha_crypt_generic(state, dst, src, bytes, nrounds);
136
+
131
137
do {
132
138
unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
133
139
···
152
142
153
143
bool chacha_is_arch_optimized(void)
154
144
{
155
-
return true;
145
+
return static_key_enabled(&chacha_use_simd);
156
146
}
157
147
EXPORT_SYMBOL(chacha_is_arch_optimized);
158
148
···
160
150
{
161
151
if (!boot_cpu_has(X86_FEATURE_SSSE3))
162
152
return 0;
153
+
154
+
static_branch_enable(&chacha_use_simd);
163
155
164
156
if (boot_cpu_has(X86_FEATURE_AVX) &&
165
157
boot_cpu_has(X86_FEATURE_AVX2) &&