Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
crypto: powerpc/poly1305 - Add SIMD fallback
Add a SIMD fallback path for poly1305-p10 by converting the 2^64
based hash state into a 2^44 base. In order to ensure that the
generic fallback is actually 2^44, add ARCH_SUPPORTS_INT128 to
powerpc and make poly1305-p10 depend on it.
Fixes: ba8f8624fde2 ("crypto: poly1305-p10 - Glue code for optmized Poly1305 implementation for ppc64le")
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
+40
-4
+1
arch/powerpc/Kconfig
+1
arch/powerpc/Kconfig
···
173
173
select ARCH_STACKWALK
174
174
select ARCH_SUPPORTS_ATOMIC_RMW
175
175
select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC_BOOK3S || PPC_8xx
176
+
select ARCH_SUPPORTS_INT128 if PPC64 && CC_HAS_INT128
176
177
select ARCH_USE_BUILTIN_BSWAP
177
178
select ARCH_USE_CMPXCHG_LOCKREF if PPC64
178
179
select ARCH_USE_MEMTEST
+1
-1
arch/powerpc/lib/crypto/Kconfig
+1
-1
arch/powerpc/lib/crypto/Kconfig
+38
-3
arch/powerpc/lib/crypto/poly1305-p10-glue.c
+38
-3
arch/powerpc/lib/crypto/poly1305-p10-glue.c
···
6
6
*/
7
7
#include <asm/switch_to.h>
8
8
#include <crypto/internal/poly1305.h>
9
+
#include <crypto/internal/simd.h>
9
10
#include <linux/cpufeature.h>
10
11
#include <linux/jump_label.h>
11
12
#include <linux/kernel.h>
···
18
17
asmlinkage void poly1305_emit_64(const struct poly1305_state *state, const u32 nonce[4], u8 digest[POLY1305_DIGEST_SIZE]);
19
18
20
19
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10);
20
+
21
+
static inline bool is_state_base64(struct poly1305_block_state *state)
22
+
{
23
+
return state->core_r.precomputed_s.r64[2];
24
+
}
21
25
22
26
static void vsx_begin(void)
23
27
{
···
36
30
preempt_enable();
37
31
}
38
32
33
+
static void convert_to_base2_44(struct poly1305_block_state *state)
34
+
{
35
+
u8 raw_key[POLY1305_BLOCK_SIZE];
36
+
u64 h0, h1, h2;
37
+
38
+
if (!is_state_base64(state))
39
+
return;
40
+
41
+
state->core_r.precomputed_s.r64[2] = 0;
42
+
put_unaligned_le64(state->core_r.key.r64[0], raw_key + 0);
43
+
put_unaligned_le64(state->core_r.key.r64[1], raw_key + 8);
44
+
poly1305_core_setkey(&state->core_r, raw_key);
45
+
46
+
h0 = state->h.h64[0];
47
+
h1 = state->h.h64[1];
48
+
h2 = state->h.h64[2];
49
+
state->h.h64[0] = h0 & 0xfffffffffffULL;
50
+
state->h.h64[1] = h0 >> 44 | (h1 & 0xffffffULL) << 20;
51
+
state->h.h64[2] = h1 >> 24 | h2 << 40;
52
+
}
53
+
39
54
void poly1305_block_init_arch(struct poly1305_block_state *dctx,
40
55
const u8 raw_key[POLY1305_BLOCK_SIZE])
41
56
{
42
-
if (!static_key_enabled(&have_p10))
57
+
dctx->core_r.precomputed_s.r64[2] = 0;
58
+
if (!static_key_enabled(&have_p10) || !crypto_simd_usable())
43
59
return poly1305_block_init_generic(dctx, raw_key);
44
60
61
+
dctx->core_r.precomputed_s.r64[2] = 1;
45
62
dctx->h = (struct poly1305_state){};
46
63
dctx->core_r.key.r64[0] = get_unaligned_le64(raw_key + 0);
47
64
dctx->core_r.key.r64[1] = get_unaligned_le64(raw_key + 8);
···
74
45
void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
75
46
unsigned int len, u32 padbit)
76
47
{
77
-
if (!static_key_enabled(&have_p10))
48
+
if (!static_key_enabled(&have_p10) || !is_state_base64(state) ||
49
+
!crypto_simd_usable()) {
50
+
convert_to_base2_44(state);
78
51
return poly1305_blocks_generic(state, src, len, padbit);
52
+
}
79
53
vsx_begin();
80
54
if (len >= POLY1305_BLOCK_SIZE * 4) {
81
55
poly1305_p10le_4blocks(state, src, len);
···
98
66
u8 digest[POLY1305_DIGEST_SIZE],
99
67
const u32 nonce[4])
100
68
{
101
-
if (!static_key_enabled(&have_p10))
69
+
struct poly1305_block_state *dctx =
70
+
container_of(state, struct poly1305_block_state, h);
71
+
72
+
if (!static_key_enabled(&have_p10) || !is_state_base64(dctx))
102
73
return poly1305_emit_generic(state, digest, nonce);
103
74
poly1305_emit_64(state, nonce, digest);
104
75
}