Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib/crypto: arm64: Move remaining algorithms to scoped ksimd API
Move the arm64 implementations of SHA-3 and POLYVAL to the newly
introduced scoped ksimd API, which replaces kernel_neon_begin() and
kernel_neon_end(). On arm64, this is needed because the latter API
will change in an incompatible manner.
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
authored by
Ard Biesheuvel
and committed by
Eric Biggers
8dcac98a
c0d597e0
+16
-21
+11
-13
lib/crypto/arm64/polyval.h
+11
-13
lib/crypto/arm64/polyval.h
···
4
4
*
5
5
* Copyright 2025 Google LLC
6
6
*/
7
-
#include <asm/neon.h>
8
7
#include <asm/simd.h>
9
8
#include <linux/cpufeature.h>
10
9
···
23
24
static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS);
24
25
memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE);
25
26
if (static_branch_likely(&have_pmull) && may_use_simd()) {
26
-
kernel_neon_begin();
27
-
for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
28
-
key->h_powers[i] = key->h_powers[i + 1];
29
-
polyval_mul_pmull(&key->h_powers[i],
30
-
&key->h_powers[NUM_H_POWERS - 1]);
27
+
scoped_ksimd() {
28
+
for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
29
+
key->h_powers[i] = key->h_powers[i + 1];
30
+
polyval_mul_pmull(
31
+
&key->h_powers[i],
32
+
&key->h_powers[NUM_H_POWERS - 1]);
33
+
}
31
34
}
32
-
kernel_neon_end();
33
35
} else {
34
36
for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
35
37
key->h_powers[i] = key->h_powers[i + 1];
···
44
44
const struct polyval_key *key)
45
45
{
46
46
if (static_branch_likely(&have_pmull) && may_use_simd()) {
47
-
kernel_neon_begin();
48
-
polyval_mul_pmull(acc, &key->h_powers[NUM_H_POWERS - 1]);
49
-
kernel_neon_end();
47
+
scoped_ksimd()
48
+
polyval_mul_pmull(acc, &key->h_powers[NUM_H_POWERS - 1]);
50
49
} else {
51
50
polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]);
52
51
}
···
61
62
size_t n = min_t(size_t, nblocks,
62
63
4096 / POLYVAL_BLOCK_SIZE);
63
64
64
-
kernel_neon_begin();
65
-
polyval_blocks_pmull(acc, key, data, n);
66
-
kernel_neon_end();
65
+
scoped_ksimd()
66
+
polyval_blocks_pmull(acc, key, data, n);
67
67
data += n * POLYVAL_BLOCK_SIZE;
68
68
nblocks -= n;
69
69
} while (nblocks);
+5
-8
lib/crypto/arm64/sha3.h
+5
-8
lib/crypto/arm64/sha3.h
···
7
7
* published by the Free Software Foundation.
8
8
*/
9
9
10
-
#include <asm/neon.h>
11
10
#include <asm/simd.h>
12
11
#include <linux/cpufeature.h>
13
12
···
22
23
do {
23
24
size_t rem;
24
25
25
-
kernel_neon_begin();
26
-
rem = sha3_ce_transform(state, data, nblocks,
27
-
block_size);
28
-
kernel_neon_end();
26
+
scoped_ksimd()
27
+
rem = sha3_ce_transform(state, data, nblocks,
28
+
block_size);
29
29
data += (nblocks - rem) * block_size;
30
30
nblocks = rem;
31
31
} while (nblocks);
···
44
46
*/
45
47
static const u8 zeroes[SHA3_512_BLOCK_SIZE];
46
48
47
-
kernel_neon_begin();
48
-
sha3_ce_transform(state, zeroes, 1, sizeof(zeroes));
49
-
kernel_neon_end();
49
+
scoped_ksimd()
50
+
sha3_ce_transform(state, zeroes, 1, sizeof(zeroes));
50
51
} else {
51
52
sha3_keccakf_generic(state);
52
53
}