Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib/crypto: arm/ghash: Migrate optimized code into library
Remove the "ghash-neon" crypto_shash algorithm. Move the corresponding
assembly code into lib/crypto/, and wire it up to the GHASH library.
This makes the GHASH library be optimized on arm (though only with NEON,
not PMULL; for now the goal is just parity with crypto_shash). It
greatly reduces the amount of arm-specific glue code that is needed, and
it fixes the issue where this optimization was disabled by default.
To integrate the assembly code correctly with the library, make the
following tweaks:
- Change the type of 'blocks' from int to size_t.
- Change the types of 'dg' and 'h' to polyval_elem. Note that this
simply reflects the format that the code was already using, at least
on little endian CPUs. For big endian CPUs, add byte-swaps.
- Remove the 'head' argument, which is no longer needed.
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260319061723.1140720-8-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
+66
-162
+2
-11
arch/arm/crypto/Kconfig
+2
-11
arch/arm/crypto/Kconfig
···
3
3
menu "Accelerated Cryptographic Algorithms for CPU (arm)"
4
4
5
5
config CRYPTO_GHASH_ARM_CE
6
-
tristate "Hash functions: GHASH (PMULL/NEON/ARMv8 Crypto Extensions)"
6
+
tristate "AEAD cipher: AES in GCM mode (ARMv8 Crypto Extensions)"
7
7
depends on KERNEL_MODE_NEON
8
8
select CRYPTO_AEAD
9
-
select CRYPTO_HASH
10
-
select CRYPTO_CRYPTD
11
9
select CRYPTO_LIB_AES
12
10
select CRYPTO_LIB_GF128MUL
13
11
help
14
-
GCM GHASH function (NIST SP800-38D)
12
+
AEAD cipher: AES-GCM
15
13
16
14
Architecture: arm using
17
-
- PMULL (Polynomial Multiply Long) instructions
18
-
- NEON (Advanced SIMD) extensions
19
15
- ARMv8 Crypto Extensions
20
-
21
-
Use an implementation of GHASH (used by the GCM AEAD chaining mode)
22
-
that uses the 64x64 to 128 bit polynomial multiplication (vmull.p64)
23
-
that is part of the ARMv8 Crypto Extensions, or a slower variant that
24
-
uses the vmull.p8 instruction that is part of the basic NEON ISA.
25
16
26
17
config CRYPTO_AES_ARM_BS
27
18
tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (bit-sliced NEON)"
+1
-1
arch/arm/crypto/Makefile
+1
-1
arch/arm/crypto/Makefile
+5
-139
arch/arm/crypto/ghash-ce-glue.c
+5
-139
arch/arm/crypto/ghash-ce-glue.c
···
1
1
// SPDX-License-Identifier: GPL-2.0-only
2
2
/*
3
-
* Accelerated GHASH implementation with ARMv8 vmull.p64 instructions.
3
+
* AES-GCM using ARMv8 Crypto Extensions
4
4
*
5
5
* Copyright (C) 2015 - 2018 Linaro Ltd.
6
6
* Copyright (C) 2023 Google LLC.
···
14
14
#include <crypto/gf128mul.h>
15
15
#include <crypto/ghash.h>
16
16
#include <crypto/internal/aead.h>
17
-
#include <crypto/internal/hash.h>
18
17
#include <crypto/internal/skcipher.h>
19
18
#include <crypto/scatterwalk.h>
20
19
#include <linux/cpufeature.h>
···
24
25
#include <linux/string.h>
25
26
#include <linux/unaligned.h>
26
27
27
-
MODULE_DESCRIPTION("GHASH hash function using ARMv8 Crypto Extensions");
28
+
MODULE_DESCRIPTION("AES-GCM using ARMv8 Crypto Extensions");
28
29
MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>");
29
30
MODULE_LICENSE("GPL");
30
-
MODULE_ALIAS_CRYPTO("ghash");
31
31
MODULE_ALIAS_CRYPTO("gcm(aes)");
32
32
MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))");
33
33
34
34
#define RFC4106_NONCE_SIZE 4
35
-
36
-
struct ghash_key {
37
-
be128 k;
38
-
u64 h[1][2];
39
-
};
40
35
41
36
struct gcm_key {
42
37
u64 h[4][2];
···
39
46
u8 nonce[]; // for RFC4106 nonce
40
47
};
41
48
42
-
struct arm_ghash_desc_ctx {
43
-
u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
44
-
};
45
-
46
49
asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
47
50
u64 const h[4][2], const char *head);
48
-
49
-
asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
50
-
u64 const h[1][2], const char *head);
51
-
52
-
static int ghash_init(struct shash_desc *desc)
53
-
{
54
-
struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
55
-
56
-
*ctx = (struct arm_ghash_desc_ctx){};
57
-
return 0;
58
-
}
59
-
60
-
static void ghash_do_update(int blocks, u64 dg[], const char *src,
61
-
struct ghash_key *key, const char *head)
62
-
{
63
-
kernel_neon_begin();
64
-
pmull_ghash_update_p8(blocks, dg, src, key->h, head);
65
-
kernel_neon_end();
66
-
}
67
-
68
-
static int ghash_update(struct shash_desc *desc, const u8 *src,
69
-
unsigned int len)
70
-
{
71
-
struct ghash_key *key = crypto_shash_ctx(desc->tfm);
72
-
struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
73
-
int blocks;
74
-
75
-
blocks = len / GHASH_BLOCK_SIZE;
76
-
ghash_do_update(blocks, ctx->digest, src, key, NULL);
77
-
return len - blocks * GHASH_BLOCK_SIZE;
78
-
}
79
-
80
-
static int ghash_export(struct shash_desc *desc, void *out)
81
-
{
82
-
struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
83
-
u8 *dst = out;
84
-
85
-
put_unaligned_be64(ctx->digest[1], dst);
86
-
put_unaligned_be64(ctx->digest[0], dst + 8);
87
-
return 0;
88
-
}
89
-
90
-
static int ghash_import(struct shash_desc *desc, const void *in)
91
-
{
92
-
struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
93
-
const u8 *src = in;
94
-
95
-
ctx->digest[1] = get_unaligned_be64(src);
96
-
ctx->digest[0] = get_unaligned_be64(src + 8);
97
-
return 0;
98
-
}
99
-
100
-
static int ghash_finup(struct shash_desc *desc, const u8 *src,
101
-
unsigned int len, u8 *dst)
102
-
{
103
-
struct ghash_key *key = crypto_shash_ctx(desc->tfm);
104
-
struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
105
-
106
-
if (len) {
107
-
u8 buf[GHASH_BLOCK_SIZE] = {};
108
-
109
-
memcpy(buf, src, len);
110
-
ghash_do_update(1, ctx->digest, buf, key, NULL);
111
-
memzero_explicit(buf, sizeof(buf));
112
-
}
113
-
return ghash_export(desc, dst);
114
-
}
115
51
116
52
static void ghash_reflect(u64 h[], const be128 *k)
117
53
{
···
52
130
if (carry)
53
131
h[1] ^= 0xc200000000000000UL;
54
132
}
55
-
56
-
static int ghash_setkey(struct crypto_shash *tfm,
57
-
const u8 *inkey, unsigned int keylen)
58
-
{
59
-
struct ghash_key *key = crypto_shash_ctx(tfm);
60
-
61
-
if (keylen != GHASH_BLOCK_SIZE)
62
-
return -EINVAL;
63
-
64
-
/* needed for the fallback */
65
-
memcpy(&key->k, inkey, GHASH_BLOCK_SIZE);
66
-
ghash_reflect(key->h[0], &key->k);
67
-
return 0;
68
-
}
69
-
70
-
static struct shash_alg ghash_alg = {
71
-
.digestsize = GHASH_DIGEST_SIZE,
72
-
.init = ghash_init,
73
-
.update = ghash_update,
74
-
.finup = ghash_finup,
75
-
.setkey = ghash_setkey,
76
-
.export = ghash_export,
77
-
.import = ghash_import,
78
-
.descsize = sizeof(struct arm_ghash_desc_ctx),
79
-
.statesize = sizeof(struct ghash_desc_ctx),
80
-
81
-
.base.cra_name = "ghash",
82
-
.base.cra_driver_name = "ghash-neon",
83
-
.base.cra_priority = 300,
84
-
.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
85
-
.base.cra_blocksize = GHASH_BLOCK_SIZE,
86
-
.base.cra_ctxsize = sizeof(struct ghash_key),
87
-
.base.cra_module = THIS_MODULE,
88
-
};
89
133
90
134
void pmull_gcm_encrypt(int blocks, u64 dg[], const char *src,
91
135
struct gcm_key const *k, char *dst,
···
431
543
432
544
static int __init ghash_ce_mod_init(void)
433
545
{
434
-
int err;
435
-
436
-
if (!(elf_hwcap & HWCAP_NEON))
546
+
if (!(elf_hwcap & HWCAP_NEON) || !(elf_hwcap2 & HWCAP2_PMULL))
437
547
return -ENODEV;
438
548
439
-
if (elf_hwcap2 & HWCAP2_PMULL) {
440
-
err = crypto_register_aeads(gcm_aes_algs,
441
-
ARRAY_SIZE(gcm_aes_algs));
442
-
if (err)
443
-
return err;
444
-
}
445
-
446
-
err = crypto_register_shash(&ghash_alg);
447
-
if (err)
448
-
goto err_aead;
449
-
450
-
return 0;
451
-
452
-
err_aead:
453
-
if (elf_hwcap2 & HWCAP2_PMULL)
454
-
crypto_unregister_aeads(gcm_aes_algs,
455
-
ARRAY_SIZE(gcm_aes_algs));
456
-
return err;
549
+
return crypto_register_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs));
457
550
}
458
551
459
552
static void __exit ghash_ce_mod_exit(void)
460
553
{
461
-
crypto_unregister_shash(&ghash_alg);
462
-
if (elf_hwcap2 & HWCAP2_PMULL)
463
-
crypto_unregister_aeads(gcm_aes_algs,
464
-
ARRAY_SIZE(gcm_aes_algs));
554
+
crypto_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs));
465
555
}
466
556
467
557
module_init(ghash_ce_mod_init);
+13
-11
arch/arm/crypto/ghash-neon-core.S
lib/crypto/arm/ghash-neon-core.S
+13
-11
arch/arm/crypto/ghash-neon-core.S
lib/crypto/arm/ghash-neon-core.S
···
141
141
vshr.u64 XL, XL, #1
142
142
.endm
143
143
144
+
.macro vrev64_if_be a
145
+
#ifdef CONFIG_CPU_BIG_ENDIAN
146
+
vrev64.8 \a, \a
147
+
#endif
148
+
.endm
149
+
144
150
.macro ghash_update
145
151
vld1.64 {XL}, [r1]
146
-
147
-
/* do the head block first, if supplied */
148
-
ldr ip, [sp]
149
-
teq ip, #0
150
-
beq 0f
151
-
vld1.64 {T1}, [ip]
152
-
teq r0, #0
153
-
b 3f
152
+
vrev64_if_be XL
154
153
155
154
0:
156
155
vld1.8 {T1}, [r2]!
157
156
subs r0, r0, #1
158
157
159
-
3: /* multiply XL by SHASH in GF(2^128) */
158
+
/* multiply XL by SHASH in GF(2^128) */
160
159
vrev64.8 T1, T1
161
160
162
161
vext.8 IN1, T1, T1, #8
···
179
180
.endm
180
181
181
182
/*
182
-
* void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
183
-
* u64 const h[1][2], const char *head)
183
+
* void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg,
184
+
* const u8 *src,
185
+
* const struct polyval_elem *h)
184
186
*/
185
187
ENTRY(pmull_ghash_update_p8)
186
188
vld1.64 {SHASH}, [r3]
189
+
vrev64_if_be SHASH
187
190
veor SHASH2_p8, SHASH_L, SHASH_H
188
191
189
192
vext.8 s1l, SHASH_L, SHASH_L, #1
···
202
201
vmov.i64 k48, #0xffffffffffff
203
202
204
203
ghash_update
204
+
vrev64_if_be XL
205
205
vst1.64 {XL}, [r1]
206
206
207
207
bx lr
+1
lib/crypto/Kconfig
+1
lib/crypto/Kconfig
+1
lib/crypto/Makefile
+1
lib/crypto/Makefile
···
158
158
libgf128hash-y := gf128hash.o
159
159
ifeq ($(CONFIG_CRYPTO_LIB_GF128HASH_ARCH),y)
160
160
CFLAGS_gf128hash.o += -I$(src)/$(SRCARCH)
161
+
libgf128hash-$(CONFIG_ARM) += arm/ghash-neon-core.o
161
162
libgf128hash-$(CONFIG_ARM64) += arm64/polyval-ce-core.o
162
163
libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o
163
164
endif
+43
lib/crypto/arm/gf128hash.h
+43
lib/crypto/arm/gf128hash.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-or-later */
2
+
/*
3
+
* GHASH, arm optimized
4
+
*
5
+
* Copyright 2026 Google LLC
6
+
*/
7
+
8
+
#include <asm/hwcap.h>
9
+
#include <asm/neon.h>
10
+
#include <asm/simd.h>
11
+
12
+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
13
+
14
+
void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg,
15
+
const u8 *src, const struct polyval_elem *h);
16
+
17
+
#define ghash_blocks_arch ghash_blocks_arch
18
+
static void ghash_blocks_arch(struct polyval_elem *acc,
19
+
const struct ghash_key *key,
20
+
const u8 *data, size_t nblocks)
21
+
{
22
+
if (static_branch_likely(&have_neon) && may_use_simd()) {
23
+
do {
24
+
/* Allow rescheduling every 4 KiB. */
25
+
size_t n =
26
+
min_t(size_t, nblocks, 4096 / GHASH_BLOCK_SIZE);
27
+
28
+
scoped_ksimd()
29
+
pmull_ghash_update_p8(n, acc, data, &key->h);
30
+
data += n * GHASH_BLOCK_SIZE;
31
+
nblocks -= n;
32
+
} while (nblocks);
33
+
} else {
34
+
ghash_blocks_generic(acc, &key->h, data, nblocks);
35
+
}
36
+
}
37
+
38
+
#define gf128hash_mod_init_arch gf128hash_mod_init_arch
39
+
static void gf128hash_mod_init_arch(void)
40
+
{
41
+
if (elf_hwcap & HWCAP_NEON)
42
+
static_branch_enable(&have_neon);
43
+
}