Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib/crypto: arm/nh: Migrate optimized code into library
Migrate the arm32 NEON implementation of NH into lib/crypto/. This
makes the nh() function be optimized on arm32 kernels.
Note: this temporarily makes the adiantum template not utilize the arm32
optimized NH code. This is resolved in a later commit that converts the
adiantum template to use nh() instead of "nhpoly1305".
Link: https://lore.kernel.org/r/20251211011846.8179-4-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
+35
-92
-10
arch/arm/crypto/Kconfig
-10
arch/arm/crypto/Kconfig
···
23
23
that is part of the ARMv8 Crypto Extensions, or a slower variant that
24
24
uses the vmull.p8 instruction that is part of the basic NEON ISA.
25
25
26
-
config CRYPTO_NHPOLY1305_NEON
27
-
tristate "Hash functions: NHPoly1305 (NEON)"
28
-
depends on KERNEL_MODE_NEON
29
-
select CRYPTO_NHPOLY1305
30
-
help
31
-
NHPoly1305 hash function (Adiantum)
32
-
33
-
Architecture: arm using:
34
-
- NEON (Advanced SIMD) extensions
35
-
36
26
config CRYPTO_AES_ARM
37
27
tristate "Ciphers: AES"
38
28
select CRYPTO_ALGAPI
-2
arch/arm/crypto/Makefile
-2
arch/arm/crypto/Makefile
···
5
5
6
6
obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o
7
7
obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o
8
-
obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o
9
8
10
9
obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o
11
10
obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) += ghash-arm-ce.o
···
13
14
aes-arm-bs-y := aes-neonbs-core.o aes-neonbs-glue.o
14
15
aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o
15
16
ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o
16
-
nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o
arch/arm/crypto/nh-neon-core.S
lib/crypto/arm/nh-neon-core.S
arch/arm/crypto/nh-neon-core.S
lib/crypto/arm/nh-neon-core.S
-80
arch/arm/crypto/nhpoly1305-neon-glue.c
-80
arch/arm/crypto/nhpoly1305-neon-glue.c
···
1
-
// SPDX-License-Identifier: GPL-2.0
2
-
/*
3
-
* NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
4
-
* (NEON accelerated version)
5
-
*
6
-
* Copyright 2018 Google LLC
7
-
*/
8
-
9
-
#include <asm/neon.h>
10
-
#include <asm/simd.h>
11
-
#include <crypto/internal/hash.h>
12
-
#include <crypto/internal/simd.h>
13
-
#include <crypto/nhpoly1305.h>
14
-
#include <linux/module.h>
15
-
16
-
asmlinkage void nh_neon(const u32 *key, const u8 *message, size_t message_len,
17
-
__le64 hash[NH_NUM_PASSES]);
18
-
19
-
static int nhpoly1305_neon_update(struct shash_desc *desc,
20
-
const u8 *src, unsigned int srclen)
21
-
{
22
-
if (srclen < 64 || !crypto_simd_usable())
23
-
return crypto_nhpoly1305_update(desc, src, srclen);
24
-
25
-
do {
26
-
unsigned int n = min_t(unsigned int, srclen, SZ_4K);
27
-
28
-
kernel_neon_begin();
29
-
crypto_nhpoly1305_update_helper(desc, src, n, nh_neon);
30
-
kernel_neon_end();
31
-
src += n;
32
-
srclen -= n;
33
-
} while (srclen);
34
-
return 0;
35
-
}
36
-
37
-
static int nhpoly1305_neon_digest(struct shash_desc *desc,
38
-
const u8 *src, unsigned int srclen, u8 *out)
39
-
{
40
-
return crypto_nhpoly1305_init(desc) ?:
41
-
nhpoly1305_neon_update(desc, src, srclen) ?:
42
-
crypto_nhpoly1305_final(desc, out);
43
-
}
44
-
45
-
static struct shash_alg nhpoly1305_alg = {
46
-
.base.cra_name = "nhpoly1305",
47
-
.base.cra_driver_name = "nhpoly1305-neon",
48
-
.base.cra_priority = 200,
49
-
.base.cra_ctxsize = sizeof(struct nhpoly1305_key),
50
-
.base.cra_module = THIS_MODULE,
51
-
.digestsize = POLY1305_DIGEST_SIZE,
52
-
.init = crypto_nhpoly1305_init,
53
-
.update = nhpoly1305_neon_update,
54
-
.final = crypto_nhpoly1305_final,
55
-
.digest = nhpoly1305_neon_digest,
56
-
.setkey = crypto_nhpoly1305_setkey,
57
-
.descsize = sizeof(struct nhpoly1305_state),
58
-
};
59
-
60
-
static int __init nhpoly1305_mod_init(void)
61
-
{
62
-
if (!(elf_hwcap & HWCAP_NEON))
63
-
return -ENODEV;
64
-
65
-
return crypto_register_shash(&nhpoly1305_alg);
66
-
}
67
-
68
-
static void __exit nhpoly1305_mod_exit(void)
69
-
{
70
-
crypto_unregister_shash(&nhpoly1305_alg);
71
-
}
72
-
73
-
module_init(nhpoly1305_mod_init);
74
-
module_exit(nhpoly1305_mod_exit);
75
-
76
-
MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (NEON-accelerated)");
77
-
MODULE_LICENSE("GPL v2");
78
-
MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
79
-
MODULE_ALIAS_CRYPTO("nhpoly1305");
80
-
MODULE_ALIAS_CRYPTO("nhpoly1305-neon");
+1
lib/crypto/Kconfig
+1
lib/crypto/Kconfig
+1
lib/crypto/Makefile
+1
lib/crypto/Makefile
+33
lib/crypto/arm/nh.h
+33
lib/crypto/arm/nh.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/*
3
+
* ARM32 accelerated implementation of NH
4
+
*
5
+
* Copyright 2018 Google LLC
6
+
*/
7
+
8
+
#include <asm/neon.h>
9
+
#include <asm/simd.h>
10
+
11
+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
12
+
13
+
asmlinkage void nh_neon(const u32 *key, const u8 *message, size_t message_len,
14
+
__le64 hash[NH_NUM_PASSES]);
15
+
16
+
static bool nh_arch(const u32 *key, const u8 *message, size_t message_len,
17
+
__le64 hash[NH_NUM_PASSES])
18
+
{
19
+
if (static_branch_likely(&have_neon) && message_len >= 64 &&
20
+
may_use_simd()) {
21
+
scoped_ksimd()
22
+
nh_neon(key, message, message_len, hash);
23
+
return true;
24
+
}
25
+
return false;
26
+
}
27
+
28
+
#define nh_mod_init_arch nh_mod_init_arch
29
+
static void nh_mod_init_arch(void)
30
+
{
31
+
if (elf_hwcap & HWCAP_NEON)
32
+
static_branch_enable(&have_neon);
33
+
}