Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib/crypto: x86/nh: Migrate optimized code into library
Migrate the x86_64 implementations of NH into lib/crypto/. This makes
the nh() function be optimized on x86_64 kernels.
Note: this temporarily makes the adiantum template not utilize the
x86_64 optimized NH code. This is resolved in a later commit that
converts the adiantum template to use nh() instead of "nhpoly1305".
Link: https://lore.kernel.org/r/20251211011846.8179-6-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
+49
-190
-20
arch/x86/crypto/Kconfig
-20
arch/x86/crypto/Kconfig
···
333
333
- AES-NI (AES New Instructions)
334
334
- SSE4.1 (Streaming SIMD Extensions 4.1)
335
335
336
-
config CRYPTO_NHPOLY1305_SSE2
337
-
tristate "Hash functions: NHPoly1305 (SSE2)"
338
-
depends on 64BIT
339
-
select CRYPTO_NHPOLY1305
340
-
help
341
-
NHPoly1305 hash function for Adiantum
342
-
343
-
Architecture: x86_64 using:
344
-
- SSE2 (Streaming SIMD Extensions 2)
345
-
346
-
config CRYPTO_NHPOLY1305_AVX2
347
-
tristate "Hash functions: NHPoly1305 (AVX2)"
348
-
depends on 64BIT
349
-
select CRYPTO_NHPOLY1305
350
-
help
351
-
NHPoly1305 hash function for Adiantum
352
-
353
-
Architecture: x86_64 using:
354
-
- AVX2 (Advanced Vector Extensions 2)
355
-
356
336
config CRYPTO_SM3_AVX_X86_64
357
337
tristate "Hash functions: SM3 (AVX)"
358
338
depends on 64BIT
-5
arch/x86/crypto/Makefile
-5
arch/x86/crypto/Makefile
···
53
53
obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
54
54
ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
55
55
56
-
obj-$(CONFIG_CRYPTO_NHPOLY1305_SSE2) += nhpoly1305-sse2.o
57
-
nhpoly1305-sse2-y := nh-sse2-x86_64.o nhpoly1305-sse2-glue.o
58
-
obj-$(CONFIG_CRYPTO_NHPOLY1305_AVX2) += nhpoly1305-avx2.o
59
-
nhpoly1305-avx2-y := nh-avx2-x86_64.o nhpoly1305-avx2-glue.o
60
-
61
56
obj-$(CONFIG_CRYPTO_SM3_AVX_X86_64) += sm3-avx-x86_64.o
62
57
sm3-avx-x86_64-y := sm3-avx-asm_64.o sm3_avx_glue.o
63
58
+1
-2
arch/x86/crypto/nh-avx2-x86_64.S
lib/crypto/x86/nh-avx2.S
+1
-2
arch/x86/crypto/nh-avx2-x86_64.S
lib/crypto/x86/nh-avx2.S
···
8
8
*/
9
9
10
10
#include <linux/linkage.h>
11
-
#include <linux/cfi_types.h>
12
11
13
12
#define PASS0_SUMS %ymm0
14
13
#define PASS1_SUMS %ymm1
···
69
70
*
70
71
* It's guaranteed that message_len % 16 == 0.
71
72
*/
72
-
SYM_TYPED_FUNC_START(nh_avx2)
73
+
SYM_FUNC_START(nh_avx2)
73
74
74
75
vmovdqu 0x00(KEY), K0
75
76
vmovdqu 0x10(KEY), K1
+1
-2
arch/x86/crypto/nh-sse2-x86_64.S
lib/crypto/x86/nh-sse2.S
+1
-2
arch/x86/crypto/nh-sse2-x86_64.S
lib/crypto/x86/nh-sse2.S
···
8
8
*/
9
9
10
10
#include <linux/linkage.h>
11
-
#include <linux/cfi_types.h>
12
11
13
12
#define PASS0_SUMS %xmm0
14
13
#define PASS1_SUMS %xmm1
···
71
72
*
72
73
* It's guaranteed that message_len % 16 == 0.
73
74
*/
74
-
SYM_TYPED_FUNC_START(nh_sse2)
75
+
SYM_FUNC_START(nh_sse2)
75
76
76
77
movdqu 0x00(KEY), K0
77
78
movdqu 0x10(KEY), K1
-81
arch/x86/crypto/nhpoly1305-avx2-glue.c
-81
arch/x86/crypto/nhpoly1305-avx2-glue.c
···
1
-
// SPDX-License-Identifier: GPL-2.0
2
-
/*
3
-
* NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
4
-
* (AVX2 accelerated version)
5
-
*
6
-
* Copyright 2018 Google LLC
7
-
*/
8
-
9
-
#include <crypto/internal/hash.h>
10
-
#include <crypto/internal/simd.h>
11
-
#include <crypto/nhpoly1305.h>
12
-
#include <linux/module.h>
13
-
#include <linux/sizes.h>
14
-
#include <asm/simd.h>
15
-
16
-
asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
17
-
__le64 hash[NH_NUM_PASSES]);
18
-
19
-
static int nhpoly1305_avx2_update(struct shash_desc *desc,
20
-
const u8 *src, unsigned int srclen)
21
-
{
22
-
if (srclen < 64 || !crypto_simd_usable())
23
-
return crypto_nhpoly1305_update(desc, src, srclen);
24
-
25
-
do {
26
-
unsigned int n = min_t(unsigned int, srclen, SZ_4K);
27
-
28
-
kernel_fpu_begin();
29
-
crypto_nhpoly1305_update_helper(desc, src, n, nh_avx2);
30
-
kernel_fpu_end();
31
-
src += n;
32
-
srclen -= n;
33
-
} while (srclen);
34
-
return 0;
35
-
}
36
-
37
-
static int nhpoly1305_avx2_digest(struct shash_desc *desc,
38
-
const u8 *src, unsigned int srclen, u8 *out)
39
-
{
40
-
return crypto_nhpoly1305_init(desc) ?:
41
-
nhpoly1305_avx2_update(desc, src, srclen) ?:
42
-
crypto_nhpoly1305_final(desc, out);
43
-
}
44
-
45
-
static struct shash_alg nhpoly1305_alg = {
46
-
.base.cra_name = "nhpoly1305",
47
-
.base.cra_driver_name = "nhpoly1305-avx2",
48
-
.base.cra_priority = 300,
49
-
.base.cra_ctxsize = sizeof(struct nhpoly1305_key),
50
-
.base.cra_module = THIS_MODULE,
51
-
.digestsize = POLY1305_DIGEST_SIZE,
52
-
.init = crypto_nhpoly1305_init,
53
-
.update = nhpoly1305_avx2_update,
54
-
.final = crypto_nhpoly1305_final,
55
-
.digest = nhpoly1305_avx2_digest,
56
-
.setkey = crypto_nhpoly1305_setkey,
57
-
.descsize = sizeof(struct nhpoly1305_state),
58
-
};
59
-
60
-
static int __init nhpoly1305_mod_init(void)
61
-
{
62
-
if (!boot_cpu_has(X86_FEATURE_AVX2) ||
63
-
!boot_cpu_has(X86_FEATURE_OSXSAVE))
64
-
return -ENODEV;
65
-
66
-
return crypto_register_shash(&nhpoly1305_alg);
67
-
}
68
-
69
-
static void __exit nhpoly1305_mod_exit(void)
70
-
{
71
-
crypto_unregister_shash(&nhpoly1305_alg);
72
-
}
73
-
74
-
module_init(nhpoly1305_mod_init);
75
-
module_exit(nhpoly1305_mod_exit);
76
-
77
-
MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (AVX2-accelerated)");
78
-
MODULE_LICENSE("GPL v2");
79
-
MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
80
-
MODULE_ALIAS_CRYPTO("nhpoly1305");
81
-
MODULE_ALIAS_CRYPTO("nhpoly1305-avx2");
-80
arch/x86/crypto/nhpoly1305-sse2-glue.c
-80
arch/x86/crypto/nhpoly1305-sse2-glue.c
···
1
-
// SPDX-License-Identifier: GPL-2.0
2
-
/*
3
-
* NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
4
-
* (SSE2 accelerated version)
5
-
*
6
-
* Copyright 2018 Google LLC
7
-
*/
8
-
9
-
#include <crypto/internal/hash.h>
10
-
#include <crypto/internal/simd.h>
11
-
#include <crypto/nhpoly1305.h>
12
-
#include <linux/module.h>
13
-
#include <linux/sizes.h>
14
-
#include <asm/simd.h>
15
-
16
-
asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
17
-
__le64 hash[NH_NUM_PASSES]);
18
-
19
-
static int nhpoly1305_sse2_update(struct shash_desc *desc,
20
-
const u8 *src, unsigned int srclen)
21
-
{
22
-
if (srclen < 64 || !crypto_simd_usable())
23
-
return crypto_nhpoly1305_update(desc, src, srclen);
24
-
25
-
do {
26
-
unsigned int n = min_t(unsigned int, srclen, SZ_4K);
27
-
28
-
kernel_fpu_begin();
29
-
crypto_nhpoly1305_update_helper(desc, src, n, nh_sse2);
30
-
kernel_fpu_end();
31
-
src += n;
32
-
srclen -= n;
33
-
} while (srclen);
34
-
return 0;
35
-
}
36
-
37
-
static int nhpoly1305_sse2_digest(struct shash_desc *desc,
38
-
const u8 *src, unsigned int srclen, u8 *out)
39
-
{
40
-
return crypto_nhpoly1305_init(desc) ?:
41
-
nhpoly1305_sse2_update(desc, src, srclen) ?:
42
-
crypto_nhpoly1305_final(desc, out);
43
-
}
44
-
45
-
static struct shash_alg nhpoly1305_alg = {
46
-
.base.cra_name = "nhpoly1305",
47
-
.base.cra_driver_name = "nhpoly1305-sse2",
48
-
.base.cra_priority = 200,
49
-
.base.cra_ctxsize = sizeof(struct nhpoly1305_key),
50
-
.base.cra_module = THIS_MODULE,
51
-
.digestsize = POLY1305_DIGEST_SIZE,
52
-
.init = crypto_nhpoly1305_init,
53
-
.update = nhpoly1305_sse2_update,
54
-
.final = crypto_nhpoly1305_final,
55
-
.digest = nhpoly1305_sse2_digest,
56
-
.setkey = crypto_nhpoly1305_setkey,
57
-
.descsize = sizeof(struct nhpoly1305_state),
58
-
};
59
-
60
-
static int __init nhpoly1305_mod_init(void)
61
-
{
62
-
if (!boot_cpu_has(X86_FEATURE_XMM2))
63
-
return -ENODEV;
64
-
65
-
return crypto_register_shash(&nhpoly1305_alg);
66
-
}
67
-
68
-
static void __exit nhpoly1305_mod_exit(void)
69
-
{
70
-
crypto_unregister_shash(&nhpoly1305_alg);
71
-
}
72
-
73
-
module_init(nhpoly1305_mod_init);
74
-
module_exit(nhpoly1305_mod_exit);
75
-
76
-
MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (SSE2-accelerated)");
77
-
MODULE_LICENSE("GPL v2");
78
-
MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
79
-
MODULE_ALIAS_CRYPTO("nhpoly1305");
80
-
MODULE_ALIAS_CRYPTO("nhpoly1305-sse2");
+1
lib/crypto/Kconfig
+1
lib/crypto/Kconfig
+1
lib/crypto/Makefile
+1
lib/crypto/Makefile
···
137
137
CFLAGS_nh.o += -I$(src)/$(SRCARCH)
138
138
libnh-$(CONFIG_ARM) += arm/nh-neon-core.o
139
139
libnh-$(CONFIG_ARM64) += arm64/nh-neon-core.o
140
+
libnh-$(CONFIG_X86) += x86/nh-sse2.o x86/nh-avx2.o
140
141
endif
141
142
142
143
################################################################################
+45
lib/crypto/x86/nh.h
+45
lib/crypto/x86/nh.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/*
3
+
* x86_64 accelerated implementation of NH
4
+
*
5
+
* Copyright 2018 Google LLC
6
+
*/
7
+
8
+
#include <asm/fpu/api.h>
9
+
#include <linux/static_call.h>
10
+
11
+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sse2);
12
+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_avx2);
13
+
14
+
asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
15
+
__le64 hash[NH_NUM_PASSES]);
16
+
asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
17
+
__le64 hash[NH_NUM_PASSES]);
18
+
19
+
static bool nh_arch(const u32 *key, const u8 *message, size_t message_len,
20
+
__le64 hash[NH_NUM_PASSES])
21
+
{
22
+
if (message_len >= 64 && static_branch_likely(&have_sse2) &&
23
+
irq_fpu_usable()) {
24
+
kernel_fpu_begin();
25
+
if (static_branch_likely(&have_avx2))
26
+
nh_avx2(key, message, message_len, hash);
27
+
else
28
+
nh_sse2(key, message, message_len, hash);
29
+
kernel_fpu_end();
30
+
return true;
31
+
}
32
+
return false;
33
+
}
34
+
35
+
#define nh_mod_init_arch nh_mod_init_arch
36
+
static void nh_mod_init_arch(void)
37
+
{
38
+
if (boot_cpu_has(X86_FEATURE_XMM2)) {
39
+
static_branch_enable(&have_sse2);
40
+
if (boot_cpu_has(X86_FEATURE_AVX2) &&
41
+
cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
42
+
NULL))
43
+
static_branch_enable(&have_avx2);
44
+
}
45
+
}