Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib/crypto: poly1305: Consolidate into single module
Consolidate the Poly1305 code into a single module, similar to various
other algorithms (SHA-1, SHA-256, SHA-512, etc.):
- Each arch now provides a header file lib/crypto/$(SRCARCH)/poly1305.h,
replacing lib/crypto/$(SRCARCH)/poly1305*.c. The header defines
poly1305_block_init(), poly1305_blocks(), poly1305_emit(), and
optionally poly1305_mod_init_arch(). It is included by
lib/crypto/poly1305.c, and thus the code gets built into the single
libpoly1305 module, with improved inlining in some cases.
- Whether arch-optimized Poly1305 is buildable is now controlled
centrally by lib/crypto/Kconfig instead of by
lib/crypto/$(SRCARCH)/Kconfig. The conditions for enabling it remain
the same as before, and it remains enabled by default. (The PPC64 one
remains unconditionally disabled due to 'depends on BROKEN'.)
- Any additional arch-specific translation units for the optimized
Poly1305 code, such as assembly files, are now compiled by
lib/crypto/Makefile instead of lib/crypto/$(SRCARCH)/Makefile.
A special consideration is needed because the Adiantum code uses the
poly1305_core_*() functions directly. For now, just carry forward that
approach. This means retaining the CRYPTO_LIB_POLY1305_GENERIC kconfig
symbol, and keeping the poly1305_core_*() functions in separate
translation units. So it's not quite as streamlined I've done with the
other hash functions, but we still get a single libpoly1305 module.
Note: to see the diff from the arm, arm64, and x86 .c files to the new
.h files, view this commit with 'git show -M10'.
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20250829152513.92459-3-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
+299
-424
+2
crypto/Kconfig
+2
crypto/Kconfig
···
609
609
config CRYPTO_ADIANTUM
610
610
tristate "Adiantum"
611
611
select CRYPTO_CHACHA20
612
+
select CRYPTO_LIB_POLY1305
612
613
select CRYPTO_LIB_POLY1305_GENERIC
613
614
select CRYPTO_NHPOLY1305
614
615
select CRYPTO_MANAGER
···
771
770
config CRYPTO_NHPOLY1305
772
771
tristate
773
772
select CRYPTO_HASH
773
+
select CRYPTO_LIB_POLY1305
774
774
select CRYPTO_LIB_POLY1305_GENERIC
775
775
776
776
endmenu
+7
-9
include/crypto/internal/poly1305.h
+7
-9
include/crypto/internal/poly1305.h
···
30
30
void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4],
31
31
void *dst);
32
32
33
-
void poly1305_block_init_arch(struct poly1305_block_state *state,
34
-
const u8 raw_key[POLY1305_BLOCK_SIZE]);
35
-
void poly1305_block_init_generic(struct poly1305_block_state *state,
36
-
const u8 raw_key[POLY1305_BLOCK_SIZE]);
37
-
void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
38
-
unsigned int len, u32 padbit);
33
+
static inline void
34
+
poly1305_block_init_generic(struct poly1305_block_state *desc,
35
+
const u8 raw_key[POLY1305_BLOCK_SIZE])
36
+
{
37
+
poly1305_core_init(&desc->h);
38
+
poly1305_core_setkey(&desc->core_r, raw_key);
39
+
}
39
40
40
41
static inline void poly1305_blocks_generic(struct poly1305_block_state *state,
41
42
const u8 *src, unsigned int len,
···
45
44
poly1305_core_blocks(&state->h, &state->core_r, src,
46
45
len / POLY1305_BLOCK_SIZE, padbit);
47
46
}
48
-
49
-
void poly1305_emit_arch(const struct poly1305_state *state,
50
-
u8 digest[POLY1305_DIGEST_SIZE], const u32 nonce[4]);
51
47
52
48
static inline void poly1305_emit_generic(const struct poly1305_state *state,
53
49
u8 digest[POLY1305_DIGEST_SIZE],
+27
-23
lib/crypto/Kconfig
+27
-23
lib/crypto/Kconfig
···
114
114
default y if PPC
115
115
default y if SPARC64
116
116
117
+
config CRYPTO_LIB_POLY1305
118
+
tristate
119
+
help
120
+
The Poly1305 library functions. Select this if your module uses any
121
+
of the functions from <crypto/poly1305.h>.
122
+
123
+
config CRYPTO_LIB_POLY1305_ARCH
124
+
bool
125
+
depends on CRYPTO_LIB_POLY1305 && !UML
126
+
default y if ARM
127
+
default y if ARM64 && KERNEL_MODE_NEON
128
+
default y if MIPS
129
+
# The PPC64 code needs to be fixed to work in softirq context.
130
+
default y if PPC64 && CPU_LITTLE_ENDIAN && VSX && BROKEN
131
+
default y if X86_64
132
+
133
+
# This symbol controls the inclusion of the Poly1305 generic code. This differs
134
+
# from most of the other algorithms, which handle the generic code
135
+
# "automatically" via __maybe_unused. This is needed so that the Adiantum code,
136
+
# which calls the poly1305_core_*() functions directly, can enable them.
137
+
config CRYPTO_LIB_POLY1305_GENERIC
138
+
bool
139
+
depends on CRYPTO_LIB_POLY1305
140
+
# Enable if there's no arch impl or the arch impl requires the generic
141
+
# impl as a fallback. (Or if selected explicitly.)
142
+
default y if !CRYPTO_LIB_POLY1305_ARCH || PPC64
143
+
117
144
config CRYPTO_LIB_POLY1305_RSIZE
118
145
int
119
146
default 2 if MIPS
120
147
default 11 if X86_64
121
148
default 9 if ARM || ARM64
122
149
default 1
123
-
124
-
config CRYPTO_ARCH_HAVE_LIB_POLY1305
125
-
bool
126
-
help
127
-
Declares whether the architecture provides an arch-specific
128
-
accelerated implementation of the Poly1305 library interface,
129
-
either builtin or as a module.
130
-
131
-
config CRYPTO_LIB_POLY1305_GENERIC
132
-
tristate
133
-
default CRYPTO_LIB_POLY1305 if !CRYPTO_ARCH_HAVE_LIB_POLY1305
134
-
help
135
-
This symbol can be selected by arch implementations of the Poly1305
136
-
library interface that require the generic code as a fallback, e.g.,
137
-
for SIMD implementations. If no arch specific implementation is
138
-
enabled, this implementation serves the users of CRYPTO_LIB_POLY1305.
139
-
140
-
config CRYPTO_LIB_POLY1305
141
-
tristate
142
-
help
143
-
Enable the Poly1305 library interface. This interface may be fulfilled
144
-
by either the generic implementation or an arch-specific one, if one
145
-
is available and enabled.
146
150
147
151
config CRYPTO_LIB_CHACHA20POLY1305
148
152
tristate
+53
-6
lib/crypto/Makefile
+53
-6
lib/crypto/Makefile
···
71
71
72
72
################################################################################
73
73
74
-
obj-$(CONFIG_CRYPTO_LIB_POLY1305) += libpoly1305.o
75
-
libpoly1305-y += poly1305.o
74
+
obj-$(CONFIG_CRYPTO_LIB_POLY1305) += libpoly1305.o
75
+
libpoly1305-y := poly1305.o
76
+
ifeq ($(CONFIG_ARCH_SUPPORTS_INT128),y)
77
+
libpoly1305-$(CONFIG_CRYPTO_LIB_POLY1305_GENERIC) += poly1305-donna64.o
78
+
else
79
+
libpoly1305-$(CONFIG_CRYPTO_LIB_POLY1305_GENERIC) += poly1305-donna32.o
80
+
endif
76
81
77
-
obj-$(CONFIG_CRYPTO_LIB_POLY1305_GENERIC) += libpoly1305-generic.o
78
-
libpoly1305-generic-y := poly1305-donna32.o
79
-
libpoly1305-generic-$(CONFIG_ARCH_SUPPORTS_INT128) := poly1305-donna64.o
80
-
libpoly1305-generic-y += poly1305-generic.o
82
+
ifeq ($(CONFIG_CRYPTO_LIB_POLY1305_ARCH),y)
83
+
CFLAGS_poly1305.o += -I$(src)/$(SRCARCH)
84
+
85
+
ifeq ($(CONFIG_ARM),y)
86
+
libpoly1305-y += arm/poly1305-core.o
87
+
$(obj)/arm/poly1305-core.S: $(src)/arm/poly1305-armv4.pl
88
+
$(call cmd,perlasm)
89
+
# massage the perlasm code a bit so we only get the NEON routine if we need it
90
+
poly1305-aflags-$(CONFIG_CPU_V7) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=5
91
+
poly1305-aflags-$(CONFIG_KERNEL_MODE_NEON) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=7
92
+
AFLAGS_arm/poly1305-core.o += $(poly1305-aflags-y) $(aflags-thumb2-y)
93
+
endif
94
+
95
+
ifeq ($(CONFIG_ARM64),y)
96
+
libpoly1305-y += arm64/poly1305-core.o
97
+
$(obj)/arm64/poly1305-core.S: $(src)/arm64/poly1305-armv8.pl
98
+
$(call cmd,perlasm_with_args)
99
+
endif
100
+
101
+
ifeq ($(CONFIG_MIPS),y)
102
+
libpoly1305-y += mips/poly1305-core.o
103
+
poly1305-perlasm-flavour-$(CONFIG_32BIT) := o32
104
+
poly1305-perlasm-flavour-$(CONFIG_64BIT) := 64
105
+
quiet_cmd_perlasm_poly1305 = PERLASM $@
106
+
cmd_perlasm_poly1305 = $(PERL) $< $(poly1305-perlasm-flavour-y) $@
107
+
# Use if_changed instead of cmd, in case the flavour changed.
108
+
$(obj)/mips/poly1305-core.S: $(src)/mips/poly1305-mips.pl FORCE
109
+
$(call if_changed,perlasm_poly1305)
110
+
targets += mips/poly1305-core.S
111
+
endif
112
+
113
+
libpoly1305-$(CONFIG_PPC) += powerpc/poly1305-p10le_64.o
114
+
115
+
ifeq ($(CONFIG_X86),y)
116
+
libpoly1305-y += x86/poly1305-x86_64-cryptogams.o
117
+
$(obj)/x86/poly1305-x86_64-cryptogams.S: $(src)/x86/poly1305-x86_64-cryptogams.pl
118
+
$(call cmd,perlasm)
119
+
endif
120
+
121
+
endif # CONFIG_CRYPTO_LIB_POLY1305_ARCH
122
+
123
+
# clean-files must be defined unconditionally
124
+
clean-files += arm/poly1305-core.S \
125
+
arm64/poly1305-core.S \
126
+
mips/poly1305-core.S \
127
+
x86/poly1305-x86_64-cryptogams.S
81
128
82
129
################################################################################
83
130
-5
lib/crypto/arm/Kconfig
-5
lib/crypto/arm/Kconfig
-18
lib/crypto/arm/Makefile
-18
lib/crypto/arm/Makefile
···
6
6
obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha-neon.o
7
7
chacha-neon-y := chacha-scalar-core.o chacha-glue.o
8
8
chacha-neon-$(CONFIG_KERNEL_MODE_NEON) += chacha-neon-core.o
9
-
10
-
obj-$(CONFIG_CRYPTO_POLY1305_ARM) += poly1305-arm.o
11
-
poly1305-arm-y := poly1305-core.o poly1305-glue.o
12
-
13
-
quiet_cmd_perl = PERL $@
14
-
cmd_perl = $(PERL) $(<) > $(@)
15
-
16
-
$(obj)/%-core.S: $(src)/%-armv4.pl
17
-
$(call cmd,perl)
18
-
19
-
clean-files += poly1305-core.S
20
-
21
-
aflags-thumb2-$(CONFIG_THUMB2_KERNEL) := -U__thumb2__ -D__thumb2__=1
22
-
23
-
# massage the perlasm code a bit so we only get the NEON routine if we need it
24
-
poly1305-aflags-$(CONFIG_CPU_V7) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=5
25
-
poly1305-aflags-$(CONFIG_KERNEL_MODE_NEON) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=7
26
-
AFLAGS_poly1305-core.o += $(poly1305-aflags-y) $(aflags-thumb2-y)
+1
-2
lib/crypto/arm/poly1305-armv4.pl
+1
-2
lib/crypto/arm/poly1305-armv4.pl
···
43
43
#else
44
44
# define __ARM_ARCH__ __LINUX_ARM_ARCH__
45
45
# define __ARM_MAX_ARCH__ __LINUX_ARM_ARCH__
46
-
# define poly1305_init poly1305_block_init_arch
46
+
# define poly1305_init poly1305_block_init
47
47
# define poly1305_blocks poly1305_blocks_arm
48
-
# define poly1305_emit poly1305_emit_arch
49
48
#endif
50
49
51
50
#if defined(__thumb2__)
-69
lib/crypto/arm/poly1305-glue.c
-69
lib/crypto/arm/poly1305-glue.c
···
1
-
// SPDX-License-Identifier: GPL-2.0
2
-
/*
3
-
* OpenSSL/Cryptogams accelerated Poly1305 transform for ARM
4
-
*
5
-
* Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org>
6
-
*/
7
-
8
-
#include <asm/hwcap.h>
9
-
#include <asm/neon.h>
10
-
#include <asm/simd.h>
11
-
#include <crypto/internal/poly1305.h>
12
-
#include <linux/cpufeature.h>
13
-
#include <linux/jump_label.h>
14
-
#include <linux/kernel.h>
15
-
#include <linux/module.h>
16
-
#include <linux/unaligned.h>
17
-
18
-
asmlinkage void poly1305_block_init_arch(
19
-
struct poly1305_block_state *state,
20
-
const u8 raw_key[POLY1305_BLOCK_SIZE]);
21
-
EXPORT_SYMBOL_GPL(poly1305_block_init_arch);
22
-
asmlinkage void poly1305_blocks_arm(struct poly1305_block_state *state,
23
-
const u8 *src, u32 len, u32 hibit);
24
-
asmlinkage void poly1305_blocks_neon(struct poly1305_block_state *state,
25
-
const u8 *src, u32 len, u32 hibit);
26
-
asmlinkage void poly1305_emit_arch(const struct poly1305_state *state,
27
-
u8 digest[POLY1305_DIGEST_SIZE],
28
-
const u32 nonce[4]);
29
-
EXPORT_SYMBOL_GPL(poly1305_emit_arch);
30
-
31
-
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
32
-
33
-
void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
34
-
unsigned int len, u32 padbit)
35
-
{
36
-
len = round_down(len, POLY1305_BLOCK_SIZE);
37
-
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
38
-
static_branch_likely(&have_neon) && likely(may_use_simd())) {
39
-
do {
40
-
unsigned int todo = min_t(unsigned int, len, SZ_4K);
41
-
42
-
kernel_neon_begin();
43
-
poly1305_blocks_neon(state, src, todo, padbit);
44
-
kernel_neon_end();
45
-
46
-
len -= todo;
47
-
src += todo;
48
-
} while (len);
49
-
} else
50
-
poly1305_blocks_arm(state, src, len, padbit);
51
-
}
52
-
EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
53
-
54
-
static int __init arm_poly1305_mod_init(void)
55
-
{
56
-
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
57
-
(elf_hwcap & HWCAP_NEON))
58
-
static_branch_enable(&have_neon);
59
-
return 0;
60
-
}
61
-
subsys_initcall(arm_poly1305_mod_init);
62
-
63
-
static void __exit arm_poly1305_mod_exit(void)
64
-
{
65
-
}
66
-
module_exit(arm_poly1305_mod_exit);
67
-
68
-
MODULE_DESCRIPTION("Accelerated Poly1305 transform for ARM");
69
-
MODULE_LICENSE("GPL v2");
+53
lib/crypto/arm/poly1305.h
+53
lib/crypto/arm/poly1305.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/*
3
+
* OpenSSL/Cryptogams accelerated Poly1305 transform for ARM
4
+
*
5
+
* Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org>
6
+
*/
7
+
8
+
#include <asm/hwcap.h>
9
+
#include <asm/neon.h>
10
+
#include <asm/simd.h>
11
+
#include <linux/cpufeature.h>
12
+
#include <linux/jump_label.h>
13
+
#include <linux/kernel.h>
14
+
15
+
asmlinkage void poly1305_block_init(struct poly1305_block_state *state,
16
+
const u8 raw_key[POLY1305_BLOCK_SIZE]);
17
+
asmlinkage void poly1305_blocks_arm(struct poly1305_block_state *state,
18
+
const u8 *src, u32 len, u32 hibit);
19
+
asmlinkage void poly1305_blocks_neon(struct poly1305_block_state *state,
20
+
const u8 *src, u32 len, u32 hibit);
21
+
asmlinkage void poly1305_emit(const struct poly1305_state *state,
22
+
u8 digest[POLY1305_DIGEST_SIZE],
23
+
const u32 nonce[4]);
24
+
25
+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
26
+
27
+
static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src,
28
+
unsigned int len, u32 padbit)
29
+
{
30
+
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
31
+
static_branch_likely(&have_neon) && likely(may_use_simd())) {
32
+
do {
33
+
unsigned int todo = min_t(unsigned int, len, SZ_4K);
34
+
35
+
kernel_neon_begin();
36
+
poly1305_blocks_neon(state, src, todo, padbit);
37
+
kernel_neon_end();
38
+
39
+
len -= todo;
40
+
src += todo;
41
+
} while (len);
42
+
} else
43
+
poly1305_blocks_arm(state, src, len, padbit);
44
+
}
45
+
46
+
#ifdef CONFIG_KERNEL_MODE_NEON
47
+
#define poly1305_mod_init_arch poly1305_mod_init_arch
48
+
static void poly1305_mod_init_arch(void)
49
+
{
50
+
if (elf_hwcap & HWCAP_NEON)
51
+
static_branch_enable(&have_neon);
52
+
}
53
+
#endif /* CONFIG_KERNEL_MODE_NEON */
-6
lib/crypto/arm64/Kconfig
-6
lib/crypto/arm64/Kconfig
-13
lib/crypto/arm64/Makefile
-13
lib/crypto/arm64/Makefile
···
2
2
3
3
obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha-neon.o
4
4
chacha-neon-y := chacha-neon-core.o chacha-neon-glue.o
5
-
6
-
obj-$(CONFIG_CRYPTO_POLY1305_NEON) += poly1305-neon.o
7
-
poly1305-neon-y := poly1305-core.o poly1305-glue.o
8
-
AFLAGS_poly1305-core.o += -Dpoly1305_init=poly1305_block_init_arch
9
-
AFLAGS_poly1305-core.o += -Dpoly1305_emit=poly1305_emit_arch
10
-
11
-
quiet_cmd_perlasm = PERLASM $@
12
-
cmd_perlasm = $(PERL) $(<) void $(@)
13
-
14
-
$(obj)/%-core.S: $(src)/%-armv8.pl
15
-
$(call cmd,perlasm)
16
-
17
-
clean-files += poly1305-core.S
+3
lib/crypto/arm64/poly1305-armv8.pl
+3
lib/crypto/arm64/poly1305-armv8.pl
-67
lib/crypto/arm64/poly1305-glue.c
-67
lib/crypto/arm64/poly1305-glue.c
···
1
-
// SPDX-License-Identifier: GPL-2.0
2
-
/*
3
-
* OpenSSL/Cryptogams accelerated Poly1305 transform for arm64
4
-
*
5
-
* Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org>
6
-
*/
7
-
8
-
#include <asm/hwcap.h>
9
-
#include <asm/neon.h>
10
-
#include <asm/simd.h>
11
-
#include <crypto/internal/poly1305.h>
12
-
#include <linux/cpufeature.h>
13
-
#include <linux/jump_label.h>
14
-
#include <linux/kernel.h>
15
-
#include <linux/module.h>
16
-
#include <linux/unaligned.h>
17
-
18
-
asmlinkage void poly1305_block_init_arch(
19
-
struct poly1305_block_state *state,
20
-
const u8 raw_key[POLY1305_BLOCK_SIZE]);
21
-
EXPORT_SYMBOL_GPL(poly1305_block_init_arch);
22
-
asmlinkage void poly1305_blocks(struct poly1305_block_state *state,
23
-
const u8 *src, u32 len, u32 hibit);
24
-
asmlinkage void poly1305_blocks_neon(struct poly1305_block_state *state,
25
-
const u8 *src, u32 len, u32 hibit);
26
-
asmlinkage void poly1305_emit_arch(const struct poly1305_state *state,
27
-
u8 digest[POLY1305_DIGEST_SIZE],
28
-
const u32 nonce[4]);
29
-
EXPORT_SYMBOL_GPL(poly1305_emit_arch);
30
-
31
-
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
32
-
33
-
void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
34
-
unsigned int len, u32 padbit)
35
-
{
36
-
len = round_down(len, POLY1305_BLOCK_SIZE);
37
-
if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
38
-
do {
39
-
unsigned int todo = min_t(unsigned int, len, SZ_4K);
40
-
41
-
kernel_neon_begin();
42
-
poly1305_blocks_neon(state, src, todo, padbit);
43
-
kernel_neon_end();
44
-
45
-
len -= todo;
46
-
src += todo;
47
-
} while (len);
48
-
} else
49
-
poly1305_blocks(state, src, len, padbit);
50
-
}
51
-
EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
52
-
53
-
static int __init neon_poly1305_mod_init(void)
54
-
{
55
-
if (cpu_have_named_feature(ASIMD))
56
-
static_branch_enable(&have_neon);
57
-
return 0;
58
-
}
59
-
subsys_initcall(neon_poly1305_mod_init);
60
-
61
-
static void __exit neon_poly1305_mod_exit(void)
62
-
{
63
-
}
64
-
module_exit(neon_poly1305_mod_exit);
65
-
66
-
MODULE_DESCRIPTION("Poly1305 authenticator (ARM64 optimized)");
67
-
MODULE_LICENSE("GPL v2");
+50
lib/crypto/arm64/poly1305.h
+50
lib/crypto/arm64/poly1305.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/*
3
+
* OpenSSL/Cryptogams accelerated Poly1305 transform for arm64
4
+
*
5
+
* Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org>
6
+
*/
7
+
8
+
#include <asm/hwcap.h>
9
+
#include <asm/neon.h>
10
+
#include <asm/simd.h>
11
+
#include <linux/cpufeature.h>
12
+
#include <linux/jump_label.h>
13
+
#include <linux/kernel.h>
14
+
15
+
asmlinkage void poly1305_block_init(struct poly1305_block_state *state,
16
+
const u8 raw_key[POLY1305_BLOCK_SIZE]);
17
+
asmlinkage void poly1305_blocks_arm64(struct poly1305_block_state *state,
18
+
const u8 *src, u32 len, u32 hibit);
19
+
asmlinkage void poly1305_blocks_neon(struct poly1305_block_state *state,
20
+
const u8 *src, u32 len, u32 hibit);
21
+
asmlinkage void poly1305_emit(const struct poly1305_state *state,
22
+
u8 digest[POLY1305_DIGEST_SIZE],
23
+
const u32 nonce[4]);
24
+
25
+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
26
+
27
+
static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src,
28
+
unsigned int len, u32 padbit)
29
+
{
30
+
if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
31
+
do {
32
+
unsigned int todo = min_t(unsigned int, len, SZ_4K);
33
+
34
+
kernel_neon_begin();
35
+
poly1305_blocks_neon(state, src, todo, padbit);
36
+
kernel_neon_end();
37
+
38
+
len -= todo;
39
+
src += todo;
40
+
} while (len);
41
+
} else
42
+
poly1305_blocks_arm64(state, src, len, padbit);
43
+
}
44
+
45
+
#define poly1305_mod_init_arch poly1305_mod_init_arch
46
+
static void poly1305_mod_init_arch(void)
47
+
{
48
+
if (cpu_have_named_feature(ASIMD))
49
+
static_branch_enable(&have_neon);
50
+
}
-5
lib/crypto/mips/Kconfig
-5
lib/crypto/mips/Kconfig
-14
lib/crypto/mips/Makefile
-14
lib/crypto/mips/Makefile
···
3
3
obj-$(CONFIG_CRYPTO_CHACHA_MIPS) += chacha-mips.o
4
4
chacha-mips-y := chacha-core.o chacha-glue.o
5
5
AFLAGS_chacha-core.o += -O2 # needed to fill branch delay slots
6
-
7
-
obj-$(CONFIG_CRYPTO_POLY1305_MIPS) += poly1305-mips.o
8
-
poly1305-mips-y := poly1305-core.o poly1305-glue.o
9
-
10
-
perlasm-flavour-$(CONFIG_32BIT) := o32
11
-
perlasm-flavour-$(CONFIG_64BIT) := 64
12
-
13
-
quiet_cmd_perlasm = PERLASM $@
14
-
cmd_perlasm = $(PERL) $(<) $(perlasm-flavour-y) $(@)
15
-
16
-
$(obj)/poly1305-core.S: $(src)/poly1305-mips.pl FORCE
17
-
$(call if_changed,perlasm)
18
-
19
-
targets += poly1305-core.S
-27
lib/crypto/mips/poly1305-glue.c
-27
lib/crypto/mips/poly1305-glue.c
···
1
-
// SPDX-License-Identifier: GPL-2.0
2
-
/*
3
-
* OpenSSL/Cryptogams accelerated Poly1305 transform for MIPS
4
-
*
5
-
* Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org>
6
-
*/
7
-
8
-
#include <crypto/internal/poly1305.h>
9
-
#include <linux/cpufeature.h>
10
-
#include <linux/kernel.h>
11
-
#include <linux/module.h>
12
-
#include <linux/unaligned.h>
13
-
14
-
asmlinkage void poly1305_block_init_arch(
15
-
struct poly1305_block_state *state,
16
-
const u8 raw_key[POLY1305_BLOCK_SIZE]);
17
-
EXPORT_SYMBOL_GPL(poly1305_block_init_arch);
18
-
asmlinkage void poly1305_blocks_arch(struct poly1305_block_state *state,
19
-
const u8 *src, u32 len, u32 hibit);
20
-
EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
21
-
asmlinkage void poly1305_emit_arch(const struct poly1305_state *state,
22
-
u8 digest[POLY1305_DIGEST_SIZE],
23
-
const u32 nonce[4]);
24
-
EXPORT_SYMBOL_GPL(poly1305_emit_arch);
25
-
26
-
MODULE_DESCRIPTION("Poly1305 transform (MIPS accelerated");
27
-
MODULE_LICENSE("GPL v2");
+2
-6
lib/crypto/mips/poly1305-mips.pl
+2
-6
lib/crypto/mips/poly1305-mips.pl
···
93
93
#endif
94
94
95
95
#ifdef __KERNEL__
96
-
# define poly1305_init poly1305_block_init_arch
97
-
# define poly1305_blocks poly1305_blocks_arch
98
-
# define poly1305_emit poly1305_emit_arch
96
+
# define poly1305_init poly1305_block_init
99
97
#endif
100
98
101
99
#if defined(__MIPSEB__) && !defined(MIPSEB)
···
563
565
#endif
564
566
565
567
#ifdef __KERNEL__
566
-
# define poly1305_init poly1305_block_init_arch
567
-
# define poly1305_blocks poly1305_blocks_arch
568
-
# define poly1305_emit poly1305_emit_arch
568
+
# define poly1305_init poly1305_block_init
569
569
#endif
570
570
571
571
#if defined(__MIPSEB__) && !defined(MIPSEB)
+14
lib/crypto/mips/poly1305.h
+14
lib/crypto/mips/poly1305.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/*
3
+
* OpenSSL/Cryptogams accelerated Poly1305 transform for MIPS
4
+
*
5
+
* Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org>
6
+
*/
7
+
8
+
asmlinkage void poly1305_block_init(struct poly1305_block_state *state,
9
+
const u8 raw_key[POLY1305_BLOCK_SIZE]);
10
+
asmlinkage void poly1305_blocks(struct poly1305_block_state *state,
11
+
const u8 *src, u32 len, u32 hibit);
12
+
asmlinkage void poly1305_emit(const struct poly1305_state *state,
13
+
u8 digest[POLY1305_DIGEST_SIZE],
14
+
const u32 nonce[4]);
-25
lib/crypto/poly1305-generic.c
-25
lib/crypto/poly1305-generic.c
···
1
-
// SPDX-License-Identifier: GPL-2.0-or-later
2
-
/*
3
-
* Poly1305 authenticator algorithm, RFC7539
4
-
*
5
-
* Copyright (C) 2015 Martin Willi
6
-
*
7
-
* Based on public domain code by Andrew Moon and Daniel J. Bernstein.
8
-
*/
9
-
10
-
#include <crypto/internal/poly1305.h>
11
-
#include <linux/export.h>
12
-
#include <linux/kernel.h>
13
-
#include <linux/module.h>
14
-
15
-
void poly1305_block_init_generic(struct poly1305_block_state *desc,
16
-
const u8 raw_key[POLY1305_BLOCK_SIZE])
17
-
{
18
-
poly1305_core_init(&desc->h);
19
-
poly1305_core_setkey(&desc->core_r, raw_key);
20
-
}
21
-
EXPORT_SYMBOL_GPL(poly1305_block_init_generic);
22
-
23
-
MODULE_LICENSE("GPL");
24
-
MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
25
-
MODULE_DESCRIPTION("Poly1305 algorithm (generic implementation)");
+53
-28
lib/crypto/poly1305.c
+53
-28
lib/crypto/poly1305.c
···
7
7
* Based on public domain code by Andrew Moon and Daniel J. Bernstein.
8
8
*/
9
9
10
-
#include <crypto/internal/blockhash.h>
11
10
#include <crypto/internal/poly1305.h>
12
11
#include <linux/export.h>
13
12
#include <linux/kernel.h>
14
13
#include <linux/module.h>
15
14
#include <linux/string.h>
16
15
#include <linux/unaligned.h>
16
+
17
+
#ifdef CONFIG_CRYPTO_LIB_POLY1305_ARCH
18
+
#include "poly1305.h" /* $(SRCARCH)/poly1305.h */
19
+
#else
20
+
#define poly1305_block_init poly1305_block_init_generic
21
+
#define poly1305_blocks poly1305_blocks_generic
22
+
#define poly1305_emit poly1305_emit_generic
23
+
#endif
17
24
18
25
void poly1305_init(struct poly1305_desc_ctx *desc,
19
26
const u8 key[POLY1305_KEY_SIZE])
···
30
23
desc->s[2] = get_unaligned_le32(key + 24);
31
24
desc->s[3] = get_unaligned_le32(key + 28);
32
25
desc->buflen = 0;
33
-
if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305))
34
-
poly1305_block_init_arch(&desc->state, key);
35
-
else
36
-
poly1305_block_init_generic(&desc->state, key);
26
+
poly1305_block_init(&desc->state, key);
37
27
}
38
28
EXPORT_SYMBOL(poly1305_init);
39
-
40
-
static inline void poly1305_blocks(struct poly1305_block_state *state,
41
-
const u8 *src, unsigned int len)
42
-
{
43
-
if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305))
44
-
poly1305_blocks_arch(state, src, len, 1);
45
-
else
46
-
poly1305_blocks_generic(state, src, len, 1);
47
-
}
48
29
49
30
void poly1305_update(struct poly1305_desc_ctx *desc,
50
31
const u8 *src, unsigned int nbytes)
51
32
{
52
-
desc->buflen = BLOCK_HASH_UPDATE(poly1305_blocks, &desc->state,
53
-
src, nbytes, POLY1305_BLOCK_SIZE,
54
-
desc->buf, desc->buflen);
33
+
if (desc->buflen + nbytes >= POLY1305_BLOCK_SIZE) {
34
+
unsigned int bulk_len;
35
+
36
+
if (desc->buflen) {
37
+
unsigned int l = POLY1305_BLOCK_SIZE - desc->buflen;
38
+
39
+
memcpy(&desc->buf[desc->buflen], src, l);
40
+
src += l;
41
+
nbytes -= l;
42
+
43
+
poly1305_blocks(&desc->state, desc->buf,
44
+
POLY1305_BLOCK_SIZE, 1);
45
+
desc->buflen = 0;
46
+
}
47
+
48
+
bulk_len = round_down(nbytes, POLY1305_BLOCK_SIZE);
49
+
nbytes %= POLY1305_BLOCK_SIZE;
50
+
51
+
if (bulk_len) {
52
+
poly1305_blocks(&desc->state, src, bulk_len, 1);
53
+
src += bulk_len;
54
+
}
55
+
}
56
+
if (nbytes) {
57
+
memcpy(&desc->buf[desc->buflen], src, nbytes);
58
+
desc->buflen += nbytes;
59
+
}
55
60
}
56
61
EXPORT_SYMBOL(poly1305_update);
57
62
···
73
54
desc->buf[desc->buflen++] = 1;
74
55
memset(desc->buf + desc->buflen, 0,
75
56
POLY1305_BLOCK_SIZE - desc->buflen);
76
-
if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305))
77
-
poly1305_blocks_arch(&desc->state, desc->buf,
78
-
POLY1305_BLOCK_SIZE, 0);
79
-
else
80
-
poly1305_blocks_generic(&desc->state, desc->buf,
81
-
POLY1305_BLOCK_SIZE, 0);
57
+
poly1305_blocks(&desc->state, desc->buf, POLY1305_BLOCK_SIZE,
58
+
0);
82
59
}
83
60
84
-
if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305))
85
-
poly1305_emit_arch(&desc->state.h, dst, desc->s);
86
-
else
87
-
poly1305_emit_generic(&desc->state.h, dst, desc->s);
61
+
poly1305_emit(&desc->state.h, dst, desc->s);
88
62
*desc = (struct poly1305_desc_ctx){};
89
63
}
90
64
EXPORT_SYMBOL(poly1305_final);
91
65
66
+
#ifdef poly1305_mod_init_arch
67
+
static int __init poly1305_mod_init(void)
68
+
{
69
+
poly1305_mod_init_arch();
70
+
return 0;
71
+
}
72
+
subsys_initcall(poly1305_mod_init);
73
+
74
+
static void __exit poly1305_mod_exit(void)
75
+
{
76
+
}
77
+
module_exit(poly1305_mod_exit);
78
+
#endif
79
+
92
80
MODULE_LICENSE("GPL");
93
-
MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
94
81
MODULE_DESCRIPTION("Poly1305 authenticator algorithm, RFC7539");
-8
lib/crypto/powerpc/Kconfig
-8
lib/crypto/powerpc/Kconfig
···
6
6
default CRYPTO_LIB_CHACHA
7
7
select CRYPTO_LIB_CHACHA_GENERIC
8
8
select CRYPTO_ARCH_HAVE_LIB_CHACHA
9
-
10
-
config CRYPTO_POLY1305_P10
11
-
tristate
12
-
depends on PPC64 && CPU_LITTLE_ENDIAN && VSX
13
-
depends on BROKEN # Needs to be fixed to work in softirq context
14
-
default CRYPTO_LIB_POLY1305
15
-
select CRYPTO_ARCH_HAVE_LIB_POLY1305
16
-
select CRYPTO_LIB_POLY1305_GENERIC
-3
lib/crypto/powerpc/Makefile
-3
lib/crypto/powerpc/Makefile
+9
-25
lib/crypto/powerpc/poly1305-p10-glue.c
lib/crypto/powerpc/poly1305.h
+9
-25
lib/crypto/powerpc/poly1305-p10-glue.c
lib/crypto/powerpc/poly1305.h
···
1
-
// SPDX-License-Identifier: GPL-2.0
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
2
/*
3
3
* Poly1305 authenticator algorithm, RFC7539.
4
4
*
5
5
* Copyright 2023- IBM Corp. All rights reserved.
6
6
*/
7
7
#include <asm/switch_to.h>
8
-
#include <crypto/internal/poly1305.h>
9
8
#include <linux/cpufeature.h>
10
9
#include <linux/jump_label.h>
11
10
#include <linux/kernel.h>
12
-
#include <linux/module.h>
13
11
#include <linux/unaligned.h>
14
12
15
13
asmlinkage void poly1305_p10le_4blocks(struct poly1305_block_state *state, const u8 *m, u32 mlen);
···
28
30
preempt_enable();
29
31
}
30
32
31
-
void poly1305_block_init_arch(struct poly1305_block_state *dctx,
32
-
const u8 raw_key[POLY1305_BLOCK_SIZE])
33
+
static void poly1305_block_init(struct poly1305_block_state *dctx,
34
+
const u8 raw_key[POLY1305_BLOCK_SIZE])
33
35
{
34
36
if (!static_key_enabled(&have_p10))
35
37
return poly1305_block_init_generic(dctx, raw_key);
···
38
40
dctx->core_r.key.r64[0] = get_unaligned_le64(raw_key + 0);
39
41
dctx->core_r.key.r64[1] = get_unaligned_le64(raw_key + 8);
40
42
}
41
-
EXPORT_SYMBOL_GPL(poly1305_block_init_arch);
42
43
43
-
void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
44
-
unsigned int len, u32 padbit)
44
+
static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src,
45
+
unsigned int len, u32 padbit)
45
46
{
46
47
if (!static_key_enabled(&have_p10))
47
48
return poly1305_blocks_generic(state, src, len, padbit);
···
57
60
}
58
61
vsx_end();
59
62
}
60
-
EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
61
63
62
-
void poly1305_emit_arch(const struct poly1305_state *state,
63
-
u8 digest[POLY1305_DIGEST_SIZE],
64
-
const u32 nonce[4])
64
+
static void poly1305_emit(const struct poly1305_state *state,
65
+
u8 digest[POLY1305_DIGEST_SIZE], const u32 nonce[4])
65
66
{
66
67
if (!static_key_enabled(&have_p10))
67
68
return poly1305_emit_generic(state, digest, nonce);
68
69
poly1305_emit_64(state, nonce, digest);
69
70
}
70
-
EXPORT_SYMBOL_GPL(poly1305_emit_arch);
71
71
72
-
static int __init poly1305_p10_init(void)
72
+
#define poly1305_mod_init_arch poly1305_mod_init_arch
73
+
static void poly1305_mod_init_arch(void)
73
74
{
74
75
if (cpu_has_feature(CPU_FTR_ARCH_31))
75
76
static_branch_enable(&have_p10);
76
-
return 0;
77
77
}
78
-
subsys_initcall(poly1305_p10_init);
79
-
80
-
static void __exit poly1305_p10_exit(void)
81
-
{
82
-
}
83
-
module_exit(poly1305_p10_exit);
84
-
85
-
MODULE_LICENSE("GPL");
86
-
MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com>");
87
-
MODULE_DESCRIPTION("Optimized Poly1305 for P10");
-6
lib/crypto/x86/Kconfig
-6
lib/crypto/x86/Kconfig
-10
lib/crypto/x86/Makefile
-10
lib/crypto/x86/Makefile
···
5
5
6
6
obj-$(CONFIG_CRYPTO_CHACHA20_X86_64) += chacha-x86_64.o
7
7
chacha-x86_64-y := chacha-avx2-x86_64.o chacha-ssse3-x86_64.o chacha-avx512vl-x86_64.o chacha_glue.o
8
-
9
-
obj-$(CONFIG_CRYPTO_POLY1305_X86_64) += poly1305-x86_64.o
10
-
poly1305-x86_64-y := poly1305-x86_64-cryptogams.o poly1305_glue.o
11
-
targets += poly1305-x86_64-cryptogams.S
12
-
13
-
quiet_cmd_perlasm = PERLASM $@
14
-
cmd_perlasm = $(PERL) $< > $@
15
-
16
-
$(obj)/%.S: $(src)/%.pl FORCE
17
-
$(call if_changed,perlasm)
+10
-23
lib/crypto/x86/poly1305-x86_64-cryptogams.pl
+10
-23
lib/crypto/x86/poly1305-x86_64-cryptogams.pl
···
118
118
}
119
119
}
120
120
121
-
sub declare_typed_function() {
122
-
my ($name, $align, $nargs) = @_;
123
-
if($kernel) {
124
-
$code .= "SYM_TYPED_FUNC_START($name)\n";
125
-
$code .= ".L$name:\n";
126
-
} else {
127
-
$code .= ".globl $name\n";
128
-
$code .= ".type $name,\@function,$nargs\n";
129
-
$code .= ".align $align\n";
130
-
$code .= "$name:\n";
131
-
}
132
-
}
133
-
134
121
sub end_function() {
135
122
my ($name) = @_;
136
123
if($kernel) {
···
128
141
}
129
142
130
143
$code.=<<___ if $kernel;
131
-
#include <linux/cfi_types.h>
144
+
#include <linux/linkage.h>
132
145
___
133
146
134
147
if ($avx) {
···
236
249
$code.=<<___ if (!$kernel);
237
250
.extern OPENSSL_ia32cap_P
238
251
239
-
.globl poly1305_block_init_arch
240
-
.hidden poly1305_block_init_arch
252
+
.globl poly1305_init_x86_64
253
+
.hidden poly1305_init_x86_64
241
254
.globl poly1305_blocks_x86_64
242
255
.hidden poly1305_blocks_x86_64
243
256
.globl poly1305_emit_x86_64
244
257
.hidden poly1305_emit_x86_64
245
258
___
246
-
&declare_typed_function("poly1305_block_init_arch", 32, 3);
259
+
&declare_function("poly1305_init_x86_64", 32, 3);
247
260
$code.=<<___;
248
261
xor %eax,%eax
249
262
mov %rax,0($ctx) # initialize hash value
···
298
311
.Lno_key:
299
312
RET
300
313
___
301
-
&end_function("poly1305_block_init_arch");
314
+
&end_function("poly1305_init_x86_64");
302
315
303
316
&declare_function("poly1305_blocks_x86_64", 32, 4);
304
317
$code.=<<___;
···
4105
4118
4106
4119
.section .pdata
4107
4120
.align 4
4108
-
.rva .LSEH_begin_poly1305_block_init_arch
4109
-
.rva .LSEH_end_poly1305_block_init_arch
4110
-
.rva .LSEH_info_poly1305_block_init_arch
4121
+
.rva .LSEH_begin_poly1305_init_x86_64
4122
+
.rva .LSEH_end_poly1305_init_x86_64
4123
+
.rva .LSEH_info_poly1305_init_x86_64
4111
4124
4112
4125
.rva .LSEH_begin_poly1305_blocks_x86_64
4113
4126
.rva .LSEH_end_poly1305_blocks_x86_64
···
4155
4168
$code.=<<___;
4156
4169
.section .xdata
4157
4170
.align 8
4158
-
.LSEH_info_poly1305_block_init_arch:
4171
+
.LSEH_info_poly1305_init_x86_64:
4159
4172
.byte 9,0,0,0
4160
4173
.rva se_handler
4161
-
.rva .LSEH_begin_poly1305_block_init_arch,.LSEH_begin_poly1305_block_init_arch
4174
+
.rva .LSEH_begin_poly1305_init_x86_64,.LSEH_begin_poly1305_init_x86_64
4162
4175
4163
4176
.LSEH_info_poly1305_blocks_x86_64:
4164
4177
.byte 9,0,0,0
+15
-26
lib/crypto/x86/poly1305_glue.c
lib/crypto/x86/poly1305.h
+15
-26
lib/crypto/x86/poly1305_glue.c
lib/crypto/x86/poly1305.h
···
1
-
// SPDX-License-Identifier: GPL-2.0 OR MIT
1
+
/* SPDX-License-Identifier: GPL-2.0 OR MIT */
2
2
/*
3
3
* Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4
4
*/
5
5
6
6
#include <asm/cpu_device_id.h>
7
7
#include <asm/fpu/api.h>
8
-
#include <crypto/internal/poly1305.h>
9
8
#include <linux/jump_label.h>
10
9
#include <linux/kernel.h>
11
-
#include <linux/module.h>
12
10
#include <linux/sizes.h>
13
-
#include <linux/unaligned.h>
14
11
15
12
struct poly1305_arch_internal {
16
13
union {
···
58
61
state->is_base2_26 = 0;
59
62
}
60
63
61
-
asmlinkage void poly1305_block_init_arch(
62
-
struct poly1305_block_state *state,
63
-
const u8 raw_key[POLY1305_BLOCK_SIZE]);
64
-
EXPORT_SYMBOL_GPL(poly1305_block_init_arch);
64
+
asmlinkage void poly1305_init_x86_64(struct poly1305_block_state *state,
65
+
const u8 raw_key[POLY1305_BLOCK_SIZE]);
65
66
asmlinkage void poly1305_blocks_x86_64(struct poly1305_arch_internal *ctx,
66
67
const u8 *inp,
67
68
const size_t len, const u32 padbit);
···
83
88
static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx2);
84
89
static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx512);
85
90
86
-
void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *inp,
87
-
unsigned int len, u32 padbit)
91
+
static void poly1305_block_init(struct poly1305_block_state *state,
92
+
const u8 raw_key[POLY1305_BLOCK_SIZE])
93
+
{
94
+
poly1305_init_x86_64(state, raw_key);
95
+
}
96
+
97
+
static void poly1305_blocks(struct poly1305_block_state *state, const u8 *inp,
98
+
unsigned int len, u32 padbit)
88
99
{
89
100
struct poly1305_arch_internal *ctx =
90
101
container_of(&state->h.h, struct poly1305_arch_internal, h);
···
130
129
inp += bytes;
131
130
} while (len);
132
131
}
133
-
EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
134
132
135
-
void poly1305_emit_arch(const struct poly1305_state *ctx,
136
-
u8 mac[POLY1305_DIGEST_SIZE], const u32 nonce[4])
133
+
static void poly1305_emit(const struct poly1305_state *ctx,
134
+
u8 mac[POLY1305_DIGEST_SIZE], const u32 nonce[4])
137
135
{
138
136
if (!static_branch_likely(&poly1305_use_avx))
139
137
poly1305_emit_x86_64(ctx, mac, nonce);
140
138
else
141
139
poly1305_emit_avx(ctx, mac, nonce);
142
140
}
143
-
EXPORT_SYMBOL_GPL(poly1305_emit_arch);
144
141
145
-
static int __init poly1305_simd_mod_init(void)
142
+
#define poly1305_mod_init_arch poly1305_mod_init_arch
143
+
static void poly1305_mod_init_arch(void)
146
144
{
147
145
if (boot_cpu_has(X86_FEATURE_AVX) &&
148
146
cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
···
155
155
/* Skylake downclocks unacceptably much when using zmm, but later generations are fast. */
156
156
boot_cpu_data.x86_vfm != INTEL_SKYLAKE_X)
157
157
static_branch_enable(&poly1305_use_avx512);
158
-
return 0;
159
158
}
160
-
subsys_initcall(poly1305_simd_mod_init);
161
-
162
-
static void __exit poly1305_simd_mod_exit(void)
163
-
{
164
-
}
165
-
module_exit(poly1305_simd_mod_exit);
166
-
167
-
MODULE_LICENSE("GPL");
168
-
MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
169
-
MODULE_DESCRIPTION("Poly1305 authenticator");