Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
x86/crc32: expose CRC32 functions through lib
Move the x86 CRC32 assembly code into the lib directory and wire it up
to the library interface. This allows it to be used without going
through the crypto API. It remains usable via the crypto API too via
the shash algorithms that use the library interface. Thus all the
arch-specific "shash" code becomes unnecessary and is removed.
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20241202010844.144356-14-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@google.com>
+129
-481
+1
arch/x86/Kconfig
+1
arch/x86/Kconfig
-22
arch/x86/crypto/Kconfig
-22
arch/x86/crypto/Kconfig
···
492
492
Architecture: x86_64 using:
493
493
- CLMUL-NI (carry-less multiplication new instructions)
494
494
495
-
config CRYPTO_CRC32C_INTEL
496
-
tristate "CRC32c (SSE4.2/PCLMULQDQ)"
497
-
depends on X86
498
-
select CRYPTO_HASH
499
-
help
500
-
CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720)
501
-
502
-
Architecture: x86 (32-bit and 64-bit) using:
503
-
- SSE4.2 (Streaming SIMD Extensions 4.2) CRC32 instruction
504
-
- PCLMULQDQ (carry-less multiplication)
505
-
506
-
config CRYPTO_CRC32_PCLMUL
507
-
tristate "CRC32 (PCLMULQDQ)"
508
-
depends on X86
509
-
select CRYPTO_HASH
510
-
select CRC32
511
-
help
512
-
CRC32 CRC algorithm (IEEE 802.3)
513
-
514
-
Architecture: x86 (32-bit and 64-bit) using:
515
-
- PCLMULQDQ (carry-less multiplication)
516
-
517
495
config CRYPTO_CRCT10DIF_PCLMUL
518
496
tristate "CRCT10DIF (PCLMULQDQ)"
519
497
depends on X86 && 64BIT && CRC_T10DIF
-7
arch/x86/crypto/Makefile
-7
arch/x86/crypto/Makefile
···
75
75
obj-$(CONFIG_CRYPTO_POLYVAL_CLMUL_NI) += polyval-clmulni.o
76
76
polyval-clmulni-y := polyval-clmulni_asm.o polyval-clmulni_glue.o
77
77
78
-
obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o
79
-
crc32c-intel-y := crc32c-intel_glue.o
80
-
crc32c-intel-$(CONFIG_64BIT) += crc32c-pcl-intel-asm_64.o
81
-
82
-
obj-$(CONFIG_CRYPTO_CRC32_PCLMUL) += crc32-pclmul.o
83
-
crc32-pclmul-y := crc32-pclmul_asm.o crc32-pclmul_glue.o
84
-
85
78
obj-$(CONFIG_CRYPTO_CRCT10DIF_PCLMUL) += crct10dif-pclmul.o
86
79
crct10dif-pclmul-y := crct10dif-pcl-asm_64.o crct10dif-pclmul_glue.o
87
80
arch/x86/crypto/crc32-pclmul_asm.S
arch/x86/lib/crc32-pclmul.S
arch/x86/crypto/crc32-pclmul_asm.S
arch/x86/lib/crc32-pclmul.S
-202
arch/x86/crypto/crc32-pclmul_glue.c
-202
arch/x86/crypto/crc32-pclmul_glue.c
···
1
-
/* GPL HEADER START
2
-
*
3
-
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4
-
*
5
-
* This program is free software; you can redistribute it and/or modify
6
-
* it under the terms of the GNU General Public License version 2 only,
7
-
* as published by the Free Software Foundation.
8
-
*
9
-
* This program is distributed in the hope that it will be useful, but
10
-
* WITHOUT ANY WARRANTY; without even the implied warranty of
11
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
-
* General Public License version 2 for more details (a copy is included
13
-
* in the LICENSE file that accompanied this code).
14
-
*
15
-
* You should have received a copy of the GNU General Public License
16
-
* version 2 along with this program; If not, see http://www.gnu.org/licenses
17
-
*
18
-
* Please visit http://www.xyratex.com/contact if you need additional
19
-
* information or have any questions.
20
-
*
21
-
* GPL HEADER END
22
-
*/
23
-
24
-
/*
25
-
* Copyright 2012 Xyratex Technology Limited
26
-
*
27
-
* Wrappers for kernel crypto shash api to pclmulqdq crc32 implementation.
28
-
*/
29
-
#include <linux/init.h>
30
-
#include <linux/module.h>
31
-
#include <linux/string.h>
32
-
#include <linux/kernel.h>
33
-
#include <linux/crc32.h>
34
-
#include <crypto/internal/hash.h>
35
-
#include <crypto/internal/simd.h>
36
-
37
-
#include <asm/cpufeatures.h>
38
-
#include <asm/cpu_device_id.h>
39
-
#include <asm/simd.h>
40
-
41
-
#define CHKSUM_BLOCK_SIZE 1
42
-
#define CHKSUM_DIGEST_SIZE 4
43
-
44
-
#define PCLMUL_MIN_LEN 64L /* minimum size of buffer
45
-
* for crc32_pclmul_le_16 */
46
-
#define SCALE_F 16L /* size of xmm register */
47
-
#define SCALE_F_MASK (SCALE_F - 1)
48
-
49
-
u32 crc32_pclmul_le_16(u32 crc, const u8 *buffer, size_t len);
50
-
51
-
static u32 __attribute__((pure))
52
-
crc32_pclmul_le(u32 crc, unsigned char const *p, size_t len)
53
-
{
54
-
unsigned int iquotient;
55
-
unsigned int iremainder;
56
-
unsigned int prealign;
57
-
58
-
if (len < PCLMUL_MIN_LEN + SCALE_F_MASK || !crypto_simd_usable())
59
-
return crc32_le(crc, p, len);
60
-
61
-
if ((long)p & SCALE_F_MASK) {
62
-
/* align p to 16 byte */
63
-
prealign = SCALE_F - ((long)p & SCALE_F_MASK);
64
-
65
-
crc = crc32_le(crc, p, prealign);
66
-
len -= prealign;
67
-
p = (unsigned char *)(((unsigned long)p + SCALE_F_MASK) &
68
-
~SCALE_F_MASK);
69
-
}
70
-
iquotient = len & (~SCALE_F_MASK);
71
-
iremainder = len & SCALE_F_MASK;
72
-
73
-
kernel_fpu_begin();
74
-
crc = crc32_pclmul_le_16(crc, p, iquotient);
75
-
kernel_fpu_end();
76
-
77
-
if (iremainder)
78
-
crc = crc32_le(crc, p + iquotient, iremainder);
79
-
80
-
return crc;
81
-
}
82
-
83
-
static int crc32_pclmul_cra_init(struct crypto_tfm *tfm)
84
-
{
85
-
u32 *key = crypto_tfm_ctx(tfm);
86
-
87
-
*key = 0;
88
-
89
-
return 0;
90
-
}
91
-
92
-
static int crc32_pclmul_setkey(struct crypto_shash *hash, const u8 *key,
93
-
unsigned int keylen)
94
-
{
95
-
u32 *mctx = crypto_shash_ctx(hash);
96
-
97
-
if (keylen != sizeof(u32))
98
-
return -EINVAL;
99
-
*mctx = le32_to_cpup((__le32 *)key);
100
-
return 0;
101
-
}
102
-
103
-
static int crc32_pclmul_init(struct shash_desc *desc)
104
-
{
105
-
u32 *mctx = crypto_shash_ctx(desc->tfm);
106
-
u32 *crcp = shash_desc_ctx(desc);
107
-
108
-
*crcp = *mctx;
109
-
110
-
return 0;
111
-
}
112
-
113
-
static int crc32_pclmul_update(struct shash_desc *desc, const u8 *data,
114
-
unsigned int len)
115
-
{
116
-
u32 *crcp = shash_desc_ctx(desc);
117
-
118
-
*crcp = crc32_pclmul_le(*crcp, data, len);
119
-
return 0;
120
-
}
121
-
122
-
/* No final XOR 0xFFFFFFFF, like crc32_le */
123
-
static int __crc32_pclmul_finup(u32 *crcp, const u8 *data, unsigned int len,
124
-
u8 *out)
125
-
{
126
-
*(__le32 *)out = cpu_to_le32(crc32_pclmul_le(*crcp, data, len));
127
-
return 0;
128
-
}
129
-
130
-
static int crc32_pclmul_finup(struct shash_desc *desc, const u8 *data,
131
-
unsigned int len, u8 *out)
132
-
{
133
-
return __crc32_pclmul_finup(shash_desc_ctx(desc), data, len, out);
134
-
}
135
-
136
-
static int crc32_pclmul_final(struct shash_desc *desc, u8 *out)
137
-
{
138
-
u32 *crcp = shash_desc_ctx(desc);
139
-
140
-
*(__le32 *)out = cpu_to_le32p(crcp);
141
-
return 0;
142
-
}
143
-
144
-
static int crc32_pclmul_digest(struct shash_desc *desc, const u8 *data,
145
-
unsigned int len, u8 *out)
146
-
{
147
-
return __crc32_pclmul_finup(crypto_shash_ctx(desc->tfm), data, len,
148
-
out);
149
-
}
150
-
151
-
static struct shash_alg alg = {
152
-
.setkey = crc32_pclmul_setkey,
153
-
.init = crc32_pclmul_init,
154
-
.update = crc32_pclmul_update,
155
-
.final = crc32_pclmul_final,
156
-
.finup = crc32_pclmul_finup,
157
-
.digest = crc32_pclmul_digest,
158
-
.descsize = sizeof(u32),
159
-
.digestsize = CHKSUM_DIGEST_SIZE,
160
-
.base = {
161
-
.cra_name = "crc32",
162
-
.cra_driver_name = "crc32-pclmul",
163
-
.cra_priority = 200,
164
-
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
165
-
.cra_blocksize = CHKSUM_BLOCK_SIZE,
166
-
.cra_ctxsize = sizeof(u32),
167
-
.cra_module = THIS_MODULE,
168
-
.cra_init = crc32_pclmul_cra_init,
169
-
}
170
-
};
171
-
172
-
static const struct x86_cpu_id crc32pclmul_cpu_id[] = {
173
-
X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL),
174
-
{}
175
-
};
176
-
MODULE_DEVICE_TABLE(x86cpu, crc32pclmul_cpu_id);
177
-
178
-
179
-
static int __init crc32_pclmul_mod_init(void)
180
-
{
181
-
182
-
if (!x86_match_cpu(crc32pclmul_cpu_id)) {
183
-
pr_info("PCLMULQDQ-NI instructions are not detected.\n");
184
-
return -ENODEV;
185
-
}
186
-
return crypto_register_shash(&alg);
187
-
}
188
-
189
-
static void __exit crc32_pclmul_mod_fini(void)
190
-
{
191
-
crypto_unregister_shash(&alg);
192
-
}
193
-
194
-
module_init(crc32_pclmul_mod_init);
195
-
module_exit(crc32_pclmul_mod_fini);
196
-
197
-
MODULE_AUTHOR("Alexander Boyko <alexander_boyko@xyratex.com>");
198
-
MODULE_DESCRIPTION("CRC32 algorithm (IEEE 802.3) accelerated with PCLMULQDQ");
199
-
MODULE_LICENSE("GPL");
200
-
201
-
MODULE_ALIAS_CRYPTO("crc32");
202
-
MODULE_ALIAS_CRYPTO("crc32-pclmul");
-249
arch/x86/crypto/crc32c-intel_glue.c
-249
arch/x86/crypto/crc32c-intel_glue.c
···
1
-
// SPDX-License-Identifier: GPL-2.0-only
2
-
/*
3
-
* Using hardware provided CRC32 instruction to accelerate the CRC32 disposal.
4
-
* CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE)
5
-
* CRC32 is a new instruction in Intel SSE4.2, the reference can be found at:
6
-
* http://www.intel.com/products/processor/manuals/
7
-
* Intel(R) 64 and IA-32 Architectures Software Developer's Manual
8
-
* Volume 2A: Instruction Set Reference, A-M
9
-
*
10
-
* Copyright (C) 2008 Intel Corporation
11
-
* Authors: Austin Zhang <austin_zhang@linux.intel.com>
12
-
* Kent Liu <kent.liu@intel.com>
13
-
*/
14
-
#include <linux/init.h>
15
-
#include <linux/module.h>
16
-
#include <linux/string.h>
17
-
#include <linux/kernel.h>
18
-
#include <crypto/internal/hash.h>
19
-
#include <crypto/internal/simd.h>
20
-
21
-
#include <asm/cpufeatures.h>
22
-
#include <asm/cpu_device_id.h>
23
-
#include <asm/simd.h>
24
-
25
-
#define CHKSUM_BLOCK_SIZE 1
26
-
#define CHKSUM_DIGEST_SIZE 4
27
-
28
-
#define SCALE_F sizeof(unsigned long)
29
-
30
-
#ifdef CONFIG_X86_64
31
-
#define CRC32_INST "crc32q %1, %q0"
32
-
#else
33
-
#define CRC32_INST "crc32l %1, %0"
34
-
#endif
35
-
36
-
#ifdef CONFIG_X86_64
37
-
/*
38
-
* use carryless multiply version of crc32c when buffer
39
-
* size is >= 512 to account
40
-
* for fpu state save/restore overhead.
41
-
*/
42
-
#define CRC32C_PCL_BREAKEVEN 512
43
-
44
-
asmlinkage u32 crc32c_x86_3way(u32 crc, const u8 *buffer, size_t len);
45
-
#endif /* CONFIG_X86_64 */
46
-
47
-
static u32 crc32c_intel_le_hw_byte(u32 crc, unsigned char const *data, size_t length)
48
-
{
49
-
while (length--) {
50
-
asm("crc32b %1, %0"
51
-
: "+r" (crc) : "rm" (*data));
52
-
data++;
53
-
}
54
-
55
-
return crc;
56
-
}
57
-
58
-
static u32 __pure crc32c_intel_le_hw(u32 crc, unsigned char const *p, size_t len)
59
-
{
60
-
unsigned int iquotient = len / SCALE_F;
61
-
unsigned int iremainder = len % SCALE_F;
62
-
unsigned long *ptmp = (unsigned long *)p;
63
-
64
-
while (iquotient--) {
65
-
asm(CRC32_INST
66
-
: "+r" (crc) : "rm" (*ptmp));
67
-
ptmp++;
68
-
}
69
-
70
-
if (iremainder)
71
-
crc = crc32c_intel_le_hw_byte(crc, (unsigned char *)ptmp,
72
-
iremainder);
73
-
74
-
return crc;
75
-
}
76
-
77
-
/*
78
-
* Setting the seed allows arbitrary accumulators and flexible XOR policy
79
-
* If your algorithm starts with ~0, then XOR with ~0 before you set
80
-
* the seed.
81
-
*/
82
-
static int crc32c_intel_setkey(struct crypto_shash *hash, const u8 *key,
83
-
unsigned int keylen)
84
-
{
85
-
u32 *mctx = crypto_shash_ctx(hash);
86
-
87
-
if (keylen != sizeof(u32))
88
-
return -EINVAL;
89
-
*mctx = le32_to_cpup((__le32 *)key);
90
-
return 0;
91
-
}
92
-
93
-
static int crc32c_intel_init(struct shash_desc *desc)
94
-
{
95
-
u32 *mctx = crypto_shash_ctx(desc->tfm);
96
-
u32 *crcp = shash_desc_ctx(desc);
97
-
98
-
*crcp = *mctx;
99
-
100
-
return 0;
101
-
}
102
-
103
-
static int crc32c_intel_update(struct shash_desc *desc, const u8 *data,
104
-
unsigned int len)
105
-
{
106
-
u32 *crcp = shash_desc_ctx(desc);
107
-
108
-
*crcp = crc32c_intel_le_hw(*crcp, data, len);
109
-
return 0;
110
-
}
111
-
112
-
static int __crc32c_intel_finup(u32 *crcp, const u8 *data, unsigned int len,
113
-
u8 *out)
114
-
{
115
-
*(__le32 *)out = ~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len));
116
-
return 0;
117
-
}
118
-
119
-
static int crc32c_intel_finup(struct shash_desc *desc, const u8 *data,
120
-
unsigned int len, u8 *out)
121
-
{
122
-
return __crc32c_intel_finup(shash_desc_ctx(desc), data, len, out);
123
-
}
124
-
125
-
static int crc32c_intel_final(struct shash_desc *desc, u8 *out)
126
-
{
127
-
u32 *crcp = shash_desc_ctx(desc);
128
-
129
-
*(__le32 *)out = ~cpu_to_le32p(crcp);
130
-
return 0;
131
-
}
132
-
133
-
static int crc32c_intel_digest(struct shash_desc *desc, const u8 *data,
134
-
unsigned int len, u8 *out)
135
-
{
136
-
return __crc32c_intel_finup(crypto_shash_ctx(desc->tfm), data, len,
137
-
out);
138
-
}
139
-
140
-
static int crc32c_intel_cra_init(struct crypto_tfm *tfm)
141
-
{
142
-
u32 *key = crypto_tfm_ctx(tfm);
143
-
144
-
*key = ~0;
145
-
146
-
return 0;
147
-
}
148
-
149
-
#ifdef CONFIG_X86_64
150
-
static int crc32c_pcl_intel_update(struct shash_desc *desc, const u8 *data,
151
-
unsigned int len)
152
-
{
153
-
u32 *crcp = shash_desc_ctx(desc);
154
-
155
-
/*
156
-
* use faster PCL version if datasize is large enough to
157
-
* overcome kernel fpu state save/restore overhead
158
-
*/
159
-
if (len >= CRC32C_PCL_BREAKEVEN && crypto_simd_usable()) {
160
-
kernel_fpu_begin();
161
-
*crcp = crc32c_x86_3way(*crcp, data, len);
162
-
kernel_fpu_end();
163
-
} else
164
-
*crcp = crc32c_intel_le_hw(*crcp, data, len);
165
-
return 0;
166
-
}
167
-
168
-
static int __crc32c_pcl_intel_finup(u32 *crcp, const u8 *data, unsigned int len,
169
-
u8 *out)
170
-
{
171
-
if (len >= CRC32C_PCL_BREAKEVEN && crypto_simd_usable()) {
172
-
kernel_fpu_begin();
173
-
*(__le32 *)out = ~cpu_to_le32(crc32c_x86_3way(*crcp, data, len));
174
-
kernel_fpu_end();
175
-
} else
176
-
*(__le32 *)out =
177
-
~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len));
178
-
return 0;
179
-
}
180
-
181
-
static int crc32c_pcl_intel_finup(struct shash_desc *desc, const u8 *data,
182
-
unsigned int len, u8 *out)
183
-
{
184
-
return __crc32c_pcl_intel_finup(shash_desc_ctx(desc), data, len, out);
185
-
}
186
-
187
-
static int crc32c_pcl_intel_digest(struct shash_desc *desc, const u8 *data,
188
-
unsigned int len, u8 *out)
189
-
{
190
-
return __crc32c_pcl_intel_finup(crypto_shash_ctx(desc->tfm), data, len,
191
-
out);
192
-
}
193
-
#endif /* CONFIG_X86_64 */
194
-
195
-
static struct shash_alg alg = {
196
-
.setkey = crc32c_intel_setkey,
197
-
.init = crc32c_intel_init,
198
-
.update = crc32c_intel_update,
199
-
.final = crc32c_intel_final,
200
-
.finup = crc32c_intel_finup,
201
-
.digest = crc32c_intel_digest,
202
-
.descsize = sizeof(u32),
203
-
.digestsize = CHKSUM_DIGEST_SIZE,
204
-
.base = {
205
-
.cra_name = "crc32c",
206
-
.cra_driver_name = "crc32c-intel",
207
-
.cra_priority = 200,
208
-
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
209
-
.cra_blocksize = CHKSUM_BLOCK_SIZE,
210
-
.cra_ctxsize = sizeof(u32),
211
-
.cra_module = THIS_MODULE,
212
-
.cra_init = crc32c_intel_cra_init,
213
-
}
214
-
};
215
-
216
-
static const struct x86_cpu_id crc32c_cpu_id[] = {
217
-
X86_MATCH_FEATURE(X86_FEATURE_XMM4_2, NULL),
218
-
{}
219
-
};
220
-
MODULE_DEVICE_TABLE(x86cpu, crc32c_cpu_id);
221
-
222
-
static int __init crc32c_intel_mod_init(void)
223
-
{
224
-
if (!x86_match_cpu(crc32c_cpu_id))
225
-
return -ENODEV;
226
-
#ifdef CONFIG_X86_64
227
-
if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
228
-
alg.update = crc32c_pcl_intel_update;
229
-
alg.finup = crc32c_pcl_intel_finup;
230
-
alg.digest = crc32c_pcl_intel_digest;
231
-
}
232
-
#endif
233
-
return crypto_register_shash(&alg);
234
-
}
235
-
236
-
static void __exit crc32c_intel_mod_fini(void)
237
-
{
238
-
crypto_unregister_shash(&alg);
239
-
}
240
-
241
-
module_init(crc32c_intel_mod_init);
242
-
module_exit(crc32c_intel_mod_fini);
243
-
244
-
MODULE_AUTHOR("Austin Zhang <austin.zhang@intel.com>, Kent Liu <kent.liu@intel.com>");
245
-
MODULE_DESCRIPTION("CRC32c (Castagnoli) optimization using Intel Hardware.");
246
-
MODULE_LICENSE("GPL");
247
-
248
-
MODULE_ALIAS_CRYPTO("crc32c");
249
-
MODULE_ALIAS_CRYPTO("crc32c-intel");
arch/x86/crypto/crc32c-pcl-intel-asm_64.S
arch/x86/lib/crc32c-3way.S
arch/x86/crypto/crc32c-pcl-intel-asm_64.S
arch/x86/lib/crc32c-3way.S
+4
arch/x86/lib/Makefile
+4
arch/x86/lib/Makefile
···
38
38
lib-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
39
39
lib-$(CONFIG_MITIGATION_RETPOLINE) += retpoline.o
40
40
41
+
obj-$(CONFIG_CRC32_ARCH) += crc32-x86.o
42
+
crc32-x86-y := crc32-glue.o crc32-pclmul.o
43
+
crc32-x86-$(CONFIG_64BIT) += crc32c-3way.o
44
+
41
45
obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
42
46
obj-y += iomem.o
43
47
+124
arch/x86/lib/crc32-glue.c
+124
arch/x86/lib/crc32-glue.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/*
3
+
* x86-optimized CRC32 functions
4
+
*
5
+
* Copyright (C) 2008 Intel Corporation
6
+
* Copyright 2012 Xyratex Technology Limited
7
+
* Copyright 2024 Google LLC
8
+
*/
9
+
10
+
#include <asm/cpufeatures.h>
11
+
#include <asm/simd.h>
12
+
#include <crypto/internal/simd.h>
13
+
#include <linux/crc32.h>
14
+
#include <linux/linkage.h>
15
+
#include <linux/module.h>
16
+
17
+
/* minimum size of buffer for crc32_pclmul_le_16 */
18
+
#define CRC32_PCLMUL_MIN_LEN 64
19
+
20
+
static DEFINE_STATIC_KEY_FALSE(have_crc32);
21
+
static DEFINE_STATIC_KEY_FALSE(have_pclmulqdq);
22
+
23
+
u32 crc32_pclmul_le_16(u32 crc, const u8 *buffer, size_t len);
24
+
25
+
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
26
+
{
27
+
if (len >= CRC32_PCLMUL_MIN_LEN + 15 &&
28
+
static_branch_likely(&have_pclmulqdq) && crypto_simd_usable()) {
29
+
size_t n = -(uintptr_t)p & 15;
30
+
31
+
/* align p to 16-byte boundary */
32
+
if (n) {
33
+
crc = crc32_le_base(crc, p, n);
34
+
p += n;
35
+
len -= n;
36
+
}
37
+
n = round_down(len, 16);
38
+
kernel_fpu_begin();
39
+
crc = crc32_pclmul_le_16(crc, p, n);
40
+
kernel_fpu_end();
41
+
p += n;
42
+
len -= n;
43
+
}
44
+
if (len)
45
+
crc = crc32_le_base(crc, p, len);
46
+
return crc;
47
+
}
48
+
EXPORT_SYMBOL(crc32_le_arch);
49
+
50
+
#ifdef CONFIG_X86_64
51
+
#define CRC32_INST "crc32q %1, %q0"
52
+
#else
53
+
#define CRC32_INST "crc32l %1, %0"
54
+
#endif
55
+
56
+
/*
57
+
* Use carryless multiply version of crc32c when buffer size is >= 512 to
58
+
* account for FPU state save/restore overhead.
59
+
*/
60
+
#define CRC32C_PCLMUL_BREAKEVEN 512
61
+
62
+
asmlinkage u32 crc32c_x86_3way(u32 crc, const u8 *buffer, size_t len);
63
+
64
+
u32 crc32c_le_arch(u32 crc, const u8 *p, size_t len)
65
+
{
66
+
size_t num_longs;
67
+
68
+
if (!static_branch_likely(&have_crc32))
69
+
return crc32c_le_base(crc, p, len);
70
+
71
+
if (IS_ENABLED(CONFIG_X86_64) && len >= CRC32C_PCLMUL_BREAKEVEN &&
72
+
static_branch_likely(&have_pclmulqdq) && crypto_simd_usable()) {
73
+
kernel_fpu_begin();
74
+
crc = crc32c_x86_3way(crc, p, len);
75
+
kernel_fpu_end();
76
+
return crc;
77
+
}
78
+
79
+
for (num_longs = len / sizeof(unsigned long);
80
+
num_longs != 0; num_longs--, p += sizeof(unsigned long))
81
+
asm(CRC32_INST : "+r" (crc) : "rm" (*(unsigned long *)p));
82
+
83
+
for (len %= sizeof(unsigned long); len; len--, p++)
84
+
asm("crc32b %1, %0" : "+r" (crc) : "rm" (*p));
85
+
86
+
return crc;
87
+
}
88
+
EXPORT_SYMBOL(crc32c_le_arch);
89
+
90
+
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
91
+
{
92
+
return crc32_be_base(crc, p, len);
93
+
}
94
+
EXPORT_SYMBOL(crc32_be_arch);
95
+
96
+
static int __init crc32_x86_init(void)
97
+
{
98
+
if (boot_cpu_has(X86_FEATURE_XMM4_2))
99
+
static_branch_enable(&have_crc32);
100
+
if (boot_cpu_has(X86_FEATURE_PCLMULQDQ))
101
+
static_branch_enable(&have_pclmulqdq);
102
+
return 0;
103
+
}
104
+
arch_initcall(crc32_x86_init);
105
+
106
+
static void __exit crc32_x86_exit(void)
107
+
{
108
+
}
109
+
module_exit(crc32_x86_exit);
110
+
111
+
u32 crc32_optimizations(void)
112
+
{
113
+
u32 optimizations = 0;
114
+
115
+
if (static_key_enabled(&have_crc32))
116
+
optimizations |= CRC32C_OPTIMIZATION;
117
+
if (static_key_enabled(&have_pclmulqdq))
118
+
optimizations |= CRC32_LE_OPTIMIZATION;
119
+
return optimizations;
120
+
}
121
+
EXPORT_SYMBOL(crc32_optimizations);
122
+
123
+
MODULE_DESCRIPTION("x86-optimized CRC32 functions");
124
+
MODULE_LICENSE("GPL");