Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'crc-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux
Pull CRC updates from Eric Biggers:
- Reorganize the architecture-optimized CRC32 and CRC-T10DIF code to be
directly accessible via the library API, instead of requiring the
crypto API. This is much simpler and more efficient.
- Convert some users such as ext4 to use the CRC32 library API instead
of the crypto API. More conversions like this will come later.
- Add a KUnit test that tests and benchmarks multiple CRC variants.
Remove older, less-comprehensive tests that are made redundant by
this.
- Add an entry to MAINTAINERS for the kernel's CRC library code. I'm
volunteering to maintain it. I have additional cleanups and
optimizations planned for future cycles.
* tag 'crc-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux: (31 commits)
MAINTAINERS: add entry for CRC library
powerpc/crc: delete obsolete crc-vpmsum_test.c
lib/crc32test: delete obsolete crc32test.c
lib/crc16_kunit: delete obsolete crc16_kunit.c
lib/crc_kunit.c: add KUnit test suite for CRC library functions
powerpc/crc-t10dif: expose CRC-T10DIF function through lib
arm64/crc-t10dif: expose CRC-T10DIF function through lib
arm/crc-t10dif: expose CRC-T10DIF function through lib
x86/crc-t10dif: expose CRC-T10DIF function through lib
crypto: crct10dif - expose arch-optimized lib function
lib/crc-t10dif: add support for arch overrides
lib/crc-t10dif: stop wrapping the crypto API
scsi: target: iscsi: switch to using the crc32c library
f2fs: switch to using the crc32 library
jbd2: switch to using the crc32c library
ext4: switch to using the crc32c library
lib/crc32: make crc32c() go directly to lib
bcachefs: Explicitly select CRYPTO from BCACHEFS_FS
x86/crc32: expose CRC32 functions through lib
x86/crc32: update prototype for crc32_pclmul_le_16()
...
+2035
-4555
+11
MAINTAINERS
+11
MAINTAINERS
···
6092
6092
F: Documentation/filesystems/cramfs.rst
6093
6093
F: fs/cramfs/
6094
6094
6095
+
CRC LIBRARY
6096
+
M: Eric Biggers <ebiggers@kernel.org>
6097
+
R: Ard Biesheuvel <ardb@kernel.org>
6098
+
L: linux-crypto@vger.kernel.org
6099
+
S: Maintained
6100
+
T: git https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git crc-next
6101
+
F: Documentation/staging/crc*
6102
+
F: arch/*/lib/crc*
6103
+
F: include/linux/crc*
6104
+
F: lib/crc*
6105
+
6095
6106
CREATIVE SB0540
6096
6107
M: Bastien Nocera <hadess@hadess.net>
6097
6108
L: linux-input@vger.kernel.org
+2
arch/arm/Kconfig
+2
arch/arm/Kconfig
···
7
7
select ARCH_HAS_BINFMT_FLAT
8
8
select ARCH_HAS_CPU_CACHE_ALIASING
9
9
select ARCH_HAS_CPU_FINALIZE_INIT if MMU
10
+
select ARCH_HAS_CRC32 if KERNEL_MODE_NEON
11
+
select ARCH_HAS_CRC_T10DIF if KERNEL_MODE_NEON
10
12
select ARCH_HAS_CURRENT_STACK_POINTER
11
13
select ARCH_HAS_DEBUG_VIRTUAL if MMU
12
14
select ARCH_HAS_DMA_ALLOC if MMU
-1
arch/arm/configs/milbeaut_m10v_defconfig
-1
arch/arm/configs/milbeaut_m10v_defconfig
-1
arch/arm/configs/multi_v7_defconfig
-1
arch/arm/configs/multi_v7_defconfig
-25
arch/arm/crypto/Kconfig
-25
arch/arm/crypto/Kconfig
···
222
222
Architecture: arm using:
223
223
- NEON (Advanced SIMD) extensions
224
224
225
-
config CRYPTO_CRC32_ARM_CE
226
-
tristate "CRC32C and CRC32"
227
-
depends on KERNEL_MODE_NEON
228
-
depends on CRC32
229
-
select CRYPTO_HASH
230
-
help
231
-
CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720)
232
-
and CRC32 CRC algorithm (IEEE 802.3)
233
-
234
-
Architecture: arm using:
235
-
- CRC and/or PMULL instructions
236
-
237
-
Drivers: crc32-arm-ce and crc32c-arm-ce
238
-
239
-
config CRYPTO_CRCT10DIF_ARM_CE
240
-
tristate "CRCT10DIF"
241
-
depends on KERNEL_MODE_NEON
242
-
depends on CRC_T10DIF
243
-
select CRYPTO_HASH
244
-
help
245
-
CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF)
246
-
247
-
Architecture: arm using:
248
-
- PMULL (Polynomial Multiply Long) instructions
249
-
250
225
endmenu
251
226
-4
arch/arm/crypto/Makefile
-4
arch/arm/crypto/Makefile
···
20
20
obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o
21
21
obj-$(CONFIG_CRYPTO_SHA2_ARM_CE) += sha2-arm-ce.o
22
22
obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) += ghash-arm-ce.o
23
-
obj-$(CONFIG_CRYPTO_CRCT10DIF_ARM_CE) += crct10dif-arm-ce.o
24
-
obj-$(CONFIG_CRYPTO_CRC32_ARM_CE) += crc32-arm-ce.o
25
23
26
24
aes-arm-y := aes-cipher-core.o aes-cipher-glue.o
27
25
aes-arm-bs-y := aes-neonbs-core.o aes-neonbs-glue.o
···
35
37
sha2-arm-ce-y := sha2-ce-core.o sha2-ce-glue.o
36
38
aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o
37
39
ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o
38
-
crct10dif-arm-ce-y := crct10dif-ce-core.o crct10dif-ce-glue.o
39
-
crc32-arm-ce-y:= crc32-ce-core.o crc32-ce-glue.o
40
40
chacha-neon-y := chacha-scalar-core.o chacha-glue.o
41
41
chacha-neon-$(CONFIG_KERNEL_MODE_NEON) += chacha-neon-core.o
42
42
poly1305-arm-y := poly1305-core.o poly1305-glue.o
+2
-3
arch/arm/crypto/crc32-ce-core.S
arch/arm/lib/crc32-core.S
+2
-3
arch/arm/crypto/crc32-ce-core.S
arch/arm/lib/crc32-core.S
···
48
48
*/
49
49
50
50
#include <linux/linkage.h>
51
-
#include <linux/cfi_types.h>
52
51
#include <asm/assembler.h>
53
52
54
53
.text
···
296
297
.endm
297
298
298
299
.align 5
299
-
SYM_TYPED_FUNC_START(crc32_armv8_le)
300
+
SYM_FUNC_START(crc32_armv8_le)
300
301
__crc32
301
302
SYM_FUNC_END(crc32_armv8_le)
302
303
303
304
.align 5
304
-
SYM_TYPED_FUNC_START(crc32c_armv8_le)
305
+
SYM_FUNC_START(crc32c_armv8_le)
305
306
__crc32 c
306
307
SYM_FUNC_END(crc32c_armv8_le)
-247
arch/arm/crypto/crc32-ce-glue.c
-247
arch/arm/crypto/crc32-ce-glue.c
···
1
-
// SPDX-License-Identifier: GPL-2.0-only
2
-
/*
3
-
* Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions
4
-
*
5
-
* Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
6
-
*/
7
-
8
-
#include <linux/cpufeature.h>
9
-
#include <linux/crc32.h>
10
-
#include <linux/init.h>
11
-
#include <linux/kernel.h>
12
-
#include <linux/module.h>
13
-
#include <linux/string.h>
14
-
15
-
#include <crypto/internal/hash.h>
16
-
#include <crypto/internal/simd.h>
17
-
18
-
#include <asm/hwcap.h>
19
-
#include <asm/neon.h>
20
-
#include <asm/simd.h>
21
-
#include <linux/unaligned.h>
22
-
23
-
#define PMULL_MIN_LEN 64L /* minimum size of buffer
24
-
* for crc32_pmull_le_16 */
25
-
#define SCALE_F 16L /* size of NEON register */
26
-
27
-
asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc);
28
-
asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len);
29
-
30
-
asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc);
31
-
asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len);
32
-
33
-
static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], u32 len);
34
-
static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], u32 len);
35
-
36
-
static int crc32_cra_init(struct crypto_tfm *tfm)
37
-
{
38
-
u32 *key = crypto_tfm_ctx(tfm);
39
-
40
-
*key = 0;
41
-
return 0;
42
-
}
43
-
44
-
static int crc32c_cra_init(struct crypto_tfm *tfm)
45
-
{
46
-
u32 *key = crypto_tfm_ctx(tfm);
47
-
48
-
*key = ~0;
49
-
return 0;
50
-
}
51
-
52
-
static int crc32_setkey(struct crypto_shash *hash, const u8 *key,
53
-
unsigned int keylen)
54
-
{
55
-
u32 *mctx = crypto_shash_ctx(hash);
56
-
57
-
if (keylen != sizeof(u32))
58
-
return -EINVAL;
59
-
*mctx = le32_to_cpup((__le32 *)key);
60
-
return 0;
61
-
}
62
-
63
-
static int crc32_init(struct shash_desc *desc)
64
-
{
65
-
u32 *mctx = crypto_shash_ctx(desc->tfm);
66
-
u32 *crc = shash_desc_ctx(desc);
67
-
68
-
*crc = *mctx;
69
-
return 0;
70
-
}
71
-
72
-
static int crc32_update(struct shash_desc *desc, const u8 *data,
73
-
unsigned int length)
74
-
{
75
-
u32 *crc = shash_desc_ctx(desc);
76
-
77
-
*crc = crc32_armv8_le(*crc, data, length);
78
-
return 0;
79
-
}
80
-
81
-
static int crc32c_update(struct shash_desc *desc, const u8 *data,
82
-
unsigned int length)
83
-
{
84
-
u32 *crc = shash_desc_ctx(desc);
85
-
86
-
*crc = crc32c_armv8_le(*crc, data, length);
87
-
return 0;
88
-
}
89
-
90
-
static int crc32_final(struct shash_desc *desc, u8 *out)
91
-
{
92
-
u32 *crc = shash_desc_ctx(desc);
93
-
94
-
put_unaligned_le32(*crc, out);
95
-
return 0;
96
-
}
97
-
98
-
static int crc32c_final(struct shash_desc *desc, u8 *out)
99
-
{
100
-
u32 *crc = shash_desc_ctx(desc);
101
-
102
-
put_unaligned_le32(~*crc, out);
103
-
return 0;
104
-
}
105
-
106
-
static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
107
-
unsigned int length)
108
-
{
109
-
u32 *crc = shash_desc_ctx(desc);
110
-
unsigned int l;
111
-
112
-
if (crypto_simd_usable()) {
113
-
if ((u32)data % SCALE_F) {
114
-
l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F));
115
-
116
-
*crc = fallback_crc32(*crc, data, l);
117
-
118
-
data += l;
119
-
length -= l;
120
-
}
121
-
122
-
if (length >= PMULL_MIN_LEN) {
123
-
l = round_down(length, SCALE_F);
124
-
125
-
kernel_neon_begin();
126
-
*crc = crc32_pmull_le(data, l, *crc);
127
-
kernel_neon_end();
128
-
129
-
data += l;
130
-
length -= l;
131
-
}
132
-
}
133
-
134
-
if (length > 0)
135
-
*crc = fallback_crc32(*crc, data, length);
136
-
137
-
return 0;
138
-
}
139
-
140
-
static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data,
141
-
unsigned int length)
142
-
{
143
-
u32 *crc = shash_desc_ctx(desc);
144
-
unsigned int l;
145
-
146
-
if (crypto_simd_usable()) {
147
-
if ((u32)data % SCALE_F) {
148
-
l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F));
149
-
150
-
*crc = fallback_crc32c(*crc, data, l);
151
-
152
-
data += l;
153
-
length -= l;
154
-
}
155
-
156
-
if (length >= PMULL_MIN_LEN) {
157
-
l = round_down(length, SCALE_F);
158
-
159
-
kernel_neon_begin();
160
-
*crc = crc32c_pmull_le(data, l, *crc);
161
-
kernel_neon_end();
162
-
163
-
data += l;
164
-
length -= l;
165
-
}
166
-
}
167
-
168
-
if (length > 0)
169
-
*crc = fallback_crc32c(*crc, data, length);
170
-
171
-
return 0;
172
-
}
173
-
174
-
static struct shash_alg crc32_pmull_algs[] = { {
175
-
.setkey = crc32_setkey,
176
-
.init = crc32_init,
177
-
.update = crc32_update,
178
-
.final = crc32_final,
179
-
.descsize = sizeof(u32),
180
-
.digestsize = sizeof(u32),
181
-
182
-
.base.cra_ctxsize = sizeof(u32),
183
-
.base.cra_init = crc32_cra_init,
184
-
.base.cra_name = "crc32",
185
-
.base.cra_driver_name = "crc32-arm-ce",
186
-
.base.cra_priority = 200,
187
-
.base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
188
-
.base.cra_blocksize = 1,
189
-
.base.cra_module = THIS_MODULE,
190
-
}, {
191
-
.setkey = crc32_setkey,
192
-
.init = crc32_init,
193
-
.update = crc32c_update,
194
-
.final = crc32c_final,
195
-
.descsize = sizeof(u32),
196
-
.digestsize = sizeof(u32),
197
-
198
-
.base.cra_ctxsize = sizeof(u32),
199
-
.base.cra_init = crc32c_cra_init,
200
-
.base.cra_name = "crc32c",
201
-
.base.cra_driver_name = "crc32c-arm-ce",
202
-
.base.cra_priority = 200,
203
-
.base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
204
-
.base.cra_blocksize = 1,
205
-
.base.cra_module = THIS_MODULE,
206
-
} };
207
-
208
-
static int __init crc32_pmull_mod_init(void)
209
-
{
210
-
if (elf_hwcap2 & HWCAP2_PMULL) {
211
-
crc32_pmull_algs[0].update = crc32_pmull_update;
212
-
crc32_pmull_algs[1].update = crc32c_pmull_update;
213
-
214
-
if (elf_hwcap2 & HWCAP2_CRC32) {
215
-
fallback_crc32 = crc32_armv8_le;
216
-
fallback_crc32c = crc32c_armv8_le;
217
-
} else {
218
-
fallback_crc32 = crc32_le;
219
-
fallback_crc32c = __crc32c_le;
220
-
}
221
-
} else if (!(elf_hwcap2 & HWCAP2_CRC32)) {
222
-
return -ENODEV;
223
-
}
224
-
225
-
return crypto_register_shashes(crc32_pmull_algs,
226
-
ARRAY_SIZE(crc32_pmull_algs));
227
-
}
228
-
229
-
static void __exit crc32_pmull_mod_exit(void)
230
-
{
231
-
crypto_unregister_shashes(crc32_pmull_algs,
232
-
ARRAY_SIZE(crc32_pmull_algs));
233
-
}
234
-
235
-
static const struct cpu_feature __maybe_unused crc32_cpu_feature[] = {
236
-
{ cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { }
237
-
};
238
-
MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature);
239
-
240
-
module_init(crc32_pmull_mod_init);
241
-
module_exit(crc32_pmull_mod_exit);
242
-
243
-
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
244
-
MODULE_DESCRIPTION("Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions");
245
-
MODULE_LICENSE("GPL v2");
246
-
MODULE_ALIAS_CRYPTO("crc32");
247
-
MODULE_ALIAS_CRYPTO("crc32c");
arch/arm/crypto/crct10dif-ce-core.S
arch/arm/lib/crc-t10dif-core.S
arch/arm/crypto/crct10dif-ce-core.S
arch/arm/lib/crc-t10dif-core.S
-124
arch/arm/crypto/crct10dif-ce-glue.c
-124
arch/arm/crypto/crct10dif-ce-glue.c
···
1
-
// SPDX-License-Identifier: GPL-2.0-only
2
-
/*
3
-
* Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions
4
-
*
5
-
* Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
6
-
*/
7
-
8
-
#include <linux/crc-t10dif.h>
9
-
#include <linux/init.h>
10
-
#include <linux/kernel.h>
11
-
#include <linux/module.h>
12
-
#include <linux/string.h>
13
-
14
-
#include <crypto/internal/hash.h>
15
-
#include <crypto/internal/simd.h>
16
-
17
-
#include <asm/neon.h>
18
-
#include <asm/simd.h>
19
-
20
-
#define CRC_T10DIF_PMULL_CHUNK_SIZE 16U
21
-
22
-
asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len);
23
-
asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len,
24
-
u8 out[16]);
25
-
26
-
static int crct10dif_init(struct shash_desc *desc)
27
-
{
28
-
u16 *crc = shash_desc_ctx(desc);
29
-
30
-
*crc = 0;
31
-
return 0;
32
-
}
33
-
34
-
static int crct10dif_update_ce(struct shash_desc *desc, const u8 *data,
35
-
unsigned int length)
36
-
{
37
-
u16 *crc = shash_desc_ctx(desc);
38
-
39
-
if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
40
-
kernel_neon_begin();
41
-
*crc = crc_t10dif_pmull64(*crc, data, length);
42
-
kernel_neon_end();
43
-
} else {
44
-
*crc = crc_t10dif_generic(*crc, data, length);
45
-
}
46
-
47
-
return 0;
48
-
}
49
-
50
-
static int crct10dif_update_neon(struct shash_desc *desc, const u8 *data,
51
-
unsigned int length)
52
-
{
53
-
u16 *crcp = shash_desc_ctx(desc);
54
-
u8 buf[16] __aligned(16);
55
-
u16 crc = *crcp;
56
-
57
-
if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
58
-
kernel_neon_begin();
59
-
crc_t10dif_pmull8(crc, data, length, buf);
60
-
kernel_neon_end();
61
-
62
-
crc = 0;
63
-
data = buf;
64
-
length = sizeof(buf);
65
-
}
66
-
67
-
*crcp = crc_t10dif_generic(crc, data, length);
68
-
return 0;
69
-
}
70
-
71
-
static int crct10dif_final(struct shash_desc *desc, u8 *out)
72
-
{
73
-
u16 *crc = shash_desc_ctx(desc);
74
-
75
-
*(u16 *)out = *crc;
76
-
return 0;
77
-
}
78
-
79
-
static struct shash_alg algs[] = {{
80
-
.digestsize = CRC_T10DIF_DIGEST_SIZE,
81
-
.init = crct10dif_init,
82
-
.update = crct10dif_update_neon,
83
-
.final = crct10dif_final,
84
-
.descsize = CRC_T10DIF_DIGEST_SIZE,
85
-
86
-
.base.cra_name = "crct10dif",
87
-
.base.cra_driver_name = "crct10dif-arm-neon",
88
-
.base.cra_priority = 150,
89
-
.base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
90
-
.base.cra_module = THIS_MODULE,
91
-
}, {
92
-
.digestsize = CRC_T10DIF_DIGEST_SIZE,
93
-
.init = crct10dif_init,
94
-
.update = crct10dif_update_ce,
95
-
.final = crct10dif_final,
96
-
.descsize = CRC_T10DIF_DIGEST_SIZE,
97
-
98
-
.base.cra_name = "crct10dif",
99
-
.base.cra_driver_name = "crct10dif-arm-ce",
100
-
.base.cra_priority = 200,
101
-
.base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
102
-
.base.cra_module = THIS_MODULE,
103
-
}};
104
-
105
-
static int __init crc_t10dif_mod_init(void)
106
-
{
107
-
if (!(elf_hwcap & HWCAP_NEON))
108
-
return -ENODEV;
109
-
110
-
return crypto_register_shashes(algs, 1 + !!(elf_hwcap2 & HWCAP2_PMULL));
111
-
}
112
-
113
-
static void __exit crc_t10dif_mod_exit(void)
114
-
{
115
-
crypto_unregister_shashes(algs, 1 + !!(elf_hwcap2 & HWCAP2_PMULL));
116
-
}
117
-
118
-
module_init(crc_t10dif_mod_init);
119
-
module_exit(crc_t10dif_mod_exit);
120
-
121
-
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
122
-
MODULE_DESCRIPTION("Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions");
123
-
MODULE_LICENSE("GPL v2");
124
-
MODULE_ALIAS_CRYPTO("crct10dif");
+6
arch/arm/lib/Makefile
+6
arch/arm/lib/Makefile
···
45
45
endif
46
46
47
47
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
48
+
49
+
obj-$(CONFIG_CRC32_ARCH) += crc32-arm.o
50
+
crc32-arm-y := crc32-glue.o crc32-core.o
51
+
52
+
obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-arm.o
53
+
crc-t10dif-arm-y := crc-t10dif-glue.o crc-t10dif-core.o
+80
arch/arm/lib/crc-t10dif-glue.c
+80
arch/arm/lib/crc-t10dif-glue.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/*
3
+
* Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions
4
+
*
5
+
* Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
6
+
*/
7
+
8
+
#include <linux/crc-t10dif.h>
9
+
#include <linux/init.h>
10
+
#include <linux/kernel.h>
11
+
#include <linux/module.h>
12
+
#include <linux/string.h>
13
+
14
+
#include <crypto/internal/simd.h>
15
+
16
+
#include <asm/neon.h>
17
+
#include <asm/simd.h>
18
+
19
+
static DEFINE_STATIC_KEY_FALSE(have_neon);
20
+
static DEFINE_STATIC_KEY_FALSE(have_pmull);
21
+
22
+
#define CRC_T10DIF_PMULL_CHUNK_SIZE 16U
23
+
24
+
asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len);
25
+
asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len,
26
+
u8 out[16]);
27
+
28
+
u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
29
+
{
30
+
if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) {
31
+
if (static_branch_likely(&have_pmull)) {
32
+
if (crypto_simd_usable()) {
33
+
kernel_neon_begin();
34
+
crc = crc_t10dif_pmull64(crc, data, length);
35
+
kernel_neon_end();
36
+
return crc;
37
+
}
38
+
} else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE &&
39
+
static_branch_likely(&have_neon) &&
40
+
crypto_simd_usable()) {
41
+
u8 buf[16] __aligned(16);
42
+
43
+
kernel_neon_begin();
44
+
crc_t10dif_pmull8(crc, data, length, buf);
45
+
kernel_neon_end();
46
+
47
+
crc = 0;
48
+
data = buf;
49
+
length = sizeof(buf);
50
+
}
51
+
}
52
+
return crc_t10dif_generic(crc, data, length);
53
+
}
54
+
EXPORT_SYMBOL(crc_t10dif_arch);
55
+
56
+
static int __init crc_t10dif_arm_init(void)
57
+
{
58
+
if (elf_hwcap & HWCAP_NEON) {
59
+
static_branch_enable(&have_neon);
60
+
if (elf_hwcap2 & HWCAP2_PMULL)
61
+
static_branch_enable(&have_pmull);
62
+
}
63
+
return 0;
64
+
}
65
+
arch_initcall(crc_t10dif_arm_init);
66
+
67
+
static void __exit crc_t10dif_arm_exit(void)
68
+
{
69
+
}
70
+
module_exit(crc_t10dif_arm_exit);
71
+
72
+
bool crc_t10dif_is_optimized(void)
73
+
{
74
+
return static_key_enabled(&have_neon);
75
+
}
76
+
EXPORT_SYMBOL(crc_t10dif_is_optimized);
77
+
78
+
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
79
+
MODULE_DESCRIPTION("Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions");
80
+
MODULE_LICENSE("GPL v2");
+123
arch/arm/lib/crc32-glue.c
+123
arch/arm/lib/crc32-glue.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/*
3
+
* Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions
4
+
*
5
+
* Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
6
+
*/
7
+
8
+
#include <linux/cpufeature.h>
9
+
#include <linux/crc32.h>
10
+
#include <linux/init.h>
11
+
#include <linux/kernel.h>
12
+
#include <linux/module.h>
13
+
#include <linux/string.h>
14
+
15
+
#include <crypto/internal/simd.h>
16
+
17
+
#include <asm/hwcap.h>
18
+
#include <asm/neon.h>
19
+
#include <asm/simd.h>
20
+
21
+
static DEFINE_STATIC_KEY_FALSE(have_crc32);
22
+
static DEFINE_STATIC_KEY_FALSE(have_pmull);
23
+
24
+
#define PMULL_MIN_LEN 64 /* min size of buffer for pmull functions */
25
+
26
+
asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc);
27
+
asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len);
28
+
29
+
asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc);
30
+
asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len);
31
+
32
+
static u32 crc32_le_scalar(u32 crc, const u8 *p, size_t len)
33
+
{
34
+
if (static_branch_likely(&have_crc32))
35
+
return crc32_armv8_le(crc, p, len);
36
+
return crc32_le_base(crc, p, len);
37
+
}
38
+
39
+
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
40
+
{
41
+
if (len >= PMULL_MIN_LEN + 15 &&
42
+
static_branch_likely(&have_pmull) && crypto_simd_usable()) {
43
+
size_t n = -(uintptr_t)p & 15;
44
+
45
+
/* align p to 16-byte boundary */
46
+
if (n) {
47
+
crc = crc32_le_scalar(crc, p, n);
48
+
p += n;
49
+
len -= n;
50
+
}
51
+
n = round_down(len, 16);
52
+
kernel_neon_begin();
53
+
crc = crc32_pmull_le(p, n, crc);
54
+
kernel_neon_end();
55
+
p += n;
56
+
len -= n;
57
+
}
58
+
return crc32_le_scalar(crc, p, len);
59
+
}
60
+
EXPORT_SYMBOL(crc32_le_arch);
61
+
62
+
static u32 crc32c_le_scalar(u32 crc, const u8 *p, size_t len)
63
+
{
64
+
if (static_branch_likely(&have_crc32))
65
+
return crc32c_armv8_le(crc, p, len);
66
+
return crc32c_le_base(crc, p, len);
67
+
}
68
+
69
+
u32 crc32c_le_arch(u32 crc, const u8 *p, size_t len)
70
+
{
71
+
if (len >= PMULL_MIN_LEN + 15 &&
72
+
static_branch_likely(&have_pmull) && crypto_simd_usable()) {
73
+
size_t n = -(uintptr_t)p & 15;
74
+
75
+
/* align p to 16-byte boundary */
76
+
if (n) {
77
+
crc = crc32c_le_scalar(crc, p, n);
78
+
p += n;
79
+
len -= n;
80
+
}
81
+
n = round_down(len, 16);
82
+
kernel_neon_begin();
83
+
crc = crc32c_pmull_le(p, n, crc);
84
+
kernel_neon_end();
85
+
p += n;
86
+
len -= n;
87
+
}
88
+
return crc32c_le_scalar(crc, p, len);
89
+
}
90
+
EXPORT_SYMBOL(crc32c_le_arch);
91
+
92
+
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
93
+
{
94
+
return crc32_be_base(crc, p, len);
95
+
}
96
+
EXPORT_SYMBOL(crc32_be_arch);
97
+
98
+
static int __init crc32_arm_init(void)
99
+
{
100
+
if (elf_hwcap2 & HWCAP2_CRC32)
101
+
static_branch_enable(&have_crc32);
102
+
if (elf_hwcap2 & HWCAP2_PMULL)
103
+
static_branch_enable(&have_pmull);
104
+
return 0;
105
+
}
106
+
arch_initcall(crc32_arm_init);
107
+
108
+
static void __exit crc32_arm_exit(void)
109
+
{
110
+
}
111
+
module_exit(crc32_arm_exit);
112
+
113
+
u32 crc32_optimizations(void)
114
+
{
115
+
if (elf_hwcap2 & (HWCAP2_CRC32 | HWCAP2_PMULL))
116
+
return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION;
117
+
return 0;
118
+
}
119
+
EXPORT_SYMBOL(crc32_optimizations);
120
+
121
+
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
122
+
MODULE_DESCRIPTION("Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions");
123
+
MODULE_LICENSE("GPL v2");
+2
arch/arm64/Kconfig
+2
arch/arm64/Kconfig
···
21
21
select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
22
22
select ARCH_HAS_CACHE_LINE_SIZE
23
23
select ARCH_HAS_CC_PLATFORM
24
+
select ARCH_HAS_CRC32
25
+
select ARCH_HAS_CRC_T10DIF if KERNEL_MODE_NEON
24
26
select ARCH_HAS_CURRENT_STACK_POINTER
25
27
select ARCH_HAS_DEBUG_VIRTUAL
26
28
select ARCH_HAS_DEBUG_VM_PGTABLE
-1
arch/arm64/configs/defconfig
-1
arch/arm64/configs/defconfig
-10
arch/arm64/crypto/Kconfig
-10
arch/arm64/crypto/Kconfig
···
312
312
- PMULL (Polynomial Multiply Long) instructions
313
313
- NEON (Advanced SIMD) extensions
314
314
315
-
config CRYPTO_CRCT10DIF_ARM64_CE
316
-
tristate "CRCT10DIF (PMULL)"
317
-
depends on KERNEL_MODE_NEON && CRC_T10DIF
318
-
select CRYPTO_HASH
319
-
help
320
-
CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF)
321
-
322
-
Architecture: arm64 using
323
-
- PMULL (Polynomial Multiply Long) instructions
324
-
325
315
endmenu
326
316
-3
arch/arm64/crypto/Makefile
-3
arch/arm64/crypto/Makefile
···
44
44
obj-$(CONFIG_CRYPTO_POLYVAL_ARM64_CE) += polyval-ce.o
45
45
polyval-ce-y := polyval-ce-glue.o polyval-ce-core.o
46
46
47
-
obj-$(CONFIG_CRYPTO_CRCT10DIF_ARM64_CE) += crct10dif-ce.o
48
-
crct10dif-ce-y := crct10dif-ce-core.o crct10dif-ce-glue.o
49
-
50
47
obj-$(CONFIG_CRYPTO_AES_ARM64_CE) += aes-ce-cipher.o
51
48
aes-ce-cipher-y := aes-ce-core.o aes-ce-glue.o
52
49
arch/arm64/crypto/crct10dif-ce-core.S
arch/arm64/lib/crc-t10dif-core.S
arch/arm64/crypto/crct10dif-ce-core.S
arch/arm64/lib/crc-t10dif-core.S
-132
arch/arm64/crypto/crct10dif-ce-glue.c
-132
arch/arm64/crypto/crct10dif-ce-glue.c
···
1
-
// SPDX-License-Identifier: GPL-2.0-only
2
-
/*
3
-
* Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
4
-
*
5
-
* Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
6
-
*/
7
-
8
-
#include <linux/cpufeature.h>
9
-
#include <linux/crc-t10dif.h>
10
-
#include <linux/init.h>
11
-
#include <linux/kernel.h>
12
-
#include <linux/module.h>
13
-
#include <linux/string.h>
14
-
15
-
#include <crypto/internal/hash.h>
16
-
#include <crypto/internal/simd.h>
17
-
18
-
#include <asm/neon.h>
19
-
#include <asm/simd.h>
20
-
21
-
#define CRC_T10DIF_PMULL_CHUNK_SIZE 16U
22
-
23
-
asmlinkage void crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len,
24
-
u8 out[16]);
25
-
asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len);
26
-
27
-
static int crct10dif_init(struct shash_desc *desc)
28
-
{
29
-
u16 *crc = shash_desc_ctx(desc);
30
-
31
-
*crc = 0;
32
-
return 0;
33
-
}
34
-
35
-
static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data,
36
-
unsigned int length)
37
-
{
38
-
u16 *crcp = shash_desc_ctx(desc);
39
-
u16 crc = *crcp;
40
-
u8 buf[16];
41
-
42
-
if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
43
-
kernel_neon_begin();
44
-
crc_t10dif_pmull_p8(crc, data, length, buf);
45
-
kernel_neon_end();
46
-
47
-
crc = 0;
48
-
data = buf;
49
-
length = sizeof(buf);
50
-
}
51
-
52
-
*crcp = crc_t10dif_generic(crc, data, length);
53
-
return 0;
54
-
}
55
-
56
-
static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data,
57
-
unsigned int length)
58
-
{
59
-
u16 *crc = shash_desc_ctx(desc);
60
-
61
-
if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
62
-
kernel_neon_begin();
63
-
*crc = crc_t10dif_pmull_p64(*crc, data, length);
64
-
kernel_neon_end();
65
-
} else {
66
-
*crc = crc_t10dif_generic(*crc, data, length);
67
-
}
68
-
69
-
return 0;
70
-
}
71
-
72
-
static int crct10dif_final(struct shash_desc *desc, u8 *out)
73
-
{
74
-
u16 *crc = shash_desc_ctx(desc);
75
-
76
-
*(u16 *)out = *crc;
77
-
return 0;
78
-
}
79
-
80
-
static struct shash_alg crc_t10dif_alg[] = {{
81
-
.digestsize = CRC_T10DIF_DIGEST_SIZE,
82
-
.init = crct10dif_init,
83
-
.update = crct10dif_update_pmull_p8,
84
-
.final = crct10dif_final,
85
-
.descsize = CRC_T10DIF_DIGEST_SIZE,
86
-
87
-
.base.cra_name = "crct10dif",
88
-
.base.cra_driver_name = "crct10dif-arm64-neon",
89
-
.base.cra_priority = 150,
90
-
.base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
91
-
.base.cra_module = THIS_MODULE,
92
-
}, {
93
-
.digestsize = CRC_T10DIF_DIGEST_SIZE,
94
-
.init = crct10dif_init,
95
-
.update = crct10dif_update_pmull_p64,
96
-
.final = crct10dif_final,
97
-
.descsize = CRC_T10DIF_DIGEST_SIZE,
98
-
99
-
.base.cra_name = "crct10dif",
100
-
.base.cra_driver_name = "crct10dif-arm64-ce",
101
-
.base.cra_priority = 200,
102
-
.base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
103
-
.base.cra_module = THIS_MODULE,
104
-
}};
105
-
106
-
static int __init crc_t10dif_mod_init(void)
107
-
{
108
-
if (cpu_have_named_feature(PMULL))
109
-
return crypto_register_shashes(crc_t10dif_alg,
110
-
ARRAY_SIZE(crc_t10dif_alg));
111
-
else
112
-
/* only register the first array element */
113
-
return crypto_register_shash(crc_t10dif_alg);
114
-
}
115
-
116
-
static void __exit crc_t10dif_mod_exit(void)
117
-
{
118
-
if (cpu_have_named_feature(PMULL))
119
-
crypto_unregister_shashes(crc_t10dif_alg,
120
-
ARRAY_SIZE(crc_t10dif_alg));
121
-
else
122
-
crypto_unregister_shash(crc_t10dif_alg);
123
-
}
124
-
125
-
module_cpu_feature_match(ASIMD, crc_t10dif_mod_init);
126
-
module_exit(crc_t10dif_mod_exit);
127
-
128
-
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
129
-
MODULE_DESCRIPTION("CRC-T10DIF using arm64 NEON and Crypto Extensions");
130
-
MODULE_LICENSE("GPL v2");
131
-
MODULE_ALIAS_CRYPTO("crct10dif");
132
-
MODULE_ALIAS_CRYPTO("crct10dif-arm64-ce");
+5
-1
arch/arm64/lib/Makefile
+5
-1
arch/arm64/lib/Makefile
···
13
13
14
14
lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o
15
15
16
-
obj-$(CONFIG_CRC32) += crc32.o crc32-glue.o
16
+
obj-$(CONFIG_CRC32_ARCH) += crc32-arm64.o
17
+
crc32-arm64-y := crc32.o crc32-glue.o
18
+
19
+
obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-arm64.o
20
+
crc-t10dif-arm64-y := crc-t10dif-glue.o crc-t10dif-core.o
17
21
18
22
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
19
23
+81
arch/arm64/lib/crc-t10dif-glue.c
+81
arch/arm64/lib/crc-t10dif-glue.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/*
3
+
* Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
4
+
*
5
+
* Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
6
+
*/
7
+
8
+
#include <linux/cpufeature.h>
9
+
#include <linux/crc-t10dif.h>
10
+
#include <linux/init.h>
11
+
#include <linux/kernel.h>
12
+
#include <linux/module.h>
13
+
#include <linux/string.h>
14
+
15
+
#include <crypto/internal/simd.h>
16
+
17
+
#include <asm/neon.h>
18
+
#include <asm/simd.h>
19
+
20
+
static DEFINE_STATIC_KEY_FALSE(have_asimd);
21
+
static DEFINE_STATIC_KEY_FALSE(have_pmull);
22
+
23
+
#define CRC_T10DIF_PMULL_CHUNK_SIZE 16U
24
+
25
+
asmlinkage void crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len,
26
+
u8 out[16]);
27
+
asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len);
28
+
29
+
u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
30
+
{
31
+
if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) {
32
+
if (static_branch_likely(&have_pmull)) {
33
+
if (crypto_simd_usable()) {
34
+
kernel_neon_begin();
35
+
crc = crc_t10dif_pmull_p64(crc, data, length);
36
+
kernel_neon_end();
37
+
return crc;
38
+
}
39
+
} else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE &&
40
+
static_branch_likely(&have_asimd) &&
41
+
crypto_simd_usable()) {
42
+
u8 buf[16];
43
+
44
+
kernel_neon_begin();
45
+
crc_t10dif_pmull_p8(crc, data, length, buf);
46
+
kernel_neon_end();
47
+
48
+
crc = 0;
49
+
data = buf;
50
+
length = sizeof(buf);
51
+
}
52
+
}
53
+
return crc_t10dif_generic(crc, data, length);
54
+
}
55
+
EXPORT_SYMBOL(crc_t10dif_arch);
56
+
57
+
static int __init crc_t10dif_arm64_init(void)
58
+
{
59
+
if (cpu_have_named_feature(ASIMD)) {
60
+
static_branch_enable(&have_asimd);
61
+
if (cpu_have_named_feature(PMULL))
62
+
static_branch_enable(&have_pmull);
63
+
}
64
+
return 0;
65
+
}
66
+
arch_initcall(crc_t10dif_arm64_init);
67
+
68
+
static void __exit crc_t10dif_arm64_exit(void)
69
+
{
70
+
}
71
+
module_exit(crc_t10dif_arm64_exit);
72
+
73
+
bool crc_t10dif_is_optimized(void)
74
+
{
75
+
return static_key_enabled(&have_asimd);
76
+
}
77
+
EXPORT_SYMBOL(crc_t10dif_is_optimized);
78
+
79
+
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
80
+
MODULE_DESCRIPTION("CRC-T10DIF using arm64 NEON and Crypto Extensions");
81
+
MODULE_LICENSE("GPL v2");
+21
-4
arch/arm64/lib/crc32-glue.c
+21
-4
arch/arm64/lib/crc32-glue.c
···
2
2
3
3
#include <linux/crc32.h>
4
4
#include <linux/linkage.h>
5
+
#include <linux/module.h>
5
6
6
7
#include <asm/alternative.h>
7
8
#include <asm/cpufeature.h>
···
22
21
asmlinkage u32 crc32c_le_arm64_4way(u32 crc, unsigned char const *p, size_t len);
23
22
asmlinkage u32 crc32_be_arm64_4way(u32 crc, unsigned char const *p, size_t len);
24
23
25
-
u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len)
24
+
u32 __pure crc32_le_arch(u32 crc, const u8 *p, size_t len)
26
25
{
27
26
if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
28
27
return crc32_le_base(crc, p, len);
···
41
40
42
41
return crc32_le_arm64(crc, p, len);
43
42
}
43
+
EXPORT_SYMBOL(crc32_le_arch);
44
44
45
-
u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len)
45
+
u32 __pure crc32c_le_arch(u32 crc, const u8 *p, size_t len)
46
46
{
47
47
if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
48
-
return __crc32c_le_base(crc, p, len);
48
+
return crc32c_le_base(crc, p, len);
49
49
50
50
if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) {
51
51
kernel_neon_begin();
···
62
60
63
61
return crc32c_le_arm64(crc, p, len);
64
62
}
63
+
EXPORT_SYMBOL(crc32c_le_arch);
65
64
66
-
u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len)
65
+
u32 __pure crc32_be_arch(u32 crc, const u8 *p, size_t len)
67
66
{
68
67
if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
69
68
return crc32_be_base(crc, p, len);
···
83
80
84
81
return crc32_be_arm64(crc, p, len);
85
82
}
83
+
EXPORT_SYMBOL(crc32_be_arch);
84
+
85
+
u32 crc32_optimizations(void)
86
+
{
87
+
if (alternative_has_cap_likely(ARM64_HAS_CRC32))
88
+
return CRC32_LE_OPTIMIZATION |
89
+
CRC32_BE_OPTIMIZATION |
90
+
CRC32C_OPTIMIZATION;
91
+
return 0;
92
+
}
93
+
EXPORT_SYMBOL(crc32_optimizations);
94
+
95
+
MODULE_LICENSE("GPL");
96
+
MODULE_DESCRIPTION("arm64-optimized CRC32 functions");
+1
arch/loongarch/Kconfig
+1
arch/loongarch/Kconfig
···
15
15
select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
16
16
select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI
17
17
select ARCH_HAS_CPU_FINALIZE_INIT
18
+
select ARCH_HAS_CRC32
18
19
select ARCH_HAS_CURRENT_STACK_POINTER
19
20
select ARCH_HAS_DEBUG_VM_PGTABLE
20
21
select ARCH_HAS_FAST_MULTIPLIER
-1
arch/loongarch/configs/loongson3_defconfig
-1
arch/loongarch/configs/loongson3_defconfig
-9
arch/loongarch/crypto/Kconfig
-9
arch/loongarch/crypto/Kconfig
···
2
2
3
3
menu "Accelerated Cryptographic Algorithms for CPU (loongarch)"
4
4
5
-
config CRYPTO_CRC32_LOONGARCH
6
-
tristate "CRC32c and CRC32"
7
-
select CRC32
8
-
select CRYPTO_HASH
9
-
help
10
-
CRC32c and CRC32 CRC algorithms
11
-
12
-
Architecture: LoongArch with CRC32 instructions
13
-
14
5
endmenu
-2
arch/loongarch/crypto/Makefile
-2
arch/loongarch/crypto/Makefile
-300
arch/loongarch/crypto/crc32-loongarch.c
-300
arch/loongarch/crypto/crc32-loongarch.c
···
1
-
// SPDX-License-Identifier: GPL-2.0
2
-
/*
3
-
* crc32.c - CRC32 and CRC32C using LoongArch crc* instructions
4
-
*
5
-
* Module based on mips/crypto/crc32-mips.c
6
-
*
7
-
* Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
8
-
* Copyright (C) 2018 MIPS Tech, LLC
9
-
* Copyright (C) 2020-2023 Loongson Technology Corporation Limited
10
-
*/
11
-
12
-
#include <linux/module.h>
13
-
#include <crypto/internal/hash.h>
14
-
15
-
#include <asm/cpu-features.h>
16
-
#include <linux/unaligned.h>
17
-
18
-
#define _CRC32(crc, value, size, type) \
19
-
do { \
20
-
__asm__ __volatile__( \
21
-
#type ".w." #size ".w" " %0, %1, %0\n\t"\
22
-
: "+r" (crc) \
23
-
: "r" (value) \
24
-
: "memory"); \
25
-
} while (0)
26
-
27
-
#define CRC32(crc, value, size) _CRC32(crc, value, size, crc)
28
-
#define CRC32C(crc, value, size) _CRC32(crc, value, size, crcc)
29
-
30
-
static u32 crc32_loongarch_hw(u32 crc_, const u8 *p, unsigned int len)
31
-
{
32
-
u32 crc = crc_;
33
-
34
-
while (len >= sizeof(u64)) {
35
-
u64 value = get_unaligned_le64(p);
36
-
37
-
CRC32(crc, value, d);
38
-
p += sizeof(u64);
39
-
len -= sizeof(u64);
40
-
}
41
-
42
-
if (len & sizeof(u32)) {
43
-
u32 value = get_unaligned_le32(p);
44
-
45
-
CRC32(crc, value, w);
46
-
p += sizeof(u32);
47
-
}
48
-
49
-
if (len & sizeof(u16)) {
50
-
u16 value = get_unaligned_le16(p);
51
-
52
-
CRC32(crc, value, h);
53
-
p += sizeof(u16);
54
-
}
55
-
56
-
if (len & sizeof(u8)) {
57
-
u8 value = *p++;
58
-
59
-
CRC32(crc, value, b);
60
-
}
61
-
62
-
return crc;
63
-
}
64
-
65
-
static u32 crc32c_loongarch_hw(u32 crc_, const u8 *p, unsigned int len)
66
-
{
67
-
u32 crc = crc_;
68
-
69
-
while (len >= sizeof(u64)) {
70
-
u64 value = get_unaligned_le64(p);
71
-
72
-
CRC32C(crc, value, d);
73
-
p += sizeof(u64);
74
-
len -= sizeof(u64);
75
-
}
76
-
77
-
if (len & sizeof(u32)) {
78
-
u32 value = get_unaligned_le32(p);
79
-
80
-
CRC32C(crc, value, w);
81
-
p += sizeof(u32);
82
-
}
83
-
84
-
if (len & sizeof(u16)) {
85
-
u16 value = get_unaligned_le16(p);
86
-
87
-
CRC32C(crc, value, h);
88
-
p += sizeof(u16);
89
-
}
90
-
91
-
if (len & sizeof(u8)) {
92
-
u8 value = *p++;
93
-
94
-
CRC32C(crc, value, b);
95
-
}
96
-
97
-
return crc;
98
-
}
99
-
100
-
#define CHKSUM_BLOCK_SIZE 1
101
-
#define CHKSUM_DIGEST_SIZE 4
102
-
103
-
struct chksum_ctx {
104
-
u32 key;
105
-
};
106
-
107
-
struct chksum_desc_ctx {
108
-
u32 crc;
109
-
};
110
-
111
-
static int chksum_init(struct shash_desc *desc)
112
-
{
113
-
struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
114
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
115
-
116
-
ctx->crc = mctx->key;
117
-
118
-
return 0;
119
-
}
120
-
121
-
/*
122
-
* Setting the seed allows arbitrary accumulators and flexible XOR policy
123
-
* If your algorithm starts with ~0, then XOR with ~0 before you set the seed.
124
-
*/
125
-
static int chksum_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen)
126
-
{
127
-
struct chksum_ctx *mctx = crypto_shash_ctx(tfm);
128
-
129
-
if (keylen != sizeof(mctx->key))
130
-
return -EINVAL;
131
-
132
-
mctx->key = get_unaligned_le32(key);
133
-
134
-
return 0;
135
-
}
136
-
137
-
static int chksum_update(struct shash_desc *desc, const u8 *data, unsigned int length)
138
-
{
139
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
140
-
141
-
ctx->crc = crc32_loongarch_hw(ctx->crc, data, length);
142
-
return 0;
143
-
}
144
-
145
-
static int chksumc_update(struct shash_desc *desc, const u8 *data, unsigned int length)
146
-
{
147
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
148
-
149
-
ctx->crc = crc32c_loongarch_hw(ctx->crc, data, length);
150
-
return 0;
151
-
}
152
-
153
-
static int chksum_final(struct shash_desc *desc, u8 *out)
154
-
{
155
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
156
-
157
-
put_unaligned_le32(ctx->crc, out);
158
-
return 0;
159
-
}
160
-
161
-
static int chksumc_final(struct shash_desc *desc, u8 *out)
162
-
{
163
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
164
-
165
-
put_unaligned_le32(~ctx->crc, out);
166
-
return 0;
167
-
}
168
-
169
-
static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
170
-
{
171
-
put_unaligned_le32(crc32_loongarch_hw(crc, data, len), out);
172
-
return 0;
173
-
}
174
-
175
-
static int __chksumc_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
176
-
{
177
-
put_unaligned_le32(~crc32c_loongarch_hw(crc, data, len), out);
178
-
return 0;
179
-
}
180
-
181
-
static int chksum_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out)
182
-
{
183
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
184
-
185
-
return __chksum_finup(ctx->crc, data, len, out);
186
-
}
187
-
188
-
static int chksumc_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out)
189
-
{
190
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
191
-
192
-
return __chksumc_finup(ctx->crc, data, len, out);
193
-
}
194
-
195
-
static int chksum_digest(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out)
196
-
{
197
-
struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
198
-
199
-
return __chksum_finup(mctx->key, data, length, out);
200
-
}
201
-
202
-
static int chksumc_digest(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out)
203
-
{
204
-
struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
205
-
206
-
return __chksumc_finup(mctx->key, data, length, out);
207
-
}
208
-
209
-
static int chksum_cra_init(struct crypto_tfm *tfm)
210
-
{
211
-
struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
212
-
213
-
mctx->key = 0;
214
-
return 0;
215
-
}
216
-
217
-
static int chksumc_cra_init(struct crypto_tfm *tfm)
218
-
{
219
-
struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
220
-
221
-
mctx->key = ~0;
222
-
return 0;
223
-
}
224
-
225
-
static struct shash_alg crc32_alg = {
226
-
.digestsize = CHKSUM_DIGEST_SIZE,
227
-
.setkey = chksum_setkey,
228
-
.init = chksum_init,
229
-
.update = chksum_update,
230
-
.final = chksum_final,
231
-
.finup = chksum_finup,
232
-
.digest = chksum_digest,
233
-
.descsize = sizeof(struct chksum_desc_ctx),
234
-
.base = {
235
-
.cra_name = "crc32",
236
-
.cra_driver_name = "crc32-loongarch",
237
-
.cra_priority = 300,
238
-
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
239
-
.cra_blocksize = CHKSUM_BLOCK_SIZE,
240
-
.cra_ctxsize = sizeof(struct chksum_ctx),
241
-
.cra_module = THIS_MODULE,
242
-
.cra_init = chksum_cra_init,
243
-
}
244
-
};
245
-
246
-
static struct shash_alg crc32c_alg = {
247
-
.digestsize = CHKSUM_DIGEST_SIZE,
248
-
.setkey = chksum_setkey,
249
-
.init = chksum_init,
250
-
.update = chksumc_update,
251
-
.final = chksumc_final,
252
-
.finup = chksumc_finup,
253
-
.digest = chksumc_digest,
254
-
.descsize = sizeof(struct chksum_desc_ctx),
255
-
.base = {
256
-
.cra_name = "crc32c",
257
-
.cra_driver_name = "crc32c-loongarch",
258
-
.cra_priority = 300,
259
-
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
260
-
.cra_blocksize = CHKSUM_BLOCK_SIZE,
261
-
.cra_ctxsize = sizeof(struct chksum_ctx),
262
-
.cra_module = THIS_MODULE,
263
-
.cra_init = chksumc_cra_init,
264
-
}
265
-
};
266
-
267
-
static int __init crc32_mod_init(void)
268
-
{
269
-
int err;
270
-
271
-
if (!cpu_has(CPU_FEATURE_CRC32))
272
-
return 0;
273
-
274
-
err = crypto_register_shash(&crc32_alg);
275
-
if (err)
276
-
return err;
277
-
278
-
err = crypto_register_shash(&crc32c_alg);
279
-
if (err)
280
-
return err;
281
-
282
-
return 0;
283
-
}
284
-
285
-
static void __exit crc32_mod_exit(void)
286
-
{
287
-
if (!cpu_has(CPU_FEATURE_CRC32))
288
-
return;
289
-
290
-
crypto_unregister_shash(&crc32_alg);
291
-
crypto_unregister_shash(&crc32c_alg);
292
-
}
293
-
294
-
module_init(crc32_mod_init);
295
-
module_exit(crc32_mod_exit);
296
-
297
-
MODULE_AUTHOR("Min Zhou <zhoumin@loongson.cn>");
298
-
MODULE_AUTHOR("Huacai Chen <chenhuacai@loongson.cn>");
299
-
MODULE_DESCRIPTION("CRC32 and CRC32C using LoongArch crc* instructions");
300
-
MODULE_LICENSE("GPL v2");
+2
arch/loongarch/lib/Makefile
+2
arch/loongarch/lib/Makefile
+135
arch/loongarch/lib/crc32-loongarch.c
+135
arch/loongarch/lib/crc32-loongarch.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/*
3
+
* CRC32 and CRC32C using LoongArch crc* instructions
4
+
*
5
+
* Module based on mips/crypto/crc32-mips.c
6
+
*
7
+
* Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
8
+
* Copyright (C) 2018 MIPS Tech, LLC
9
+
* Copyright (C) 2020-2023 Loongson Technology Corporation Limited
10
+
*/
11
+
12
+
#include <asm/cpu-features.h>
13
+
#include <linux/crc32.h>
14
+
#include <linux/module.h>
15
+
#include <linux/unaligned.h>
16
+
17
+
#define _CRC32(crc, value, size, type) \
18
+
do { \
19
+
__asm__ __volatile__( \
20
+
#type ".w." #size ".w" " %0, %1, %0\n\t"\
21
+
: "+r" (crc) \
22
+
: "r" (value) \
23
+
: "memory"); \
24
+
} while (0)
25
+
26
+
#define CRC32(crc, value, size) _CRC32(crc, value, size, crc)
27
+
#define CRC32C(crc, value, size) _CRC32(crc, value, size, crcc)
28
+
29
+
static DEFINE_STATIC_KEY_FALSE(have_crc32);
30
+
31
+
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
32
+
{
33
+
if (!static_branch_likely(&have_crc32))
34
+
return crc32_le_base(crc, p, len);
35
+
36
+
while (len >= sizeof(u64)) {
37
+
u64 value = get_unaligned_le64(p);
38
+
39
+
CRC32(crc, value, d);
40
+
p += sizeof(u64);
41
+
len -= sizeof(u64);
42
+
}
43
+
44
+
if (len & sizeof(u32)) {
45
+
u32 value = get_unaligned_le32(p);
46
+
47
+
CRC32(crc, value, w);
48
+
p += sizeof(u32);
49
+
}
50
+
51
+
if (len & sizeof(u16)) {
52
+
u16 value = get_unaligned_le16(p);
53
+
54
+
CRC32(crc, value, h);
55
+
p += sizeof(u16);
56
+
}
57
+
58
+
if (len & sizeof(u8)) {
59
+
u8 value = *p++;
60
+
61
+
CRC32(crc, value, b);
62
+
}
63
+
64
+
return crc;
65
+
}
66
+
EXPORT_SYMBOL(crc32_le_arch);
67
+
68
+
u32 crc32c_le_arch(u32 crc, const u8 *p, size_t len)
69
+
{
70
+
if (!static_branch_likely(&have_crc32))
71
+
return crc32c_le_base(crc, p, len);
72
+
73
+
while (len >= sizeof(u64)) {
74
+
u64 value = get_unaligned_le64(p);
75
+
76
+
CRC32C(crc, value, d);
77
+
p += sizeof(u64);
78
+
len -= sizeof(u64);
79
+
}
80
+
81
+
if (len & sizeof(u32)) {
82
+
u32 value = get_unaligned_le32(p);
83
+
84
+
CRC32C(crc, value, w);
85
+
p += sizeof(u32);
86
+
}
87
+
88
+
if (len & sizeof(u16)) {
89
+
u16 value = get_unaligned_le16(p);
90
+
91
+
CRC32C(crc, value, h);
92
+
p += sizeof(u16);
93
+
}
94
+
95
+
if (len & sizeof(u8)) {
96
+
u8 value = *p++;
97
+
98
+
CRC32C(crc, value, b);
99
+
}
100
+
101
+
return crc;
102
+
}
103
+
EXPORT_SYMBOL(crc32c_le_arch);
104
+
105
+
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
106
+
{
107
+
return crc32_be_base(crc, p, len);
108
+
}
109
+
EXPORT_SYMBOL(crc32_be_arch);
110
+
111
+
static int __init crc32_loongarch_init(void)
112
+
{
113
+
if (cpu_has_crc32)
114
+
static_branch_enable(&have_crc32);
115
+
return 0;
116
+
}
117
+
arch_initcall(crc32_loongarch_init);
118
+
119
+
static void __exit crc32_loongarch_exit(void)
120
+
{
121
+
}
122
+
module_exit(crc32_loongarch_exit);
123
+
124
+
u32 crc32_optimizations(void)
125
+
{
126
+
if (static_key_enabled(&have_crc32))
127
+
return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION;
128
+
return 0;
129
+
}
130
+
EXPORT_SYMBOL(crc32_optimizations);
131
+
132
+
MODULE_AUTHOR("Min Zhou <zhoumin@loongson.cn>");
133
+
MODULE_AUTHOR("Huacai Chen <chenhuacai@loongson.cn>");
134
+
MODULE_DESCRIPTION("CRC32 and CRC32C using LoongArch crc* instructions");
135
+
MODULE_LICENSE("GPL v2");
-1
arch/m68k/configs/amiga_defconfig
-1
arch/m68k/configs/amiga_defconfig
-1
arch/m68k/configs/apollo_defconfig
-1
arch/m68k/configs/apollo_defconfig
-1
arch/m68k/configs/atari_defconfig
-1
arch/m68k/configs/atari_defconfig
-1
arch/m68k/configs/bvme6000_defconfig
-1
arch/m68k/configs/bvme6000_defconfig
-1
arch/m68k/configs/hp300_defconfig
-1
arch/m68k/configs/hp300_defconfig
-1
arch/m68k/configs/mac_defconfig
-1
arch/m68k/configs/mac_defconfig
-1
arch/m68k/configs/multi_defconfig
-1
arch/m68k/configs/multi_defconfig
-1
arch/m68k/configs/mvme147_defconfig
-1
arch/m68k/configs/mvme147_defconfig
-1
arch/m68k/configs/mvme16x_defconfig
-1
arch/m68k/configs/mvme16x_defconfig
-1
arch/m68k/configs/q40_defconfig
-1
arch/m68k/configs/q40_defconfig
-1
arch/m68k/configs/sun3_defconfig
-1
arch/m68k/configs/sun3_defconfig
-1
arch/m68k/configs/sun3x_defconfig
-1
arch/m68k/configs/sun3x_defconfig
+1
-4
arch/mips/Kconfig
+1
-4
arch/mips/Kconfig
···
1995
1995
config CPU_MIPSR6
1996
1996
bool
1997
1997
default y if CPU_MIPS32_R6 || CPU_MIPS64_R6
1998
+
select ARCH_HAS_CRC32
1998
1999
select CPU_HAS_RIXI
1999
2000
select CPU_HAS_DIEI if !CPU_DIEI_BROKEN
2000
2001
select HAVE_ARCH_BITREVERSE
2001
2002
select MIPS_ASID_BITS_VARIABLE
2002
-
select MIPS_CRC_SUPPORT
2003
2003
select MIPS_SPRAM
2004
2004
2005
2005
config TARGET_ISA_REV
···
2473
2473
default 8
2474
2474
2475
2475
config MIPS_ASID_BITS_VARIABLE
2476
-
bool
2477
-
2478
-
config MIPS_CRC_SUPPORT
2479
2476
bool
2480
2477
2481
2478
# R4600 erratum. Due to the lack of errata information the exact
-1
arch/mips/configs/eyeq5_defconfig
-1
arch/mips/configs/eyeq5_defconfig
-1
arch/mips/configs/eyeq6_defconfig
-1
arch/mips/configs/eyeq6_defconfig
-2
arch/mips/configs/generic/32r6.config
-2
arch/mips/configs/generic/32r6.config
-1
arch/mips/configs/generic/64r6.config
-1
arch/mips/configs/generic/64r6.config
-9
arch/mips/crypto/Kconfig
-9
arch/mips/crypto/Kconfig
···
2
2
3
3
menu "Accelerated Cryptographic Algorithms for CPU (mips)"
4
4
5
-
config CRYPTO_CRC32_MIPS
6
-
tristate "CRC32c and CRC32"
7
-
depends on MIPS_CRC_SUPPORT
8
-
select CRYPTO_HASH
9
-
help
10
-
CRC32c and CRC32 CRC algorithms
11
-
12
-
Architecture: mips
13
-
14
5
config CRYPTO_POLY1305_MIPS
15
6
tristate "Hash functions: Poly1305"
16
7
depends on MIPS
-2
arch/mips/crypto/Makefile
-2
arch/mips/crypto/Makefile
-354
arch/mips/crypto/crc32-mips.c
-354
arch/mips/crypto/crc32-mips.c
···
1
-
// SPDX-License-Identifier: GPL-2.0
2
-
/*
3
-
* crc32-mips.c - CRC32 and CRC32C using optional MIPSr6 instructions
4
-
*
5
-
* Module based on arm64/crypto/crc32-arm.c
6
-
*
7
-
* Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
8
-
* Copyright (C) 2018 MIPS Tech, LLC
9
-
*/
10
-
11
-
#include <linux/cpufeature.h>
12
-
#include <linux/init.h>
13
-
#include <linux/kernel.h>
14
-
#include <linux/module.h>
15
-
#include <linux/string.h>
16
-
#include <asm/mipsregs.h>
17
-
#include <linux/unaligned.h>
18
-
19
-
#include <crypto/internal/hash.h>
20
-
21
-
enum crc_op_size {
22
-
b, h, w, d,
23
-
};
24
-
25
-
enum crc_type {
26
-
crc32,
27
-
crc32c,
28
-
};
29
-
30
-
#ifndef TOOLCHAIN_SUPPORTS_CRC
31
-
#define _ASM_SET_CRC(OP, SZ, TYPE) \
32
-
_ASM_MACRO_3R(OP, rt, rs, rt2, \
33
-
".ifnc \\rt, \\rt2\n\t" \
34
-
".error \"invalid operands \\\"" #OP " \\rt,\\rs,\\rt2\\\"\"\n\t" \
35
-
".endif\n\t" \
36
-
_ASM_INSN_IF_MIPS(0x7c00000f | (__rt << 16) | (__rs << 21) | \
37
-
((SZ) << 6) | ((TYPE) << 8)) \
38
-
_ASM_INSN32_IF_MM(0x00000030 | (__rs << 16) | (__rt << 21) | \
39
-
((SZ) << 14) | ((TYPE) << 3)))
40
-
#define _ASM_UNSET_CRC(op, SZ, TYPE) ".purgem " #op "\n\t"
41
-
#else /* !TOOLCHAIN_SUPPORTS_CRC */
42
-
#define _ASM_SET_CRC(op, SZ, TYPE) ".set\tcrc\n\t"
43
-
#define _ASM_UNSET_CRC(op, SZ, TYPE)
44
-
#endif
45
-
46
-
#define __CRC32(crc, value, op, SZ, TYPE) \
47
-
do { \
48
-
__asm__ __volatile__( \
49
-
".set push\n\t" \
50
-
_ASM_SET_CRC(op, SZ, TYPE) \
51
-
#op " %0, %1, %0\n\t" \
52
-
_ASM_UNSET_CRC(op, SZ, TYPE) \
53
-
".set pop" \
54
-
: "+r" (crc) \
55
-
: "r" (value)); \
56
-
} while (0)
57
-
58
-
#define _CRC32_crc32b(crc, value) __CRC32(crc, value, crc32b, 0, 0)
59
-
#define _CRC32_crc32h(crc, value) __CRC32(crc, value, crc32h, 1, 0)
60
-
#define _CRC32_crc32w(crc, value) __CRC32(crc, value, crc32w, 2, 0)
61
-
#define _CRC32_crc32d(crc, value) __CRC32(crc, value, crc32d, 3, 0)
62
-
#define _CRC32_crc32cb(crc, value) __CRC32(crc, value, crc32cb, 0, 1)
63
-
#define _CRC32_crc32ch(crc, value) __CRC32(crc, value, crc32ch, 1, 1)
64
-
#define _CRC32_crc32cw(crc, value) __CRC32(crc, value, crc32cw, 2, 1)
65
-
#define _CRC32_crc32cd(crc, value) __CRC32(crc, value, crc32cd, 3, 1)
66
-
67
-
#define _CRC32(crc, value, size, op) \
68
-
_CRC32_##op##size(crc, value)
69
-
70
-
#define CRC32(crc, value, size) \
71
-
_CRC32(crc, value, size, crc32)
72
-
73
-
#define CRC32C(crc, value, size) \
74
-
_CRC32(crc, value, size, crc32c)
75
-
76
-
static u32 crc32_mips_le_hw(u32 crc_, const u8 *p, unsigned int len)
77
-
{
78
-
u32 crc = crc_;
79
-
80
-
if (IS_ENABLED(CONFIG_64BIT)) {
81
-
for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) {
82
-
u64 value = get_unaligned_le64(p);
83
-
84
-
CRC32(crc, value, d);
85
-
}
86
-
87
-
if (len & sizeof(u32)) {
88
-
u32 value = get_unaligned_le32(p);
89
-
90
-
CRC32(crc, value, w);
91
-
p += sizeof(u32);
92
-
}
93
-
} else {
94
-
for (; len >= sizeof(u32); len -= sizeof(u32)) {
95
-
u32 value = get_unaligned_le32(p);
96
-
97
-
CRC32(crc, value, w);
98
-
p += sizeof(u32);
99
-
}
100
-
}
101
-
102
-
if (len & sizeof(u16)) {
103
-
u16 value = get_unaligned_le16(p);
104
-
105
-
CRC32(crc, value, h);
106
-
p += sizeof(u16);
107
-
}
108
-
109
-
if (len & sizeof(u8)) {
110
-
u8 value = *p++;
111
-
112
-
CRC32(crc, value, b);
113
-
}
114
-
115
-
return crc;
116
-
}
117
-
118
-
static u32 crc32c_mips_le_hw(u32 crc_, const u8 *p, unsigned int len)
119
-
{
120
-
u32 crc = crc_;
121
-
122
-
if (IS_ENABLED(CONFIG_64BIT)) {
123
-
for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) {
124
-
u64 value = get_unaligned_le64(p);
125
-
126
-
CRC32C(crc, value, d);
127
-
}
128
-
129
-
if (len & sizeof(u32)) {
130
-
u32 value = get_unaligned_le32(p);
131
-
132
-
CRC32C(crc, value, w);
133
-
p += sizeof(u32);
134
-
}
135
-
} else {
136
-
for (; len >= sizeof(u32); len -= sizeof(u32)) {
137
-
u32 value = get_unaligned_le32(p);
138
-
139
-
CRC32C(crc, value, w);
140
-
p += sizeof(u32);
141
-
}
142
-
}
143
-
144
-
if (len & sizeof(u16)) {
145
-
u16 value = get_unaligned_le16(p);
146
-
147
-
CRC32C(crc, value, h);
148
-
p += sizeof(u16);
149
-
}
150
-
151
-
if (len & sizeof(u8)) {
152
-
u8 value = *p++;
153
-
154
-
CRC32C(crc, value, b);
155
-
}
156
-
return crc;
157
-
}
158
-
159
-
#define CHKSUM_BLOCK_SIZE 1
160
-
#define CHKSUM_DIGEST_SIZE 4
161
-
162
-
struct chksum_ctx {
163
-
u32 key;
164
-
};
165
-
166
-
struct chksum_desc_ctx {
167
-
u32 crc;
168
-
};
169
-
170
-
static int chksum_init(struct shash_desc *desc)
171
-
{
172
-
struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
173
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
174
-
175
-
ctx->crc = mctx->key;
176
-
177
-
return 0;
178
-
}
179
-
180
-
/*
181
-
* Setting the seed allows arbitrary accumulators and flexible XOR policy
182
-
* If your algorithm starts with ~0, then XOR with ~0 before you set
183
-
* the seed.
184
-
*/
185
-
static int chksum_setkey(struct crypto_shash *tfm, const u8 *key,
186
-
unsigned int keylen)
187
-
{
188
-
struct chksum_ctx *mctx = crypto_shash_ctx(tfm);
189
-
190
-
if (keylen != sizeof(mctx->key))
191
-
return -EINVAL;
192
-
mctx->key = get_unaligned_le32(key);
193
-
return 0;
194
-
}
195
-
196
-
static int chksum_update(struct shash_desc *desc, const u8 *data,
197
-
unsigned int length)
198
-
{
199
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
200
-
201
-
ctx->crc = crc32_mips_le_hw(ctx->crc, data, length);
202
-
return 0;
203
-
}
204
-
205
-
static int chksumc_update(struct shash_desc *desc, const u8 *data,
206
-
unsigned int length)
207
-
{
208
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
209
-
210
-
ctx->crc = crc32c_mips_le_hw(ctx->crc, data, length);
211
-
return 0;
212
-
}
213
-
214
-
static int chksum_final(struct shash_desc *desc, u8 *out)
215
-
{
216
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
217
-
218
-
put_unaligned_le32(ctx->crc, out);
219
-
return 0;
220
-
}
221
-
222
-
static int chksumc_final(struct shash_desc *desc, u8 *out)
223
-
{
224
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
225
-
226
-
put_unaligned_le32(~ctx->crc, out);
227
-
return 0;
228
-
}
229
-
230
-
static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
231
-
{
232
-
put_unaligned_le32(crc32_mips_le_hw(crc, data, len), out);
233
-
return 0;
234
-
}
235
-
236
-
static int __chksumc_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
237
-
{
238
-
put_unaligned_le32(~crc32c_mips_le_hw(crc, data, len), out);
239
-
return 0;
240
-
}
241
-
242
-
static int chksum_finup(struct shash_desc *desc, const u8 *data,
243
-
unsigned int len, u8 *out)
244
-
{
245
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
246
-
247
-
return __chksum_finup(ctx->crc, data, len, out);
248
-
}
249
-
250
-
static int chksumc_finup(struct shash_desc *desc, const u8 *data,
251
-
unsigned int len, u8 *out)
252
-
{
253
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
254
-
255
-
return __chksumc_finup(ctx->crc, data, len, out);
256
-
}
257
-
258
-
static int chksum_digest(struct shash_desc *desc, const u8 *data,
259
-
unsigned int length, u8 *out)
260
-
{
261
-
struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
262
-
263
-
return __chksum_finup(mctx->key, data, length, out);
264
-
}
265
-
266
-
static int chksumc_digest(struct shash_desc *desc, const u8 *data,
267
-
unsigned int length, u8 *out)
268
-
{
269
-
struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
270
-
271
-
return __chksumc_finup(mctx->key, data, length, out);
272
-
}
273
-
274
-
static int chksum_cra_init(struct crypto_tfm *tfm)
275
-
{
276
-
struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
277
-
278
-
mctx->key = ~0;
279
-
return 0;
280
-
}
281
-
282
-
static struct shash_alg crc32_alg = {
283
-
.digestsize = CHKSUM_DIGEST_SIZE,
284
-
.setkey = chksum_setkey,
285
-
.init = chksum_init,
286
-
.update = chksum_update,
287
-
.final = chksum_final,
288
-
.finup = chksum_finup,
289
-
.digest = chksum_digest,
290
-
.descsize = sizeof(struct chksum_desc_ctx),
291
-
.base = {
292
-
.cra_name = "crc32",
293
-
.cra_driver_name = "crc32-mips-hw",
294
-
.cra_priority = 300,
295
-
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
296
-
.cra_blocksize = CHKSUM_BLOCK_SIZE,
297
-
.cra_ctxsize = sizeof(struct chksum_ctx),
298
-
.cra_module = THIS_MODULE,
299
-
.cra_init = chksum_cra_init,
300
-
}
301
-
};
302
-
303
-
static struct shash_alg crc32c_alg = {
304
-
.digestsize = CHKSUM_DIGEST_SIZE,
305
-
.setkey = chksum_setkey,
306
-
.init = chksum_init,
307
-
.update = chksumc_update,
308
-
.final = chksumc_final,
309
-
.finup = chksumc_finup,
310
-
.digest = chksumc_digest,
311
-
.descsize = sizeof(struct chksum_desc_ctx),
312
-
.base = {
313
-
.cra_name = "crc32c",
314
-
.cra_driver_name = "crc32c-mips-hw",
315
-
.cra_priority = 300,
316
-
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
317
-
.cra_blocksize = CHKSUM_BLOCK_SIZE,
318
-
.cra_ctxsize = sizeof(struct chksum_ctx),
319
-
.cra_module = THIS_MODULE,
320
-
.cra_init = chksum_cra_init,
321
-
}
322
-
};
323
-
324
-
static int __init crc32_mod_init(void)
325
-
{
326
-
int err;
327
-
328
-
err = crypto_register_shash(&crc32_alg);
329
-
330
-
if (err)
331
-
return err;
332
-
333
-
err = crypto_register_shash(&crc32c_alg);
334
-
335
-
if (err) {
336
-
crypto_unregister_shash(&crc32_alg);
337
-
return err;
338
-
}
339
-
340
-
return 0;
341
-
}
342
-
343
-
static void __exit crc32_mod_exit(void)
344
-
{
345
-
crypto_unregister_shash(&crc32_alg);
346
-
crypto_unregister_shash(&crc32c_alg);
347
-
}
348
-
349
-
MODULE_AUTHOR("Marcin Nowakowski <marcin.nowakowski@mips.com");
350
-
MODULE_DESCRIPTION("CRC32 and CRC32C using optional MIPS instructions");
351
-
MODULE_LICENSE("GPL v2");
352
-
353
-
module_cpu_feature_match(MIPS_CRC32, crc32_mod_init);
354
-
module_exit(crc32_mod_exit);
+2
arch/mips/lib/Makefile
+2
arch/mips/lib/Makefile
+192
arch/mips/lib/crc32-mips.c
+192
arch/mips/lib/crc32-mips.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/*
3
+
* crc32-mips.c - CRC32 and CRC32C using optional MIPSr6 instructions
4
+
*
5
+
* Module based on arm64/crypto/crc32-arm.c
6
+
*
7
+
* Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
8
+
* Copyright (C) 2018 MIPS Tech, LLC
9
+
*/
10
+
11
+
#include <linux/cpufeature.h>
12
+
#include <linux/crc32.h>
13
+
#include <linux/init.h>
14
+
#include <linux/kernel.h>
15
+
#include <linux/module.h>
16
+
#include <asm/mipsregs.h>
17
+
#include <linux/unaligned.h>
18
+
19
+
enum crc_op_size {
20
+
b, h, w, d,
21
+
};
22
+
23
+
enum crc_type {
24
+
crc32,
25
+
crc32c,
26
+
};
27
+
28
+
#ifndef TOOLCHAIN_SUPPORTS_CRC
29
+
#define _ASM_SET_CRC(OP, SZ, TYPE) \
30
+
_ASM_MACRO_3R(OP, rt, rs, rt2, \
31
+
".ifnc \\rt, \\rt2\n\t" \
32
+
".error \"invalid operands \\\"" #OP " \\rt,\\rs,\\rt2\\\"\"\n\t" \
33
+
".endif\n\t" \
34
+
_ASM_INSN_IF_MIPS(0x7c00000f | (__rt << 16) | (__rs << 21) | \
35
+
((SZ) << 6) | ((TYPE) << 8)) \
36
+
_ASM_INSN32_IF_MM(0x00000030 | (__rs << 16) | (__rt << 21) | \
37
+
((SZ) << 14) | ((TYPE) << 3)))
38
+
#define _ASM_UNSET_CRC(op, SZ, TYPE) ".purgem " #op "\n\t"
39
+
#else /* !TOOLCHAIN_SUPPORTS_CRC */
40
+
#define _ASM_SET_CRC(op, SZ, TYPE) ".set\tcrc\n\t"
41
+
#define _ASM_UNSET_CRC(op, SZ, TYPE)
42
+
#endif
43
+
44
+
#define __CRC32(crc, value, op, SZ, TYPE) \
45
+
do { \
46
+
__asm__ __volatile__( \
47
+
".set push\n\t" \
48
+
_ASM_SET_CRC(op, SZ, TYPE) \
49
+
#op " %0, %1, %0\n\t" \
50
+
_ASM_UNSET_CRC(op, SZ, TYPE) \
51
+
".set pop" \
52
+
: "+r" (crc) \
53
+
: "r" (value)); \
54
+
} while (0)
55
+
56
+
#define _CRC32_crc32b(crc, value) __CRC32(crc, value, crc32b, 0, 0)
57
+
#define _CRC32_crc32h(crc, value) __CRC32(crc, value, crc32h, 1, 0)
58
+
#define _CRC32_crc32w(crc, value) __CRC32(crc, value, crc32w, 2, 0)
59
+
#define _CRC32_crc32d(crc, value) __CRC32(crc, value, crc32d, 3, 0)
60
+
#define _CRC32_crc32cb(crc, value) __CRC32(crc, value, crc32cb, 0, 1)
61
+
#define _CRC32_crc32ch(crc, value) __CRC32(crc, value, crc32ch, 1, 1)
62
+
#define _CRC32_crc32cw(crc, value) __CRC32(crc, value, crc32cw, 2, 1)
63
+
#define _CRC32_crc32cd(crc, value) __CRC32(crc, value, crc32cd, 3, 1)
64
+
65
+
#define _CRC32(crc, value, size, op) \
66
+
_CRC32_##op##size(crc, value)
67
+
68
+
#define CRC32(crc, value, size) \
69
+
_CRC32(crc, value, size, crc32)
70
+
71
+
#define CRC32C(crc, value, size) \
72
+
_CRC32(crc, value, size, crc32c)
73
+
74
+
static DEFINE_STATIC_KEY_FALSE(have_crc32);
75
+
76
+
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
77
+
{
78
+
if (!static_branch_likely(&have_crc32))
79
+
return crc32_le_base(crc, p, len);
80
+
81
+
if (IS_ENABLED(CONFIG_64BIT)) {
82
+
for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) {
83
+
u64 value = get_unaligned_le64(p);
84
+
85
+
CRC32(crc, value, d);
86
+
}
87
+
88
+
if (len & sizeof(u32)) {
89
+
u32 value = get_unaligned_le32(p);
90
+
91
+
CRC32(crc, value, w);
92
+
p += sizeof(u32);
93
+
}
94
+
} else {
95
+
for (; len >= sizeof(u32); len -= sizeof(u32)) {
96
+
u32 value = get_unaligned_le32(p);
97
+
98
+
CRC32(crc, value, w);
99
+
p += sizeof(u32);
100
+
}
101
+
}
102
+
103
+
if (len & sizeof(u16)) {
104
+
u16 value = get_unaligned_le16(p);
105
+
106
+
CRC32(crc, value, h);
107
+
p += sizeof(u16);
108
+
}
109
+
110
+
if (len & sizeof(u8)) {
111
+
u8 value = *p++;
112
+
113
+
CRC32(crc, value, b);
114
+
}
115
+
116
+
return crc;
117
+
}
118
+
EXPORT_SYMBOL(crc32_le_arch);
119
+
120
+
u32 crc32c_le_arch(u32 crc, const u8 *p, size_t len)
121
+
{
122
+
if (!static_branch_likely(&have_crc32))
123
+
return crc32c_le_base(crc, p, len);
124
+
125
+
if (IS_ENABLED(CONFIG_64BIT)) {
126
+
for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) {
127
+
u64 value = get_unaligned_le64(p);
128
+
129
+
CRC32C(crc, value, d);
130
+
}
131
+
132
+
if (len & sizeof(u32)) {
133
+
u32 value = get_unaligned_le32(p);
134
+
135
+
CRC32C(crc, value, w);
136
+
p += sizeof(u32);
137
+
}
138
+
} else {
139
+
for (; len >= sizeof(u32); len -= sizeof(u32)) {
140
+
u32 value = get_unaligned_le32(p);
141
+
142
+
CRC32C(crc, value, w);
143
+
p += sizeof(u32);
144
+
}
145
+
}
146
+
147
+
if (len & sizeof(u16)) {
148
+
u16 value = get_unaligned_le16(p);
149
+
150
+
CRC32C(crc, value, h);
151
+
p += sizeof(u16);
152
+
}
153
+
154
+
if (len & sizeof(u8)) {
155
+
u8 value = *p++;
156
+
157
+
CRC32C(crc, value, b);
158
+
}
159
+
return crc;
160
+
}
161
+
EXPORT_SYMBOL(crc32c_le_arch);
162
+
163
+
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
164
+
{
165
+
return crc32_be_base(crc, p, len);
166
+
}
167
+
EXPORT_SYMBOL(crc32_be_arch);
168
+
169
+
static int __init crc32_mips_init(void)
170
+
{
171
+
if (cpu_have_feature(cpu_feature(MIPS_CRC32)))
172
+
static_branch_enable(&have_crc32);
173
+
return 0;
174
+
}
175
+
arch_initcall(crc32_mips_init);
176
+
177
+
static void __exit crc32_mips_exit(void)
178
+
{
179
+
}
180
+
module_exit(crc32_mips_exit);
181
+
182
+
u32 crc32_optimizations(void)
183
+
{
184
+
if (static_key_enabled(&have_crc32))
185
+
return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION;
186
+
return 0;
187
+
}
188
+
EXPORT_SYMBOL(crc32_optimizations);
189
+
190
+
MODULE_AUTHOR("Marcin Nowakowski <marcin.nowakowski@mips.com");
191
+
MODULE_DESCRIPTION("CRC32 and CRC32C using optional MIPS instructions");
192
+
MODULE_LICENSE("GPL v2");
+2
arch/powerpc/Kconfig
+2
arch/powerpc/Kconfig
···
127
127
select ARCH_ENABLE_MEMORY_HOTPLUG
128
128
select ARCH_ENABLE_MEMORY_HOTREMOVE
129
129
select ARCH_HAS_COPY_MC if PPC64
130
+
select ARCH_HAS_CRC32 if PPC64 && ALTIVEC
131
+
select ARCH_HAS_CRC_T10DIF if PPC64 && ALTIVEC
130
132
select ARCH_HAS_CURRENT_STACK_POINTER
131
133
select ARCH_HAS_DEBUG_VIRTUAL
132
134
select ARCH_HAS_DEBUG_VM_PGTABLE
-2
arch/powerpc/configs/powernv_defconfig
-2
arch/powerpc/configs/powernv_defconfig
-3
arch/powerpc/configs/ppc64_defconfig
-3
arch/powerpc/configs/ppc64_defconfig
···
389
389
CONFIG_CRYPTO_SHA256=y
390
390
CONFIG_CRYPTO_WP512=m
391
391
CONFIG_CRYPTO_LZO=m
392
-
CONFIG_CRYPTO_CRC32C_VPMSUM=m
393
-
CONFIG_CRYPTO_CRCT10DIF_VPMSUM=m
394
-
CONFIG_CRYPTO_VPMSUM_TESTER=m
395
392
CONFIG_CRYPTO_MD5_PPC=m
396
393
CONFIG_CRYPTO_SHA1_PPC=m
397
394
CONFIG_CRYPTO_AES_GCM_P10=m
-33
arch/powerpc/crypto/Kconfig
-33
arch/powerpc/crypto/Kconfig
···
13
13
Architecture: PowerPC64
14
14
- Little-endian
15
15
16
-
config CRYPTO_CRC32C_VPMSUM
17
-
tristate "CRC32c"
18
-
depends on PPC64 && ALTIVEC
19
-
select CRYPTO_HASH
20
-
select CRC32
21
-
help
22
-
CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720)
23
-
24
-
Architecture: powerpc64 using
25
-
- AltiVec extensions
26
-
27
-
Enable on POWER8 and newer processors for improved performance.
28
-
29
-
config CRYPTO_CRCT10DIF_VPMSUM
30
-
tristate "CRC32T10DIF"
31
-
depends on PPC64 && ALTIVEC && CRC_T10DIF
32
-
select CRYPTO_HASH
33
-
help
34
-
CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF)
35
-
36
-
Architecture: powerpc64 using
37
-
- AltiVec extensions
38
-
39
-
Enable on POWER8 and newer processors for improved performance.
40
-
41
-
config CRYPTO_VPMSUM_TESTER
42
-
tristate "CRC32c and CRC32T10DIF hardware acceleration tester"
43
-
depends on CRYPTO_CRCT10DIF_VPMSUM && CRYPTO_CRC32C_VPMSUM
44
-
help
45
-
Stress test for CRC32c and CRCT10DIF algorithms implemented with
46
-
powerpc64 AltiVec extensions (POWER8 vpmsum instructions).
47
-
Unless you are testing these algorithms, you don't need this.
48
-
49
16
config CRYPTO_MD5_PPC
50
17
tristate "Digests: MD5"
51
18
depends on PPC
-5
arch/powerpc/crypto/Makefile
-5
arch/powerpc/crypto/Makefile
···
10
10
obj-$(CONFIG_CRYPTO_SHA1_PPC) += sha1-powerpc.o
11
11
obj-$(CONFIG_CRYPTO_SHA1_PPC_SPE) += sha1-ppc-spe.o
12
12
obj-$(CONFIG_CRYPTO_SHA256_PPC_SPE) += sha256-ppc-spe.o
13
-
obj-$(CONFIG_CRYPTO_CRC32C_VPMSUM) += crc32c-vpmsum.o
14
-
obj-$(CONFIG_CRYPTO_CRCT10DIF_VPMSUM) += crct10dif-vpmsum.o
15
-
obj-$(CONFIG_CRYPTO_VPMSUM_TESTER) += crc-vpmsum_test.o
16
13
obj-$(CONFIG_CRYPTO_AES_GCM_P10) += aes-gcm-p10-crypto.o
17
14
obj-$(CONFIG_CRYPTO_CHACHA20_P10) += chacha-p10-crypto.o
18
15
obj-$(CONFIG_CRYPTO_POLY1305_P10) += poly1305-p10-crypto.o
···
21
24
sha1-powerpc-y := sha1-powerpc-asm.o sha1.o
22
25
sha1-ppc-spe-y := sha1-spe-asm.o sha1-spe-glue.o
23
26
sha256-ppc-spe-y := sha256-spe-asm.o sha256-spe-glue.o
24
-
crc32c-vpmsum-y := crc32c-vpmsum_asm.o crc32c-vpmsum_glue.o
25
-
crct10dif-vpmsum-y := crct10dif-vpmsum_asm.o crct10dif-vpmsum_glue.o
26
27
aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o
27
28
chacha-p10-crypto-y := chacha-p10-glue.o chacha-p10le-8x.o
28
29
poly1305-p10-crypto-y := poly1305-p10-glue.o poly1305-p10le_64.o
-133
arch/powerpc/crypto/crc-vpmsum_test.c
-133
arch/powerpc/crypto/crc-vpmsum_test.c
···
1
-
// SPDX-License-Identifier: GPL-2.0-only
2
-
/*
3
-
* CRC vpmsum tester
4
-
* Copyright 2017 Daniel Axtens, IBM Corporation.
5
-
*/
6
-
7
-
#include <linux/crc-t10dif.h>
8
-
#include <linux/crc32.h>
9
-
#include <crypto/internal/hash.h>
10
-
#include <linux/init.h>
11
-
#include <linux/module.h>
12
-
#include <linux/random.h>
13
-
#include <linux/string.h>
14
-
#include <linux/kernel.h>
15
-
#include <linux/cpufeature.h>
16
-
#include <asm/switch_to.h>
17
-
18
-
static unsigned long iterations = 10000;
19
-
20
-
#define MAX_CRC_LENGTH 65535
21
-
22
-
23
-
static int __init crc_test_init(void)
24
-
{
25
-
u16 crc16 = 0, verify16 = 0;
26
-
__le32 verify32le = 0;
27
-
unsigned char *data;
28
-
u32 verify32 = 0;
29
-
unsigned long i;
30
-
__le32 crc32;
31
-
int ret;
32
-
33
-
struct crypto_shash *crct10dif_tfm;
34
-
struct crypto_shash *crc32c_tfm;
35
-
36
-
if (!cpu_has_feature(CPU_FTR_ARCH_207S))
37
-
return -ENODEV;
38
-
39
-
data = kmalloc(MAX_CRC_LENGTH, GFP_KERNEL);
40
-
if (!data)
41
-
return -ENOMEM;
42
-
43
-
crct10dif_tfm = crypto_alloc_shash("crct10dif", 0, 0);
44
-
45
-
if (IS_ERR(crct10dif_tfm)) {
46
-
pr_err("Error allocating crc-t10dif\n");
47
-
goto free_buf;
48
-
}
49
-
50
-
crc32c_tfm = crypto_alloc_shash("crc32c", 0, 0);
51
-
52
-
if (IS_ERR(crc32c_tfm)) {
53
-
pr_err("Error allocating crc32c\n");
54
-
goto free_16;
55
-
}
56
-
57
-
do {
58
-
SHASH_DESC_ON_STACK(crct10dif_shash, crct10dif_tfm);
59
-
SHASH_DESC_ON_STACK(crc32c_shash, crc32c_tfm);
60
-
61
-
crct10dif_shash->tfm = crct10dif_tfm;
62
-
ret = crypto_shash_init(crct10dif_shash);
63
-
64
-
if (ret) {
65
-
pr_err("Error initing crc-t10dif\n");
66
-
goto free_32;
67
-
}
68
-
69
-
70
-
crc32c_shash->tfm = crc32c_tfm;
71
-
ret = crypto_shash_init(crc32c_shash);
72
-
73
-
if (ret) {
74
-
pr_err("Error initing crc32c\n");
75
-
goto free_32;
76
-
}
77
-
78
-
pr_info("crc-vpmsum_test begins, %lu iterations\n", iterations);
79
-
for (i=0; i<iterations; i++) {
80
-
size_t offset = get_random_u32_below(16);
81
-
size_t len = get_random_u32_below(MAX_CRC_LENGTH);
82
-
83
-
if (len <= offset)
84
-
continue;
85
-
get_random_bytes(data, len);
86
-
len -= offset;
87
-
88
-
crypto_shash_update(crct10dif_shash, data+offset, len);
89
-
crypto_shash_final(crct10dif_shash, (u8 *)(&crc16));
90
-
verify16 = crc_t10dif_generic(verify16, data+offset, len);
91
-
92
-
93
-
if (crc16 != verify16) {
94
-
pr_err("FAILURE in CRC16: got 0x%04x expected 0x%04x (len %lu)\n",
95
-
crc16, verify16, len);
96
-
break;
97
-
}
98
-
99
-
crypto_shash_update(crc32c_shash, data+offset, len);
100
-
crypto_shash_final(crc32c_shash, (u8 *)(&crc32));
101
-
verify32 = le32_to_cpu(verify32le);
102
-
verify32le = ~cpu_to_le32(__crc32c_le(~verify32, data+offset, len));
103
-
if (crc32 != verify32le) {
104
-
pr_err("FAILURE in CRC32: got 0x%08x expected 0x%08x (len %lu)\n",
105
-
crc32, verify32, len);
106
-
break;
107
-
}
108
-
cond_resched();
109
-
}
110
-
pr_info("crc-vpmsum_test done, completed %lu iterations\n", i);
111
-
} while (0);
112
-
113
-
free_32:
114
-
crypto_free_shash(crc32c_tfm);
115
-
116
-
free_16:
117
-
crypto_free_shash(crct10dif_tfm);
118
-
119
-
free_buf:
120
-
kfree(data);
121
-
122
-
return 0;
123
-
}
124
-
125
-
static void __exit crc_test_exit(void) {}
126
-
127
-
module_init(crc_test_init);
128
-
module_exit(crc_test_exit);
129
-
module_param(iterations, long, 0400);
130
-
131
-
MODULE_AUTHOR("Daniel Axtens <dja@axtens.net>");
132
-
MODULE_DESCRIPTION("Vector polynomial multiply-sum CRC tester");
133
-
MODULE_LICENSE("GPL");
arch/powerpc/crypto/crc32-vpmsum_core.S
arch/powerpc/lib/crc32-vpmsum_core.S
arch/powerpc/crypto/crc32-vpmsum_core.S
arch/powerpc/lib/crc32-vpmsum_core.S
arch/powerpc/crypto/crc32c-vpmsum_asm.S
arch/powerpc/lib/crc32c-vpmsum_asm.S
arch/powerpc/crypto/crc32c-vpmsum_asm.S
arch/powerpc/lib/crc32c-vpmsum_asm.S
-173
arch/powerpc/crypto/crc32c-vpmsum_glue.c
-173
arch/powerpc/crypto/crc32c-vpmsum_glue.c
···
1
-
// SPDX-License-Identifier: GPL-2.0-only
2
-
#include <linux/crc32.h>
3
-
#include <crypto/internal/hash.h>
4
-
#include <crypto/internal/simd.h>
5
-
#include <linux/init.h>
6
-
#include <linux/module.h>
7
-
#include <linux/string.h>
8
-
#include <linux/kernel.h>
9
-
#include <linux/cpufeature.h>
10
-
#include <asm/simd.h>
11
-
#include <asm/switch_to.h>
12
-
13
-
#define CHKSUM_BLOCK_SIZE 1
14
-
#define CHKSUM_DIGEST_SIZE 4
15
-
16
-
#define VMX_ALIGN 16
17
-
#define VMX_ALIGN_MASK (VMX_ALIGN-1)
18
-
19
-
#define VECTOR_BREAKPOINT 512
20
-
21
-
u32 __crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len);
22
-
23
-
static u32 crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len)
24
-
{
25
-
unsigned int prealign;
26
-
unsigned int tail;
27
-
28
-
if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || !crypto_simd_usable())
29
-
return __crc32c_le(crc, p, len);
30
-
31
-
if ((unsigned long)p & VMX_ALIGN_MASK) {
32
-
prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);
33
-
crc = __crc32c_le(crc, p, prealign);
34
-
len -= prealign;
35
-
p += prealign;
36
-
}
37
-
38
-
if (len & ~VMX_ALIGN_MASK) {
39
-
preempt_disable();
40
-
pagefault_disable();
41
-
enable_kernel_altivec();
42
-
crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);
43
-
disable_kernel_altivec();
44
-
pagefault_enable();
45
-
preempt_enable();
46
-
}
47
-
48
-
tail = len & VMX_ALIGN_MASK;
49
-
if (tail) {
50
-
p += len & ~VMX_ALIGN_MASK;
51
-
crc = __crc32c_le(crc, p, tail);
52
-
}
53
-
54
-
return crc;
55
-
}
56
-
57
-
static int crc32c_vpmsum_cra_init(struct crypto_tfm *tfm)
58
-
{
59
-
u32 *key = crypto_tfm_ctx(tfm);
60
-
61
-
*key = ~0;
62
-
63
-
return 0;
64
-
}
65
-
66
-
/*
67
-
* Setting the seed allows arbitrary accumulators and flexible XOR policy
68
-
* If your algorithm starts with ~0, then XOR with ~0 before you set
69
-
* the seed.
70
-
*/
71
-
static int crc32c_vpmsum_setkey(struct crypto_shash *hash, const u8 *key,
72
-
unsigned int keylen)
73
-
{
74
-
u32 *mctx = crypto_shash_ctx(hash);
75
-
76
-
if (keylen != sizeof(u32))
77
-
return -EINVAL;
78
-
*mctx = le32_to_cpup((__le32 *)key);
79
-
return 0;
80
-
}
81
-
82
-
static int crc32c_vpmsum_init(struct shash_desc *desc)
83
-
{
84
-
u32 *mctx = crypto_shash_ctx(desc->tfm);
85
-
u32 *crcp = shash_desc_ctx(desc);
86
-
87
-
*crcp = *mctx;
88
-
89
-
return 0;
90
-
}
91
-
92
-
static int crc32c_vpmsum_update(struct shash_desc *desc, const u8 *data,
93
-
unsigned int len)
94
-
{
95
-
u32 *crcp = shash_desc_ctx(desc);
96
-
97
-
*crcp = crc32c_vpmsum(*crcp, data, len);
98
-
99
-
return 0;
100
-
}
101
-
102
-
static int __crc32c_vpmsum_finup(u32 *crcp, const u8 *data, unsigned int len,
103
-
u8 *out)
104
-
{
105
-
*(__le32 *)out = ~cpu_to_le32(crc32c_vpmsum(*crcp, data, len));
106
-
107
-
return 0;
108
-
}
109
-
110
-
static int crc32c_vpmsum_finup(struct shash_desc *desc, const u8 *data,
111
-
unsigned int len, u8 *out)
112
-
{
113
-
return __crc32c_vpmsum_finup(shash_desc_ctx(desc), data, len, out);
114
-
}
115
-
116
-
static int crc32c_vpmsum_final(struct shash_desc *desc, u8 *out)
117
-
{
118
-
u32 *crcp = shash_desc_ctx(desc);
119
-
120
-
*(__le32 *)out = ~cpu_to_le32p(crcp);
121
-
122
-
return 0;
123
-
}
124
-
125
-
static int crc32c_vpmsum_digest(struct shash_desc *desc, const u8 *data,
126
-
unsigned int len, u8 *out)
127
-
{
128
-
return __crc32c_vpmsum_finup(crypto_shash_ctx(desc->tfm), data, len,
129
-
out);
130
-
}
131
-
132
-
static struct shash_alg alg = {
133
-
.setkey = crc32c_vpmsum_setkey,
134
-
.init = crc32c_vpmsum_init,
135
-
.update = crc32c_vpmsum_update,
136
-
.final = crc32c_vpmsum_final,
137
-
.finup = crc32c_vpmsum_finup,
138
-
.digest = crc32c_vpmsum_digest,
139
-
.descsize = sizeof(u32),
140
-
.digestsize = CHKSUM_DIGEST_SIZE,
141
-
.base = {
142
-
.cra_name = "crc32c",
143
-
.cra_driver_name = "crc32c-vpmsum",
144
-
.cra_priority = 200,
145
-
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
146
-
.cra_blocksize = CHKSUM_BLOCK_SIZE,
147
-
.cra_ctxsize = sizeof(u32),
148
-
.cra_module = THIS_MODULE,
149
-
.cra_init = crc32c_vpmsum_cra_init,
150
-
}
151
-
};
152
-
153
-
static int __init crc32c_vpmsum_mod_init(void)
154
-
{
155
-
if (!cpu_has_feature(CPU_FTR_ARCH_207S))
156
-
return -ENODEV;
157
-
158
-
return crypto_register_shash(&alg);
159
-
}
160
-
161
-
static void __exit crc32c_vpmsum_mod_fini(void)
162
-
{
163
-
crypto_unregister_shash(&alg);
164
-
}
165
-
166
-
module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, crc32c_vpmsum_mod_init);
167
-
module_exit(crc32c_vpmsum_mod_fini);
168
-
169
-
MODULE_AUTHOR("Anton Blanchard <anton@samba.org>");
170
-
MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions");
171
-
MODULE_LICENSE("GPL");
172
-
MODULE_ALIAS_CRYPTO("crc32c");
173
-
MODULE_ALIAS_CRYPTO("crc32c-vpmsum");
arch/powerpc/crypto/crct10dif-vpmsum_asm.S
arch/powerpc/lib/crct10dif-vpmsum_asm.S
arch/powerpc/crypto/crct10dif-vpmsum_asm.S
arch/powerpc/lib/crct10dif-vpmsum_asm.S
+16
-53
arch/powerpc/crypto/crct10dif-vpmsum_glue.c
arch/powerpc/lib/crc-t10dif-glue.c
+16
-53
arch/powerpc/crypto/crct10dif-vpmsum_glue.c
arch/powerpc/lib/crc-t10dif-glue.c
···
7
7
*/
8
8
9
9
#include <linux/crc-t10dif.h>
10
-
#include <crypto/internal/hash.h>
11
10
#include <crypto/internal/simd.h>
12
11
#include <linux/init.h>
13
12
#include <linux/module.h>
···
21
22
22
23
#define VECTOR_BREAKPOINT 64
23
24
25
+
static DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
26
+
24
27
u32 __crct10dif_vpmsum(u32 crc, unsigned char const *p, size_t len);
25
28
26
-
static u16 crct10dif_vpmsum(u16 crci, unsigned char const *p, size_t len)
29
+
u16 crc_t10dif_arch(u16 crci, const u8 *p, size_t len)
27
30
{
28
31
unsigned int prealign;
29
32
unsigned int tail;
30
33
u32 crc = crci;
31
34
32
-
if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || !crypto_simd_usable())
35
+
if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) ||
36
+
!static_branch_likely(&have_vec_crypto) || !crypto_simd_usable())
33
37
return crc_t10dif_generic(crc, p, len);
34
38
35
39
if ((unsigned long)p & VMX_ALIGN_MASK) {
···
62
60
63
61
return crc & 0xffff;
64
62
}
63
+
EXPORT_SYMBOL(crc_t10dif_arch);
65
64
66
-
static int crct10dif_vpmsum_init(struct shash_desc *desc)
65
+
static int __init crc_t10dif_powerpc_init(void)
67
66
{
68
-
u16 *crc = shash_desc_ctx(desc);
69
-
70
-
*crc = 0;
67
+
if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
68
+
(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
69
+
static_branch_enable(&have_vec_crypto);
71
70
return 0;
72
71
}
72
+
arch_initcall(crc_t10dif_powerpc_init);
73
73
74
-
static int crct10dif_vpmsum_update(struct shash_desc *desc, const u8 *data,
75
-
unsigned int length)
74
+
static void __exit crc_t10dif_powerpc_exit(void)
76
75
{
77
-
u16 *crc = shash_desc_ctx(desc);
78
-
79
-
*crc = crct10dif_vpmsum(*crc, data, length);
80
-
81
-
return 0;
82
76
}
77
+
module_exit(crc_t10dif_powerpc_exit);
83
78
84
-
85
-
static int crct10dif_vpmsum_final(struct shash_desc *desc, u8 *out)
79
+
bool crc_t10dif_is_optimized(void)
86
80
{
87
-
u16 *crcp = shash_desc_ctx(desc);
88
-
89
-
*(u16 *)out = *crcp;
90
-
return 0;
81
+
return static_key_enabled(&have_vec_crypto);
91
82
}
92
-
93
-
static struct shash_alg alg = {
94
-
.init = crct10dif_vpmsum_init,
95
-
.update = crct10dif_vpmsum_update,
96
-
.final = crct10dif_vpmsum_final,
97
-
.descsize = CRC_T10DIF_DIGEST_SIZE,
98
-
.digestsize = CRC_T10DIF_DIGEST_SIZE,
99
-
.base = {
100
-
.cra_name = "crct10dif",
101
-
.cra_driver_name = "crct10dif-vpmsum",
102
-
.cra_priority = 200,
103
-
.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
104
-
.cra_module = THIS_MODULE,
105
-
}
106
-
};
107
-
108
-
static int __init crct10dif_vpmsum_mod_init(void)
109
-
{
110
-
if (!cpu_has_feature(CPU_FTR_ARCH_207S))
111
-
return -ENODEV;
112
-
113
-
return crypto_register_shash(&alg);
114
-
}
115
-
116
-
static void __exit crct10dif_vpmsum_mod_fini(void)
117
-
{
118
-
crypto_unregister_shash(&alg);
119
-
}
120
-
121
-
module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, crct10dif_vpmsum_mod_init);
122
-
module_exit(crct10dif_vpmsum_mod_fini);
83
+
EXPORT_SYMBOL(crc_t10dif_is_optimized);
123
84
124
85
MODULE_AUTHOR("Daniel Axtens <dja@axtens.net>");
125
86
MODULE_DESCRIPTION("CRCT10DIF using vector polynomial multiply-sum instructions");
126
87
MODULE_LICENSE("GPL");
127
-
MODULE_ALIAS_CRYPTO("crct10dif");
128
-
MODULE_ALIAS_CRYPTO("crct10dif-vpmsum");
+6
arch/powerpc/lib/Makefile
+6
arch/powerpc/lib/Makefile
···
78
78
# Enable <altivec.h>
79
79
CFLAGS_xor_vmx.o += -isystem $(shell $(CC) -print-file-name=include)
80
80
81
+
obj-$(CONFIG_CRC32_ARCH) += crc32-powerpc.o
82
+
crc32-powerpc-y := crc32-glue.o crc32c-vpmsum_asm.o
83
+
84
+
obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-powerpc.o
85
+
crc-t10dif-powerpc-y := crc-t10dif-glue.o crct10dif-vpmsum_asm.o
86
+
81
87
obj-$(CONFIG_PPC64) += $(obj64-y)
+92
arch/powerpc/lib/crc32-glue.c
+92
arch/powerpc/lib/crc32-glue.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
#include <linux/crc32.h>
3
+
#include <crypto/internal/simd.h>
4
+
#include <linux/init.h>
5
+
#include <linux/module.h>
6
+
#include <linux/kernel.h>
7
+
#include <linux/cpufeature.h>
8
+
#include <asm/simd.h>
9
+
#include <asm/switch_to.h>
10
+
11
+
#define VMX_ALIGN 16
12
+
#define VMX_ALIGN_MASK (VMX_ALIGN-1)
13
+
14
+
#define VECTOR_BREAKPOINT 512
15
+
16
+
static DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
17
+
18
+
u32 __crc32c_vpmsum(u32 crc, const u8 *p, size_t len);
19
+
20
+
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
21
+
{
22
+
return crc32_le_base(crc, p, len);
23
+
}
24
+
EXPORT_SYMBOL(crc32_le_arch);
25
+
26
+
u32 crc32c_le_arch(u32 crc, const u8 *p, size_t len)
27
+
{
28
+
unsigned int prealign;
29
+
unsigned int tail;
30
+
31
+
if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) ||
32
+
!static_branch_likely(&have_vec_crypto) || !crypto_simd_usable())
33
+
return crc32c_le_base(crc, p, len);
34
+
35
+
if ((unsigned long)p & VMX_ALIGN_MASK) {
36
+
prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);
37
+
crc = crc32c_le_base(crc, p, prealign);
38
+
len -= prealign;
39
+
p += prealign;
40
+
}
41
+
42
+
if (len & ~VMX_ALIGN_MASK) {
43
+
preempt_disable();
44
+
pagefault_disable();
45
+
enable_kernel_altivec();
46
+
crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);
47
+
disable_kernel_altivec();
48
+
pagefault_enable();
49
+
preempt_enable();
50
+
}
51
+
52
+
tail = len & VMX_ALIGN_MASK;
53
+
if (tail) {
54
+
p += len & ~VMX_ALIGN_MASK;
55
+
crc = crc32c_le_base(crc, p, tail);
56
+
}
57
+
58
+
return crc;
59
+
}
60
+
EXPORT_SYMBOL(crc32c_le_arch);
61
+
62
+
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
63
+
{
64
+
return crc32_be_base(crc, p, len);
65
+
}
66
+
EXPORT_SYMBOL(crc32_be_arch);
67
+
68
+
static int __init crc32_powerpc_init(void)
69
+
{
70
+
if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
71
+
(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
72
+
static_branch_enable(&have_vec_crypto);
73
+
return 0;
74
+
}
75
+
arch_initcall(crc32_powerpc_init);
76
+
77
+
static void __exit crc32_powerpc_exit(void)
78
+
{
79
+
}
80
+
module_exit(crc32_powerpc_exit);
81
+
82
+
u32 crc32_optimizations(void)
83
+
{
84
+
if (static_key_enabled(&have_vec_crypto))
85
+
return CRC32C_OPTIMIZATION;
86
+
return 0;
87
+
}
88
+
EXPORT_SYMBOL(crc32_optimizations);
89
+
90
+
MODULE_AUTHOR("Anton Blanchard <anton@samba.org>");
91
+
MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions");
92
+
MODULE_LICENSE("GPL");
+1
arch/riscv/Kconfig
+1
arch/riscv/Kconfig
···
24
24
select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2
25
25
select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
26
26
select ARCH_HAS_BINFMT_FLAT
27
+
select ARCH_HAS_CRC32 if RISCV_ISA_ZBC
27
28
select ARCH_HAS_CURRENT_STACK_POINTER
28
29
select ARCH_HAS_DEBUG_VIRTUAL if MMU
29
30
select ARCH_HAS_DEBUG_VM_PGTABLE
+1
-2
arch/riscv/lib/Makefile
+1
-2
arch/riscv/lib/Makefile
···
15
15
lib-$(CONFIG_MMU) += uaccess.o
16
16
lib-$(CONFIG_64BIT) += tishift.o
17
17
lib-$(CONFIG_RISCV_ISA_ZICBOZ) += clear_page.o
18
-
lib-$(CONFIG_RISCV_ISA_ZBC) += crc32.o
19
-
18
+
obj-$(CONFIG_CRC32_ARCH) += crc32-riscv.o
20
19
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
21
20
lib-$(CONFIG_RISCV_ISA_V) += xor.o
22
21
lib-$(CONFIG_RISCV_ISA_V) += riscv_v_helpers.o
+21
-4
arch/riscv/lib/crc32.c
arch/riscv/lib/crc32-riscv.c
+21
-4
arch/riscv/lib/crc32.c
arch/riscv/lib/crc32-riscv.c
···
14
14
#include <linux/crc32poly.h>
15
15
#include <linux/crc32.h>
16
16
#include <linux/byteorder/generic.h>
17
+
#include <linux/module.h>
17
18
18
19
/*
19
20
* Refer to https://www.corsix.org/content/barrett-reduction-polynomials for
···
218
217
return crc_fb(crc, p, len);
219
218
}
220
219
221
-
u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len)
220
+
u32 __pure crc32_le_arch(u32 crc, const u8 *p, size_t len)
222
221
{
223
222
return crc32_le_generic(crc, p, len, CRC32_POLY_LE, CRC32_POLY_QT_LE,
224
223
crc32_le_base);
225
224
}
225
+
EXPORT_SYMBOL(crc32_le_arch);
226
226
227
-
u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len)
227
+
u32 __pure crc32c_le_arch(u32 crc, const u8 *p, size_t len)
228
228
{
229
229
return crc32_le_generic(crc, p, len, CRC32C_POLY_LE,
230
-
CRC32C_POLY_QT_LE, __crc32c_le_base);
230
+
CRC32C_POLY_QT_LE, crc32c_le_base);
231
231
}
232
+
EXPORT_SYMBOL(crc32c_le_arch);
232
233
233
234
static inline u32 crc32_be_unaligned(u32 crc, unsigned char const *p,
234
235
size_t len)
···
256
253
return crc;
257
254
}
258
255
259
-
u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len)
256
+
u32 __pure crc32_be_arch(u32 crc, const u8 *p, size_t len)
260
257
{
261
258
size_t offset, head_len, tail_len;
262
259
unsigned long const *p_ul;
···
295
292
legacy:
296
293
return crc32_be_base(crc, p, len);
297
294
}
295
+
EXPORT_SYMBOL(crc32_be_arch);
296
+
297
+
u32 crc32_optimizations(void)
298
+
{
299
+
if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC))
300
+
return CRC32_LE_OPTIMIZATION |
301
+
CRC32_BE_OPTIMIZATION |
302
+
CRC32C_OPTIMIZATION;
303
+
return 0;
304
+
}
305
+
EXPORT_SYMBOL(crc32_optimizations);
306
+
307
+
MODULE_LICENSE("GPL");
308
+
MODULE_DESCRIPTION("Accelerated CRC32 implementation with Zbc extension");
+1
arch/s390/Kconfig
+1
arch/s390/Kconfig
···
72
72
select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM
73
73
select ARCH_ENABLE_MEMORY_HOTREMOVE
74
74
select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2
75
+
select ARCH_HAS_CRC32
75
76
select ARCH_HAS_CURRENT_STACK_POINTER
76
77
select ARCH_HAS_DEBUG_VIRTUAL
77
78
select ARCH_HAS_DEBUG_VM_PGTABLE
-2
arch/s390/configs/debug_defconfig
-2
arch/s390/configs/debug_defconfig
···
795
795
CONFIG_CRYPTO_USER_API_SKCIPHER=m
796
796
CONFIG_CRYPTO_USER_API_RNG=m
797
797
CONFIG_CRYPTO_USER_API_AEAD=m
798
-
CONFIG_CRYPTO_CRC32_S390=y
799
798
CONFIG_CRYPTO_SHA512_S390=m
800
799
CONFIG_CRYPTO_SHA1_S390=m
801
800
CONFIG_CRYPTO_SHA256_S390=m
···
817
818
CONFIG_CORDIC=m
818
819
CONFIG_CRYPTO_LIB_CURVE25519=m
819
820
CONFIG_CRYPTO_LIB_CHACHA20POLY1305=m
820
-
CONFIG_CRC32_SELFTEST=y
821
821
CONFIG_CRC4=m
822
822
CONFIG_CRC7=m
823
823
CONFIG_CRC8=m
-1
arch/s390/configs/defconfig
-1
arch/s390/configs/defconfig
-12
arch/s390/crypto/Kconfig
-12
arch/s390/crypto/Kconfig
···
2
2
3
3
menu "Accelerated Cryptographic Algorithms for CPU (s390)"
4
4
5
-
config CRYPTO_CRC32_S390
6
-
tristate "CRC32c and CRC32"
7
-
depends on S390
8
-
select CRYPTO_HASH
9
-
select CRC32
10
-
help
11
-
CRC32c and CRC32 CRC algorithms
12
-
13
-
Architecture: s390
14
-
15
-
It is available with IBM z13 or later.
16
-
17
5
config CRYPTO_SHA512_S390
18
6
tristate "Hash functions: SHA-384 and SHA-512"
19
7
depends on S390
-2
arch/s390/crypto/Makefile
-2
arch/s390/crypto/Makefile
···
14
14
obj-$(CONFIG_CRYPTO_CHACHA_S390) += chacha_s390.o
15
15
obj-$(CONFIG_S390_PRNG) += prng.o
16
16
obj-$(CONFIG_CRYPTO_GHASH_S390) += ghash_s390.o
17
-
obj-$(CONFIG_CRYPTO_CRC32_S390) += crc32-vx_s390.o
18
17
obj-$(CONFIG_CRYPTO_HMAC_S390) += hmac_s390.o
19
18
obj-y += arch_random.o
20
19
21
-
crc32-vx_s390-y := crc32-vx.o crc32le-vx.o crc32be-vx.o
22
20
chacha_s390-y := chacha-glue.o chacha-s390.o
-306
arch/s390/crypto/crc32-vx.c
-306
arch/s390/crypto/crc32-vx.c
···
1
-
// SPDX-License-Identifier: GPL-2.0
2
-
/*
3
-
* Crypto-API module for CRC-32 algorithms implemented with the
4
-
* z/Architecture Vector Extension Facility.
5
-
*
6
-
* Copyright IBM Corp. 2015
7
-
* Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
8
-
*/
9
-
#define KMSG_COMPONENT "crc32-vx"
10
-
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
11
-
12
-
#include <linux/module.h>
13
-
#include <linux/cpufeature.h>
14
-
#include <linux/crc32.h>
15
-
#include <crypto/internal/hash.h>
16
-
#include <asm/fpu.h>
17
-
#include "crc32-vx.h"
18
-
19
-
#define CRC32_BLOCK_SIZE 1
20
-
#define CRC32_DIGEST_SIZE 4
21
-
22
-
#define VX_MIN_LEN 64
23
-
#define VX_ALIGNMENT 16L
24
-
#define VX_ALIGN_MASK (VX_ALIGNMENT - 1)
25
-
26
-
struct crc_ctx {
27
-
u32 key;
28
-
};
29
-
30
-
struct crc_desc_ctx {
31
-
u32 crc;
32
-
};
33
-
34
-
/*
35
-
* DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
36
-
*
37
-
* Creates a function to perform a particular CRC-32 computation. Depending
38
-
* on the message buffer, the hardware-accelerated or software implementation
39
-
* is used. Note that the message buffer is aligned to improve fetch
40
-
* operations of VECTOR LOAD MULTIPLE instructions.
41
-
*
42
-
*/
43
-
#define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \
44
-
static u32 __pure ___fname(u32 crc, \
45
-
unsigned char const *data, size_t datalen) \
46
-
{ \
47
-
unsigned long prealign, aligned, remaining; \
48
-
DECLARE_KERNEL_FPU_ONSTACK16(vxstate); \
49
-
\
50
-
if (datalen < VX_MIN_LEN + VX_ALIGN_MASK) \
51
-
return ___crc32_sw(crc, data, datalen); \
52
-
\
53
-
if ((unsigned long)data & VX_ALIGN_MASK) { \
54
-
prealign = VX_ALIGNMENT - \
55
-
((unsigned long)data & VX_ALIGN_MASK); \
56
-
datalen -= prealign; \
57
-
crc = ___crc32_sw(crc, data, prealign); \
58
-
data = (void *)((unsigned long)data + prealign); \
59
-
} \
60
-
\
61
-
aligned = datalen & ~VX_ALIGN_MASK; \
62
-
remaining = datalen & VX_ALIGN_MASK; \
63
-
\
64
-
kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \
65
-
crc = ___crc32_vx(crc, data, aligned); \
66
-
kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \
67
-
\
68
-
if (remaining) \
69
-
crc = ___crc32_sw(crc, data + aligned, remaining); \
70
-
\
71
-
return crc; \
72
-
}
73
-
74
-
DEFINE_CRC32_VX(crc32_le_vx, crc32_le_vgfm_16, crc32_le)
75
-
DEFINE_CRC32_VX(crc32_be_vx, crc32_be_vgfm_16, crc32_be)
76
-
DEFINE_CRC32_VX(crc32c_le_vx, crc32c_le_vgfm_16, __crc32c_le)
77
-
78
-
79
-
static int crc32_vx_cra_init_zero(struct crypto_tfm *tfm)
80
-
{
81
-
struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
82
-
83
-
mctx->key = 0;
84
-
return 0;
85
-
}
86
-
87
-
static int crc32_vx_cra_init_invert(struct crypto_tfm *tfm)
88
-
{
89
-
struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
90
-
91
-
mctx->key = ~0;
92
-
return 0;
93
-
}
94
-
95
-
static int crc32_vx_init(struct shash_desc *desc)
96
-
{
97
-
struct crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
98
-
struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
99
-
100
-
ctx->crc = mctx->key;
101
-
return 0;
102
-
}
103
-
104
-
static int crc32_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
105
-
unsigned int newkeylen)
106
-
{
107
-
struct crc_ctx *mctx = crypto_shash_ctx(tfm);
108
-
109
-
if (newkeylen != sizeof(mctx->key))
110
-
return -EINVAL;
111
-
mctx->key = le32_to_cpu(*(__le32 *)newkey);
112
-
return 0;
113
-
}
114
-
115
-
static int crc32be_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
116
-
unsigned int newkeylen)
117
-
{
118
-
struct crc_ctx *mctx = crypto_shash_ctx(tfm);
119
-
120
-
if (newkeylen != sizeof(mctx->key))
121
-
return -EINVAL;
122
-
mctx->key = be32_to_cpu(*(__be32 *)newkey);
123
-
return 0;
124
-
}
125
-
126
-
static int crc32le_vx_final(struct shash_desc *desc, u8 *out)
127
-
{
128
-
struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
129
-
130
-
*(__le32 *)out = cpu_to_le32p(&ctx->crc);
131
-
return 0;
132
-
}
133
-
134
-
static int crc32be_vx_final(struct shash_desc *desc, u8 *out)
135
-
{
136
-
struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
137
-
138
-
*(__be32 *)out = cpu_to_be32p(&ctx->crc);
139
-
return 0;
140
-
}
141
-
142
-
static int crc32c_vx_final(struct shash_desc *desc, u8 *out)
143
-
{
144
-
struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
145
-
146
-
/*
147
-
* Perform a final XOR with 0xFFFFFFFF to be in sync
148
-
* with the generic crc32c shash implementation.
149
-
*/
150
-
*(__le32 *)out = ~cpu_to_le32p(&ctx->crc);
151
-
return 0;
152
-
}
153
-
154
-
static int __crc32le_vx_finup(u32 *crc, const u8 *data, unsigned int len,
155
-
u8 *out)
156
-
{
157
-
*(__le32 *)out = cpu_to_le32(crc32_le_vx(*crc, data, len));
158
-
return 0;
159
-
}
160
-
161
-
static int __crc32be_vx_finup(u32 *crc, const u8 *data, unsigned int len,
162
-
u8 *out)
163
-
{
164
-
*(__be32 *)out = cpu_to_be32(crc32_be_vx(*crc, data, len));
165
-
return 0;
166
-
}
167
-
168
-
static int __crc32c_vx_finup(u32 *crc, const u8 *data, unsigned int len,
169
-
u8 *out)
170
-
{
171
-
/*
172
-
* Perform a final XOR with 0xFFFFFFFF to be in sync
173
-
* with the generic crc32c shash implementation.
174
-
*/
175
-
*(__le32 *)out = ~cpu_to_le32(crc32c_le_vx(*crc, data, len));
176
-
return 0;
177
-
}
178
-
179
-
180
-
#define CRC32_VX_FINUP(alg, func) \
181
-
static int alg ## _vx_finup(struct shash_desc *desc, const u8 *data, \
182
-
unsigned int datalen, u8 *out) \
183
-
{ \
184
-
return __ ## alg ## _vx_finup(shash_desc_ctx(desc), \
185
-
data, datalen, out); \
186
-
}
187
-
188
-
CRC32_VX_FINUP(crc32le, crc32_le_vx)
189
-
CRC32_VX_FINUP(crc32be, crc32_be_vx)
190
-
CRC32_VX_FINUP(crc32c, crc32c_le_vx)
191
-
192
-
#define CRC32_VX_DIGEST(alg, func) \
193
-
static int alg ## _vx_digest(struct shash_desc *desc, const u8 *data, \
194
-
unsigned int len, u8 *out) \
195
-
{ \
196
-
return __ ## alg ## _vx_finup(crypto_shash_ctx(desc->tfm), \
197
-
data, len, out); \
198
-
}
199
-
200
-
CRC32_VX_DIGEST(crc32le, crc32_le_vx)
201
-
CRC32_VX_DIGEST(crc32be, crc32_be_vx)
202
-
CRC32_VX_DIGEST(crc32c, crc32c_le_vx)
203
-
204
-
#define CRC32_VX_UPDATE(alg, func) \
205
-
static int alg ## _vx_update(struct shash_desc *desc, const u8 *data, \
206
-
unsigned int datalen) \
207
-
{ \
208
-
struct crc_desc_ctx *ctx = shash_desc_ctx(desc); \
209
-
ctx->crc = func(ctx->crc, data, datalen); \
210
-
return 0; \
211
-
}
212
-
213
-
CRC32_VX_UPDATE(crc32le, crc32_le_vx)
214
-
CRC32_VX_UPDATE(crc32be, crc32_be_vx)
215
-
CRC32_VX_UPDATE(crc32c, crc32c_le_vx)
216
-
217
-
218
-
static struct shash_alg crc32_vx_algs[] = {
219
-
/* CRC-32 LE */
220
-
{
221
-
.init = crc32_vx_init,
222
-
.setkey = crc32_vx_setkey,
223
-
.update = crc32le_vx_update,
224
-
.final = crc32le_vx_final,
225
-
.finup = crc32le_vx_finup,
226
-
.digest = crc32le_vx_digest,
227
-
.descsize = sizeof(struct crc_desc_ctx),
228
-
.digestsize = CRC32_DIGEST_SIZE,
229
-
.base = {
230
-
.cra_name = "crc32",
231
-
.cra_driver_name = "crc32-vx",
232
-
.cra_priority = 200,
233
-
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
234
-
.cra_blocksize = CRC32_BLOCK_SIZE,
235
-
.cra_ctxsize = sizeof(struct crc_ctx),
236
-
.cra_module = THIS_MODULE,
237
-
.cra_init = crc32_vx_cra_init_zero,
238
-
},
239
-
},
240
-
/* CRC-32 BE */
241
-
{
242
-
.init = crc32_vx_init,
243
-
.setkey = crc32be_vx_setkey,
244
-
.update = crc32be_vx_update,
245
-
.final = crc32be_vx_final,
246
-
.finup = crc32be_vx_finup,
247
-
.digest = crc32be_vx_digest,
248
-
.descsize = sizeof(struct crc_desc_ctx),
249
-
.digestsize = CRC32_DIGEST_SIZE,
250
-
.base = {
251
-
.cra_name = "crc32be",
252
-
.cra_driver_name = "crc32be-vx",
253
-
.cra_priority = 200,
254
-
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
255
-
.cra_blocksize = CRC32_BLOCK_SIZE,
256
-
.cra_ctxsize = sizeof(struct crc_ctx),
257
-
.cra_module = THIS_MODULE,
258
-
.cra_init = crc32_vx_cra_init_zero,
259
-
},
260
-
},
261
-
/* CRC-32C LE */
262
-
{
263
-
.init = crc32_vx_init,
264
-
.setkey = crc32_vx_setkey,
265
-
.update = crc32c_vx_update,
266
-
.final = crc32c_vx_final,
267
-
.finup = crc32c_vx_finup,
268
-
.digest = crc32c_vx_digest,
269
-
.descsize = sizeof(struct crc_desc_ctx),
270
-
.digestsize = CRC32_DIGEST_SIZE,
271
-
.base = {
272
-
.cra_name = "crc32c",
273
-
.cra_driver_name = "crc32c-vx",
274
-
.cra_priority = 200,
275
-
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
276
-
.cra_blocksize = CRC32_BLOCK_SIZE,
277
-
.cra_ctxsize = sizeof(struct crc_ctx),
278
-
.cra_module = THIS_MODULE,
279
-
.cra_init = crc32_vx_cra_init_invert,
280
-
},
281
-
},
282
-
};
283
-
284
-
285
-
static int __init crc_vx_mod_init(void)
286
-
{
287
-
return crypto_register_shashes(crc32_vx_algs,
288
-
ARRAY_SIZE(crc32_vx_algs));
289
-
}
290
-
291
-
static void __exit crc_vx_mod_exit(void)
292
-
{
293
-
crypto_unregister_shashes(crc32_vx_algs, ARRAY_SIZE(crc32_vx_algs));
294
-
}
295
-
296
-
module_cpu_feature_match(S390_CPU_FEATURE_VXRS, crc_vx_mod_init);
297
-
module_exit(crc_vx_mod_exit);
298
-
299
-
MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
300
-
MODULE_DESCRIPTION("CRC-32 algorithms using z/Architecture Vector Extension Facility");
301
-
MODULE_LICENSE("GPL");
302
-
303
-
MODULE_ALIAS_CRYPTO("crc32");
304
-
MODULE_ALIAS_CRYPTO("crc32-vx");
305
-
MODULE_ALIAS_CRYPTO("crc32c");
306
-
MODULE_ALIAS_CRYPTO("crc32c-vx");
arch/s390/crypto/crc32-vx.h
arch/s390/lib/crc32-vx.h
arch/s390/crypto/crc32-vx.h
arch/s390/lib/crc32-vx.h
arch/s390/crypto/crc32be-vx.c
arch/s390/lib/crc32be-vx.c
arch/s390/crypto/crc32be-vx.c
arch/s390/lib/crc32be-vx.c
arch/s390/crypto/crc32le-vx.c
arch/s390/lib/crc32le-vx.c
arch/s390/crypto/crc32le-vx.c
arch/s390/lib/crc32le-vx.c
+3
arch/s390/lib/Makefile
+3
arch/s390/lib/Makefile
+92
arch/s390/lib/crc32-glue.c
+92
arch/s390/lib/crc32-glue.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/*
3
+
* CRC-32 implemented with the z/Architecture Vector Extension Facility.
4
+
*
5
+
* Copyright IBM Corp. 2015
6
+
* Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
7
+
*/
8
+
#define KMSG_COMPONENT "crc32-vx"
9
+
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
10
+
11
+
#include <linux/module.h>
12
+
#include <linux/cpufeature.h>
13
+
#include <linux/crc32.h>
14
+
#include <asm/fpu.h>
15
+
#include "crc32-vx.h"
16
+
17
+
#define VX_MIN_LEN 64
18
+
#define VX_ALIGNMENT 16L
19
+
#define VX_ALIGN_MASK (VX_ALIGNMENT - 1)
20
+
21
+
static DEFINE_STATIC_KEY_FALSE(have_vxrs);
22
+
23
+
/*
24
+
* DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
25
+
*
26
+
* Creates a function to perform a particular CRC-32 computation. Depending
27
+
* on the message buffer, the hardware-accelerated or software implementation
28
+
* is used. Note that the message buffer is aligned to improve fetch
29
+
* operations of VECTOR LOAD MULTIPLE instructions.
30
+
*/
31
+
#define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \
32
+
u32 ___fname(u32 crc, const u8 *data, size_t datalen) \
33
+
{ \
34
+
unsigned long prealign, aligned, remaining; \
35
+
DECLARE_KERNEL_FPU_ONSTACK16(vxstate); \
36
+
\
37
+
if (datalen < VX_MIN_LEN + VX_ALIGN_MASK || \
38
+
!static_branch_likely(&have_vxrs)) \
39
+
return ___crc32_sw(crc, data, datalen); \
40
+
\
41
+
if ((unsigned long)data & VX_ALIGN_MASK) { \
42
+
prealign = VX_ALIGNMENT - \
43
+
((unsigned long)data & VX_ALIGN_MASK); \
44
+
datalen -= prealign; \
45
+
crc = ___crc32_sw(crc, data, prealign); \
46
+
data = (void *)((unsigned long)data + prealign); \
47
+
} \
48
+
\
49
+
aligned = datalen & ~VX_ALIGN_MASK; \
50
+
remaining = datalen & VX_ALIGN_MASK; \
51
+
\
52
+
kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \
53
+
crc = ___crc32_vx(crc, data, aligned); \
54
+
kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \
55
+
\
56
+
if (remaining) \
57
+
crc = ___crc32_sw(crc, data + aligned, remaining); \
58
+
\
59
+
return crc; \
60
+
} \
61
+
EXPORT_SYMBOL(___fname);
62
+
63
+
DEFINE_CRC32_VX(crc32_le_arch, crc32_le_vgfm_16, crc32_le_base)
64
+
DEFINE_CRC32_VX(crc32_be_arch, crc32_be_vgfm_16, crc32_be_base)
65
+
DEFINE_CRC32_VX(crc32c_le_arch, crc32c_le_vgfm_16, crc32c_le_base)
66
+
67
+
static int __init crc32_s390_init(void)
68
+
{
69
+
if (cpu_have_feature(S390_CPU_FEATURE_VXRS))
70
+
static_branch_enable(&have_vxrs);
71
+
return 0;
72
+
}
73
+
arch_initcall(crc32_s390_init);
74
+
75
+
static void __exit crc32_s390_exit(void)
76
+
{
77
+
}
78
+
module_exit(crc32_s390_exit);
79
+
80
+
u32 crc32_optimizations(void)
81
+
{
82
+
if (static_key_enabled(&have_vxrs))
83
+
return CRC32_LE_OPTIMIZATION |
84
+
CRC32_BE_OPTIMIZATION |
85
+
CRC32C_OPTIMIZATION;
86
+
return 0;
87
+
}
88
+
EXPORT_SYMBOL(crc32_optimizations);
89
+
90
+
MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
91
+
MODULE_DESCRIPTION("CRC-32 algorithms using z/Architecture Vector Extension Facility");
92
+
MODULE_LICENSE("GPL");
+1
arch/sparc/Kconfig
+1
arch/sparc/Kconfig
-10
arch/sparc/crypto/Kconfig
-10
arch/sparc/crypto/Kconfig
···
16
16
17
17
Architecture: sparc64
18
18
19
-
config CRYPTO_CRC32C_SPARC64
20
-
tristate "CRC32c"
21
-
depends on SPARC64
22
-
select CRYPTO_HASH
23
-
select CRC32
24
-
help
25
-
CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720)
26
-
27
-
Architecture: sparc64
28
-
29
19
config CRYPTO_MD5_SPARC64
30
20
tristate "Digests: MD5"
31
21
depends on SPARC64
-4
arch/sparc/crypto/Makefile
-4
arch/sparc/crypto/Makefile
···
12
12
obj-$(CONFIG_CRYPTO_DES_SPARC64) += des-sparc64.o
13
13
obj-$(CONFIG_CRYPTO_CAMELLIA_SPARC64) += camellia-sparc64.o
14
14
15
-
obj-$(CONFIG_CRYPTO_CRC32C_SPARC64) += crc32c-sparc64.o
16
-
17
15
sha1-sparc64-y := sha1_asm.o sha1_glue.o
18
16
sha256-sparc64-y := sha256_asm.o sha256_glue.o
19
17
sha512-sparc64-y := sha512_asm.o sha512_glue.o
···
20
22
aes-sparc64-y := aes_asm.o aes_glue.o
21
23
des-sparc64-y := des_asm.o des_glue.o
22
24
camellia-sparc64-y := camellia_asm.o camellia_glue.o
23
-
24
-
crc32c-sparc64-y := crc32c_asm.o crc32c_glue.o
+1
-1
arch/sparc/crypto/crc32c_asm.S
arch/sparc/lib/crc32c_asm.S
+1
-1
arch/sparc/crypto/crc32c_asm.S
arch/sparc/lib/crc32c_asm.S
-184
arch/sparc/crypto/crc32c_glue.c
-184
arch/sparc/crypto/crc32c_glue.c
···
1
-
// SPDX-License-Identifier: GPL-2.0-only
2
-
/* Glue code for CRC32C optimized for sparc64 crypto opcodes.
3
-
*
4
-
* This is based largely upon arch/x86/crypto/crc32c-intel.c
5
-
*
6
-
* Copyright (C) 2008 Intel Corporation
7
-
* Authors: Austin Zhang <austin_zhang@linux.intel.com>
8
-
* Kent Liu <kent.liu@intel.com>
9
-
*/
10
-
11
-
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
-
13
-
#include <linux/init.h>
14
-
#include <linux/module.h>
15
-
#include <linux/string.h>
16
-
#include <linux/kernel.h>
17
-
#include <linux/crc32.h>
18
-
19
-
#include <crypto/internal/hash.h>
20
-
21
-
#include <asm/pstate.h>
22
-
#include <asm/elf.h>
23
-
#include <linux/unaligned.h>
24
-
25
-
#include "opcodes.h"
26
-
27
-
/*
28
-
* Setting the seed allows arbitrary accumulators and flexible XOR policy
29
-
* If your algorithm starts with ~0, then XOR with ~0 before you set
30
-
* the seed.
31
-
*/
32
-
static int crc32c_sparc64_setkey(struct crypto_shash *hash, const u8 *key,
33
-
unsigned int keylen)
34
-
{
35
-
u32 *mctx = crypto_shash_ctx(hash);
36
-
37
-
if (keylen != sizeof(u32))
38
-
return -EINVAL;
39
-
*mctx = get_unaligned_le32(key);
40
-
return 0;
41
-
}
42
-
43
-
static int crc32c_sparc64_init(struct shash_desc *desc)
44
-
{
45
-
u32 *mctx = crypto_shash_ctx(desc->tfm);
46
-
u32 *crcp = shash_desc_ctx(desc);
47
-
48
-
*crcp = *mctx;
49
-
50
-
return 0;
51
-
}
52
-
53
-
extern void crc32c_sparc64(u32 *crcp, const u64 *data, unsigned int len);
54
-
55
-
static u32 crc32c_compute(u32 crc, const u8 *data, unsigned int len)
56
-
{
57
-
unsigned int n = -(uintptr_t)data & 7;
58
-
59
-
if (n) {
60
-
/* Data isn't 8-byte aligned. Align it. */
61
-
n = min(n, len);
62
-
crc = __crc32c_le(crc, data, n);
63
-
data += n;
64
-
len -= n;
65
-
}
66
-
n = len & ~7U;
67
-
if (n) {
68
-
crc32c_sparc64(&crc, (const u64 *)data, n);
69
-
data += n;
70
-
len -= n;
71
-
}
72
-
if (len)
73
-
crc = __crc32c_le(crc, data, len);
74
-
return crc;
75
-
}
76
-
77
-
static int crc32c_sparc64_update(struct shash_desc *desc, const u8 *data,
78
-
unsigned int len)
79
-
{
80
-
u32 *crcp = shash_desc_ctx(desc);
81
-
82
-
*crcp = crc32c_compute(*crcp, data, len);
83
-
return 0;
84
-
}
85
-
86
-
static int __crc32c_sparc64_finup(const u32 *crcp, const u8 *data,
87
-
unsigned int len, u8 *out)
88
-
{
89
-
put_unaligned_le32(~crc32c_compute(*crcp, data, len), out);
90
-
return 0;
91
-
}
92
-
93
-
static int crc32c_sparc64_finup(struct shash_desc *desc, const u8 *data,
94
-
unsigned int len, u8 *out)
95
-
{
96
-
return __crc32c_sparc64_finup(shash_desc_ctx(desc), data, len, out);
97
-
}
98
-
99
-
static int crc32c_sparc64_final(struct shash_desc *desc, u8 *out)
100
-
{
101
-
u32 *crcp = shash_desc_ctx(desc);
102
-
103
-
put_unaligned_le32(~*crcp, out);
104
-
return 0;
105
-
}
106
-
107
-
static int crc32c_sparc64_digest(struct shash_desc *desc, const u8 *data,
108
-
unsigned int len, u8 *out)
109
-
{
110
-
return __crc32c_sparc64_finup(crypto_shash_ctx(desc->tfm), data, len,
111
-
out);
112
-
}
113
-
114
-
static int crc32c_sparc64_cra_init(struct crypto_tfm *tfm)
115
-
{
116
-
u32 *key = crypto_tfm_ctx(tfm);
117
-
118
-
*key = ~0;
119
-
120
-
return 0;
121
-
}
122
-
123
-
#define CHKSUM_BLOCK_SIZE 1
124
-
#define CHKSUM_DIGEST_SIZE 4
125
-
126
-
static struct shash_alg alg = {
127
-
.setkey = crc32c_sparc64_setkey,
128
-
.init = crc32c_sparc64_init,
129
-
.update = crc32c_sparc64_update,
130
-
.final = crc32c_sparc64_final,
131
-
.finup = crc32c_sparc64_finup,
132
-
.digest = crc32c_sparc64_digest,
133
-
.descsize = sizeof(u32),
134
-
.digestsize = CHKSUM_DIGEST_SIZE,
135
-
.base = {
136
-
.cra_name = "crc32c",
137
-
.cra_driver_name = "crc32c-sparc64",
138
-
.cra_priority = SPARC_CR_OPCODE_PRIORITY,
139
-
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
140
-
.cra_blocksize = CHKSUM_BLOCK_SIZE,
141
-
.cra_ctxsize = sizeof(u32),
142
-
.cra_module = THIS_MODULE,
143
-
.cra_init = crc32c_sparc64_cra_init,
144
-
}
145
-
};
146
-
147
-
static bool __init sparc64_has_crc32c_opcode(void)
148
-
{
149
-
unsigned long cfr;
150
-
151
-
if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
152
-
return false;
153
-
154
-
__asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
155
-
if (!(cfr & CFR_CRC32C))
156
-
return false;
157
-
158
-
return true;
159
-
}
160
-
161
-
static int __init crc32c_sparc64_mod_init(void)
162
-
{
163
-
if (sparc64_has_crc32c_opcode()) {
164
-
pr_info("Using sparc64 crc32c opcode optimized CRC32C implementation\n");
165
-
return crypto_register_shash(&alg);
166
-
}
167
-
pr_info("sparc64 crc32c opcode not available.\n");
168
-
return -ENODEV;
169
-
}
170
-
171
-
static void __exit crc32c_sparc64_mod_fini(void)
172
-
{
173
-
crypto_unregister_shash(&alg);
174
-
}
175
-
176
-
module_init(crc32c_sparc64_mod_init);
177
-
module_exit(crc32c_sparc64_mod_fini);
178
-
179
-
MODULE_LICENSE("GPL");
180
-
MODULE_DESCRIPTION("CRC32c (Castagnoli), sparc64 crc32c opcode accelerated");
181
-
182
-
MODULE_ALIAS_CRYPTO("crc32c");
183
-
184
-
#include "crop_devid.c"
+2
arch/sparc/lib/Makefile
+2
arch/sparc/lib/Makefile
+93
arch/sparc/lib/crc32_glue.c
+93
arch/sparc/lib/crc32_glue.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/* Glue code for CRC32C optimized for sparc64 crypto opcodes.
3
+
*
4
+
* This is based largely upon arch/x86/crypto/crc32c-intel.c
5
+
*
6
+
* Copyright (C) 2008 Intel Corporation
7
+
* Authors: Austin Zhang <austin_zhang@linux.intel.com>
8
+
* Kent Liu <kent.liu@intel.com>
9
+
*/
10
+
11
+
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
+
13
+
#include <linux/init.h>
14
+
#include <linux/module.h>
15
+
#include <linux/kernel.h>
16
+
#include <linux/crc32.h>
17
+
#include <asm/pstate.h>
18
+
#include <asm/elf.h>
19
+
20
+
static DEFINE_STATIC_KEY_FALSE(have_crc32c_opcode);
21
+
22
+
u32 crc32_le_arch(u32 crc, const u8 *data, size_t len)
23
+
{
24
+
return crc32_le_base(crc, data, len);
25
+
}
26
+
EXPORT_SYMBOL(crc32_le_arch);
27
+
28
+
void crc32c_sparc64(u32 *crcp, const u64 *data, size_t len);
29
+
30
+
u32 crc32c_le_arch(u32 crc, const u8 *data, size_t len)
31
+
{
32
+
size_t n = -(uintptr_t)data & 7;
33
+
34
+
if (!static_branch_likely(&have_crc32c_opcode))
35
+
return crc32c_le_base(crc, data, len);
36
+
37
+
if (n) {
38
+
/* Data isn't 8-byte aligned. Align it. */
39
+
n = min(n, len);
40
+
crc = crc32c_le_base(crc, data, n);
41
+
data += n;
42
+
len -= n;
43
+
}
44
+
n = len & ~7U;
45
+
if (n) {
46
+
crc32c_sparc64(&crc, (const u64 *)data, n);
47
+
data += n;
48
+
len -= n;
49
+
}
50
+
if (len)
51
+
crc = crc32c_le_base(crc, data, len);
52
+
return crc;
53
+
}
54
+
EXPORT_SYMBOL(crc32c_le_arch);
55
+
56
+
u32 crc32_be_arch(u32 crc, const u8 *data, size_t len)
57
+
{
58
+
return crc32_be_base(crc, data, len);
59
+
}
60
+
EXPORT_SYMBOL(crc32_be_arch);
61
+
62
+
static int __init crc32_sparc_init(void)
63
+
{
64
+
unsigned long cfr;
65
+
66
+
if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
67
+
return 0;
68
+
69
+
__asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
70
+
if (!(cfr & CFR_CRC32C))
71
+
return 0;
72
+
73
+
static_branch_enable(&have_crc32c_opcode);
74
+
pr_info("Using sparc64 crc32c opcode optimized CRC32C implementation\n");
75
+
return 0;
76
+
}
77
+
arch_initcall(crc32_sparc_init);
78
+
79
+
static void __exit crc32_sparc_exit(void)
80
+
{
81
+
}
82
+
module_exit(crc32_sparc_exit);
83
+
84
+
u32 crc32_optimizations(void)
85
+
{
86
+
if (static_key_enabled(&have_crc32c_opcode))
87
+
return CRC32C_OPTIMIZATION;
88
+
return 0;
89
+
}
90
+
EXPORT_SYMBOL(crc32_optimizations);
91
+
92
+
MODULE_LICENSE("GPL");
93
+
MODULE_DESCRIPTION("CRC32c (Castagnoli), sparc64 crc32c opcode accelerated");
+2
arch/x86/Kconfig
+2
arch/x86/Kconfig
···
76
76
select ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION
77
77
select ARCH_HAS_CPU_FINALIZE_INIT
78
78
select ARCH_HAS_CPU_PASID if IOMMU_SVA
79
+
select ARCH_HAS_CRC32
80
+
select ARCH_HAS_CRC_T10DIF if X86_64
79
81
select ARCH_HAS_CURRENT_STACK_POINTER
80
82
select ARCH_HAS_DEBUG_VIRTUAL
81
83
select ARCH_HAS_DEBUG_VM_PGTABLE if !X86_PAE
-32
arch/x86/crypto/Kconfig
-32
arch/x86/crypto/Kconfig
···
492
492
Architecture: x86_64 using:
493
493
- CLMUL-NI (carry-less multiplication new instructions)
494
494
495
-
config CRYPTO_CRC32C_INTEL
496
-
tristate "CRC32c (SSE4.2/PCLMULQDQ)"
497
-
depends on X86
498
-
select CRYPTO_HASH
499
-
help
500
-
CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720)
501
-
502
-
Architecture: x86 (32-bit and 64-bit) using:
503
-
- SSE4.2 (Streaming SIMD Extensions 4.2) CRC32 instruction
504
-
- PCLMULQDQ (carry-less multiplication)
505
-
506
-
config CRYPTO_CRC32_PCLMUL
507
-
tristate "CRC32 (PCLMULQDQ)"
508
-
depends on X86
509
-
select CRYPTO_HASH
510
-
select CRC32
511
-
help
512
-
CRC32 CRC algorithm (IEEE 802.3)
513
-
514
-
Architecture: x86 (32-bit and 64-bit) using:
515
-
- PCLMULQDQ (carry-less multiplication)
516
-
517
-
config CRYPTO_CRCT10DIF_PCLMUL
518
-
tristate "CRCT10DIF (PCLMULQDQ)"
519
-
depends on X86 && 64BIT && CRC_T10DIF
520
-
select CRYPTO_HASH
521
-
help
522
-
CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF)
523
-
524
-
Architecture: x86_64 using:
525
-
- PCLMULQDQ (carry-less multiplication)
526
-
527
495
endmenu
-10
arch/x86/crypto/Makefile
-10
arch/x86/crypto/Makefile
···
75
75
obj-$(CONFIG_CRYPTO_POLYVAL_CLMUL_NI) += polyval-clmulni.o
76
76
polyval-clmulni-y := polyval-clmulni_asm.o polyval-clmulni_glue.o
77
77
78
-
obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o
79
-
crc32c-intel-y := crc32c-intel_glue.o
80
-
crc32c-intel-$(CONFIG_64BIT) += crc32c-pcl-intel-asm_64.o
81
-
82
-
obj-$(CONFIG_CRYPTO_CRC32_PCLMUL) += crc32-pclmul.o
83
-
crc32-pclmul-y := crc32-pclmul_asm.o crc32-pclmul_glue.o
84
-
85
-
obj-$(CONFIG_CRYPTO_CRCT10DIF_PCLMUL) += crct10dif-pclmul.o
86
-
crct10dif-pclmul-y := crct10dif-pcl-asm_64.o crct10dif-pclmul_glue.o
87
-
88
78
obj-$(CONFIG_CRYPTO_POLY1305_X86_64) += poly1305-x86_64.o
89
79
poly1305-x86_64-y := poly1305-x86_64-cryptogams.o poly1305_glue.o
90
80
targets += poly1305-x86_64-cryptogams.S
+9
-10
arch/x86/crypto/crc32-pclmul_asm.S
arch/x86/lib/crc32-pclmul.S
+9
-10
arch/x86/crypto/crc32-pclmul_asm.S
arch/x86/lib/crc32-pclmul.S
···
58
58
#define CONSTANT %xmm0
59
59
60
60
#ifdef __x86_64__
61
-
#define BUF %rdi
62
-
#define LEN %rsi
63
-
#define CRC %edx
61
+
#define CRC %edi
62
+
#define BUF %rsi
63
+
#define LEN %rdx
64
64
#else
65
-
#define BUF %eax
66
-
#define LEN %edx
67
-
#define CRC %ecx
65
+
#define CRC %eax
66
+
#define BUF %edx
67
+
#define LEN %ecx
68
68
#endif
69
69
70
70
···
72
72
.text
73
73
/**
74
74
* Calculate crc32
75
-
* BUF - buffer (16 bytes aligned)
76
-
* LEN - sizeof buffer (16 bytes aligned), LEN should be grater than 63
77
75
* CRC - initial crc32
76
+
* BUF - buffer (16 bytes aligned)
77
+
* LEN - sizeof buffer (16 bytes aligned), LEN should be greater than 63
78
78
* return %eax crc32
79
-
* uint crc32_pclmul_le_16(unsigned char const *buffer,
80
-
* size_t len, uint crc32)
79
+
* u32 crc32_pclmul_le_16(u32 crc, const u8 *buffer, size_t len);
81
80
*/
82
81
83
82
SYM_FUNC_START(crc32_pclmul_le_16) /* buffer and buffer size are 16 bytes aligned */
-202
arch/x86/crypto/crc32-pclmul_glue.c
-202
arch/x86/crypto/crc32-pclmul_glue.c
···
1
-
/* GPL HEADER START
2
-
*
3
-
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4
-
*
5
-
* This program is free software; you can redistribute it and/or modify
6
-
* it under the terms of the GNU General Public License version 2 only,
7
-
* as published by the Free Software Foundation.
8
-
*
9
-
* This program is distributed in the hope that it will be useful, but
10
-
* WITHOUT ANY WARRANTY; without even the implied warranty of
11
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
-
* General Public License version 2 for more details (a copy is included
13
-
* in the LICENSE file that accompanied this code).
14
-
*
15
-
* You should have received a copy of the GNU General Public License
16
-
* version 2 along with this program; If not, see http://www.gnu.org/licenses
17
-
*
18
-
* Please visit http://www.xyratex.com/contact if you need additional
19
-
* information or have any questions.
20
-
*
21
-
* GPL HEADER END
22
-
*/
23
-
24
-
/*
25
-
* Copyright 2012 Xyratex Technology Limited
26
-
*
27
-
* Wrappers for kernel crypto shash api to pclmulqdq crc32 implementation.
28
-
*/
29
-
#include <linux/init.h>
30
-
#include <linux/module.h>
31
-
#include <linux/string.h>
32
-
#include <linux/kernel.h>
33
-
#include <linux/crc32.h>
34
-
#include <crypto/internal/hash.h>
35
-
#include <crypto/internal/simd.h>
36
-
37
-
#include <asm/cpufeatures.h>
38
-
#include <asm/cpu_device_id.h>
39
-
#include <asm/simd.h>
40
-
41
-
#define CHKSUM_BLOCK_SIZE 1
42
-
#define CHKSUM_DIGEST_SIZE 4
43
-
44
-
#define PCLMUL_MIN_LEN 64L /* minimum size of buffer
45
-
* for crc32_pclmul_le_16 */
46
-
#define SCALE_F 16L /* size of xmm register */
47
-
#define SCALE_F_MASK (SCALE_F - 1)
48
-
49
-
u32 crc32_pclmul_le_16(unsigned char const *buffer, size_t len, u32 crc32);
50
-
51
-
static u32 __attribute__((pure))
52
-
crc32_pclmul_le(u32 crc, unsigned char const *p, size_t len)
53
-
{
54
-
unsigned int iquotient;
55
-
unsigned int iremainder;
56
-
unsigned int prealign;
57
-
58
-
if (len < PCLMUL_MIN_LEN + SCALE_F_MASK || !crypto_simd_usable())
59
-
return crc32_le(crc, p, len);
60
-
61
-
if ((long)p & SCALE_F_MASK) {
62
-
/* align p to 16 byte */
63
-
prealign = SCALE_F - ((long)p & SCALE_F_MASK);
64
-
65
-
crc = crc32_le(crc, p, prealign);
66
-
len -= prealign;
67
-
p = (unsigned char *)(((unsigned long)p + SCALE_F_MASK) &
68
-
~SCALE_F_MASK);
69
-
}
70
-
iquotient = len & (~SCALE_F_MASK);
71
-
iremainder = len & SCALE_F_MASK;
72
-
73
-
kernel_fpu_begin();
74
-
crc = crc32_pclmul_le_16(p, iquotient, crc);
75
-
kernel_fpu_end();
76
-
77
-
if (iremainder)
78
-
crc = crc32_le(crc, p + iquotient, iremainder);
79
-
80
-
return crc;
81
-
}
82
-
83
-
static int crc32_pclmul_cra_init(struct crypto_tfm *tfm)
84
-
{
85
-
u32 *key = crypto_tfm_ctx(tfm);
86
-
87
-
*key = 0;
88
-
89
-
return 0;
90
-
}
91
-
92
-
static int crc32_pclmul_setkey(struct crypto_shash *hash, const u8 *key,
93
-
unsigned int keylen)
94
-
{
95
-
u32 *mctx = crypto_shash_ctx(hash);
96
-
97
-
if (keylen != sizeof(u32))
98
-
return -EINVAL;
99
-
*mctx = le32_to_cpup((__le32 *)key);
100
-
return 0;
101
-
}
102
-
103
-
static int crc32_pclmul_init(struct shash_desc *desc)
104
-
{
105
-
u32 *mctx = crypto_shash_ctx(desc->tfm);
106
-
u32 *crcp = shash_desc_ctx(desc);
107
-
108
-
*crcp = *mctx;
109
-
110
-
return 0;
111
-
}
112
-
113
-
static int crc32_pclmul_update(struct shash_desc *desc, const u8 *data,
114
-
unsigned int len)
115
-
{
116
-
u32 *crcp = shash_desc_ctx(desc);
117
-
118
-
*crcp = crc32_pclmul_le(*crcp, data, len);
119
-
return 0;
120
-
}
121
-
122
-
/* No final XOR 0xFFFFFFFF, like crc32_le */
123
-
static int __crc32_pclmul_finup(u32 *crcp, const u8 *data, unsigned int len,
124
-
u8 *out)
125
-
{
126
-
*(__le32 *)out = cpu_to_le32(crc32_pclmul_le(*crcp, data, len));
127
-
return 0;
128
-
}
129
-
130
-
static int crc32_pclmul_finup(struct shash_desc *desc, const u8 *data,
131
-
unsigned int len, u8 *out)
132
-
{
133
-
return __crc32_pclmul_finup(shash_desc_ctx(desc), data, len, out);
134
-
}
135
-
136
-
static int crc32_pclmul_final(struct shash_desc *desc, u8 *out)
137
-
{
138
-
u32 *crcp = shash_desc_ctx(desc);
139
-
140
-
*(__le32 *)out = cpu_to_le32p(crcp);
141
-
return 0;
142
-
}
143
-
144
-
static int crc32_pclmul_digest(struct shash_desc *desc, const u8 *data,
145
-
unsigned int len, u8 *out)
146
-
{
147
-
return __crc32_pclmul_finup(crypto_shash_ctx(desc->tfm), data, len,
148
-
out);
149
-
}
150
-
151
-
static struct shash_alg alg = {
152
-
.setkey = crc32_pclmul_setkey,
153
-
.init = crc32_pclmul_init,
154
-
.update = crc32_pclmul_update,
155
-
.final = crc32_pclmul_final,
156
-
.finup = crc32_pclmul_finup,
157
-
.digest = crc32_pclmul_digest,
158
-
.descsize = sizeof(u32),
159
-
.digestsize = CHKSUM_DIGEST_SIZE,
160
-
.base = {
161
-
.cra_name = "crc32",
162
-
.cra_driver_name = "crc32-pclmul",
163
-
.cra_priority = 200,
164
-
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
165
-
.cra_blocksize = CHKSUM_BLOCK_SIZE,
166
-
.cra_ctxsize = sizeof(u32),
167
-
.cra_module = THIS_MODULE,
168
-
.cra_init = crc32_pclmul_cra_init,
169
-
}
170
-
};
171
-
172
-
static const struct x86_cpu_id crc32pclmul_cpu_id[] = {
173
-
X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL),
174
-
{}
175
-
};
176
-
MODULE_DEVICE_TABLE(x86cpu, crc32pclmul_cpu_id);
177
-
178
-
179
-
static int __init crc32_pclmul_mod_init(void)
180
-
{
181
-
182
-
if (!x86_match_cpu(crc32pclmul_cpu_id)) {
183
-
pr_info("PCLMULQDQ-NI instructions are not detected.\n");
184
-
return -ENODEV;
185
-
}
186
-
return crypto_register_shash(&alg);
187
-
}
188
-
189
-
static void __exit crc32_pclmul_mod_fini(void)
190
-
{
191
-
crypto_unregister_shash(&alg);
192
-
}
193
-
194
-
module_init(crc32_pclmul_mod_init);
195
-
module_exit(crc32_pclmul_mod_fini);
196
-
197
-
MODULE_AUTHOR("Alexander Boyko <alexander_boyko@xyratex.com>");
198
-
MODULE_DESCRIPTION("CRC32 algorithm (IEEE 802.3) accelerated with PCLMULQDQ");
199
-
MODULE_LICENSE("GPL");
200
-
201
-
MODULE_ALIAS_CRYPTO("crc32");
202
-
MODULE_ALIAS_CRYPTO("crc32-pclmul");
-250
arch/x86/crypto/crc32c-intel_glue.c
-250
arch/x86/crypto/crc32c-intel_glue.c
···
1
-
// SPDX-License-Identifier: GPL-2.0-only
2
-
/*
3
-
* Using hardware provided CRC32 instruction to accelerate the CRC32 disposal.
4
-
* CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE)
5
-
* CRC32 is a new instruction in Intel SSE4.2, the reference can be found at:
6
-
* http://www.intel.com/products/processor/manuals/
7
-
* Intel(R) 64 and IA-32 Architectures Software Developer's Manual
8
-
* Volume 2A: Instruction Set Reference, A-M
9
-
*
10
-
* Copyright (C) 2008 Intel Corporation
11
-
* Authors: Austin Zhang <austin_zhang@linux.intel.com>
12
-
* Kent Liu <kent.liu@intel.com>
13
-
*/
14
-
#include <linux/init.h>
15
-
#include <linux/module.h>
16
-
#include <linux/string.h>
17
-
#include <linux/kernel.h>
18
-
#include <crypto/internal/hash.h>
19
-
#include <crypto/internal/simd.h>
20
-
21
-
#include <asm/cpufeatures.h>
22
-
#include <asm/cpu_device_id.h>
23
-
#include <asm/simd.h>
24
-
25
-
#define CHKSUM_BLOCK_SIZE 1
26
-
#define CHKSUM_DIGEST_SIZE 4
27
-
28
-
#define SCALE_F sizeof(unsigned long)
29
-
30
-
#ifdef CONFIG_X86_64
31
-
#define CRC32_INST "crc32q %1, %q0"
32
-
#else
33
-
#define CRC32_INST "crc32l %1, %0"
34
-
#endif
35
-
36
-
#ifdef CONFIG_X86_64
37
-
/*
38
-
* use carryless multiply version of crc32c when buffer
39
-
* size is >= 512 to account
40
-
* for fpu state save/restore overhead.
41
-
*/
42
-
#define CRC32C_PCL_BREAKEVEN 512
43
-
44
-
asmlinkage unsigned int crc_pcl(const u8 *buffer, unsigned int len,
45
-
unsigned int crc_init);
46
-
#endif /* CONFIG_X86_64 */
47
-
48
-
static u32 crc32c_intel_le_hw_byte(u32 crc, unsigned char const *data, size_t length)
49
-
{
50
-
while (length--) {
51
-
asm("crc32b %1, %0"
52
-
: "+r" (crc) : "rm" (*data));
53
-
data++;
54
-
}
55
-
56
-
return crc;
57
-
}
58
-
59
-
static u32 __pure crc32c_intel_le_hw(u32 crc, unsigned char const *p, size_t len)
60
-
{
61
-
unsigned int iquotient = len / SCALE_F;
62
-
unsigned int iremainder = len % SCALE_F;
63
-
unsigned long *ptmp = (unsigned long *)p;
64
-
65
-
while (iquotient--) {
66
-
asm(CRC32_INST
67
-
: "+r" (crc) : "rm" (*ptmp));
68
-
ptmp++;
69
-
}
70
-
71
-
if (iremainder)
72
-
crc = crc32c_intel_le_hw_byte(crc, (unsigned char *)ptmp,
73
-
iremainder);
74
-
75
-
return crc;
76
-
}
77
-
78
-
/*
79
-
* Setting the seed allows arbitrary accumulators and flexible XOR policy
80
-
* If your algorithm starts with ~0, then XOR with ~0 before you set
81
-
* the seed.
82
-
*/
83
-
static int crc32c_intel_setkey(struct crypto_shash *hash, const u8 *key,
84
-
unsigned int keylen)
85
-
{
86
-
u32 *mctx = crypto_shash_ctx(hash);
87
-
88
-
if (keylen != sizeof(u32))
89
-
return -EINVAL;
90
-
*mctx = le32_to_cpup((__le32 *)key);
91
-
return 0;
92
-
}
93
-
94
-
static int crc32c_intel_init(struct shash_desc *desc)
95
-
{
96
-
u32 *mctx = crypto_shash_ctx(desc->tfm);
97
-
u32 *crcp = shash_desc_ctx(desc);
98
-
99
-
*crcp = *mctx;
100
-
101
-
return 0;
102
-
}
103
-
104
-
static int crc32c_intel_update(struct shash_desc *desc, const u8 *data,
105
-
unsigned int len)
106
-
{
107
-
u32 *crcp = shash_desc_ctx(desc);
108
-
109
-
*crcp = crc32c_intel_le_hw(*crcp, data, len);
110
-
return 0;
111
-
}
112
-
113
-
static int __crc32c_intel_finup(u32 *crcp, const u8 *data, unsigned int len,
114
-
u8 *out)
115
-
{
116
-
*(__le32 *)out = ~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len));
117
-
return 0;
118
-
}
119
-
120
-
static int crc32c_intel_finup(struct shash_desc *desc, const u8 *data,
121
-
unsigned int len, u8 *out)
122
-
{
123
-
return __crc32c_intel_finup(shash_desc_ctx(desc), data, len, out);
124
-
}
125
-
126
-
static int crc32c_intel_final(struct shash_desc *desc, u8 *out)
127
-
{
128
-
u32 *crcp = shash_desc_ctx(desc);
129
-
130
-
*(__le32 *)out = ~cpu_to_le32p(crcp);
131
-
return 0;
132
-
}
133
-
134
-
static int crc32c_intel_digest(struct shash_desc *desc, const u8 *data,
135
-
unsigned int len, u8 *out)
136
-
{
137
-
return __crc32c_intel_finup(crypto_shash_ctx(desc->tfm), data, len,
138
-
out);
139
-
}
140
-
141
-
static int crc32c_intel_cra_init(struct crypto_tfm *tfm)
142
-
{
143
-
u32 *key = crypto_tfm_ctx(tfm);
144
-
145
-
*key = ~0;
146
-
147
-
return 0;
148
-
}
149
-
150
-
#ifdef CONFIG_X86_64
151
-
static int crc32c_pcl_intel_update(struct shash_desc *desc, const u8 *data,
152
-
unsigned int len)
153
-
{
154
-
u32 *crcp = shash_desc_ctx(desc);
155
-
156
-
/*
157
-
* use faster PCL version if datasize is large enough to
158
-
* overcome kernel fpu state save/restore overhead
159
-
*/
160
-
if (len >= CRC32C_PCL_BREAKEVEN && crypto_simd_usable()) {
161
-
kernel_fpu_begin();
162
-
*crcp = crc_pcl(data, len, *crcp);
163
-
kernel_fpu_end();
164
-
} else
165
-
*crcp = crc32c_intel_le_hw(*crcp, data, len);
166
-
return 0;
167
-
}
168
-
169
-
static int __crc32c_pcl_intel_finup(u32 *crcp, const u8 *data, unsigned int len,
170
-
u8 *out)
171
-
{
172
-
if (len >= CRC32C_PCL_BREAKEVEN && crypto_simd_usable()) {
173
-
kernel_fpu_begin();
174
-
*(__le32 *)out = ~cpu_to_le32(crc_pcl(data, len, *crcp));
175
-
kernel_fpu_end();
176
-
} else
177
-
*(__le32 *)out =
178
-
~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len));
179
-
return 0;
180
-
}
181
-
182
-
static int crc32c_pcl_intel_finup(struct shash_desc *desc, const u8 *data,
183
-
unsigned int len, u8 *out)
184
-
{
185
-
return __crc32c_pcl_intel_finup(shash_desc_ctx(desc), data, len, out);
186
-
}
187
-
188
-
static int crc32c_pcl_intel_digest(struct shash_desc *desc, const u8 *data,
189
-
unsigned int len, u8 *out)
190
-
{
191
-
return __crc32c_pcl_intel_finup(crypto_shash_ctx(desc->tfm), data, len,
192
-
out);
193
-
}
194
-
#endif /* CONFIG_X86_64 */
195
-
196
-
static struct shash_alg alg = {
197
-
.setkey = crc32c_intel_setkey,
198
-
.init = crc32c_intel_init,
199
-
.update = crc32c_intel_update,
200
-
.final = crc32c_intel_final,
201
-
.finup = crc32c_intel_finup,
202
-
.digest = crc32c_intel_digest,
203
-
.descsize = sizeof(u32),
204
-
.digestsize = CHKSUM_DIGEST_SIZE,
205
-
.base = {
206
-
.cra_name = "crc32c",
207
-
.cra_driver_name = "crc32c-intel",
208
-
.cra_priority = 200,
209
-
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
210
-
.cra_blocksize = CHKSUM_BLOCK_SIZE,
211
-
.cra_ctxsize = sizeof(u32),
212
-
.cra_module = THIS_MODULE,
213
-
.cra_init = crc32c_intel_cra_init,
214
-
}
215
-
};
216
-
217
-
static const struct x86_cpu_id crc32c_cpu_id[] = {
218
-
X86_MATCH_FEATURE(X86_FEATURE_XMM4_2, NULL),
219
-
{}
220
-
};
221
-
MODULE_DEVICE_TABLE(x86cpu, crc32c_cpu_id);
222
-
223
-
static int __init crc32c_intel_mod_init(void)
224
-
{
225
-
if (!x86_match_cpu(crc32c_cpu_id))
226
-
return -ENODEV;
227
-
#ifdef CONFIG_X86_64
228
-
if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
229
-
alg.update = crc32c_pcl_intel_update;
230
-
alg.finup = crc32c_pcl_intel_finup;
231
-
alg.digest = crc32c_pcl_intel_digest;
232
-
}
233
-
#endif
234
-
return crypto_register_shash(&alg);
235
-
}
236
-
237
-
static void __exit crc32c_intel_mod_fini(void)
238
-
{
239
-
crypto_unregister_shash(&alg);
240
-
}
241
-
242
-
module_init(crc32c_intel_mod_init);
243
-
module_exit(crc32c_intel_mod_fini);
244
-
245
-
MODULE_AUTHOR("Austin Zhang <austin.zhang@intel.com>, Kent Liu <kent.liu@intel.com>");
246
-
MODULE_DESCRIPTION("CRC32c (Castagnoli) optimization using Intel Hardware.");
247
-
MODULE_LICENSE("GPL");
248
-
249
-
MODULE_ALIAS_CRYPTO("crc32c");
250
-
MODULE_ALIAS_CRYPTO("crc32c-intel");
+32
-31
arch/x86/crypto/crc32c-pcl-intel-asm_64.S
arch/x86/lib/crc32c-3way.S
+32
-31
arch/x86/crypto/crc32c-pcl-intel-asm_64.S
arch/x86/lib/crc32c-3way.S
···
52
52
# regular CRC code that does not interleave the CRC instructions.
53
53
#define SMALL_SIZE 200
54
54
55
-
# unsigned int crc_pcl(const u8 *buffer, unsigned int len, unsigned int crc_init);
55
+
# u32 crc32c_x86_3way(u32 crc, const u8 *buffer, size_t len);
56
56
57
57
.text
58
-
SYM_FUNC_START(crc_pcl)
59
-
#define bufp %rdi
60
-
#define bufp_d %edi
61
-
#define len %esi
62
-
#define crc_init %edx
63
-
#define crc_init_q %rdx
58
+
SYM_FUNC_START(crc32c_x86_3way)
59
+
#define crc0 %edi
60
+
#define crc0_q %rdi
61
+
#define bufp %rsi
62
+
#define bufp_d %esi
63
+
#define len %rdx
64
+
#define len_dw %edx
64
65
#define n_misaligned %ecx /* overlaps chunk_bytes! */
65
66
#define n_misaligned_q %rcx
66
67
#define chunk_bytes %ecx /* overlaps n_misaligned! */
···
86
85
.Ldo_align:
87
86
movq (bufp), %rax
88
87
add n_misaligned_q, bufp
89
-
sub n_misaligned, len
88
+
sub n_misaligned_q, len
90
89
.Lalign_loop:
91
-
crc32b %al, crc_init # compute crc32 of 1-byte
90
+
crc32b %al, crc0 # compute crc32 of 1-byte
92
91
shr $8, %rax # get next byte
93
92
dec n_misaligned
94
93
jne .Lalign_loop
···
103
102
104
103
.Lpartial_block:
105
104
# Compute floor(len / 24) to get num qwords to process from each lane.
106
-
imul $2731, len, %eax # 2731 = ceil(2^16 / 24)
105
+
imul $2731, len_dw, %eax # 2731 = ceil(2^16 / 24)
107
106
shr $16, %eax
108
107
jmp .Lcrc_3lanes
109
108
···
126
125
# Unroll the loop by a factor of 4 to reduce the overhead of the loop
127
126
# bookkeeping instructions, which can compete with crc32q for the ALUs.
128
127
.Lcrc_3lanes_4x_loop:
129
-
crc32q (bufp), crc_init_q
128
+
crc32q (bufp), crc0_q
130
129
crc32q (bufp,chunk_bytes_q), crc1
131
130
crc32q (bufp,chunk_bytes_q,2), crc2
132
-
crc32q 8(bufp), crc_init_q
131
+
crc32q 8(bufp), crc0_q
133
132
crc32q 8(bufp,chunk_bytes_q), crc1
134
133
crc32q 8(bufp,chunk_bytes_q,2), crc2
135
-
crc32q 16(bufp), crc_init_q
134
+
crc32q 16(bufp), crc0_q
136
135
crc32q 16(bufp,chunk_bytes_q), crc1
137
136
crc32q 16(bufp,chunk_bytes_q,2), crc2
138
-
crc32q 24(bufp), crc_init_q
137
+
crc32q 24(bufp), crc0_q
139
138
crc32q 24(bufp,chunk_bytes_q), crc1
140
139
crc32q 24(bufp,chunk_bytes_q,2), crc2
141
140
add $32, bufp
···
147
146
jz .Lcrc_3lanes_last_qword
148
147
149
148
.Lcrc_3lanes_1x_loop:
150
-
crc32q (bufp), crc_init_q
149
+
crc32q (bufp), crc0_q
151
150
crc32q (bufp,chunk_bytes_q), crc1
152
151
crc32q (bufp,chunk_bytes_q,2), crc2
153
152
add $8, bufp
···
155
154
jnz .Lcrc_3lanes_1x_loop
156
155
157
156
.Lcrc_3lanes_last_qword:
158
-
crc32q (bufp), crc_init_q
157
+
crc32q (bufp), crc0_q
159
158
crc32q (bufp,chunk_bytes_q), crc1
160
159
# SKIP crc32q (bufp,chunk_bytes_q,2), crc2 ; Don't do this one yet
161
160
···
166
165
lea (K_table-8)(%rip), %rax # first entry is for idx 1
167
166
pmovzxdq (%rax,chunk_bytes_q), %xmm0 # 2 consts: K1:K2
168
167
lea (chunk_bytes,chunk_bytes,2), %eax # chunk_bytes * 3
169
-
sub %eax, len # len -= chunk_bytes * 3
168
+
sub %rax, len # len -= chunk_bytes * 3
170
169
171
-
movq crc_init_q, %xmm1 # CRC for block 1
170
+
movq crc0_q, %xmm1 # CRC for block 1
172
171
pclmulqdq $0x00, %xmm0, %xmm1 # Multiply by K2
173
172
174
173
movq crc1, %xmm2 # CRC for block 2
···
177
176
pxor %xmm2,%xmm1
178
177
movq %xmm1, %rax
179
178
xor (bufp,chunk_bytes_q,2), %rax
180
-
mov crc2, crc_init_q
181
-
crc32 %rax, crc_init_q
179
+
mov crc2, crc0_q
180
+
crc32 %rax, crc0_q
182
181
lea 8(bufp,chunk_bytes_q,2), bufp
183
182
184
183
################################################################
···
194
193
## 6) Process any remainder without interleaving:
195
194
#######################################################################
196
195
.Lsmall:
197
-
test len, len
196
+
test len_dw, len_dw
198
197
jz .Ldone
199
-
mov len, %eax
198
+
mov len_dw, %eax
200
199
shr $3, %eax
201
200
jz .Ldo_dword
202
201
.Ldo_qwords:
203
-
crc32q (bufp), crc_init_q
202
+
crc32q (bufp), crc0_q
204
203
add $8, bufp
205
204
dec %eax
206
205
jnz .Ldo_qwords
207
206
.Ldo_dword:
208
-
test $4, len
207
+
test $4, len_dw
209
208
jz .Ldo_word
210
-
crc32l (bufp), crc_init
209
+
crc32l (bufp), crc0
211
210
add $4, bufp
212
211
.Ldo_word:
213
-
test $2, len
212
+
test $2, len_dw
214
213
jz .Ldo_byte
215
-
crc32w (bufp), crc_init
214
+
crc32w (bufp), crc0
216
215
add $2, bufp
217
216
.Ldo_byte:
218
-
test $1, len
217
+
test $1, len_dw
219
218
jz .Ldone
220
-
crc32b (bufp), crc_init
219
+
crc32b (bufp), crc0
221
220
.Ldone:
222
-
mov crc_init, %eax
221
+
mov crc0, %eax
223
222
RET
224
-
SYM_FUNC_END(crc_pcl)
223
+
SYM_FUNC_END(crc32c_x86_3way)
225
224
226
225
.section .rodata, "a", @progbits
227
226
################################################################
arch/x86/crypto/crct10dif-pcl-asm_64.S
arch/x86/lib/crct10dif-pcl-asm_64.S
arch/x86/crypto/crct10dif-pcl-asm_64.S
arch/x86/lib/crct10dif-pcl-asm_64.S
-143
arch/x86/crypto/crct10dif-pclmul_glue.c
-143
arch/x86/crypto/crct10dif-pclmul_glue.c
···
1
-
/*
2
-
* Cryptographic API.
3
-
*
4
-
* T10 Data Integrity Field CRC16 Crypto Transform using PCLMULQDQ Instructions
5
-
*
6
-
* Copyright (C) 2013 Intel Corporation
7
-
* Author: Tim Chen <tim.c.chen@linux.intel.com>
8
-
*
9
-
* This program is free software; you can redistribute it and/or modify it
10
-
* under the terms of the GNU General Public License as published by the Free
11
-
* Software Foundation; either version 2 of the License, or (at your option)
12
-
* any later version.
13
-
*
14
-
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
-
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
-
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
-
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
18
-
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
19
-
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20
-
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
-
* SOFTWARE.
22
-
*
23
-
*/
24
-
25
-
#include <linux/types.h>
26
-
#include <linux/module.h>
27
-
#include <linux/crc-t10dif.h>
28
-
#include <crypto/internal/hash.h>
29
-
#include <crypto/internal/simd.h>
30
-
#include <linux/init.h>
31
-
#include <linux/string.h>
32
-
#include <linux/kernel.h>
33
-
#include <asm/cpufeatures.h>
34
-
#include <asm/cpu_device_id.h>
35
-
#include <asm/simd.h>
36
-
37
-
asmlinkage u16 crc_t10dif_pcl(u16 init_crc, const u8 *buf, size_t len);
38
-
39
-
struct chksum_desc_ctx {
40
-
__u16 crc;
41
-
};
42
-
43
-
static int chksum_init(struct shash_desc *desc)
44
-
{
45
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
46
-
47
-
ctx->crc = 0;
48
-
49
-
return 0;
50
-
}
51
-
52
-
static int chksum_update(struct shash_desc *desc, const u8 *data,
53
-
unsigned int length)
54
-
{
55
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
56
-
57
-
if (length >= 16 && crypto_simd_usable()) {
58
-
kernel_fpu_begin();
59
-
ctx->crc = crc_t10dif_pcl(ctx->crc, data, length);
60
-
kernel_fpu_end();
61
-
} else
62
-
ctx->crc = crc_t10dif_generic(ctx->crc, data, length);
63
-
return 0;
64
-
}
65
-
66
-
static int chksum_final(struct shash_desc *desc, u8 *out)
67
-
{
68
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
69
-
70
-
*(__u16 *)out = ctx->crc;
71
-
return 0;
72
-
}
73
-
74
-
static int __chksum_finup(__u16 crc, const u8 *data, unsigned int len, u8 *out)
75
-
{
76
-
if (len >= 16 && crypto_simd_usable()) {
77
-
kernel_fpu_begin();
78
-
*(__u16 *)out = crc_t10dif_pcl(crc, data, len);
79
-
kernel_fpu_end();
80
-
} else
81
-
*(__u16 *)out = crc_t10dif_generic(crc, data, len);
82
-
return 0;
83
-
}
84
-
85
-
static int chksum_finup(struct shash_desc *desc, const u8 *data,
86
-
unsigned int len, u8 *out)
87
-
{
88
-
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
89
-
90
-
return __chksum_finup(ctx->crc, data, len, out);
91
-
}
92
-
93
-
static int chksum_digest(struct shash_desc *desc, const u8 *data,
94
-
unsigned int length, u8 *out)
95
-
{
96
-
return __chksum_finup(0, data, length, out);
97
-
}
98
-
99
-
static struct shash_alg alg = {
100
-
.digestsize = CRC_T10DIF_DIGEST_SIZE,
101
-
.init = chksum_init,
102
-
.update = chksum_update,
103
-
.final = chksum_final,
104
-
.finup = chksum_finup,
105
-
.digest = chksum_digest,
106
-
.descsize = sizeof(struct chksum_desc_ctx),
107
-
.base = {
108
-
.cra_name = "crct10dif",
109
-
.cra_driver_name = "crct10dif-pclmul",
110
-
.cra_priority = 200,
111
-
.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
112
-
.cra_module = THIS_MODULE,
113
-
}
114
-
};
115
-
116
-
static const struct x86_cpu_id crct10dif_cpu_id[] = {
117
-
X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL),
118
-
{}
119
-
};
120
-
MODULE_DEVICE_TABLE(x86cpu, crct10dif_cpu_id);
121
-
122
-
static int __init crct10dif_intel_mod_init(void)
123
-
{
124
-
if (!x86_match_cpu(crct10dif_cpu_id))
125
-
return -ENODEV;
126
-
127
-
return crypto_register_shash(&alg);
128
-
}
129
-
130
-
static void __exit crct10dif_intel_mod_fini(void)
131
-
{
132
-
crypto_unregister_shash(&alg);
133
-
}
134
-
135
-
module_init(crct10dif_intel_mod_init);
136
-
module_exit(crct10dif_intel_mod_fini);
137
-
138
-
MODULE_AUTHOR("Tim Chen <tim.c.chen@linux.intel.com>");
139
-
MODULE_DESCRIPTION("T10 DIF CRC calculation accelerated with PCLMULQDQ.");
140
-
MODULE_LICENSE("GPL");
141
-
142
-
MODULE_ALIAS_CRYPTO("crct10dif");
143
-
MODULE_ALIAS_CRYPTO("crct10dif-pclmul");
+7
arch/x86/lib/Makefile
+7
arch/x86/lib/Makefile
···
38
38
lib-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
39
39
lib-$(CONFIG_MITIGATION_RETPOLINE) += retpoline.o
40
40
41
+
obj-$(CONFIG_CRC32_ARCH) += crc32-x86.o
42
+
crc32-x86-y := crc32-glue.o crc32-pclmul.o
43
+
crc32-x86-$(CONFIG_64BIT) += crc32c-3way.o
44
+
45
+
obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-x86.o
46
+
crc-t10dif-x86-y := crc-t10dif-glue.o crct10dif-pcl-asm_64.o
47
+
41
48
obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
42
49
obj-y += iomem.o
43
50
+51
arch/x86/lib/crc-t10dif-glue.c
+51
arch/x86/lib/crc-t10dif-glue.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-or-later
2
+
/*
3
+
* CRC-T10DIF using PCLMULQDQ instructions
4
+
*
5
+
* Copyright 2024 Google LLC
6
+
*/
7
+
8
+
#include <asm/cpufeatures.h>
9
+
#include <asm/simd.h>
10
+
#include <crypto/internal/simd.h>
11
+
#include <linux/crc-t10dif.h>
12
+
#include <linux/module.h>
13
+
14
+
static DEFINE_STATIC_KEY_FALSE(have_pclmulqdq);
15
+
16
+
asmlinkage u16 crc_t10dif_pcl(u16 init_crc, const u8 *buf, size_t len);
17
+
18
+
u16 crc_t10dif_arch(u16 crc, const u8 *p, size_t len)
19
+
{
20
+
if (len >= 16 &&
21
+
static_key_enabled(&have_pclmulqdq) && crypto_simd_usable()) {
22
+
kernel_fpu_begin();
23
+
crc = crc_t10dif_pcl(crc, p, len);
24
+
kernel_fpu_end();
25
+
return crc;
26
+
}
27
+
return crc_t10dif_generic(crc, p, len);
28
+
}
29
+
EXPORT_SYMBOL(crc_t10dif_arch);
30
+
31
+
static int __init crc_t10dif_x86_init(void)
32
+
{
33
+
if (boot_cpu_has(X86_FEATURE_PCLMULQDQ))
34
+
static_branch_enable(&have_pclmulqdq);
35
+
return 0;
36
+
}
37
+
arch_initcall(crc_t10dif_x86_init);
38
+
39
+
static void __exit crc_t10dif_x86_exit(void)
40
+
{
41
+
}
42
+
module_exit(crc_t10dif_x86_exit);
43
+
44
+
bool crc_t10dif_is_optimized(void)
45
+
{
46
+
return static_key_enabled(&have_pclmulqdq);
47
+
}
48
+
EXPORT_SYMBOL(crc_t10dif_is_optimized);
49
+
50
+
MODULE_DESCRIPTION("CRC-T10DIF using PCLMULQDQ instructions");
51
+
MODULE_LICENSE("GPL");
+124
arch/x86/lib/crc32-glue.c
+124
arch/x86/lib/crc32-glue.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/*
3
+
* x86-optimized CRC32 functions
4
+
*
5
+
* Copyright (C) 2008 Intel Corporation
6
+
* Copyright 2012 Xyratex Technology Limited
7
+
* Copyright 2024 Google LLC
8
+
*/
9
+
10
+
#include <asm/cpufeatures.h>
11
+
#include <asm/simd.h>
12
+
#include <crypto/internal/simd.h>
13
+
#include <linux/crc32.h>
14
+
#include <linux/linkage.h>
15
+
#include <linux/module.h>
16
+
17
+
/* minimum size of buffer for crc32_pclmul_le_16 */
18
+
#define CRC32_PCLMUL_MIN_LEN 64
19
+
20
+
static DEFINE_STATIC_KEY_FALSE(have_crc32);
21
+
static DEFINE_STATIC_KEY_FALSE(have_pclmulqdq);
22
+
23
+
u32 crc32_pclmul_le_16(u32 crc, const u8 *buffer, size_t len);
24
+
25
+
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
26
+
{
27
+
if (len >= CRC32_PCLMUL_MIN_LEN + 15 &&
28
+
static_branch_likely(&have_pclmulqdq) && crypto_simd_usable()) {
29
+
size_t n = -(uintptr_t)p & 15;
30
+
31
+
/* align p to 16-byte boundary */
32
+
if (n) {
33
+
crc = crc32_le_base(crc, p, n);
34
+
p += n;
35
+
len -= n;
36
+
}
37
+
n = round_down(len, 16);
38
+
kernel_fpu_begin();
39
+
crc = crc32_pclmul_le_16(crc, p, n);
40
+
kernel_fpu_end();
41
+
p += n;
42
+
len -= n;
43
+
}
44
+
if (len)
45
+
crc = crc32_le_base(crc, p, len);
46
+
return crc;
47
+
}
48
+
EXPORT_SYMBOL(crc32_le_arch);
49
+
50
+
#ifdef CONFIG_X86_64
51
+
#define CRC32_INST "crc32q %1, %q0"
52
+
#else
53
+
#define CRC32_INST "crc32l %1, %0"
54
+
#endif
55
+
56
+
/*
57
+
* Use carryless multiply version of crc32c when buffer size is >= 512 to
58
+
* account for FPU state save/restore overhead.
59
+
*/
60
+
#define CRC32C_PCLMUL_BREAKEVEN 512
61
+
62
+
asmlinkage u32 crc32c_x86_3way(u32 crc, const u8 *buffer, size_t len);
63
+
64
+
u32 crc32c_le_arch(u32 crc, const u8 *p, size_t len)
65
+
{
66
+
size_t num_longs;
67
+
68
+
if (!static_branch_likely(&have_crc32))
69
+
return crc32c_le_base(crc, p, len);
70
+
71
+
if (IS_ENABLED(CONFIG_X86_64) && len >= CRC32C_PCLMUL_BREAKEVEN &&
72
+
static_branch_likely(&have_pclmulqdq) && crypto_simd_usable()) {
73
+
kernel_fpu_begin();
74
+
crc = crc32c_x86_3way(crc, p, len);
75
+
kernel_fpu_end();
76
+
return crc;
77
+
}
78
+
79
+
for (num_longs = len / sizeof(unsigned long);
80
+
num_longs != 0; num_longs--, p += sizeof(unsigned long))
81
+
asm(CRC32_INST : "+r" (crc) : "rm" (*(unsigned long *)p));
82
+
83
+
for (len %= sizeof(unsigned long); len; len--, p++)
84
+
asm("crc32b %1, %0" : "+r" (crc) : "rm" (*p));
85
+
86
+
return crc;
87
+
}
88
+
EXPORT_SYMBOL(crc32c_le_arch);
89
+
90
+
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
91
+
{
92
+
return crc32_be_base(crc, p, len);
93
+
}
94
+
EXPORT_SYMBOL(crc32_be_arch);
95
+
96
+
static int __init crc32_x86_init(void)
97
+
{
98
+
if (boot_cpu_has(X86_FEATURE_XMM4_2))
99
+
static_branch_enable(&have_crc32);
100
+
if (boot_cpu_has(X86_FEATURE_PCLMULQDQ))
101
+
static_branch_enable(&have_pclmulqdq);
102
+
return 0;
103
+
}
104
+
arch_initcall(crc32_x86_init);
105
+
106
+
static void __exit crc32_x86_exit(void)
107
+
{
108
+
}
109
+
module_exit(crc32_x86_exit);
110
+
111
+
u32 crc32_optimizations(void)
112
+
{
113
+
u32 optimizations = 0;
114
+
115
+
if (static_key_enabled(&have_crc32))
116
+
optimizations |= CRC32C_OPTIMIZATION;
117
+
if (static_key_enabled(&have_pclmulqdq))
118
+
optimizations |= CRC32_LE_OPTIMIZATION;
119
+
return optimizations;
120
+
}
121
+
EXPORT_SYMBOL(crc32_optimizations);
122
+
123
+
MODULE_DESCRIPTION("x86-optimized CRC32 functions");
124
+
MODULE_LICENSE("GPL");
+1
crypto/Kconfig
+1
crypto/Kconfig
+2
-1
crypto/Makefile
+2
-1
crypto/Makefile
···
157
157
obj-$(CONFIG_CRYPTO_CRC32) += crc32_generic.o
158
158
CFLAGS_crc32c_generic.o += -DARCH=$(ARCH)
159
159
CFLAGS_crc32_generic.o += -DARCH=$(ARCH)
160
-
obj-$(CONFIG_CRYPTO_CRCT10DIF) += crct10dif_common.o crct10dif_generic.o
160
+
obj-$(CONFIG_CRYPTO_CRCT10DIF) += crct10dif_generic.o
161
+
CFLAGS_crct10dif_generic.o += -DARCH=$(ARCH)
161
162
obj-$(CONFIG_CRYPTO_CRC64_ROCKSOFT) += crc64_rocksoft_generic.o
162
163
obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o
163
164
obj-$(CONFIG_CRYPTO_LZO) += lzo.o lzo-rle.o
+6
-2
crypto/crc32_generic.c
+6
-2
crypto/crc32_generic.c
···
157
157
.base.cra_init = crc32_cra_init,
158
158
}};
159
159
160
+
static int num_algs;
161
+
160
162
static int __init crc32_mod_init(void)
161
163
{
162
164
/* register the arch flavor only if it differs from the generic one */
163
-
return crypto_register_shashes(algs, 1 + (&crc32_le != &crc32_le_base));
165
+
num_algs = 1 + ((crc32_optimizations() & CRC32_LE_OPTIMIZATION) != 0);
166
+
167
+
return crypto_register_shashes(algs, num_algs);
164
168
}
165
169
166
170
static void __exit crc32_mod_fini(void)
167
171
{
168
-
crypto_unregister_shashes(algs, 1 + (&crc32_le != &crc32_le_base));
172
+
crypto_unregister_shashes(algs, num_algs);
169
173
}
170
174
171
175
subsys_initcall(crc32_mod_init);
+8
-4
crypto/crc32c_generic.c
+8
-4
crypto/crc32c_generic.c
···
85
85
{
86
86
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
87
87
88
-
ctx->crc = __crc32c_le_base(ctx->crc, data, length);
88
+
ctx->crc = crc32c_le_base(ctx->crc, data, length);
89
89
return 0;
90
90
}
91
91
···
108
108
109
109
static int __chksum_finup(u32 *crcp, const u8 *data, unsigned int len, u8 *out)
110
110
{
111
-
put_unaligned_le32(~__crc32c_le_base(*crcp, data, len), out);
111
+
put_unaligned_le32(~crc32c_le_base(*crcp, data, len), out);
112
112
return 0;
113
113
}
114
114
···
197
197
.base.cra_init = crc32c_cra_init,
198
198
}};
199
199
200
+
static int num_algs;
201
+
200
202
static int __init crc32c_mod_init(void)
201
203
{
202
204
/* register the arch flavor only if it differs from the generic one */
203
-
return crypto_register_shashes(algs, 1 + (&__crc32c_le != &__crc32c_le_base));
205
+
num_algs = 1 + ((crc32_optimizations() & CRC32C_OPTIMIZATION) != 0);
206
+
207
+
return crypto_register_shashes(algs, num_algs);
204
208
}
205
209
206
210
static void __exit crc32c_mod_fini(void)
207
211
{
208
-
crypto_unregister_shashes(algs, 1 + (&__crc32c_le != &__crc32c_le_base));
212
+
crypto_unregister_shashes(algs, num_algs);
209
213
}
210
214
211
215
subsys_initcall(crc32c_mod_init);
-82
crypto/crct10dif_common.c
-82
crypto/crct10dif_common.c
···
1
-
/*
2
-
* Cryptographic API.
3
-
*
4
-
* T10 Data Integrity Field CRC16 Crypto Transform
5
-
*
6
-
* Copyright (c) 2007 Oracle Corporation. All rights reserved.
7
-
* Written by Martin K. Petersen <martin.petersen@oracle.com>
8
-
* Copyright (C) 2013 Intel Corporation
9
-
* Author: Tim Chen <tim.c.chen@linux.intel.com>
10
-
*
11
-
* This program is free software; you can redistribute it and/or modify it
12
-
* under the terms of the GNU General Public License as published by the Free
13
-
* Software Foundation; either version 2 of the License, or (at your option)
14
-
* any later version.
15
-
*
16
-
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
-
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
-
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
-
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
20
-
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
21
-
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
22
-
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23
-
* SOFTWARE.
24
-
*
25
-
*/
26
-
27
-
#include <linux/crc-t10dif.h>
28
-
#include <linux/module.h>
29
-
#include <linux/kernel.h>
30
-
31
-
/* Table generated using the following polynomium:
32
-
* x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x + 1
33
-
* gt: 0x8bb7
34
-
*/
35
-
static const __u16 t10_dif_crc_table[256] = {
36
-
0x0000, 0x8BB7, 0x9CD9, 0x176E, 0xB205, 0x39B2, 0x2EDC, 0xA56B,
37
-
0xEFBD, 0x640A, 0x7364, 0xF8D3, 0x5DB8, 0xD60F, 0xC161, 0x4AD6,
38
-
0x54CD, 0xDF7A, 0xC814, 0x43A3, 0xE6C8, 0x6D7F, 0x7A11, 0xF1A6,
39
-
0xBB70, 0x30C7, 0x27A9, 0xAC1E, 0x0975, 0x82C2, 0x95AC, 0x1E1B,
40
-
0xA99A, 0x222D, 0x3543, 0xBEF4, 0x1B9F, 0x9028, 0x8746, 0x0CF1,
41
-
0x4627, 0xCD90, 0xDAFE, 0x5149, 0xF422, 0x7F95, 0x68FB, 0xE34C,
42
-
0xFD57, 0x76E0, 0x618E, 0xEA39, 0x4F52, 0xC4E5, 0xD38B, 0x583C,
43
-
0x12EA, 0x995D, 0x8E33, 0x0584, 0xA0EF, 0x2B58, 0x3C36, 0xB781,
44
-
0xD883, 0x5334, 0x445A, 0xCFED, 0x6A86, 0xE131, 0xF65F, 0x7DE8,
45
-
0x373E, 0xBC89, 0xABE7, 0x2050, 0x853B, 0x0E8C, 0x19E2, 0x9255,
46
-
0x8C4E, 0x07F9, 0x1097, 0x9B20, 0x3E4B, 0xB5FC, 0xA292, 0x2925,
47
-
0x63F3, 0xE844, 0xFF2A, 0x749D, 0xD1F6, 0x5A41, 0x4D2F, 0xC698,
48
-
0x7119, 0xFAAE, 0xEDC0, 0x6677, 0xC31C, 0x48AB, 0x5FC5, 0xD472,
49
-
0x9EA4, 0x1513, 0x027D, 0x89CA, 0x2CA1, 0xA716, 0xB078, 0x3BCF,
50
-
0x25D4, 0xAE63, 0xB90D, 0x32BA, 0x97D1, 0x1C66, 0x0B08, 0x80BF,
51
-
0xCA69, 0x41DE, 0x56B0, 0xDD07, 0x786C, 0xF3DB, 0xE4B5, 0x6F02,
52
-
0x3AB1, 0xB106, 0xA668, 0x2DDF, 0x88B4, 0x0303, 0x146D, 0x9FDA,
53
-
0xD50C, 0x5EBB, 0x49D5, 0xC262, 0x6709, 0xECBE, 0xFBD0, 0x7067,
54
-
0x6E7C, 0xE5CB, 0xF2A5, 0x7912, 0xDC79, 0x57CE, 0x40A0, 0xCB17,
55
-
0x81C1, 0x0A76, 0x1D18, 0x96AF, 0x33C4, 0xB873, 0xAF1D, 0x24AA,
56
-
0x932B, 0x189C, 0x0FF2, 0x8445, 0x212E, 0xAA99, 0xBDF7, 0x3640,
57
-
0x7C96, 0xF721, 0xE04F, 0x6BF8, 0xCE93, 0x4524, 0x524A, 0xD9FD,
58
-
0xC7E6, 0x4C51, 0x5B3F, 0xD088, 0x75E3, 0xFE54, 0xE93A, 0x628D,
59
-
0x285B, 0xA3EC, 0xB482, 0x3F35, 0x9A5E, 0x11E9, 0x0687, 0x8D30,
60
-
0xE232, 0x6985, 0x7EEB, 0xF55C, 0x5037, 0xDB80, 0xCCEE, 0x4759,
61
-
0x0D8F, 0x8638, 0x9156, 0x1AE1, 0xBF8A, 0x343D, 0x2353, 0xA8E4,
62
-
0xB6FF, 0x3D48, 0x2A26, 0xA191, 0x04FA, 0x8F4D, 0x9823, 0x1394,
63
-
0x5942, 0xD2F5, 0xC59B, 0x4E2C, 0xEB47, 0x60F0, 0x779E, 0xFC29,
64
-
0x4BA8, 0xC01F, 0xD771, 0x5CC6, 0xF9AD, 0x721A, 0x6574, 0xEEC3,
65
-
0xA415, 0x2FA2, 0x38CC, 0xB37B, 0x1610, 0x9DA7, 0x8AC9, 0x017E,
66
-
0x1F65, 0x94D2, 0x83BC, 0x080B, 0xAD60, 0x26D7, 0x31B9, 0xBA0E,
67
-
0xF0D8, 0x7B6F, 0x6C01, 0xE7B6, 0x42DD, 0xC96A, 0xDE04, 0x55B3
68
-
};
69
-
70
-
__u16 crc_t10dif_generic(__u16 crc, const unsigned char *buffer, size_t len)
71
-
{
72
-
unsigned int i;
73
-
74
-
for (i = 0 ; i < len ; i++)
75
-
crc = (crc << 8) ^ t10_dif_crc_table[((crc >> 8) ^ buffer[i]) & 0xff];
76
-
77
-
return crc;
78
-
}
79
-
EXPORT_SYMBOL(crc_t10dif_generic);
80
-
81
-
MODULE_DESCRIPTION("T10 DIF CRC calculation common code");
82
-
MODULE_LICENSE("GPL");
+64
-18
crypto/crct10dif_generic.c
+64
-18
crypto/crct10dif_generic.c
···
57
57
return 0;
58
58
}
59
59
60
+
static int chksum_update_arch(struct shash_desc *desc, const u8 *data,
61
+
unsigned int length)
62
+
{
63
+
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
64
+
65
+
ctx->crc = crc_t10dif_update(ctx->crc, data, length);
66
+
return 0;
67
+
}
68
+
60
69
static int chksum_final(struct shash_desc *desc, u8 *out)
61
70
{
62
71
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
···
80
71
return 0;
81
72
}
82
73
74
+
static int __chksum_finup_arch(__u16 crc, const u8 *data, unsigned int len,
75
+
u8 *out)
76
+
{
77
+
*(__u16 *)out = crc_t10dif_update(crc, data, len);
78
+
return 0;
79
+
}
80
+
83
81
static int chksum_finup(struct shash_desc *desc, const u8 *data,
84
82
unsigned int len, u8 *out)
85
83
{
···
95
79
return __chksum_finup(ctx->crc, data, len, out);
96
80
}
97
81
82
+
static int chksum_finup_arch(struct shash_desc *desc, const u8 *data,
83
+
unsigned int len, u8 *out)
84
+
{
85
+
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
86
+
87
+
return __chksum_finup_arch(ctx->crc, data, len, out);
88
+
}
89
+
98
90
static int chksum_digest(struct shash_desc *desc, const u8 *data,
99
91
unsigned int length, u8 *out)
100
92
{
101
93
return __chksum_finup(0, data, length, out);
102
94
}
103
95
104
-
static struct shash_alg alg = {
105
-
.digestsize = CRC_T10DIF_DIGEST_SIZE,
106
-
.init = chksum_init,
107
-
.update = chksum_update,
108
-
.final = chksum_final,
109
-
.finup = chksum_finup,
110
-
.digest = chksum_digest,
111
-
.descsize = sizeof(struct chksum_desc_ctx),
112
-
.base = {
113
-
.cra_name = "crct10dif",
114
-
.cra_driver_name = "crct10dif-generic",
115
-
.cra_priority = 100,
116
-
.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
117
-
.cra_module = THIS_MODULE,
118
-
}
119
-
};
96
+
static int chksum_digest_arch(struct shash_desc *desc, const u8 *data,
97
+
unsigned int length, u8 *out)
98
+
{
99
+
return __chksum_finup_arch(0, data, length, out);
100
+
}
101
+
102
+
static struct shash_alg algs[] = {{
103
+
.digestsize = CRC_T10DIF_DIGEST_SIZE,
104
+
.init = chksum_init,
105
+
.update = chksum_update,
106
+
.final = chksum_final,
107
+
.finup = chksum_finup,
108
+
.digest = chksum_digest,
109
+
.descsize = sizeof(struct chksum_desc_ctx),
110
+
.base.cra_name = "crct10dif",
111
+
.base.cra_driver_name = "crct10dif-generic",
112
+
.base.cra_priority = 100,
113
+
.base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
114
+
.base.cra_module = THIS_MODULE,
115
+
}, {
116
+
.digestsize = CRC_T10DIF_DIGEST_SIZE,
117
+
.init = chksum_init,
118
+
.update = chksum_update_arch,
119
+
.final = chksum_final,
120
+
.finup = chksum_finup_arch,
121
+
.digest = chksum_digest_arch,
122
+
.descsize = sizeof(struct chksum_desc_ctx),
123
+
.base.cra_name = "crct10dif",
124
+
.base.cra_driver_name = "crct10dif-" __stringify(ARCH),
125
+
.base.cra_priority = 150,
126
+
.base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
127
+
.base.cra_module = THIS_MODULE,
128
+
}};
129
+
130
+
static int num_algs;
120
131
121
132
static int __init crct10dif_mod_init(void)
122
133
{
123
-
return crypto_register_shash(&alg);
134
+
/* register the arch flavor only if it differs from the generic one */
135
+
num_algs = 1 + crc_t10dif_is_optimized();
136
+
137
+
return crypto_register_shashes(algs, num_algs);
124
138
}
125
139
126
140
static void __exit crct10dif_mod_fini(void)
127
141
{
128
-
crypto_unregister_shash(&alg);
142
+
crypto_unregister_shashes(algs, num_algs);
129
143
}
130
144
131
145
subsys_initcall(crct10dif_mod_init);
+2
-2
drivers/target/iscsi/Kconfig
+2
-2
drivers/target/iscsi/Kconfig
···
2
2
config ISCSI_TARGET
3
3
tristate "SCSI Target Mode Stack"
4
4
depends on INET
5
+
select CRC32
5
6
select CRYPTO
6
-
select CRYPTO_CRC32C
7
-
select CRYPTO_CRC32C_INTEL if X86
7
+
select CRYPTO_HASH
8
8
help
9
9
Say M to enable the SCSI target mode stack. A SCSI target mode stack
10
10
is software that makes local storage available over a storage network
+45
-108
drivers/target/iscsi/iscsi_target.c
+45
-108
drivers/target/iscsi/iscsi_target.c
···
8
8
*
9
9
******************************************************************************/
10
10
11
-
#include <crypto/hash.h>
11
+
#include <linux/crc32c.h>
12
12
#include <linux/string.h>
13
13
#include <linux/kthread.h>
14
14
#include <linux/completion.h>
···
490
490
}
491
491
EXPORT_SYMBOL(iscsit_aborted_task);
492
492
493
-
static void iscsit_do_crypto_hash_buf(struct ahash_request *, const void *,
494
-
u32, u32, const void *, void *);
493
+
static u32 iscsit_crc_buf(const void *buf, u32 payload_length,
494
+
u32 padding, const void *pad_bytes);
495
495
static void iscsit_tx_thread_wait_for_tcp(struct iscsit_conn *);
496
496
497
497
static int
···
510
510
if (conn->conn_ops->HeaderDigest) {
511
511
u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
512
512
513
-
iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr,
514
-
ISCSI_HDR_LEN, 0, NULL,
515
-
header_digest);
513
+
*header_digest = iscsit_crc_buf(hdr, ISCSI_HDR_LEN, 0, NULL);
516
514
517
515
iov[0].iov_len += ISCSI_CRC_LEN;
518
516
tx_size += ISCSI_CRC_LEN;
···
535
537
}
536
538
537
539
if (conn->conn_ops->DataDigest) {
538
-
iscsit_do_crypto_hash_buf(conn->conn_tx_hash,
539
-
data_buf, data_buf_len,
540
-
padding, &cmd->pad_bytes,
541
-
&cmd->data_crc);
542
-
540
+
cmd->data_crc = iscsit_crc_buf(data_buf, data_buf_len,
541
+
padding,
542
+
&cmd->pad_bytes);
543
543
iov[niov].iov_base = &cmd->data_crc;
544
544
iov[niov++].iov_len = ISCSI_CRC_LEN;
545
545
tx_size += ISCSI_CRC_LEN;
···
562
566
static int iscsit_map_iovec(struct iscsit_cmd *cmd, struct kvec *iov, int nvec,
563
567
u32 data_offset, u32 data_length);
564
568
static void iscsit_unmap_iovec(struct iscsit_cmd *);
565
-
static u32 iscsit_do_crypto_hash_sg(struct ahash_request *, struct iscsit_cmd *,
566
-
u32, u32, u32, u8 *);
569
+
static u32 iscsit_crc_sglist(const struct iscsit_cmd *cmd, u32 data_length,
570
+
u32 padding, const u8 *pad_bytes);
567
571
static int
568
572
iscsit_xmit_datain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
569
573
const struct iscsi_datain *datain)
···
580
584
if (conn->conn_ops->HeaderDigest) {
581
585
u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
582
586
583
-
iscsit_do_crypto_hash_buf(conn->conn_tx_hash, cmd->pdu,
584
-
ISCSI_HDR_LEN, 0, NULL,
585
-
header_digest);
586
-
587
+
*header_digest = iscsit_crc_buf(cmd->pdu, ISCSI_HDR_LEN, 0,
588
+
NULL);
587
589
iov[0].iov_len += ISCSI_CRC_LEN;
588
590
tx_size += ISCSI_CRC_LEN;
589
591
···
608
614
}
609
615
610
616
if (conn->conn_ops->DataDigest) {
611
-
cmd->data_crc = iscsit_do_crypto_hash_sg(conn->conn_tx_hash,
612
-
cmd, datain->offset,
613
-
datain->length,
614
-
cmd->padding,
615
-
cmd->pad_bytes);
616
-
617
+
cmd->data_crc = iscsit_crc_sglist(cmd, datain->length,
618
+
cmd->padding, cmd->pad_bytes);
617
619
iov[iov_count].iov_base = &cmd->data_crc;
618
620
iov[iov_count++].iov_len = ISCSI_CRC_LEN;
619
621
tx_size += ISCSI_CRC_LEN;
···
1394
1404
return iscsit_get_immediate_data(cmd, hdr, dump_payload);
1395
1405
}
1396
1406
1397
-
static u32 iscsit_do_crypto_hash_sg(
1398
-
struct ahash_request *hash,
1399
-
struct iscsit_cmd *cmd,
1400
-
u32 data_offset,
1401
-
u32 data_length,
1402
-
u32 padding,
1403
-
u8 *pad_bytes)
1407
+
static u32 iscsit_crc_sglist(const struct iscsit_cmd *cmd, u32 data_length,
1408
+
u32 padding, const u8 *pad_bytes)
1404
1409
{
1405
-
u32 data_crc;
1406
-
struct scatterlist *sg;
1407
-
unsigned int page_off;
1408
-
1409
-
crypto_ahash_init(hash);
1410
-
1411
-
sg = cmd->first_data_sg;
1412
-
page_off = cmd->first_data_sg_off;
1413
-
1414
-
if (data_length && page_off) {
1415
-
struct scatterlist first_sg;
1416
-
u32 len = min_t(u32, data_length, sg->length - page_off);
1417
-
1418
-
sg_init_table(&first_sg, 1);
1419
-
sg_set_page(&first_sg, sg_page(sg), len, sg->offset + page_off);
1420
-
1421
-
ahash_request_set_crypt(hash, &first_sg, NULL, len);
1422
-
crypto_ahash_update(hash);
1423
-
1424
-
data_length -= len;
1425
-
sg = sg_next(sg);
1426
-
}
1410
+
struct scatterlist *sg = cmd->first_data_sg;
1411
+
unsigned int page_off = cmd->first_data_sg_off;
1412
+
u32 crc = ~0;
1427
1413
1428
1414
while (data_length) {
1429
-
u32 cur_len = min_t(u32, data_length, sg->length);
1415
+
u32 cur_len = min_t(u32, data_length, sg->length - page_off);
1416
+
const void *virt;
1430
1417
1431
-
ahash_request_set_crypt(hash, sg, NULL, cur_len);
1432
-
crypto_ahash_update(hash);
1418
+
virt = kmap_local_page(sg_page(sg)) + sg->offset + page_off;
1419
+
crc = crc32c(crc, virt, cur_len);
1420
+
kunmap_local(virt);
1433
1421
1434
-
data_length -= cur_len;
1435
1422
/* iscsit_map_iovec has already checked for invalid sg pointers */
1436
1423
sg = sg_next(sg);
1424
+
1425
+
page_off = 0;
1426
+
data_length -= cur_len;
1437
1427
}
1438
1428
1439
-
if (padding) {
1440
-
struct scatterlist pad_sg;
1429
+
if (padding)
1430
+
crc = crc32c(crc, pad_bytes, padding);
1441
1431
1442
-
sg_init_one(&pad_sg, pad_bytes, padding);
1443
-
ahash_request_set_crypt(hash, &pad_sg, (u8 *)&data_crc,
1444
-
padding);
1445
-
crypto_ahash_finup(hash);
1446
-
} else {
1447
-
ahash_request_set_crypt(hash, NULL, (u8 *)&data_crc, 0);
1448
-
crypto_ahash_final(hash);
1449
-
}
1450
-
1451
-
return data_crc;
1432
+
return ~crc;
1452
1433
}
1453
1434
1454
-
static void iscsit_do_crypto_hash_buf(struct ahash_request *hash,
1455
-
const void *buf, u32 payload_length, u32 padding,
1456
-
const void *pad_bytes, void *data_crc)
1435
+
static u32 iscsit_crc_buf(const void *buf, u32 payload_length,
1436
+
u32 padding, const void *pad_bytes)
1457
1437
{
1458
-
struct scatterlist sg[2];
1438
+
u32 crc = ~0;
1459
1439
1460
-
sg_init_table(sg, ARRAY_SIZE(sg));
1461
-
sg_set_buf(sg, buf, payload_length);
1440
+
crc = crc32c(crc, buf, payload_length);
1441
+
1462
1442
if (padding)
1463
-
sg_set_buf(sg + 1, pad_bytes, padding);
1443
+
crc = crc32c(crc, pad_bytes, padding);
1464
1444
1465
-
ahash_request_set_crypt(hash, sg, data_crc, payload_length + padding);
1466
-
1467
-
crypto_ahash_digest(hash);
1445
+
return ~crc;
1468
1446
}
1469
1447
1470
1448
int
···
1620
1662
if (conn->conn_ops->DataDigest) {
1621
1663
u32 data_crc;
1622
1664
1623
-
data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd,
1624
-
be32_to_cpu(hdr->offset),
1625
-
payload_length, padding,
1626
-
cmd->pad_bytes);
1627
-
1665
+
data_crc = iscsit_crc_sglist(cmd, payload_length, padding,
1666
+
cmd->pad_bytes);
1628
1667
if (checksum != data_crc) {
1629
1668
pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1630
1669
" DataSN: 0x%08x, CRC32C DataDigest 0x%08x"
···
1880
1925
}
1881
1926
1882
1927
if (conn->conn_ops->DataDigest) {
1883
-
iscsit_do_crypto_hash_buf(conn->conn_rx_hash, ping_data,
1884
-
payload_length, padding,
1885
-
cmd->pad_bytes, &data_crc);
1886
-
1928
+
data_crc = iscsit_crc_buf(ping_data, payload_length,
1929
+
padding, cmd->pad_bytes);
1887
1930
if (checksum != data_crc) {
1888
1931
pr_err("Ping data CRC32C DataDigest"
1889
1932
" 0x%08x does not match computed 0x%08x\n",
···
2281
2328
goto reject;
2282
2329
2283
2330
if (conn->conn_ops->DataDigest) {
2284
-
iscsit_do_crypto_hash_buf(conn->conn_rx_hash,
2285
-
text_in, rx_size, 0, NULL,
2286
-
&data_crc);
2287
-
2331
+
data_crc = iscsit_crc_buf(text_in, rx_size, 0, NULL);
2288
2332
if (checksum != data_crc) {
2289
2333
pr_err("Text data CRC32C DataDigest"
2290
2334
" 0x%08x does not match computed"
···
2638
2688
if (conn->conn_ops->DataDigest) {
2639
2689
u32 data_crc;
2640
2690
2641
-
data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd,
2642
-
cmd->write_data_done, length, padding,
2643
-
cmd->pad_bytes);
2644
-
2691
+
data_crc = iscsit_crc_sglist(cmd, length, padding,
2692
+
cmd->pad_bytes);
2645
2693
if (checksum != data_crc) {
2646
2694
pr_err("ImmediateData CRC32C DataDigest 0x%08x"
2647
2695
" does not match computed 0x%08x\n", checksum,
···
4064
4116
break;
4065
4117
}
4066
4118
4067
-
iscsit_do_crypto_hash_buf(conn->conn_rx_hash, buffer,
4068
-
ISCSI_HDR_LEN, 0, NULL,
4069
-
&checksum);
4070
-
4119
+
checksum = iscsit_crc_buf(buffer, ISCSI_HDR_LEN, 0,
4120
+
NULL);
4071
4121
if (digest != checksum) {
4072
4122
pr_err("HeaderDigest CRC32C failed,"
4073
4123
" received 0x%08x, computed 0x%08x\n",
···
4351
4405
* must wait until they have completed.
4352
4406
*/
4353
4407
iscsit_check_conn_usage_count(conn);
4354
-
4355
-
ahash_request_free(conn->conn_tx_hash);
4356
-
if (conn->conn_rx_hash) {
4357
-
struct crypto_ahash *tfm;
4358
-
4359
-
tfm = crypto_ahash_reqtfm(conn->conn_rx_hash);
4360
-
ahash_request_free(conn->conn_rx_hash);
4361
-
crypto_free_ahash(tfm);
4362
-
}
4363
4408
4364
4409
if (conn->sock)
4365
4410
sock_release(conn->sock);
-50
drivers/target/iscsi/iscsi_target_login.c
-50
drivers/target/iscsi/iscsi_target_login.c
···
8
8
*
9
9
******************************************************************************/
10
10
11
-
#include <crypto/hash.h>
12
11
#include <linux/module.h>
13
12
#include <linux/string.h>
14
13
#include <linux/kthread.h>
···
68
69
out_login:
69
70
kfree(login);
70
71
return NULL;
71
-
}
72
-
73
-
/*
74
-
* Used by iscsi_target_nego.c:iscsi_target_locate_portal() to setup
75
-
* per struct iscsit_conn libcrypto contexts for crc32c and crc32-intel
76
-
*/
77
-
int iscsi_login_setup_crypto(struct iscsit_conn *conn)
78
-
{
79
-
struct crypto_ahash *tfm;
80
-
81
-
/*
82
-
* Setup slicing by CRC32C algorithm for RX and TX libcrypto contexts
83
-
* which will default to crc32c_intel.ko for cpu_has_xmm4_2, or fallback
84
-
* to software 1x8 byte slicing from crc32c.ko
85
-
*/
86
-
tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
87
-
if (IS_ERR(tfm)) {
88
-
pr_err("crypto_alloc_ahash() failed\n");
89
-
return -ENOMEM;
90
-
}
91
-
92
-
conn->conn_rx_hash = ahash_request_alloc(tfm, GFP_KERNEL);
93
-
if (!conn->conn_rx_hash) {
94
-
pr_err("ahash_request_alloc() failed for conn_rx_hash\n");
95
-
crypto_free_ahash(tfm);
96
-
return -ENOMEM;
97
-
}
98
-
ahash_request_set_callback(conn->conn_rx_hash, 0, NULL, NULL);
99
-
100
-
conn->conn_tx_hash = ahash_request_alloc(tfm, GFP_KERNEL);
101
-
if (!conn->conn_tx_hash) {
102
-
pr_err("ahash_request_alloc() failed for conn_tx_hash\n");
103
-
ahash_request_free(conn->conn_rx_hash);
104
-
conn->conn_rx_hash = NULL;
105
-
crypto_free_ahash(tfm);
106
-
return -ENOMEM;
107
-
}
108
-
ahash_request_set_callback(conn->conn_tx_hash, 0, NULL, NULL);
109
-
110
-
return 0;
111
72
}
112
73
113
74
static int iscsi_login_check_initiator_version(
···
1122
1163
} else
1123
1164
spin_unlock_bh(&conn->sess->conn_lock);
1124
1165
iscsit_dec_session_usage_count(conn->sess);
1125
-
}
1126
-
1127
-
ahash_request_free(conn->conn_tx_hash);
1128
-
if (conn->conn_rx_hash) {
1129
-
struct crypto_ahash *tfm;
1130
-
1131
-
tfm = crypto_ahash_reqtfm(conn->conn_rx_hash);
1132
-
ahash_request_free(conn->conn_rx_hash);
1133
-
crypto_free_ahash(tfm);
1134
1166
}
1135
1167
1136
1168
if (conn->param_list) {
-1
drivers/target/iscsi/iscsi_target_login.h
-1
drivers/target/iscsi/iscsi_target_login.h
···
9
9
struct iscsi_np;
10
10
struct sockaddr_storage;
11
11
12
-
extern int iscsi_login_setup_crypto(struct iscsit_conn *);
13
12
extern int iscsi_check_for_session_reinstatement(struct iscsit_conn *);
14
13
extern int iscsi_login_post_auth_non_zero_tsih(struct iscsit_conn *, u16, u32);
15
14
extern int iscsit_setup_np(struct iscsi_np *,
+2
-19
drivers/target/iscsi/iscsi_target_nego.c
+2
-19
drivers/target/iscsi/iscsi_target_nego.c
···
1194
1194
goto get_target;
1195
1195
1196
1196
sess->sess_ops->SessionType = 1;
1197
-
/*
1198
-
* Setup crc32c modules from libcrypto
1199
-
*/
1200
-
if (iscsi_login_setup_crypto(conn) < 0) {
1201
-
pr_err("iscsi_login_setup_crypto() failed\n");
1202
-
ret = -1;
1203
-
goto out;
1204
-
}
1197
+
1205
1198
/*
1206
1199
* Serialize access across the discovery struct iscsi_portal_group to
1207
1200
* process login attempt.
···
1251
1258
}
1252
1259
conn->tpg_np = tpg_np;
1253
1260
pr_debug("Located Portal Group Object: %hu\n", conn->tpg->tpgt);
1254
-
/*
1255
-
* Setup crc32c modules from libcrypto
1256
-
*/
1257
-
if (iscsi_login_setup_crypto(conn) < 0) {
1258
-
pr_err("iscsi_login_setup_crypto() failed\n");
1259
-
kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put);
1260
-
iscsit_put_tiqn_for_login(tiqn);
1261
-
conn->tpg = NULL;
1262
-
ret = -1;
1263
-
goto out;
1264
-
}
1261
+
1265
1262
/*
1266
1263
* Serialize access across the struct iscsi_portal_group to
1267
1264
* process login attempt.
+1
fs/bcachefs/Kconfig
+1
fs/bcachefs/Kconfig
+1
-2
fs/ext4/Kconfig
+1
-2
fs/ext4/Kconfig
+3
-22
fs/ext4/ext4.h
+3
-22
fs/ext4/ext4.h
···
33
33
#include <linux/blockgroup_lock.h>
34
34
#include <linux/percpu_counter.h>
35
35
#include <linux/ratelimit.h>
36
-
#include <crypto/hash.h>
36
+
#include <linux/crc32c.h>
37
37
#include <linux/falloc.h>
38
38
#include <linux/percpu-rwsem.h>
39
39
#include <linux/fiemap.h>
···
1662
1662
/* record the last minlen when FITRIM is called. */
1663
1663
unsigned long s_last_trim_minblks;
1664
1664
1665
-
/* Reference to checksum algorithm driver via cryptoapi */
1666
-
struct crypto_shash *s_chksum_driver;
1667
-
1668
1665
/* Precomputed FS UUID checksum for seeding other checksums */
1669
1666
__u32 s_csum_seed;
1670
1667
···
2460
2463
static inline u32 ext4_chksum(struct ext4_sb_info *sbi, u32 crc,
2461
2464
const void *address, unsigned int length)
2462
2465
{
2463
-
struct {
2464
-
struct shash_desc shash;
2465
-
char ctx[4];
2466
-
} desc;
2467
-
2468
-
BUG_ON(crypto_shash_descsize(sbi->s_chksum_driver)!=sizeof(desc.ctx));
2469
-
2470
-
desc.shash.tfm = sbi->s_chksum_driver;
2471
-
*(u32 *)desc.ctx = crc;
2472
-
2473
-
BUG_ON(crypto_shash_update(&desc.shash, address, length));
2474
-
2475
-
return *(u32 *)desc.ctx;
2466
+
return crc32c(crc, address, length);
2476
2467
}
2477
2468
2478
2469
#ifdef __KERNEL__
···
3261
3276
3262
3277
static inline int ext4_has_metadata_csum(struct super_block *sb)
3263
3278
{
3264
-
WARN_ON_ONCE(ext4_has_feature_metadata_csum(sb) &&
3265
-
!EXT4_SB(sb)->s_chksum_driver);
3266
-
3267
-
return ext4_has_feature_metadata_csum(sb) &&
3268
-
(EXT4_SB(sb)->s_chksum_driver != NULL);
3279
+
return ext4_has_feature_metadata_csum(sb);
3269
3280
}
3270
3281
3271
3282
static inline int ext4_has_group_desc_csum(struct super_block *sb)
-15
fs/ext4/super.c
-15
fs/ext4/super.c
···
1380
1380
*/
1381
1381
kobject_put(&sbi->s_kobj);
1382
1382
wait_for_completion(&sbi->s_kobj_unregister);
1383
-
if (sbi->s_chksum_driver)
1384
-
crypto_free_shash(sbi->s_chksum_driver);
1385
1383
kfree(sbi->s_blockgroup_lock);
1386
1384
fs_put_dax(sbi->s_daxdev, NULL);
1387
1385
fscrypt_free_dummy_policy(&sbi->s_dummy_enc_policy);
···
4632
4634
ext4_setup_csum_trigger(sb, EXT4_JTR_ORPHAN_FILE,
4633
4635
ext4_orphan_file_block_trigger);
4634
4636
4635
-
/* Load the checksum driver */
4636
-
sbi->s_chksum_driver = crypto_alloc_shash("crc32c", 0, 0);
4637
-
if (IS_ERR(sbi->s_chksum_driver)) {
4638
-
int ret = PTR_ERR(sbi->s_chksum_driver);
4639
-
ext4_msg(sb, KERN_ERR, "Cannot load crc32c driver.");
4640
-
sbi->s_chksum_driver = NULL;
4641
-
return ret;
4642
-
}
4643
-
4644
4637
/* Check superblock checksum */
4645
4638
if (!ext4_superblock_csum_verify(sb, es)) {
4646
4639
ext4_msg(sb, KERN_ERR, "VFS: Found ext4 filesystem with "
···
5676
5687
del_timer_sync(&sbi->s_err_report);
5677
5688
ext4_group_desc_free(sbi);
5678
5689
failed_mount:
5679
-
if (sbi->s_chksum_driver)
5680
-
crypto_free_shash(sbi->s_chksum_driver);
5681
-
5682
5690
#if IS_ENABLED(CONFIG_UNICODE)
5683
5691
utf8_unload(sb->s_encoding);
5684
5692
#endif
···
7480
7494
MODULE_AUTHOR("Remy Card, Stephen Tweedie, Andrew Morton, Andreas Dilger, Theodore Ts'o and others");
7481
7495
MODULE_DESCRIPTION("Fourth Extended Filesystem");
7482
7496
MODULE_LICENSE("GPL");
7483
-
MODULE_SOFTDEP("pre: crc32c");
7484
7497
module_init(ext4_init_fs)
7485
7498
module_exit(ext4_exit_fs)
+1
-2
fs/f2fs/Kconfig
+1
-2
fs/f2fs/Kconfig
+1
-19
fs/f2fs/f2fs.h
+1
-19
fs/f2fs/f2fs.h
···
24
24
#include <linux/quotaops.h>
25
25
#include <linux/part_stat.h>
26
26
#include <linux/rw_hint.h>
27
-
#include <crypto/hash.h>
28
27
29
28
#include <linux/fscrypt.h>
30
29
#include <linux/fsverity.h>
···
1767
1768
u64 sectors_written_start;
1768
1769
u64 kbytes_written;
1769
1770
1770
-
/* Reference to checksum algorithm driver via cryptoapi */
1771
-
struct crypto_shash *s_chksum_driver;
1772
-
1773
1771
/* Precomputed FS UUID checksum for seeding other checksums */
1774
1772
__u32 s_chksum_seed;
1775
1773
···
1944
1948
static inline u32 __f2fs_crc32(struct f2fs_sb_info *sbi, u32 crc,
1945
1949
const void *address, unsigned int length)
1946
1950
{
1947
-
struct {
1948
-
struct shash_desc shash;
1949
-
char ctx[4];
1950
-
} desc;
1951
-
int err;
1952
-
1953
-
BUG_ON(crypto_shash_descsize(sbi->s_chksum_driver) != sizeof(desc.ctx));
1954
-
1955
-
desc.shash.tfm = sbi->s_chksum_driver;
1956
-
*(u32 *)desc.ctx = crc;
1957
-
1958
-
err = crypto_shash_update(&desc.shash, address, length);
1959
-
BUG_ON(err);
1960
-
1961
-
return *(u32 *)desc.ctx;
1951
+
return crc32(crc, address, length);
1962
1952
}
1963
1953
1964
1954
static inline u32 f2fs_crc32(struct f2fs_sb_info *sbi, const void *address,
-15
fs/f2fs/super.c
-15
fs/f2fs/super.c
···
1694
1694
1695
1695
kvfree(sbi->ckpt);
1696
1696
1697
-
if (sbi->s_chksum_driver)
1698
-
crypto_free_shash(sbi->s_chksum_driver);
1699
1697
kfree(sbi->raw_super);
1700
1698
1701
1699
f2fs_destroy_page_array_cache(sbi);
···
4464
4466
}
4465
4467
mutex_init(&sbi->flush_lock);
4466
4468
4467
-
/* Load the checksum driver */
4468
-
sbi->s_chksum_driver = crypto_alloc_shash("crc32", 0, 0);
4469
-
if (IS_ERR(sbi->s_chksum_driver)) {
4470
-
f2fs_err(sbi, "Cannot load crc32 driver.");
4471
-
err = PTR_ERR(sbi->s_chksum_driver);
4472
-
sbi->s_chksum_driver = NULL;
4473
-
goto free_sbi;
4474
-
}
4475
-
4476
4469
/* set a block size */
4477
4470
if (unlikely(!sb_set_blocksize(sb, F2FS_BLKSIZE))) {
4478
4471
f2fs_err(sbi, "unable to set blocksize");
···
4908
4919
free_sb_buf:
4909
4920
kfree(raw_super);
4910
4921
free_sbi:
4911
-
if (sbi->s_chksum_driver)
4912
-
crypto_free_shash(sbi->s_chksum_driver);
4913
4922
kfree(sbi);
4914
4923
sb->s_fs_info = NULL;
4915
4924
···
5114
5127
MODULE_AUTHOR("Samsung Electronics's Praesto Team");
5115
5128
MODULE_DESCRIPTION("Flash Friendly File System");
5116
5129
MODULE_LICENSE("GPL");
5117
-
MODULE_SOFTDEP("pre: crc32");
5118
-
-2
fs/jbd2/Kconfig
-2
fs/jbd2/Kconfig
+3
-27
fs/jbd2/journal.c
+3
-27
fs/jbd2/journal.c
···
1369
1369
return err;
1370
1370
}
1371
1371
1372
-
/* Load the checksum driver */
1373
1372
if (jbd2_journal_has_csum_v2or3_feature(journal)) {
1374
1373
if (sb->s_checksum_type != JBD2_CRC32C_CHKSUM) {
1375
1374
printk(KERN_ERR "JBD2: Unknown checksum type\n");
1376
1375
return err;
1377
1376
}
1378
1377
1379
-
journal->j_chksum_driver = crypto_alloc_shash("crc32c", 0, 0);
1380
-
if (IS_ERR(journal->j_chksum_driver)) {
1381
-
printk(KERN_ERR "JBD2: Cannot load crc32c driver.\n");
1382
-
err = PTR_ERR(journal->j_chksum_driver);
1383
-
journal->j_chksum_driver = NULL;
1384
-
return err;
1385
-
}
1386
1378
/* Check superblock checksum */
1387
1379
if (sb->s_checksum != jbd2_superblock_csum(journal, sb)) {
1388
1380
printk(KERN_ERR "JBD2: journal checksum error\n");
···
1600
1608
1601
1609
err_cleanup:
1602
1610
percpu_counter_destroy(&journal->j_checkpoint_jh_count);
1603
-
if (journal->j_chksum_driver)
1604
-
crypto_free_shash(journal->j_chksum_driver);
1605
1611
kfree(journal->j_wbuf);
1606
1612
jbd2_journal_destroy_revoke(journal);
1607
1613
journal_fail_superblock(journal);
···
2181
2191
iput(journal->j_inode);
2182
2192
if (journal->j_revoke)
2183
2193
jbd2_journal_destroy_revoke(journal);
2184
-
if (journal->j_chksum_driver)
2185
-
crypto_free_shash(journal->j_chksum_driver);
2186
2194
kfree(journal->j_fc_wbuf);
2187
2195
kfree(journal->j_wbuf);
2188
2196
kfree(journal);
···
2325
2337
}
2326
2338
}
2327
2339
2328
-
/* Load the checksum driver if necessary */
2329
-
if ((journal->j_chksum_driver == NULL) &&
2330
-
INCOMPAT_FEATURE_ON(JBD2_FEATURE_INCOMPAT_CSUM_V3)) {
2331
-
journal->j_chksum_driver = crypto_alloc_shash("crc32c", 0, 0);
2332
-
if (IS_ERR(journal->j_chksum_driver)) {
2333
-
printk(KERN_ERR "JBD2: Cannot load crc32c driver.\n");
2334
-
journal->j_chksum_driver = NULL;
2335
-
return 0;
2336
-
}
2337
-
/* Precompute checksum seed for all metadata */
2338
-
journal->j_csum_seed = jbd2_chksum(journal, ~0, sb->s_uuid,
2339
-
sizeof(sb->s_uuid));
2340
-
}
2341
-
2342
2340
lock_buffer(journal->j_sb_buffer);
2343
2341
2344
-
/* If enabling v3 checksums, update superblock */
2342
+
/* If enabling v3 checksums, update superblock and precompute seed */
2345
2343
if (INCOMPAT_FEATURE_ON(JBD2_FEATURE_INCOMPAT_CSUM_V3)) {
2346
2344
sb->s_checksum_type = JBD2_CRC32C_CHKSUM;
2347
2345
sb->s_feature_compat &=
2348
2346
~cpu_to_be32(JBD2_FEATURE_COMPAT_CHECKSUM);
2347
+
journal->j_csum_seed = jbd2_chksum(journal, ~0, sb->s_uuid,
2348
+
sizeof(sb->s_uuid));
2349
2349
}
2350
2350
2351
2351
/* If enabling v1 checksums, downgrade superblock */
+23
-5
include/linux/crc-t10dif.h
+23
-5
include/linux/crc-t10dif.h
···
6
6
7
7
#define CRC_T10DIF_DIGEST_SIZE 2
8
8
#define CRC_T10DIF_BLOCK_SIZE 1
9
-
#define CRC_T10DIF_STRING "crct10dif"
10
9
11
-
extern __u16 crc_t10dif_generic(__u16 crc, const unsigned char *buffer,
12
-
size_t len);
13
-
extern __u16 crc_t10dif(unsigned char const *, size_t);
14
-
extern __u16 crc_t10dif_update(__u16 crc, unsigned char const *, size_t);
10
+
u16 crc_t10dif_arch(u16 crc, const u8 *p, size_t len);
11
+
u16 crc_t10dif_generic(u16 crc, const u8 *p, size_t len);
12
+
13
+
static inline u16 crc_t10dif_update(u16 crc, const u8 *p, size_t len)
14
+
{
15
+
if (IS_ENABLED(CONFIG_CRC_T10DIF_ARCH))
16
+
return crc_t10dif_arch(crc, p, len);
17
+
return crc_t10dif_generic(crc, p, len);
18
+
}
19
+
20
+
static inline u16 crc_t10dif(const u8 *p, size_t len)
21
+
{
22
+
return crc_t10dif_update(0, p, len);
23
+
}
24
+
25
+
#if IS_ENABLED(CONFIG_CRC_T10DIF_ARCH)
26
+
bool crc_t10dif_is_optimized(void);
27
+
#else
28
+
static inline bool crc_t10dif_is_optimized(void)
29
+
{
30
+
return false;
31
+
}
32
+
#endif
15
33
16
34
#endif
+43
-7
include/linux/crc32.h
+43
-7
include/linux/crc32.h
···
8
8
#include <linux/types.h>
9
9
#include <linux/bitrev.h>
10
10
11
-
u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len);
12
-
u32 __pure crc32_le_base(u32 crc, unsigned char const *p, size_t len);
13
-
u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len);
14
-
u32 __pure crc32_be_base(u32 crc, unsigned char const *p, size_t len);
11
+
u32 __pure crc32_le_arch(u32 crc, const u8 *p, size_t len);
12
+
u32 __pure crc32_le_base(u32 crc, const u8 *p, size_t len);
13
+
u32 __pure crc32_be_arch(u32 crc, const u8 *p, size_t len);
14
+
u32 __pure crc32_be_base(u32 crc, const u8 *p, size_t len);
15
+
u32 __pure crc32c_le_arch(u32 crc, const u8 *p, size_t len);
16
+
u32 __pure crc32c_le_base(u32 crc, const u8 *p, size_t len);
17
+
18
+
static inline u32 __pure crc32_le(u32 crc, const u8 *p, size_t len)
19
+
{
20
+
if (IS_ENABLED(CONFIG_CRC32_ARCH))
21
+
return crc32_le_arch(crc, p, len);
22
+
return crc32_le_base(crc, p, len);
23
+
}
24
+
25
+
static inline u32 __pure crc32_be(u32 crc, const u8 *p, size_t len)
26
+
{
27
+
if (IS_ENABLED(CONFIG_CRC32_ARCH))
28
+
return crc32_be_arch(crc, p, len);
29
+
return crc32_be_base(crc, p, len);
30
+
}
31
+
32
+
/* TODO: leading underscores should be dropped once callers have been updated */
33
+
static inline u32 __pure __crc32c_le(u32 crc, const u8 *p, size_t len)
34
+
{
35
+
if (IS_ENABLED(CONFIG_CRC32_ARCH))
36
+
return crc32c_le_arch(crc, p, len);
37
+
return crc32c_le_base(crc, p, len);
38
+
}
39
+
40
+
/*
41
+
* crc32_optimizations() returns flags that indicate which CRC32 library
42
+
* functions are using architecture-specific optimizations. Unlike
43
+
* IS_ENABLED(CONFIG_CRC32_ARCH) it takes into account the different CRC32
44
+
* variants and also whether any needed CPU features are available at runtime.
45
+
*/
46
+
#define CRC32_LE_OPTIMIZATION BIT(0) /* crc32_le() is optimized */
47
+
#define CRC32_BE_OPTIMIZATION BIT(1) /* crc32_be() is optimized */
48
+
#define CRC32C_OPTIMIZATION BIT(2) /* __crc32c_le() is optimized */
49
+
#if IS_ENABLED(CONFIG_CRC32_ARCH)
50
+
u32 crc32_optimizations(void);
51
+
#else
52
+
static inline u32 crc32_optimizations(void) { return 0; }
53
+
#endif
15
54
16
55
/**
17
56
* crc32_le_combine - Combine two crc32 check values into one. For two
···
76
37
{
77
38
return crc32_le_shift(crc1, len2) ^ crc2;
78
39
}
79
-
80
-
u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len);
81
-
u32 __pure __crc32c_le_base(u32 crc, unsigned char const *p, size_t len);
82
40
83
41
/**
84
42
* __crc32c_le_combine - Combine two crc32c check values into one. For two
+5
-2
include/linux/crc32c.h
+5
-2
include/linux/crc32c.h
···
2
2
#ifndef _LINUX_CRC32C_H
3
3
#define _LINUX_CRC32C_H
4
4
5
-
#include <linux/types.h>
5
+
#include <linux/crc32.h>
6
6
7
-
extern u32 crc32c(u32 crc, const void *address, unsigned int length);
7
+
static inline u32 crc32c(u32 crc, const void *address, unsigned int length)
8
+
{
9
+
return __crc32c_le(crc, address, length);
10
+
}
8
11
9
12
/* This macro exists for backwards-compatibility. */
10
13
#define crc32c_le crc32c
+3
-30
include/linux/jbd2.h
+3
-30
include/linux/jbd2.h
···
28
28
#include <linux/slab.h>
29
29
#include <linux/bit_spinlock.h>
30
30
#include <linux/blkdev.h>
31
-
#include <crypto/hash.h>
31
+
#include <linux/crc32c.h>
32
32
#endif
33
33
34
34
#define journal_oom_retry 1
···
1242
1242
void *j_private;
1243
1243
1244
1244
/**
1245
-
* @j_chksum_driver:
1246
-
*
1247
-
* Reference to checksum algorithm driver via cryptoapi.
1248
-
*/
1249
-
struct crypto_shash *j_chksum_driver;
1250
-
1251
-
/**
1252
1245
* @j_csum_seed:
1253
1246
*
1254
1247
* Precomputed journal UUID checksum for seeding other checksums.
···
1743
1750
1744
1751
static inline int jbd2_journal_has_csum_v2or3(journal_t *journal)
1745
1752
{
1746
-
WARN_ON_ONCE(jbd2_journal_has_csum_v2or3_feature(journal) &&
1747
-
journal->j_chksum_driver == NULL);
1748
-
1749
-
return journal->j_chksum_driver != NULL;
1753
+
return jbd2_journal_has_csum_v2or3_feature(journal);
1750
1754
}
1751
1755
1752
1756
static inline int jbd2_journal_get_num_fc_blks(journal_superblock_t *jsb)
···
1780
1790
#define BJ_Reserved 4 /* Buffer is reserved for access by journal */
1781
1791
#define BJ_Types 5
1782
1792
1783
-
/* JBD uses a CRC32 checksum */
1784
-
#define JBD_MAX_CHECKSUM_SIZE 4
1785
-
1786
1793
static inline u32 jbd2_chksum(journal_t *journal, u32 crc,
1787
1794
const void *address, unsigned int length)
1788
1795
{
1789
-
DEFINE_RAW_FLEX(struct shash_desc, desc, __ctx,
1790
-
DIV_ROUND_UP(JBD_MAX_CHECKSUM_SIZE,
1791
-
sizeof(*((struct shash_desc *)0)->__ctx)));
1792
-
int err;
1793
-
1794
-
BUG_ON(crypto_shash_descsize(journal->j_chksum_driver) >
1795
-
JBD_MAX_CHECKSUM_SIZE);
1796
-
1797
-
desc->tfm = journal->j_chksum_driver;
1798
-
*(u32 *)desc->__ctx = crc;
1799
-
1800
-
err = crypto_shash_update(desc, address, length);
1801
-
BUG_ON(err);
1802
-
1803
-
return *(u32 *)desc->__ctx;
1796
+
return crc32c(crc, address, length);
1804
1797
}
1805
1798
1806
1799
/* Return most recent uncommitted transaction */
-3
include/target/iscsi/iscsi_target_core.h
-3
include/target/iscsi/iscsi_target_core.h
···
576
576
spinlock_t state_lock;
577
577
spinlock_t login_timer_lock;
578
578
spinlock_t login_worker_lock;
579
-
/* libcrypto RX and TX contexts for crc32c */
580
-
struct ahash_request *conn_rx_hash;
581
-
struct ahash_request *conn_tx_hash;
582
579
/* Used for scheduling TX and RX connection kthreads */
583
580
cpumask_var_t conn_cpumask;
584
581
cpumask_var_t allowed_cpumask;
+88
-33
lib/Kconfig
+88
-33
lib/Kconfig
···
156
156
157
157
config CRC_T10DIF
158
158
tristate "CRC calculation for the T10 Data Integrity Field"
159
-
select CRYPTO
160
-
select CRYPTO_CRCT10DIF
161
159
help
162
160
This option is only needed if a module that's not in the
163
161
kernel tree needs to calculate CRC checks for use with the
164
162
SCSI data integrity subsystem.
163
+
164
+
config ARCH_HAS_CRC_T10DIF
165
+
bool
166
+
167
+
choice
168
+
prompt "CRC-T10DIF implementation"
169
+
depends on CRC_T10DIF
170
+
default CRC_T10DIF_IMPL_ARCH if ARCH_HAS_CRC_T10DIF
171
+
default CRC_T10DIF_IMPL_GENERIC if !ARCH_HAS_CRC_T10DIF
172
+
help
173
+
This option allows you to override the default choice of CRC-T10DIF
174
+
implementation.
175
+
176
+
config CRC_T10DIF_IMPL_ARCH
177
+
bool "Architecture-optimized" if ARCH_HAS_CRC_T10DIF
178
+
help
179
+
Use the optimized implementation of CRC-T10DIF for the selected
180
+
architecture. It is recommended to keep this enabled, as it can
181
+
greatly improve CRC-T10DIF performance.
182
+
183
+
config CRC_T10DIF_IMPL_GENERIC
184
+
bool "Generic implementation"
185
+
help
186
+
Use the generic table-based implementation of CRC-T10DIF. Selecting
187
+
this will reduce code size slightly but can greatly reduce CRC-T10DIF
188
+
performance.
189
+
190
+
endchoice
191
+
192
+
config CRC_T10DIF_ARCH
193
+
tristate
194
+
default CRC_T10DIF if CRC_T10DIF_IMPL_ARCH
165
195
166
196
config CRC64_ROCKSOFT
167
197
tristate "CRC calculation for the Rocksoft model CRC64"
···
220
190
the kernel tree does. Such modules that use library CRC32/CRC32c
221
191
functions require M here.
222
192
223
-
config CRC32_SELFTEST
224
-
tristate "CRC32 perform self test on init"
225
-
depends on CRC32
226
-
help
227
-
This option enables the CRC32 library functions to perform a
228
-
self test on initialization. The self test computes crc32_le
229
-
and crc32_be over byte strings with random alignment and length
230
-
and computes the total elapsed time and number of bytes processed.
193
+
config ARCH_HAS_CRC32
194
+
bool
231
195
232
196
choice
233
197
prompt "CRC32 implementation"
234
198
depends on CRC32
235
-
default CRC32_SLICEBY8
199
+
default CRC32_IMPL_ARCH_PLUS_SLICEBY8 if ARCH_HAS_CRC32
200
+
default CRC32_IMPL_SLICEBY8 if !ARCH_HAS_CRC32
236
201
help
237
-
This option allows a kernel builder to override the default choice
238
-
of CRC32 algorithm. Choose the default ("slice by 8") unless you
239
-
know that you need one of the others.
202
+
This option allows you to override the default choice of CRC32
203
+
implementation. Choose the default unless you know that you need one
204
+
of the others.
240
205
241
-
config CRC32_SLICEBY8
206
+
config CRC32_IMPL_ARCH_PLUS_SLICEBY8
207
+
bool "Arch-optimized, with fallback to slice-by-8" if ARCH_HAS_CRC32
208
+
help
209
+
Use architecture-optimized implementation of CRC32. Fall back to
210
+
slice-by-8 in cases where the arch-optimized implementation cannot be
211
+
used, e.g. if the CPU lacks support for the needed instructions.
212
+
213
+
This is the default when an arch-optimized implementation exists.
214
+
215
+
config CRC32_IMPL_ARCH_PLUS_SLICEBY1
216
+
bool "Arch-optimized, with fallback to slice-by-1" if ARCH_HAS_CRC32
217
+
help
218
+
Use architecture-optimized implementation of CRC32, but fall back to
219
+
slice-by-1 instead of slice-by-8 in order to reduce the binary size.
220
+
221
+
config CRC32_IMPL_SLICEBY8
242
222
bool "Slice by 8 bytes"
243
223
help
244
224
Calculate checksum 8 bytes at a time with a clever slicing algorithm.
245
-
This is the fastest algorithm, but comes with a 8KiB lookup table.
246
-
Most modern processors have enough cache to hold this table without
247
-
thrashing the cache.
225
+
This is much slower than the architecture-optimized implementation of
226
+
CRC32 (if the selected arch has one), but it is portable and is the
227
+
fastest implementation when no arch-optimized implementation is
228
+
available. It uses an 8KiB lookup table. Most modern processors have
229
+
enough cache to hold this table without thrashing the cache.
248
230
249
-
This is the default implementation choice. Choose this one unless
250
-
you have a good reason not to.
251
-
252
-
config CRC32_SLICEBY4
231
+
config CRC32_IMPL_SLICEBY4
253
232
bool "Slice by 4 bytes"
254
233
help
255
234
Calculate checksum 4 bytes at a time with a clever slicing algorithm.
···
267
228
268
229
Only choose this option if you know what you are doing.
269
230
270
-
config CRC32_SARWATE
271
-
bool "Sarwate's Algorithm (one byte at a time)"
231
+
config CRC32_IMPL_SLICEBY1
232
+
bool "Slice by 1 byte (Sarwate's algorithm)"
272
233
help
273
234
Calculate checksum a byte at a time using Sarwate's algorithm. This
274
-
is not particularly fast, but has a small 256 byte lookup table.
235
+
is not particularly fast, but has a small 1KiB lookup table.
275
236
276
237
Only choose this option if you know what you are doing.
277
238
278
-
config CRC32_BIT
239
+
config CRC32_IMPL_BIT
279
240
bool "Classic Algorithm (one bit at a time)"
280
241
help
281
242
Calculate checksum one bit at a time. This is VERY slow, but has
···
284
245
Only choose this option if you are debugging crc32.
285
246
286
247
endchoice
248
+
249
+
config CRC32_ARCH
250
+
tristate
251
+
default CRC32 if CRC32_IMPL_ARCH_PLUS_SLICEBY8 || CRC32_IMPL_ARCH_PLUS_SLICEBY1
252
+
253
+
config CRC32_SLICEBY8
254
+
bool
255
+
default y if CRC32_IMPL_SLICEBY8 || CRC32_IMPL_ARCH_PLUS_SLICEBY8
256
+
257
+
config CRC32_SLICEBY4
258
+
bool
259
+
default y if CRC32_IMPL_SLICEBY4
260
+
261
+
config CRC32_SARWATE
262
+
bool
263
+
default y if CRC32_IMPL_SLICEBY1 || CRC32_IMPL_ARCH_PLUS_SLICEBY1
264
+
265
+
config CRC32_BIT
266
+
bool
267
+
default y if CRC32_IMPL_BIT
287
268
288
269
config CRC64
289
270
tristate "CRC64 functions"
···
331
272
332
273
config LIBCRC32C
333
274
tristate "CRC32c (Castagnoli, et al) Cyclic Redundancy-Check"
334
-
select CRYPTO
335
-
select CRYPTO_CRC32C
275
+
select CRC32
336
276
help
337
-
This option is provided for the case where no in-kernel-tree
338
-
modules require CRC32c functions, but a module built outside the
339
-
kernel tree does. Such modules that use library CRC32c functions
340
-
require M here. See Castagnoli93.
341
-
Module will be libcrc32c.
277
+
This option just selects CRC32 and is provided for compatibility
278
+
purposes until the users are updated to select CRC32 directly.
342
279
343
280
config CRC8
344
281
tristate "CRC8 function"
+20
-9
lib/Kconfig.debug
+20
-9
lib/Kconfig.debug
···
2838
2838
2839
2839
If unsure, say N.
2840
2840
2841
+
config CRC_KUNIT_TEST
2842
+
tristate "KUnit tests for CRC functions" if !KUNIT_ALL_TESTS
2843
+
depends on KUNIT
2844
+
default KUNIT_ALL_TESTS
2845
+
select CRC16
2846
+
select CRC_T10DIF
2847
+
select CRC32
2848
+
select CRC64
2849
+
help
2850
+
Unit tests for the CRC library functions.
2851
+
2852
+
This is intended to help people writing architecture-specific
2853
+
optimized versions. If unsure, say N.
2854
+
2855
+
config CRC_BENCHMARK
2856
+
bool "Benchmark for the CRC functions"
2857
+
depends on CRC_KUNIT_TEST
2858
+
help
2859
+
Include benchmarks in the KUnit test suite for the CRC functions.
2860
+
2841
2861
config SIPHASH_KUNIT_TEST
2842
2862
tristate "Perform selftest on siphash functions" if !KUNIT_ALL_TESTS
2843
2863
depends on KUNIT
···
2877
2857
This builds the "usercopy_kunit" module that runs sanity checks
2878
2858
on the copy_to/from_user infrastructure, making sure basic
2879
2859
user/kernel boundary testing is working.
2880
-
2881
-
config CRC16_KUNIT_TEST
2882
-
tristate "KUnit tests for CRC16"
2883
-
depends on KUNIT
2884
-
default KUNIT_ALL_TESTS
2885
-
select CRC16
2886
-
help
2887
-
Enable this option to run unit tests for the kernel's CRC16
2888
-
implementation (<linux/crc16.h>).
2889
2860
2890
2861
config TEST_UDELAY
2891
2862
tristate "udelay test driver"
+1
-3
lib/Makefile
+1
-3
lib/Makefile
···
164
164
obj-$(CONFIG_CRC_ITU_T) += crc-itu-t.o
165
165
obj-$(CONFIG_CRC32) += crc32.o
166
166
obj-$(CONFIG_CRC64) += crc64.o
167
-
obj-$(CONFIG_CRC32_SELFTEST) += crc32test.o
168
167
obj-$(CONFIG_CRC4) += crc4.o
169
168
obj-$(CONFIG_CRC7) += crc7.o
170
-
obj-$(CONFIG_LIBCRC32C) += libcrc32c.o
171
169
obj-$(CONFIG_CRC8) += crc8.o
172
170
obj-$(CONFIG_CRC64_ROCKSOFT) += crc64-rocksoft.o
173
171
obj-$(CONFIG_XXHASH) += xxhash.o
···
390
392
CFLAGS_fortify_kunit.o += $(call cc-disable-warning, stringop-truncation)
391
393
CFLAGS_fortify_kunit.o += $(DISABLE_STRUCTLEAK_PLUGIN)
392
394
obj-$(CONFIG_FORTIFY_KUNIT_TEST) += fortify_kunit.o
395
+
obj-$(CONFIG_CRC_KUNIT_TEST) += crc_kunit.o
393
396
obj-$(CONFIG_SIPHASH_KUNIT_TEST) += siphash_kunit.o
394
397
obj-$(CONFIG_USERCOPY_KUNIT_TEST) += usercopy_kunit.o
395
-
obj-$(CONFIG_CRC16_KUNIT_TEST) += crc16_kunit.o
396
398
397
399
obj-$(CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED) += devmem_is_allowed.o
398
400
+45
-111
lib/crc-t10dif.c
+45
-111
lib/crc-t10dif.c
···
9
9
#include <linux/types.h>
10
10
#include <linux/module.h>
11
11
#include <linux/crc-t10dif.h>
12
-
#include <linux/err.h>
13
-
#include <linux/init.h>
14
-
#include <crypto/hash.h>
15
-
#include <crypto/algapi.h>
16
-
#include <linux/static_key.h>
17
-
#include <linux/notifier.h>
18
12
19
-
static struct crypto_shash __rcu *crct10dif_tfm;
20
-
static DEFINE_STATIC_KEY_TRUE(crct10dif_fallback);
21
-
static DEFINE_MUTEX(crc_t10dif_mutex);
22
-
static struct work_struct crct10dif_rehash_work;
23
-
24
-
static int crc_t10dif_notify(struct notifier_block *self, unsigned long val, void *data)
25
-
{
26
-
struct crypto_alg *alg = data;
27
-
28
-
if (val != CRYPTO_MSG_ALG_LOADED ||
29
-
strcmp(alg->cra_name, CRC_T10DIF_STRING))
30
-
return NOTIFY_DONE;
31
-
32
-
schedule_work(&crct10dif_rehash_work);
33
-
return NOTIFY_OK;
34
-
}
35
-
36
-
static void crc_t10dif_rehash(struct work_struct *work)
37
-
{
38
-
struct crypto_shash *new, *old;
39
-
40
-
mutex_lock(&crc_t10dif_mutex);
41
-
old = rcu_dereference_protected(crct10dif_tfm,
42
-
lockdep_is_held(&crc_t10dif_mutex));
43
-
new = crypto_alloc_shash(CRC_T10DIF_STRING, 0, 0);
44
-
if (IS_ERR(new)) {
45
-
mutex_unlock(&crc_t10dif_mutex);
46
-
return;
47
-
}
48
-
rcu_assign_pointer(crct10dif_tfm, new);
49
-
mutex_unlock(&crc_t10dif_mutex);
50
-
51
-
if (old) {
52
-
synchronize_rcu();
53
-
crypto_free_shash(old);
54
-
} else {
55
-
static_branch_disable(&crct10dif_fallback);
56
-
}
57
-
}
58
-
59
-
static struct notifier_block crc_t10dif_nb = {
60
-
.notifier_call = crc_t10dif_notify,
13
+
/*
14
+
* Table generated using the following polynomial:
15
+
* x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x + 1
16
+
* gt: 0x8bb7
17
+
*/
18
+
static const u16 t10_dif_crc_table[256] = {
19
+
0x0000, 0x8BB7, 0x9CD9, 0x176E, 0xB205, 0x39B2, 0x2EDC, 0xA56B,
20
+
0xEFBD, 0x640A, 0x7364, 0xF8D3, 0x5DB8, 0xD60F, 0xC161, 0x4AD6,
21
+
0x54CD, 0xDF7A, 0xC814, 0x43A3, 0xE6C8, 0x6D7F, 0x7A11, 0xF1A6,
22
+
0xBB70, 0x30C7, 0x27A9, 0xAC1E, 0x0975, 0x82C2, 0x95AC, 0x1E1B,
23
+
0xA99A, 0x222D, 0x3543, 0xBEF4, 0x1B9F, 0x9028, 0x8746, 0x0CF1,
24
+
0x4627, 0xCD90, 0xDAFE, 0x5149, 0xF422, 0x7F95, 0x68FB, 0xE34C,
25
+
0xFD57, 0x76E0, 0x618E, 0xEA39, 0x4F52, 0xC4E5, 0xD38B, 0x583C,
26
+
0x12EA, 0x995D, 0x8E33, 0x0584, 0xA0EF, 0x2B58, 0x3C36, 0xB781,
27
+
0xD883, 0x5334, 0x445A, 0xCFED, 0x6A86, 0xE131, 0xF65F, 0x7DE8,
28
+
0x373E, 0xBC89, 0xABE7, 0x2050, 0x853B, 0x0E8C, 0x19E2, 0x9255,
29
+
0x8C4E, 0x07F9, 0x1097, 0x9B20, 0x3E4B, 0xB5FC, 0xA292, 0x2925,
30
+
0x63F3, 0xE844, 0xFF2A, 0x749D, 0xD1F6, 0x5A41, 0x4D2F, 0xC698,
31
+
0x7119, 0xFAAE, 0xEDC0, 0x6677, 0xC31C, 0x48AB, 0x5FC5, 0xD472,
32
+
0x9EA4, 0x1513, 0x027D, 0x89CA, 0x2CA1, 0xA716, 0xB078, 0x3BCF,
33
+
0x25D4, 0xAE63, 0xB90D, 0x32BA, 0x97D1, 0x1C66, 0x0B08, 0x80BF,
34
+
0xCA69, 0x41DE, 0x56B0, 0xDD07, 0x786C, 0xF3DB, 0xE4B5, 0x6F02,
35
+
0x3AB1, 0xB106, 0xA668, 0x2DDF, 0x88B4, 0x0303, 0x146D, 0x9FDA,
36
+
0xD50C, 0x5EBB, 0x49D5, 0xC262, 0x6709, 0xECBE, 0xFBD0, 0x7067,
37
+
0x6E7C, 0xE5CB, 0xF2A5, 0x7912, 0xDC79, 0x57CE, 0x40A0, 0xCB17,
38
+
0x81C1, 0x0A76, 0x1D18, 0x96AF, 0x33C4, 0xB873, 0xAF1D, 0x24AA,
39
+
0x932B, 0x189C, 0x0FF2, 0x8445, 0x212E, 0xAA99, 0xBDF7, 0x3640,
40
+
0x7C96, 0xF721, 0xE04F, 0x6BF8, 0xCE93, 0x4524, 0x524A, 0xD9FD,
41
+
0xC7E6, 0x4C51, 0x5B3F, 0xD088, 0x75E3, 0xFE54, 0xE93A, 0x628D,
42
+
0x285B, 0xA3EC, 0xB482, 0x3F35, 0x9A5E, 0x11E9, 0x0687, 0x8D30,
43
+
0xE232, 0x6985, 0x7EEB, 0xF55C, 0x5037, 0xDB80, 0xCCEE, 0x4759,
44
+
0x0D8F, 0x8638, 0x9156, 0x1AE1, 0xBF8A, 0x343D, 0x2353, 0xA8E4,
45
+
0xB6FF, 0x3D48, 0x2A26, 0xA191, 0x04FA, 0x8F4D, 0x9823, 0x1394,
46
+
0x5942, 0xD2F5, 0xC59B, 0x4E2C, 0xEB47, 0x60F0, 0x779E, 0xFC29,
47
+
0x4BA8, 0xC01F, 0xD771, 0x5CC6, 0xF9AD, 0x721A, 0x6574, 0xEEC3,
48
+
0xA415, 0x2FA2, 0x38CC, 0xB37B, 0x1610, 0x9DA7, 0x8AC9, 0x017E,
49
+
0x1F65, 0x94D2, 0x83BC, 0x080B, 0xAD60, 0x26D7, 0x31B9, 0xBA0E,
50
+
0xF0D8, 0x7B6F, 0x6C01, 0xE7B6, 0x42DD, 0xC96A, 0xDE04, 0x55B3
61
51
};
62
52
63
-
__u16 crc_t10dif_update(__u16 crc, const unsigned char *buffer, size_t len)
53
+
u16 crc_t10dif_generic(u16 crc, const u8 *p, size_t len)
64
54
{
65
-
struct {
66
-
struct shash_desc shash;
67
-
__u16 crc;
68
-
} desc;
69
-
int err;
55
+
size_t i;
70
56
71
-
if (static_branch_unlikely(&crct10dif_fallback))
72
-
return crc_t10dif_generic(crc, buffer, len);
57
+
for (i = 0; i < len; i++)
58
+
crc = (crc << 8) ^ t10_dif_crc_table[(crc >> 8) ^ p[i]];
73
59
74
-
rcu_read_lock();
75
-
desc.shash.tfm = rcu_dereference(crct10dif_tfm);
76
-
desc.crc = crc;
77
-
err = crypto_shash_update(&desc.shash, buffer, len);
78
-
rcu_read_unlock();
79
-
80
-
BUG_ON(err);
81
-
82
-
return desc.crc;
60
+
return crc;
83
61
}
84
-
EXPORT_SYMBOL(crc_t10dif_update);
62
+
EXPORT_SYMBOL(crc_t10dif_generic);
85
63
86
-
__u16 crc_t10dif(const unsigned char *buffer, size_t len)
87
-
{
88
-
return crc_t10dif_update(0, buffer, len);
89
-
}
90
-
EXPORT_SYMBOL(crc_t10dif);
91
-
92
-
static int __init crc_t10dif_mod_init(void)
93
-
{
94
-
INIT_WORK(&crct10dif_rehash_work, crc_t10dif_rehash);
95
-
crypto_register_notifier(&crc_t10dif_nb);
96
-
crc_t10dif_rehash(&crct10dif_rehash_work);
97
-
return 0;
98
-
}
99
-
100
-
static void __exit crc_t10dif_mod_fini(void)
101
-
{
102
-
crypto_unregister_notifier(&crc_t10dif_nb);
103
-
cancel_work_sync(&crct10dif_rehash_work);
104
-
crypto_free_shash(rcu_dereference_protected(crct10dif_tfm, 1));
105
-
}
106
-
107
-
module_init(crc_t10dif_mod_init);
108
-
module_exit(crc_t10dif_mod_fini);
109
-
110
-
static int crc_t10dif_transform_show(char *buffer, const struct kernel_param *kp)
111
-
{
112
-
struct crypto_shash *tfm;
113
-
int len;
114
-
115
-
if (static_branch_unlikely(&crct10dif_fallback))
116
-
return sprintf(buffer, "fallback\n");
117
-
118
-
rcu_read_lock();
119
-
tfm = rcu_dereference(crct10dif_tfm);
120
-
len = snprintf(buffer, PAGE_SIZE, "%s\n",
121
-
crypto_shash_driver_name(tfm));
122
-
rcu_read_unlock();
123
-
124
-
return len;
125
-
}
126
-
127
-
module_param_call(transform, NULL, crc_t10dif_transform_show, NULL, 0444);
128
-
129
-
MODULE_DESCRIPTION("T10 DIF CRC calculation (library API)");
64
+
MODULE_DESCRIPTION("T10 DIF CRC calculation");
130
65
MODULE_LICENSE("GPL");
131
-
MODULE_SOFTDEP("pre: crct10dif");
-155
lib/crc16_kunit.c
-155
lib/crc16_kunit.c
···
1
-
// SPDX-License-Identifier: GPL-2.0
2
-
/*
3
-
* KUnits tests for CRC16.
4
-
*
5
-
* Copyright (C) 2024, LKCAMP
6
-
* Author: Vinicius Peixoto <vpeixoto@lkcamp.dev>
7
-
* Author: Fabricio Gasperin <fgasperin@lkcamp.dev>
8
-
* Author: Enzo Bertoloti <ebertoloti@lkcamp.dev>
9
-
*/
10
-
#include <kunit/test.h>
11
-
#include <linux/crc16.h>
12
-
#include <linux/prandom.h>
13
-
14
-
#define CRC16_KUNIT_DATA_SIZE 4096
15
-
#define CRC16_KUNIT_TEST_SIZE 100
16
-
#define CRC16_KUNIT_SEED 0x12345678
17
-
18
-
/**
19
-
* struct crc16_test - CRC16 test data
20
-
* @crc: initial input value to CRC16
21
-
* @start: Start index within the data buffer
22
-
* @length: Length of the data
23
-
*/
24
-
static struct crc16_test {
25
-
u16 crc;
26
-
u16 start;
27
-
u16 length;
28
-
} tests[CRC16_KUNIT_TEST_SIZE];
29
-
30
-
u8 data[CRC16_KUNIT_DATA_SIZE];
31
-
32
-
33
-
/* Naive implementation of CRC16 for validation purposes */
34
-
static inline u16 _crc16_naive_byte(u16 crc, u8 data)
35
-
{
36
-
u8 i = 0;
37
-
38
-
crc ^= (u16) data;
39
-
for (i = 0; i < 8; i++) {
40
-
if (crc & 0x01)
41
-
crc = (crc >> 1) ^ 0xa001;
42
-
else
43
-
crc = crc >> 1;
44
-
}
45
-
46
-
return crc;
47
-
}
48
-
49
-
50
-
static inline u16 _crc16_naive(u16 crc, u8 *buffer, size_t len)
51
-
{
52
-
while (len--)
53
-
crc = _crc16_naive_byte(crc, *buffer++);
54
-
return crc;
55
-
}
56
-
57
-
58
-
/* Small helper for generating pseudorandom 16-bit data */
59
-
static inline u16 _rand16(void)
60
-
{
61
-
static u32 rand = CRC16_KUNIT_SEED;
62
-
63
-
rand = next_pseudo_random32(rand);
64
-
return rand & 0xFFFF;
65
-
}
66
-
67
-
68
-
static int crc16_init_test_data(struct kunit_suite *suite)
69
-
{
70
-
size_t i;
71
-
72
-
/* Fill the data buffer with random bytes */
73
-
for (i = 0; i < CRC16_KUNIT_DATA_SIZE; i++)
74
-
data[i] = _rand16() & 0xFF;
75
-
76
-
/* Generate random test data while ensuring the random
77
-
* start + length values won't overflow the 4096-byte
78
-
* buffer (0x7FF * 2 = 0xFFE < 0x1000)
79
-
*/
80
-
for (size_t i = 0; i < CRC16_KUNIT_TEST_SIZE; i++) {
81
-
tests[i].crc = _rand16();
82
-
tests[i].start = _rand16() & 0x7FF;
83
-
tests[i].length = _rand16() & 0x7FF;
84
-
}
85
-
86
-
return 0;
87
-
}
88
-
89
-
static void crc16_test_empty(struct kunit *test)
90
-
{
91
-
u16 crc;
92
-
93
-
/* The result for empty data should be the same as the
94
-
* initial crc
95
-
*/
96
-
crc = crc16(0x00, data, 0);
97
-
KUNIT_EXPECT_EQ(test, crc, 0);
98
-
crc = crc16(0xFF, data, 0);
99
-
KUNIT_EXPECT_EQ(test, crc, 0xFF);
100
-
}
101
-
102
-
static void crc16_test_correctness(struct kunit *test)
103
-
{
104
-
size_t i;
105
-
u16 crc, crc_naive;
106
-
107
-
for (i = 0; i < CRC16_KUNIT_TEST_SIZE; i++) {
108
-
/* Compare results with the naive crc16 implementation */
109
-
crc = crc16(tests[i].crc, data + tests[i].start,
110
-
tests[i].length);
111
-
crc_naive = _crc16_naive(tests[i].crc, data + tests[i].start,
112
-
tests[i].length);
113
-
KUNIT_EXPECT_EQ(test, crc, crc_naive);
114
-
}
115
-
}
116
-
117
-
118
-
static void crc16_test_combine(struct kunit *test)
119
-
{
120
-
size_t i, j;
121
-
u16 crc, crc_naive;
122
-
123
-
/* Make sure that combining two consecutive crc16 calculations
124
-
* yields the same result as calculating the crc16 for the whole thing
125
-
*/
126
-
for (i = 0; i < CRC16_KUNIT_TEST_SIZE; i++) {
127
-
crc_naive = crc16(tests[i].crc, data + tests[i].start, tests[i].length);
128
-
for (j = 0; j < tests[i].length; j++) {
129
-
crc = crc16(tests[i].crc, data + tests[i].start, j);
130
-
crc = crc16(crc, data + tests[i].start + j, tests[i].length - j);
131
-
KUNIT_EXPECT_EQ(test, crc, crc_naive);
132
-
}
133
-
}
134
-
}
135
-
136
-
137
-
static struct kunit_case crc16_test_cases[] = {
138
-
KUNIT_CASE(crc16_test_empty),
139
-
KUNIT_CASE(crc16_test_combine),
140
-
KUNIT_CASE(crc16_test_correctness),
141
-
{},
142
-
};
143
-
144
-
static struct kunit_suite crc16_test_suite = {
145
-
.name = "crc16",
146
-
.test_cases = crc16_test_cases,
147
-
.suite_init = crc16_init_test_data,
148
-
};
149
-
kunit_test_suite(crc16_test_suite);
150
-
151
-
MODULE_AUTHOR("Fabricio Gasperin <fgasperin@lkcamp.dev>");
152
-
MODULE_AUTHOR("Vinicius Peixoto <vpeixoto@lkcamp.dev>");
153
-
MODULE_AUTHOR("Enzo Bertoloti <ebertoloti@lkcamp.dev>");
154
-
MODULE_DESCRIPTION("Unit tests for crc16");
155
-
MODULE_LICENSE("GPL");
+8
-16
lib/crc32.c
+8
-16
lib/crc32.c
···
183
183
}
184
184
185
185
#if CRC_LE_BITS == 1
186
-
u32 __pure __weak crc32_le(u32 crc, unsigned char const *p, size_t len)
186
+
u32 __pure crc32_le_base(u32 crc, const u8 *p, size_t len)
187
187
{
188
188
return crc32_le_generic(crc, p, len, NULL, CRC32_POLY_LE);
189
189
}
190
-
u32 __pure __weak __crc32c_le(u32 crc, unsigned char const *p, size_t len)
190
+
u32 __pure crc32c_le_base(u32 crc, const u8 *p, size_t len)
191
191
{
192
192
return crc32_le_generic(crc, p, len, NULL, CRC32C_POLY_LE);
193
193
}
194
194
#else
195
-
u32 __pure __weak crc32_le(u32 crc, unsigned char const *p, size_t len)
195
+
u32 __pure crc32_le_base(u32 crc, const u8 *p, size_t len)
196
196
{
197
197
return crc32_le_generic(crc, p, len, crc32table_le, CRC32_POLY_LE);
198
198
}
199
-
u32 __pure __weak __crc32c_le(u32 crc, unsigned char const *p, size_t len)
199
+
u32 __pure crc32c_le_base(u32 crc, const u8 *p, size_t len)
200
200
{
201
201
return crc32_le_generic(crc, p, len, crc32ctable_le, CRC32C_POLY_LE);
202
202
}
203
203
#endif
204
-
EXPORT_SYMBOL(crc32_le);
205
-
EXPORT_SYMBOL(__crc32c_le);
206
-
207
-
u32 __pure crc32_le_base(u32, unsigned char const *, size_t) __alias(crc32_le);
208
204
EXPORT_SYMBOL(crc32_le_base);
209
-
210
-
u32 __pure __crc32c_le_base(u32, unsigned char const *, size_t) __alias(__crc32c_le);
211
-
EXPORT_SYMBOL(__crc32c_le_base);
212
-
213
-
u32 __pure crc32_be_base(u32, unsigned char const *, size_t) __alias(crc32_be);
205
+
EXPORT_SYMBOL(crc32c_le_base);
214
206
215
207
/*
216
208
* This multiplies the polynomials x and y modulo the given modulus.
···
327
335
}
328
336
329
337
#if CRC_BE_BITS == 1
330
-
u32 __pure __weak crc32_be(u32 crc, unsigned char const *p, size_t len)
338
+
u32 __pure crc32_be_base(u32 crc, const u8 *p, size_t len)
331
339
{
332
340
return crc32_be_generic(crc, p, len, NULL, CRC32_POLY_BE);
333
341
}
334
342
#else
335
-
u32 __pure __weak crc32_be(u32 crc, unsigned char const *p, size_t len)
343
+
u32 __pure crc32_be_base(u32 crc, const u8 *p, size_t len)
336
344
{
337
345
return crc32_be_generic(crc, p, len, crc32table_be, CRC32_POLY_BE);
338
346
}
339
347
#endif
340
-
EXPORT_SYMBOL(crc32_be);
348
+
EXPORT_SYMBOL(crc32_be_base);
-852
lib/crc32test.c
-852
lib/crc32test.c
···
1
-
/*
2
-
* Aug 8, 2011 Bob Pearson with help from Joakim Tjernlund and George Spelvin
3
-
* cleaned up code to current version of sparse and added the slicing-by-8
4
-
* algorithm to the closely similar existing slicing-by-4 algorithm.
5
-
*
6
-
* Oct 15, 2000 Matt Domsch <Matt_Domsch@dell.com>
7
-
* Nicer crc32 functions/docs submitted by linux@horizon.com. Thanks!
8
-
* Code was from the public domain, copyright abandoned. Code was
9
-
* subsequently included in the kernel, thus was re-licensed under the
10
-
* GNU GPL v2.
11
-
*
12
-
* Oct 12, 2000 Matt Domsch <Matt_Domsch@dell.com>
13
-
* Same crc32 function was used in 5 other places in the kernel.
14
-
* I made one version, and deleted the others.
15
-
* There are various incantations of crc32(). Some use a seed of 0 or ~0.
16
-
* Some xor at the end with ~0. The generic crc32() function takes
17
-
* seed as an argument, and doesn't xor at the end. Then individual
18
-
* users can do whatever they need.
19
-
* drivers/net/smc9194.c uses seed ~0, doesn't xor with ~0.
20
-
* fs/jffs2 uses seed 0, doesn't xor with ~0.
21
-
* fs/partitions/efi.c uses seed ~0, xor's with ~0.
22
-
*
23
-
* This source code is licensed under the GNU General Public License,
24
-
* Version 2. See the file COPYING for more details.
25
-
*/
26
-
27
-
#include <linux/crc32.h>
28
-
#include <linux/module.h>
29
-
#include <linux/sched.h>
30
-
31
-
#include "crc32defs.h"
32
-
33
-
/* 4096 random bytes */
34
-
static u8 const __aligned(8) test_buf[] __initconst =
35
-
{
36
-
0x5b, 0x85, 0x21, 0xcb, 0x09, 0x68, 0x7d, 0x30,
37
-
0xc7, 0x69, 0xd7, 0x30, 0x92, 0xde, 0x59, 0xe4,
38
-
0xc9, 0x6e, 0x8b, 0xdb, 0x98, 0x6b, 0xaa, 0x60,
39
-
0xa8, 0xb5, 0xbc, 0x6c, 0xa9, 0xb1, 0x5b, 0x2c,
40
-
0xea, 0xb4, 0x92, 0x6a, 0x3f, 0x79, 0x91, 0xe4,
41
-
0xe9, 0x70, 0x51, 0x8c, 0x7f, 0x95, 0x6f, 0x1a,
42
-
0x56, 0xa1, 0x5c, 0x27, 0x03, 0x67, 0x9f, 0x3a,
43
-
0xe2, 0x31, 0x11, 0x29, 0x6b, 0x98, 0xfc, 0xc4,
44
-
0x53, 0x24, 0xc5, 0x8b, 0xce, 0x47, 0xb2, 0xb9,
45
-
0x32, 0xcb, 0xc1, 0xd0, 0x03, 0x57, 0x4e, 0xd4,
46
-
0xe9, 0x3c, 0xa1, 0x63, 0xcf, 0x12, 0x0e, 0xca,
47
-
0xe1, 0x13, 0xd1, 0x93, 0xa6, 0x88, 0x5c, 0x61,
48
-
0x5b, 0xbb, 0xf0, 0x19, 0x46, 0xb4, 0xcf, 0x9e,
49
-
0xb6, 0x6b, 0x4c, 0x3a, 0xcf, 0x60, 0xf9, 0x7a,
50
-
0x8d, 0x07, 0x63, 0xdb, 0x40, 0xe9, 0x0b, 0x6f,
51
-
0xad, 0x97, 0xf1, 0xed, 0xd0, 0x1e, 0x26, 0xfd,
52
-
0xbf, 0xb7, 0xc8, 0x04, 0x94, 0xf8, 0x8b, 0x8c,
53
-
0xf1, 0xab, 0x7a, 0xd4, 0xdd, 0xf3, 0xe8, 0x88,
54
-
0xc3, 0xed, 0x17, 0x8a, 0x9b, 0x40, 0x0d, 0x53,
55
-
0x62, 0x12, 0x03, 0x5f, 0x1b, 0x35, 0x32, 0x1f,
56
-
0xb4, 0x7b, 0x93, 0x78, 0x0d, 0xdb, 0xce, 0xa4,
57
-
0xc0, 0x47, 0xd5, 0xbf, 0x68, 0xe8, 0x5d, 0x74,
58
-
0x8f, 0x8e, 0x75, 0x1c, 0xb2, 0x4f, 0x9a, 0x60,
59
-
0xd1, 0xbe, 0x10, 0xf4, 0x5c, 0xa1, 0x53, 0x09,
60
-
0xa5, 0xe0, 0x09, 0x54, 0x85, 0x5c, 0xdc, 0x07,
61
-
0xe7, 0x21, 0x69, 0x7b, 0x8a, 0xfd, 0x90, 0xf1,
62
-
0x22, 0xd0, 0xb4, 0x36, 0x28, 0xe6, 0xb8, 0x0f,
63
-
0x39, 0xde, 0xc8, 0xf3, 0x86, 0x60, 0x34, 0xd2,
64
-
0x5e, 0xdf, 0xfd, 0xcf, 0x0f, 0xa9, 0x65, 0xf0,
65
-
0xd5, 0x4d, 0x96, 0x40, 0xe3, 0xdf, 0x3f, 0x95,
66
-
0x5a, 0x39, 0x19, 0x93, 0xf4, 0x75, 0xce, 0x22,
67
-
0x00, 0x1c, 0x93, 0xe2, 0x03, 0x66, 0xf4, 0x93,
68
-
0x73, 0x86, 0x81, 0x8e, 0x29, 0x44, 0x48, 0x86,
69
-
0x61, 0x7c, 0x48, 0xa3, 0x43, 0xd2, 0x9c, 0x8d,
70
-
0xd4, 0x95, 0xdd, 0xe1, 0x22, 0x89, 0x3a, 0x40,
71
-
0x4c, 0x1b, 0x8a, 0x04, 0xa8, 0x09, 0x69, 0x8b,
72
-
0xea, 0xc6, 0x55, 0x8e, 0x57, 0xe6, 0x64, 0x35,
73
-
0xf0, 0xc7, 0x16, 0x9f, 0x5d, 0x5e, 0x86, 0x40,
74
-
0x46, 0xbb, 0xe5, 0x45, 0x88, 0xfe, 0xc9, 0x63,
75
-
0x15, 0xfb, 0xf5, 0xbd, 0x71, 0x61, 0xeb, 0x7b,
76
-
0x78, 0x70, 0x07, 0x31, 0x03, 0x9f, 0xb2, 0xc8,
77
-
0xa7, 0xab, 0x47, 0xfd, 0xdf, 0xa0, 0x78, 0x72,
78
-
0xa4, 0x2a, 0xe4, 0xb6, 0xba, 0xc0, 0x1e, 0x86,
79
-
0x71, 0xe6, 0x3d, 0x18, 0x37, 0x70, 0xe6, 0xff,
80
-
0xe0, 0xbc, 0x0b, 0x22, 0xa0, 0x1f, 0xd3, 0xed,
81
-
0xa2, 0x55, 0x39, 0xab, 0xa8, 0x13, 0x73, 0x7c,
82
-
0x3f, 0xb2, 0xd6, 0x19, 0xac, 0xff, 0x99, 0xed,
83
-
0xe8, 0xe6, 0xa6, 0x22, 0xe3, 0x9c, 0xf1, 0x30,
84
-
0xdc, 0x01, 0x0a, 0x56, 0xfa, 0xe4, 0xc9, 0x99,
85
-
0xdd, 0xa8, 0xd8, 0xda, 0x35, 0x51, 0x73, 0xb4,
86
-
0x40, 0x86, 0x85, 0xdb, 0x5c, 0xd5, 0x85, 0x80,
87
-
0x14, 0x9c, 0xfd, 0x98, 0xa9, 0x82, 0xc5, 0x37,
88
-
0xff, 0x32, 0x5d, 0xd0, 0x0b, 0xfa, 0xdc, 0x04,
89
-
0x5e, 0x09, 0xd2, 0xca, 0x17, 0x4b, 0x1a, 0x8e,
90
-
0x15, 0xe1, 0xcc, 0x4e, 0x52, 0x88, 0x35, 0xbd,
91
-
0x48, 0xfe, 0x15, 0xa0, 0x91, 0xfd, 0x7e, 0x6c,
92
-
0x0e, 0x5d, 0x79, 0x1b, 0x81, 0x79, 0xd2, 0x09,
93
-
0x34, 0x70, 0x3d, 0x81, 0xec, 0xf6, 0x24, 0xbb,
94
-
0xfb, 0xf1, 0x7b, 0xdf, 0x54, 0xea, 0x80, 0x9b,
95
-
0xc7, 0x99, 0x9e, 0xbd, 0x16, 0x78, 0x12, 0x53,
96
-
0x5e, 0x01, 0xa7, 0x4e, 0xbd, 0x67, 0xe1, 0x9b,
97
-
0x4c, 0x0e, 0x61, 0x45, 0x97, 0xd2, 0xf0, 0x0f,
98
-
0xfe, 0x15, 0x08, 0xb7, 0x11, 0x4c, 0xe7, 0xff,
99
-
0x81, 0x53, 0xff, 0x91, 0x25, 0x38, 0x7e, 0x40,
100
-
0x94, 0xe5, 0xe0, 0xad, 0xe6, 0xd9, 0x79, 0xb6,
101
-
0x92, 0xc9, 0xfc, 0xde, 0xc3, 0x1a, 0x23, 0xbb,
102
-
0xdd, 0xc8, 0x51, 0x0c, 0x3a, 0x72, 0xfa, 0x73,
103
-
0x6f, 0xb7, 0xee, 0x61, 0x39, 0x03, 0x01, 0x3f,
104
-
0x7f, 0x94, 0x2e, 0x2e, 0xba, 0x3a, 0xbb, 0xb4,
105
-
0xfa, 0x6a, 0x17, 0xfe, 0xea, 0xef, 0x5e, 0x66,
106
-
0x97, 0x3f, 0x32, 0x3d, 0xd7, 0x3e, 0xb1, 0xf1,
107
-
0x6c, 0x14, 0x4c, 0xfd, 0x37, 0xd3, 0x38, 0x80,
108
-
0xfb, 0xde, 0xa6, 0x24, 0x1e, 0xc8, 0xca, 0x7f,
109
-
0x3a, 0x93, 0xd8, 0x8b, 0x18, 0x13, 0xb2, 0xe5,
110
-
0xe4, 0x93, 0x05, 0x53, 0x4f, 0x84, 0x66, 0xa7,
111
-
0x58, 0x5c, 0x7b, 0x86, 0x52, 0x6d, 0x0d, 0xce,
112
-
0xa4, 0x30, 0x7d, 0xb6, 0x18, 0x9f, 0xeb, 0xff,
113
-
0x22, 0xbb, 0x72, 0x29, 0xb9, 0x44, 0x0b, 0x48,
114
-
0x1e, 0x84, 0x71, 0x81, 0xe3, 0x6d, 0x73, 0x26,
115
-
0x92, 0xb4, 0x4d, 0x2a, 0x29, 0xb8, 0x1f, 0x72,
116
-
0xed, 0xd0, 0xe1, 0x64, 0x77, 0xea, 0x8e, 0x88,
117
-
0x0f, 0xef, 0x3f, 0xb1, 0x3b, 0xad, 0xf9, 0xc9,
118
-
0x8b, 0xd0, 0xac, 0xc6, 0xcc, 0xa9, 0x40, 0xcc,
119
-
0x76, 0xf6, 0x3b, 0x53, 0xb5, 0x88, 0xcb, 0xc8,
120
-
0x37, 0xf1, 0xa2, 0xba, 0x23, 0x15, 0x99, 0x09,
121
-
0xcc, 0xe7, 0x7a, 0x3b, 0x37, 0xf7, 0x58, 0xc8,
122
-
0x46, 0x8c, 0x2b, 0x2f, 0x4e, 0x0e, 0xa6, 0x5c,
123
-
0xea, 0x85, 0x55, 0xba, 0x02, 0x0e, 0x0e, 0x48,
124
-
0xbc, 0xe1, 0xb1, 0x01, 0x35, 0x79, 0x13, 0x3d,
125
-
0x1b, 0xc0, 0x53, 0x68, 0x11, 0xe7, 0x95, 0x0f,
126
-
0x9d, 0x3f, 0x4c, 0x47, 0x7b, 0x4d, 0x1c, 0xae,
127
-
0x50, 0x9b, 0xcb, 0xdd, 0x05, 0x8d, 0x9a, 0x97,
128
-
0xfd, 0x8c, 0xef, 0x0c, 0x1d, 0x67, 0x73, 0xa8,
129
-
0x28, 0x36, 0xd5, 0xb6, 0x92, 0x33, 0x40, 0x75,
130
-
0x0b, 0x51, 0xc3, 0x64, 0xba, 0x1d, 0xc2, 0xcc,
131
-
0xee, 0x7d, 0x54, 0x0f, 0x27, 0x69, 0xa7, 0x27,
132
-
0x63, 0x30, 0x29, 0xd9, 0xc8, 0x84, 0xd8, 0xdf,
133
-
0x9f, 0x68, 0x8d, 0x04, 0xca, 0xa6, 0xc5, 0xc7,
134
-
0x7a, 0x5c, 0xc8, 0xd1, 0xcb, 0x4a, 0xec, 0xd0,
135
-
0xd8, 0x20, 0x69, 0xc5, 0x17, 0xcd, 0x78, 0xc8,
136
-
0x75, 0x23, 0x30, 0x69, 0xc9, 0xd4, 0xea, 0x5c,
137
-
0x4f, 0x6b, 0x86, 0x3f, 0x8b, 0xfe, 0xee, 0x44,
138
-
0xc9, 0x7c, 0xb7, 0xdd, 0x3e, 0xe5, 0xec, 0x54,
139
-
0x03, 0x3e, 0xaa, 0x82, 0xc6, 0xdf, 0xb2, 0x38,
140
-
0x0e, 0x5d, 0xb3, 0x88, 0xd9, 0xd3, 0x69, 0x5f,
141
-
0x8f, 0x70, 0x8a, 0x7e, 0x11, 0xd9, 0x1e, 0x7b,
142
-
0x38, 0xf1, 0x42, 0x1a, 0xc0, 0x35, 0xf5, 0xc7,
143
-
0x36, 0x85, 0xf5, 0xf7, 0xb8, 0x7e, 0xc7, 0xef,
144
-
0x18, 0xf1, 0x63, 0xd6, 0x7a, 0xc6, 0xc9, 0x0e,
145
-
0x4d, 0x69, 0x4f, 0x84, 0xef, 0x26, 0x41, 0x0c,
146
-
0xec, 0xc7, 0xe0, 0x7e, 0x3c, 0x67, 0x01, 0x4c,
147
-
0x62, 0x1a, 0x20, 0x6f, 0xee, 0x47, 0x4d, 0xc0,
148
-
0x99, 0x13, 0x8d, 0x91, 0x4a, 0x26, 0xd4, 0x37,
149
-
0x28, 0x90, 0x58, 0x75, 0x66, 0x2b, 0x0a, 0xdf,
150
-
0xda, 0xee, 0x92, 0x25, 0x90, 0x62, 0x39, 0x9e,
151
-
0x44, 0x98, 0xad, 0xc1, 0x88, 0xed, 0xe4, 0xb4,
152
-
0xaf, 0xf5, 0x8c, 0x9b, 0x48, 0x4d, 0x56, 0x60,
153
-
0x97, 0x0f, 0x61, 0x59, 0x9e, 0xa6, 0x27, 0xfe,
154
-
0xc1, 0x91, 0x15, 0x38, 0xb8, 0x0f, 0xae, 0x61,
155
-
0x7d, 0x26, 0x13, 0x5a, 0x73, 0xff, 0x1c, 0xa3,
156
-
0x61, 0x04, 0x58, 0x48, 0x55, 0x44, 0x11, 0xfe,
157
-
0x15, 0xca, 0xc3, 0xbd, 0xca, 0xc5, 0xb4, 0x40,
158
-
0x5d, 0x1b, 0x7f, 0x39, 0xb5, 0x9c, 0x35, 0xec,
159
-
0x61, 0x15, 0x32, 0x32, 0xb8, 0x4e, 0x40, 0x9f,
160
-
0x17, 0x1f, 0x0a, 0x4d, 0xa9, 0x91, 0xef, 0xb7,
161
-
0xb0, 0xeb, 0xc2, 0x83, 0x9a, 0x6c, 0xd2, 0x79,
162
-
0x43, 0x78, 0x5e, 0x2f, 0xe5, 0xdd, 0x1a, 0x3c,
163
-
0x45, 0xab, 0x29, 0x40, 0x3a, 0x37, 0x5b, 0x6f,
164
-
0xd7, 0xfc, 0x48, 0x64, 0x3c, 0x49, 0xfb, 0x21,
165
-
0xbe, 0xc3, 0xff, 0x07, 0xfb, 0x17, 0xe9, 0xc9,
166
-
0x0c, 0x4c, 0x5c, 0x15, 0x9e, 0x8e, 0x22, 0x30,
167
-
0x0a, 0xde, 0x48, 0x7f, 0xdb, 0x0d, 0xd1, 0x2b,
168
-
0x87, 0x38, 0x9e, 0xcc, 0x5a, 0x01, 0x16, 0xee,
169
-
0x75, 0x49, 0x0d, 0x30, 0x01, 0x34, 0x6a, 0xb6,
170
-
0x9a, 0x5a, 0x2a, 0xec, 0xbb, 0x48, 0xac, 0xd3,
171
-
0x77, 0x83, 0xd8, 0x08, 0x86, 0x4f, 0x48, 0x09,
172
-
0x29, 0x41, 0x79, 0xa1, 0x03, 0x12, 0xc4, 0xcd,
173
-
0x90, 0x55, 0x47, 0x66, 0x74, 0x9a, 0xcc, 0x4f,
174
-
0x35, 0x8c, 0xd6, 0x98, 0xef, 0xeb, 0x45, 0xb9,
175
-
0x9a, 0x26, 0x2f, 0x39, 0xa5, 0x70, 0x6d, 0xfc,
176
-
0xb4, 0x51, 0xee, 0xf4, 0x9c, 0xe7, 0x38, 0x59,
177
-
0xad, 0xf4, 0xbc, 0x46, 0xff, 0x46, 0x8e, 0x60,
178
-
0x9c, 0xa3, 0x60, 0x1d, 0xf8, 0x26, 0x72, 0xf5,
179
-
0x72, 0x9d, 0x68, 0x80, 0x04, 0xf6, 0x0b, 0xa1,
180
-
0x0a, 0xd5, 0xa7, 0x82, 0x3a, 0x3e, 0x47, 0xa8,
181
-
0x5a, 0xde, 0x59, 0x4f, 0x7b, 0x07, 0xb3, 0xe9,
182
-
0x24, 0x19, 0x3d, 0x34, 0x05, 0xec, 0xf1, 0xab,
183
-
0x6e, 0x64, 0x8f, 0xd3, 0xe6, 0x41, 0x86, 0x80,
184
-
0x70, 0xe3, 0x8d, 0x60, 0x9c, 0x34, 0x25, 0x01,
185
-
0x07, 0x4d, 0x19, 0x41, 0x4e, 0x3d, 0x5c, 0x7e,
186
-
0xa8, 0xf5, 0xcc, 0xd5, 0x7b, 0xe2, 0x7d, 0x3d,
187
-
0x49, 0x86, 0x7d, 0x07, 0xb7, 0x10, 0xe3, 0x35,
188
-
0xb8, 0x84, 0x6d, 0x76, 0xab, 0x17, 0xc6, 0x38,
189
-
0xb4, 0xd3, 0x28, 0x57, 0xad, 0xd3, 0x88, 0x5a,
190
-
0xda, 0xea, 0xc8, 0x94, 0xcc, 0x37, 0x19, 0xac,
191
-
0x9c, 0x9f, 0x4b, 0x00, 0x15, 0xc0, 0xc8, 0xca,
192
-
0x1f, 0x15, 0xaa, 0xe0, 0xdb, 0xf9, 0x2f, 0x57,
193
-
0x1b, 0x24, 0xc7, 0x6f, 0x76, 0x29, 0xfb, 0xed,
194
-
0x25, 0x0d, 0xc0, 0xfe, 0xbd, 0x5a, 0xbf, 0x20,
195
-
0x08, 0x51, 0x05, 0xec, 0x71, 0xa3, 0xbf, 0xef,
196
-
0x5e, 0x99, 0x75, 0xdb, 0x3c, 0x5f, 0x9a, 0x8c,
197
-
0xbb, 0x19, 0x5c, 0x0e, 0x93, 0x19, 0xf8, 0x6a,
198
-
0xbc, 0xf2, 0x12, 0x54, 0x2f, 0xcb, 0x28, 0x64,
199
-
0x88, 0xb3, 0x92, 0x0d, 0x96, 0xd1, 0xa6, 0xe4,
200
-
0x1f, 0xf1, 0x4d, 0xa4, 0xab, 0x1c, 0xee, 0x54,
201
-
0xf2, 0xad, 0x29, 0x6d, 0x32, 0x37, 0xb2, 0x16,
202
-
0x77, 0x5c, 0xdc, 0x2e, 0x54, 0xec, 0x75, 0x26,
203
-
0xc6, 0x36, 0xd9, 0x17, 0x2c, 0xf1, 0x7a, 0xdc,
204
-
0x4b, 0xf1, 0xe2, 0xd9, 0x95, 0xba, 0xac, 0x87,
205
-
0xc1, 0xf3, 0x8e, 0x58, 0x08, 0xd8, 0x87, 0x60,
206
-
0xc9, 0xee, 0x6a, 0xde, 0xa4, 0xd2, 0xfc, 0x0d,
207
-
0xe5, 0x36, 0xc4, 0x5c, 0x52, 0xb3, 0x07, 0x54,
208
-
0x65, 0x24, 0xc1, 0xb1, 0xd1, 0xb1, 0x53, 0x13,
209
-
0x31, 0x79, 0x7f, 0x05, 0x76, 0xeb, 0x37, 0x59,
210
-
0x15, 0x2b, 0xd1, 0x3f, 0xac, 0x08, 0x97, 0xeb,
211
-
0x91, 0x98, 0xdf, 0x6c, 0x09, 0x0d, 0x04, 0x9f,
212
-
0xdc, 0x3b, 0x0e, 0x60, 0x68, 0x47, 0x23, 0x15,
213
-
0x16, 0xc6, 0x0b, 0x35, 0xf8, 0x77, 0xa2, 0x78,
214
-
0x50, 0xd4, 0x64, 0x22, 0x33, 0xff, 0xfb, 0x93,
215
-
0x71, 0x46, 0x50, 0x39, 0x1b, 0x9c, 0xea, 0x4e,
216
-
0x8d, 0x0c, 0x37, 0xe5, 0x5c, 0x51, 0x3a, 0x31,
217
-
0xb2, 0x85, 0x84, 0x3f, 0x41, 0xee, 0xa2, 0xc1,
218
-
0xc6, 0x13, 0x3b, 0x54, 0x28, 0xd2, 0x18, 0x37,
219
-
0xcc, 0x46, 0x9f, 0x6a, 0x91, 0x3d, 0x5a, 0x15,
220
-
0x3c, 0x89, 0xa3, 0x61, 0x06, 0x7d, 0x2e, 0x78,
221
-
0xbe, 0x7d, 0x40, 0xba, 0x2f, 0x95, 0xb1, 0x2f,
222
-
0x87, 0x3b, 0x8a, 0xbe, 0x6a, 0xf4, 0xc2, 0x31,
223
-
0x74, 0xee, 0x91, 0xe0, 0x23, 0xaa, 0x5d, 0x7f,
224
-
0xdd, 0xf0, 0x44, 0x8c, 0x0b, 0x59, 0x2b, 0xfc,
225
-
0x48, 0x3a, 0xdf, 0x07, 0x05, 0x38, 0x6c, 0xc9,
226
-
0xeb, 0x18, 0x24, 0x68, 0x8d, 0x58, 0x98, 0xd3,
227
-
0x31, 0xa3, 0xe4, 0x70, 0x59, 0xb1, 0x21, 0xbe,
228
-
0x7e, 0x65, 0x7d, 0xb8, 0x04, 0xab, 0xf6, 0xe4,
229
-
0xd7, 0xda, 0xec, 0x09, 0x8f, 0xda, 0x6d, 0x24,
230
-
0x07, 0xcc, 0x29, 0x17, 0x05, 0x78, 0x1a, 0xc1,
231
-
0xb1, 0xce, 0xfc, 0xaa, 0x2d, 0xe7, 0xcc, 0x85,
232
-
0x84, 0x84, 0x03, 0x2a, 0x0c, 0x3f, 0xa9, 0xf8,
233
-
0xfd, 0x84, 0x53, 0x59, 0x5c, 0xf0, 0xd4, 0x09,
234
-
0xf0, 0xd2, 0x6c, 0x32, 0x03, 0xb0, 0xa0, 0x8c,
235
-
0x52, 0xeb, 0x23, 0x91, 0x88, 0x43, 0x13, 0x46,
236
-
0xf6, 0x1e, 0xb4, 0x1b, 0xf5, 0x8e, 0x3a, 0xb5,
237
-
0x3d, 0x00, 0xf6, 0xe5, 0x08, 0x3d, 0x5f, 0x39,
238
-
0xd3, 0x21, 0x69, 0xbc, 0x03, 0x22, 0x3a, 0xd2,
239
-
0x5c, 0x84, 0xf8, 0x15, 0xc4, 0x80, 0x0b, 0xbc,
240
-
0x29, 0x3c, 0xf3, 0x95, 0x98, 0xcd, 0x8f, 0x35,
241
-
0xbc, 0xa5, 0x3e, 0xfc, 0xd4, 0x13, 0x9e, 0xde,
242
-
0x4f, 0xce, 0x71, 0x9d, 0x09, 0xad, 0xf2, 0x80,
243
-
0x6b, 0x65, 0x7f, 0x03, 0x00, 0x14, 0x7c, 0x15,
244
-
0x85, 0x40, 0x6d, 0x70, 0xea, 0xdc, 0xb3, 0x63,
245
-
0x35, 0x4f, 0x4d, 0xe0, 0xd9, 0xd5, 0x3c, 0x58,
246
-
0x56, 0x23, 0x80, 0xe2, 0x36, 0xdd, 0x75, 0x1d,
247
-
0x94, 0x11, 0x41, 0x8e, 0xe0, 0x81, 0x8e, 0xcf,
248
-
0xe0, 0xe5, 0xf6, 0xde, 0xd1, 0xe7, 0x04, 0x12,
249
-
0x79, 0x92, 0x2b, 0x71, 0x2a, 0x79, 0x8b, 0x7c,
250
-
0x44, 0x79, 0x16, 0x30, 0x4e, 0xf4, 0xf6, 0x9b,
251
-
0xb7, 0x40, 0xa3, 0x5a, 0xa7, 0x69, 0x3e, 0xc1,
252
-
0x3a, 0x04, 0xd0, 0x88, 0xa0, 0x3b, 0xdd, 0xc6,
253
-
0x9e, 0x7e, 0x1e, 0x1e, 0x8f, 0x44, 0xf7, 0x73,
254
-
0x67, 0x1e, 0x1a, 0x78, 0xfa, 0x62, 0xf4, 0xa9,
255
-
0xa8, 0xc6, 0x5b, 0xb8, 0xfa, 0x06, 0x7d, 0x5e,
256
-
0x38, 0x1c, 0x9a, 0x39, 0xe9, 0x39, 0x98, 0x22,
257
-
0x0b, 0xa7, 0xac, 0x0b, 0xf3, 0xbc, 0xf1, 0xeb,
258
-
0x8c, 0x81, 0xe3, 0x48, 0x8a, 0xed, 0x42, 0xc2,
259
-
0x38, 0xcf, 0x3e, 0xda, 0xd2, 0x89, 0x8d, 0x9c,
260
-
0x53, 0xb5, 0x2f, 0x41, 0x01, 0x26, 0x84, 0x9c,
261
-
0xa3, 0x56, 0xf6, 0x49, 0xc7, 0xd4, 0x9f, 0x93,
262
-
0x1b, 0x96, 0x49, 0x5e, 0xad, 0xb3, 0x84, 0x1f,
263
-
0x3c, 0xa4, 0xe0, 0x9b, 0xd1, 0x90, 0xbc, 0x38,
264
-
0x6c, 0xdd, 0x95, 0x4d, 0x9d, 0xb1, 0x71, 0x57,
265
-
0x2d, 0x34, 0xe8, 0xb8, 0x42, 0xc7, 0x99, 0x03,
266
-
0xc7, 0x07, 0x30, 0x65, 0x91, 0x55, 0xd5, 0x90,
267
-
0x70, 0x97, 0x37, 0x68, 0xd4, 0x11, 0xf9, 0xe8,
268
-
0xce, 0xec, 0xdc, 0x34, 0xd5, 0xd3, 0xb7, 0xc4,
269
-
0xb8, 0x97, 0x05, 0x92, 0xad, 0xf8, 0xe2, 0x36,
270
-
0x64, 0x41, 0xc9, 0xc5, 0x41, 0x77, 0x52, 0xd7,
271
-
0x2c, 0xa5, 0x24, 0x2f, 0xd9, 0x34, 0x0b, 0x47,
272
-
0x35, 0xa7, 0x28, 0x8b, 0xc5, 0xcd, 0xe9, 0x46,
273
-
0xac, 0x39, 0x94, 0x3c, 0x10, 0xc6, 0x29, 0x73,
274
-
0x0e, 0x0e, 0x5d, 0xe0, 0x71, 0x03, 0x8a, 0x72,
275
-
0x0e, 0x26, 0xb0, 0x7d, 0x84, 0xed, 0x95, 0x23,
276
-
0x49, 0x5a, 0x45, 0x83, 0x45, 0x60, 0x11, 0x4a,
277
-
0x46, 0x31, 0xd4, 0xd8, 0x16, 0x54, 0x98, 0x58,
278
-
0xed, 0x6d, 0xcc, 0x5d, 0xd6, 0x50, 0x61, 0x9f,
279
-
0x9d, 0xc5, 0x3e, 0x9d, 0x32, 0x47, 0xde, 0x96,
280
-
0xe1, 0x5d, 0xd8, 0xf8, 0xb4, 0x69, 0x6f, 0xb9,
281
-
0x15, 0x90, 0x57, 0x7a, 0xf6, 0xad, 0xb0, 0x5b,
282
-
0xf5, 0xa6, 0x36, 0x94, 0xfd, 0x84, 0xce, 0x1c,
283
-
0x0f, 0x4b, 0xd0, 0xc2, 0x5b, 0x6b, 0x56, 0xef,
284
-
0x73, 0x93, 0x0b, 0xc3, 0xee, 0xd9, 0xcf, 0xd3,
285
-
0xa4, 0x22, 0x58, 0xcd, 0x50, 0x6e, 0x65, 0xf4,
286
-
0xe9, 0xb7, 0x71, 0xaf, 0x4b, 0xb3, 0xb6, 0x2f,
287
-
0x0f, 0x0e, 0x3b, 0xc9, 0x85, 0x14, 0xf5, 0x17,
288
-
0xe8, 0x7a, 0x3a, 0xbf, 0x5f, 0x5e, 0xf8, 0x18,
289
-
0x48, 0xa6, 0x72, 0xab, 0x06, 0x95, 0xe9, 0xc8,
290
-
0xa7, 0xf4, 0x32, 0x44, 0x04, 0x0c, 0x84, 0x98,
291
-
0x73, 0xe3, 0x89, 0x8d, 0x5f, 0x7e, 0x4a, 0x42,
292
-
0x8f, 0xc5, 0x28, 0xb1, 0x82, 0xef, 0x1c, 0x97,
293
-
0x31, 0x3b, 0x4d, 0xe0, 0x0e, 0x10, 0x10, 0x97,
294
-
0x93, 0x49, 0x78, 0x2f, 0x0d, 0x86, 0x8b, 0xa1,
295
-
0x53, 0xa9, 0x81, 0x20, 0x79, 0xe7, 0x07, 0x77,
296
-
0xb6, 0xac, 0x5e, 0xd2, 0x05, 0xcd, 0xe9, 0xdb,
297
-
0x8a, 0x94, 0x82, 0x8a, 0x23, 0xb9, 0x3d, 0x1c,
298
-
0xa9, 0x7d, 0x72, 0x4a, 0xed, 0x33, 0xa3, 0xdb,
299
-
0x21, 0xa7, 0x86, 0x33, 0x45, 0xa5, 0xaa, 0x56,
300
-
0x45, 0xb5, 0x83, 0x29, 0x40, 0x47, 0x79, 0x04,
301
-
0x6e, 0xb9, 0x95, 0xd0, 0x81, 0x77, 0x2d, 0x48,
302
-
0x1e, 0xfe, 0xc3, 0xc2, 0x1e, 0xe5, 0xf2, 0xbe,
303
-
0xfd, 0x3b, 0x94, 0x9f, 0xc4, 0xc4, 0x26, 0x9d,
304
-
0xe4, 0x66, 0x1e, 0x19, 0xee, 0x6c, 0x79, 0x97,
305
-
0x11, 0x31, 0x4b, 0x0d, 0x01, 0xcb, 0xde, 0xa8,
306
-
0xf6, 0x6d, 0x7c, 0x39, 0x46, 0x4e, 0x7e, 0x3f,
307
-
0x94, 0x17, 0xdf, 0xa1, 0x7d, 0xd9, 0x1c, 0x8e,
308
-
0xbc, 0x7d, 0x33, 0x7d, 0xe3, 0x12, 0x40, 0xca,
309
-
0xab, 0x37, 0x11, 0x46, 0xd4, 0xae, 0xef, 0x44,
310
-
0xa2, 0xb3, 0x6a, 0x66, 0x0e, 0x0c, 0x90, 0x7f,
311
-
0xdf, 0x5c, 0x66, 0x5f, 0xf2, 0x94, 0x9f, 0xa6,
312
-
0x73, 0x4f, 0xeb, 0x0d, 0xad, 0xbf, 0xc0, 0x63,
313
-
0x5c, 0xdc, 0x46, 0x51, 0xe8, 0x8e, 0x90, 0x19,
314
-
0xa8, 0xa4, 0x3c, 0x91, 0x79, 0xfa, 0x7e, 0x58,
315
-
0x85, 0x13, 0x55, 0xc5, 0x19, 0x82, 0x37, 0x1b,
316
-
0x0a, 0x02, 0x1f, 0x99, 0x6b, 0x18, 0xf1, 0x28,
317
-
0x08, 0xa2, 0x73, 0xb8, 0x0f, 0x2e, 0xcd, 0xbf,
318
-
0xf3, 0x86, 0x7f, 0xea, 0xef, 0xd0, 0xbb, 0xa6,
319
-
0x21, 0xdf, 0x49, 0x73, 0x51, 0xcc, 0x36, 0xd3,
320
-
0x3e, 0xa0, 0xf8, 0x44, 0xdf, 0xd3, 0xa6, 0xbe,
321
-
0x8a, 0xd4, 0x57, 0xdd, 0x72, 0x94, 0x61, 0x0f,
322
-
0x82, 0xd1, 0x07, 0xb8, 0x7c, 0x18, 0x83, 0xdf,
323
-
0x3a, 0xe5, 0x50, 0x6a, 0x82, 0x20, 0xac, 0xa9,
324
-
0xa8, 0xff, 0xd9, 0xf3, 0x77, 0x33, 0x5a, 0x9e,
325
-
0x7f, 0x6d, 0xfe, 0x5d, 0x33, 0x41, 0x42, 0xe7,
326
-
0x6c, 0x19, 0xe0, 0x44, 0x8a, 0x15, 0xf6, 0x70,
327
-
0x98, 0xb7, 0x68, 0x4d, 0xfa, 0x97, 0x39, 0xb0,
328
-
0x8e, 0xe8, 0x84, 0x8b, 0x75, 0x30, 0xb7, 0x7d,
329
-
0x92, 0x69, 0x20, 0x9c, 0x81, 0xfb, 0x4b, 0xf4,
330
-
0x01, 0x50, 0xeb, 0xce, 0x0c, 0x1c, 0x6c, 0xb5,
331
-
0x4a, 0xd7, 0x27, 0x0c, 0xce, 0xbb, 0xe5, 0x85,
332
-
0xf0, 0xb6, 0xee, 0xd5, 0x70, 0xdd, 0x3b, 0xfc,
333
-
0xd4, 0x99, 0xf1, 0x33, 0xdd, 0x8b, 0xc4, 0x2f,
334
-
0xae, 0xab, 0x74, 0x96, 0x32, 0xc7, 0x4c, 0x56,
335
-
0x3c, 0x89, 0x0f, 0x96, 0x0b, 0x42, 0xc0, 0xcb,
336
-
0xee, 0x0f, 0x0b, 0x8c, 0xfb, 0x7e, 0x47, 0x7b,
337
-
0x64, 0x48, 0xfd, 0xb2, 0x00, 0x80, 0x89, 0xa5,
338
-
0x13, 0x55, 0x62, 0xfc, 0x8f, 0xe2, 0x42, 0x03,
339
-
0xb7, 0x4e, 0x2a, 0x79, 0xb4, 0x82, 0xea, 0x23,
340
-
0x49, 0xda, 0xaf, 0x52, 0x63, 0x1e, 0x60, 0x03,
341
-
0x89, 0x06, 0x44, 0x46, 0x08, 0xc3, 0xc4, 0x87,
342
-
0x70, 0x2e, 0xda, 0x94, 0xad, 0x6b, 0xe0, 0xe4,
343
-
0xd1, 0x8a, 0x06, 0xc2, 0xa8, 0xc0, 0xa7, 0x43,
344
-
0x3c, 0x47, 0x52, 0x0e, 0xc3, 0x77, 0x81, 0x11,
345
-
0x67, 0x0e, 0xa0, 0x70, 0x04, 0x47, 0x29, 0x40,
346
-
0x86, 0x0d, 0x34, 0x56, 0xa7, 0xc9, 0x35, 0x59,
347
-
0x68, 0xdc, 0x93, 0x81, 0x70, 0xee, 0x86, 0xd9,
348
-
0x80, 0x06, 0x40, 0x4f, 0x1a, 0x0d, 0x40, 0x30,
349
-
0x0b, 0xcb, 0x96, 0x47, 0xc1, 0xb7, 0x52, 0xfd,
350
-
0x56, 0xe0, 0x72, 0x4b, 0xfb, 0xbd, 0x92, 0x45,
351
-
0x61, 0x71, 0xc2, 0x33, 0x11, 0xbf, 0x52, 0x83,
352
-
0x79, 0x26, 0xe0, 0x49, 0x6b, 0xb7, 0x05, 0x8b,
353
-
0xe8, 0x0e, 0x87, 0x31, 0xd7, 0x9d, 0x8a, 0xf5,
354
-
0xc0, 0x5f, 0x2e, 0x58, 0x4a, 0xdb, 0x11, 0xb3,
355
-
0x6c, 0x30, 0x2a, 0x46, 0x19, 0xe3, 0x27, 0x84,
356
-
0x1f, 0x63, 0x6e, 0xf6, 0x57, 0xc7, 0xc9, 0xd8,
357
-
0x5e, 0xba, 0xb3, 0x87, 0xd5, 0x83, 0x26, 0x34,
358
-
0x21, 0x9e, 0x65, 0xde, 0x42, 0xd3, 0xbe, 0x7b,
359
-
0xbc, 0x91, 0x71, 0x44, 0x4d, 0x99, 0x3b, 0x31,
360
-
0xe5, 0x3f, 0x11, 0x4e, 0x7f, 0x13, 0x51, 0x3b,
361
-
0xae, 0x79, 0xc9, 0xd3, 0x81, 0x8e, 0x25, 0x40,
362
-
0x10, 0xfc, 0x07, 0x1e, 0xf9, 0x7b, 0x9a, 0x4b,
363
-
0x6c, 0xe3, 0xb3, 0xad, 0x1a, 0x0a, 0xdd, 0x9e,
364
-
0x59, 0x0c, 0xa2, 0xcd, 0xae, 0x48, 0x4a, 0x38,
365
-
0x5b, 0x47, 0x41, 0x94, 0x65, 0x6b, 0xbb, 0xeb,
366
-
0x5b, 0xe3, 0xaf, 0x07, 0x5b, 0xd4, 0x4a, 0xa2,
367
-
0xc9, 0x5d, 0x2f, 0x64, 0x03, 0xd7, 0x3a, 0x2c,
368
-
0x6e, 0xce, 0x76, 0x95, 0xb4, 0xb3, 0xc0, 0xf1,
369
-
0xe2, 0x45, 0x73, 0x7a, 0x5c, 0xab, 0xc1, 0xfc,
370
-
0x02, 0x8d, 0x81, 0x29, 0xb3, 0xac, 0x07, 0xec,
371
-
0x40, 0x7d, 0x45, 0xd9, 0x7a, 0x59, 0xee, 0x34,
372
-
0xf0, 0xe9, 0xd5, 0x7b, 0x96, 0xb1, 0x3d, 0x95,
373
-
0xcc, 0x86, 0xb5, 0xb6, 0x04, 0x2d, 0xb5, 0x92,
374
-
0x7e, 0x76, 0xf4, 0x06, 0xa9, 0xa3, 0x12, 0x0f,
375
-
0xb1, 0xaf, 0x26, 0xba, 0x7c, 0xfc, 0x7e, 0x1c,
376
-
0xbc, 0x2c, 0x49, 0x97, 0x53, 0x60, 0x13, 0x0b,
377
-
0xa6, 0x61, 0x83, 0x89, 0x42, 0xd4, 0x17, 0x0c,
378
-
0x6c, 0x26, 0x52, 0xc3, 0xb3, 0xd4, 0x67, 0xf5,
379
-
0xe3, 0x04, 0xb7, 0xf4, 0xcb, 0x80, 0xb8, 0xcb,
380
-
0x77, 0x56, 0x3e, 0xaa, 0x57, 0x54, 0xee, 0xb4,
381
-
0x2c, 0x67, 0xcf, 0xf2, 0xdc, 0xbe, 0x55, 0xf9,
382
-
0x43, 0x1f, 0x6e, 0x22, 0x97, 0x67, 0x7f, 0xc4,
383
-
0xef, 0xb1, 0x26, 0x31, 0x1e, 0x27, 0xdf, 0x41,
384
-
0x80, 0x47, 0x6c, 0xe2, 0xfa, 0xa9, 0x8c, 0x2a,
385
-
0xf6, 0xf2, 0xab, 0xf0, 0x15, 0xda, 0x6c, 0xc8,
386
-
0xfe, 0xb5, 0x23, 0xde, 0xa9, 0x05, 0x3f, 0x06,
387
-
0x54, 0x4c, 0xcd, 0xe1, 0xab, 0xfc, 0x0e, 0x62,
388
-
0x33, 0x31, 0x73, 0x2c, 0x76, 0xcb, 0xb4, 0x47,
389
-
0x1e, 0x20, 0xad, 0xd8, 0xf2, 0x31, 0xdd, 0xc4,
390
-
0x8b, 0x0c, 0x77, 0xbe, 0xe1, 0x8b, 0x26, 0x00,
391
-
0x02, 0x58, 0xd6, 0x8d, 0xef, 0xad, 0x74, 0x67,
392
-
0xab, 0x3f, 0xef, 0xcb, 0x6f, 0xb0, 0xcc, 0x81,
393
-
0x44, 0x4c, 0xaf, 0xe9, 0x49, 0x4f, 0xdb, 0xa0,
394
-
0x25, 0xa4, 0xf0, 0x89, 0xf1, 0xbe, 0xd8, 0x10,
395
-
0xff, 0xb1, 0x3b, 0x4b, 0xfa, 0x98, 0xf5, 0x79,
396
-
0x6d, 0x1e, 0x69, 0x4d, 0x57, 0xb1, 0xc8, 0x19,
397
-
0x1b, 0xbd, 0x1e, 0x8c, 0x84, 0xb7, 0x7b, 0xe8,
398
-
0xd2, 0x2d, 0x09, 0x41, 0x41, 0x37, 0x3d, 0xb1,
399
-
0x6f, 0x26, 0x5d, 0x71, 0x16, 0x3d, 0xb7, 0x83,
400
-
0x27, 0x2c, 0xa7, 0xb6, 0x50, 0xbd, 0x91, 0x86,
401
-
0xab, 0x24, 0xa1, 0x38, 0xfd, 0xea, 0x71, 0x55,
402
-
0x7e, 0x9a, 0x07, 0x77, 0x4b, 0xfa, 0x61, 0x66,
403
-
0x20, 0x1e, 0x28, 0x95, 0x18, 0x1b, 0xa4, 0xa0,
404
-
0xfd, 0xc0, 0x89, 0x72, 0x43, 0xd9, 0x3b, 0x49,
405
-
0x5a, 0x3f, 0x9d, 0xbf, 0xdb, 0xb4, 0x46, 0xea,
406
-
0x42, 0x01, 0x77, 0x23, 0x68, 0x95, 0xb6, 0x24,
407
-
0xb3, 0xa8, 0x6c, 0x28, 0x3b, 0x11, 0x40, 0x7e,
408
-
0x18, 0x65, 0x6d, 0xd8, 0x24, 0x42, 0x7d, 0x88,
409
-
0xc0, 0x52, 0xd9, 0x05, 0xe4, 0x95, 0x90, 0x87,
410
-
0x8c, 0xf4, 0xd0, 0x6b, 0xb9, 0x83, 0x99, 0x34,
411
-
0x6d, 0xfe, 0x54, 0x40, 0x94, 0x52, 0x21, 0x4f,
412
-
0x14, 0x25, 0xc5, 0xd6, 0x5e, 0x95, 0xdc, 0x0a,
413
-
0x2b, 0x89, 0x20, 0x11, 0x84, 0x48, 0xd6, 0x3a,
414
-
0xcd, 0x5c, 0x24, 0xad, 0x62, 0xe3, 0xb1, 0x93,
415
-
0x25, 0x8d, 0xcd, 0x7e, 0xfc, 0x27, 0xa3, 0x37,
416
-
0xfd, 0x84, 0xfc, 0x1b, 0xb2, 0xf1, 0x27, 0x38,
417
-
0x5a, 0xb7, 0xfc, 0xf2, 0xfa, 0x95, 0x66, 0xd4,
418
-
0xfb, 0xba, 0xa7, 0xd7, 0xa3, 0x72, 0x69, 0x48,
419
-
0x48, 0x8c, 0xeb, 0x28, 0x89, 0xfe, 0x33, 0x65,
420
-
0x5a, 0x36, 0x01, 0x7e, 0x06, 0x79, 0x0a, 0x09,
421
-
0x3b, 0x74, 0x11, 0x9a, 0x6e, 0xbf, 0xd4, 0x9e,
422
-
0x58, 0x90, 0x49, 0x4f, 0x4d, 0x08, 0xd4, 0xe5,
423
-
0x4a, 0x09, 0x21, 0xef, 0x8b, 0xb8, 0x74, 0x3b,
424
-
0x91, 0xdd, 0x36, 0x85, 0x60, 0x2d, 0xfa, 0xd4,
425
-
0x45, 0x7b, 0x45, 0x53, 0xf5, 0x47, 0x87, 0x7e,
426
-
0xa6, 0x37, 0xc8, 0x78, 0x7a, 0x68, 0x9d, 0x8d,
427
-
0x65, 0x2c, 0x0e, 0x91, 0x5c, 0xa2, 0x60, 0xf0,
428
-
0x8e, 0x3f, 0xe9, 0x1a, 0xcd, 0xaa, 0xe7, 0xd5,
429
-
0x77, 0x18, 0xaf, 0xc9, 0xbc, 0x18, 0xea, 0x48,
430
-
0x1b, 0xfb, 0x22, 0x48, 0x70, 0x16, 0x29, 0x9e,
431
-
0x5b, 0xc1, 0x2c, 0x66, 0x23, 0xbc, 0xf0, 0x1f,
432
-
0xef, 0xaf, 0xe4, 0xd6, 0x04, 0x19, 0x82, 0x7a,
433
-
0x0b, 0xba, 0x4b, 0x46, 0xb1, 0x6a, 0x85, 0x5d,
434
-
0xb4, 0x73, 0xd6, 0x21, 0xa1, 0x71, 0x60, 0x14,
435
-
0xee, 0x0a, 0x77, 0xc4, 0x66, 0x2e, 0xf9, 0x69,
436
-
0x30, 0xaf, 0x41, 0x0b, 0xc8, 0x83, 0x3c, 0x53,
437
-
0x99, 0x19, 0x27, 0x46, 0xf7, 0x41, 0x6e, 0x56,
438
-
0xdc, 0x94, 0x28, 0x67, 0x4e, 0xb7, 0x25, 0x48,
439
-
0x8a, 0xc2, 0xe0, 0x60, 0x96, 0xcc, 0x18, 0xf4,
440
-
0x84, 0xdd, 0xa7, 0x5e, 0x3e, 0x05, 0x0b, 0x26,
441
-
0x26, 0xb2, 0x5c, 0x1f, 0x57, 0x1a, 0x04, 0x7e,
442
-
0x6a, 0xe3, 0x2f, 0xb4, 0x35, 0xb6, 0x38, 0x40,
443
-
0x40, 0xcd, 0x6f, 0x87, 0x2e, 0xef, 0xa3, 0xd7,
444
-
0xa9, 0xc2, 0xe8, 0x0d, 0x27, 0xdf, 0x44, 0x62,
445
-
0x99, 0xa0, 0xfc, 0xcf, 0x81, 0x78, 0xcb, 0xfe,
446
-
0xe5, 0xa0, 0x03, 0x4e, 0x6c, 0xd7, 0xf4, 0xaf,
447
-
0x7a, 0xbb, 0x61, 0x82, 0xfe, 0x71, 0x89, 0xb2,
448
-
0x22, 0x7c, 0x8e, 0x83, 0x04, 0xce, 0xf6, 0x5d,
449
-
0x84, 0x8f, 0x95, 0x6a, 0x7f, 0xad, 0xfd, 0x32,
450
-
0x9c, 0x5e, 0xe4, 0x9c, 0x89, 0x60, 0x54, 0xaa,
451
-
0x96, 0x72, 0xd2, 0xd7, 0x36, 0x85, 0xa9, 0x45,
452
-
0xd2, 0x2a, 0xa1, 0x81, 0x49, 0x6f, 0x7e, 0x04,
453
-
0xfa, 0xe2, 0xfe, 0x90, 0x26, 0x77, 0x5a, 0x33,
454
-
0xb8, 0x04, 0x9a, 0x7a, 0xe6, 0x4c, 0x4f, 0xad,
455
-
0x72, 0x96, 0x08, 0x28, 0x58, 0x13, 0xf8, 0xc4,
456
-
0x1c, 0xf0, 0xc3, 0x45, 0x95, 0x49, 0x20, 0x8c,
457
-
0x9f, 0x39, 0x70, 0xe1, 0x77, 0xfe, 0xd5, 0x4b,
458
-
0xaf, 0x86, 0xda, 0xef, 0x22, 0x06, 0x83, 0x36,
459
-
0x29, 0x12, 0x11, 0x40, 0xbc, 0x3b, 0x86, 0xaa,
460
-
0xaa, 0x65, 0x60, 0xc3, 0x80, 0xca, 0xed, 0xa9,
461
-
0xf3, 0xb0, 0x79, 0x96, 0xa2, 0x55, 0x27, 0x28,
462
-
0x55, 0x73, 0x26, 0xa5, 0x50, 0xea, 0x92, 0x4b,
463
-
0x3c, 0x5c, 0x82, 0x33, 0xf0, 0x01, 0x3f, 0x03,
464
-
0xc1, 0x08, 0x05, 0xbf, 0x98, 0xf4, 0x9b, 0x6d,
465
-
0xa5, 0xa8, 0xb4, 0x82, 0x0c, 0x06, 0xfa, 0xff,
466
-
0x2d, 0x08, 0xf3, 0x05, 0x4f, 0x57, 0x2a, 0x39,
467
-
0xd4, 0x83, 0x0d, 0x75, 0x51, 0xd8, 0x5b, 0x1b,
468
-
0xd3, 0x51, 0x5a, 0x32, 0x2a, 0x9b, 0x32, 0xb2,
469
-
0xf2, 0xa4, 0x96, 0x12, 0xf2, 0xae, 0x40, 0x34,
470
-
0x67, 0xa8, 0xf5, 0x44, 0xd5, 0x35, 0x53, 0xfe,
471
-
0xa3, 0x60, 0x96, 0x63, 0x0f, 0x1f, 0x6e, 0xb0,
472
-
0x5a, 0x42, 0xa6, 0xfc, 0x51, 0x0b, 0x60, 0x27,
473
-
0xbc, 0x06, 0x71, 0xed, 0x65, 0x5b, 0x23, 0x86,
474
-
0x4a, 0x07, 0x3b, 0x22, 0x07, 0x46, 0xe6, 0x90,
475
-
0x3e, 0xf3, 0x25, 0x50, 0x1b, 0x4c, 0x7f, 0x03,
476
-
0x08, 0xa8, 0x36, 0x6b, 0x87, 0xe5, 0xe3, 0xdb,
477
-
0x9a, 0x38, 0x83, 0xff, 0x9f, 0x1a, 0x9f, 0x57,
478
-
0xa4, 0x2a, 0xf6, 0x37, 0xbc, 0x1a, 0xff, 0xc9,
479
-
0x1e, 0x35, 0x0c, 0xc3, 0x7c, 0xa3, 0xb2, 0xe5,
480
-
0xd2, 0xc6, 0xb4, 0x57, 0x47, 0xe4, 0x32, 0x16,
481
-
0x6d, 0xa9, 0xae, 0x64, 0xe6, 0x2d, 0x8d, 0xc5,
482
-
0x8d, 0x50, 0x8e, 0xe8, 0x1a, 0x22, 0x34, 0x2a,
483
-
0xd9, 0xeb, 0x51, 0x90, 0x4a, 0xb1, 0x41, 0x7d,
484
-
0x64, 0xf9, 0xb9, 0x0d, 0xf6, 0x23, 0x33, 0xb0,
485
-
0x33, 0xf4, 0xf7, 0x3f, 0x27, 0x84, 0xc6, 0x0f,
486
-
0x54, 0xa5, 0xc0, 0x2e, 0xec, 0x0b, 0x3a, 0x48,
487
-
0x6e, 0x80, 0x35, 0x81, 0x43, 0x9b, 0x90, 0xb1,
488
-
0xd0, 0x2b, 0xea, 0x21, 0xdc, 0xda, 0x5b, 0x09,
489
-
0xf4, 0xcc, 0x10, 0xb4, 0xc7, 0xfe, 0x79, 0x51,
490
-
0xc3, 0xc5, 0xac, 0x88, 0x74, 0x84, 0x0b, 0x4b,
491
-
0xca, 0x79, 0x16, 0x29, 0xfb, 0x69, 0x54, 0xdf,
492
-
0x41, 0x7e, 0xe9, 0xc7, 0x8e, 0xea, 0xa5, 0xfe,
493
-
0xfc, 0x76, 0x0e, 0x90, 0xc4, 0x92, 0x38, 0xad,
494
-
0x7b, 0x48, 0xe6, 0x6e, 0xf7, 0x21, 0xfd, 0x4e,
495
-
0x93, 0x0a, 0x7b, 0x41, 0x83, 0x68, 0xfb, 0x57,
496
-
0x51, 0x76, 0x34, 0xa9, 0x6c, 0x00, 0xaa, 0x4f,
497
-
0x66, 0x65, 0x98, 0x4a, 0x4f, 0xa3, 0xa0, 0xef,
498
-
0x69, 0x3f, 0xe3, 0x1c, 0x92, 0x8c, 0xfd, 0xd8,
499
-
0xe8, 0xde, 0x7c, 0x7f, 0x3e, 0x84, 0x8e, 0x69,
500
-
0x3c, 0xf1, 0xf2, 0x05, 0x46, 0xdc, 0x2f, 0x9d,
501
-
0x5e, 0x6e, 0x4c, 0xfb, 0xb5, 0x99, 0x2a, 0x59,
502
-
0x63, 0xc1, 0x34, 0xbc, 0x57, 0xc0, 0x0d, 0xb9,
503
-
0x61, 0x25, 0xf3, 0x33, 0x23, 0x51, 0xb6, 0x0d,
504
-
0x07, 0xa6, 0xab, 0x94, 0x4a, 0xb7, 0x2a, 0xea,
505
-
0xee, 0xac, 0xa3, 0xc3, 0x04, 0x8b, 0x0e, 0x56,
506
-
0xfe, 0x44, 0xa7, 0x39, 0xe2, 0xed, 0xed, 0xb4,
507
-
0x22, 0x2b, 0xac, 0x12, 0x32, 0x28, 0x91, 0xd8,
508
-
0xa5, 0xab, 0xff, 0x5f, 0xe0, 0x4b, 0xda, 0x78,
509
-
0x17, 0xda, 0xf1, 0x01, 0x5b, 0xcd, 0xe2, 0x5f,
510
-
0x50, 0x45, 0x73, 0x2b, 0xe4, 0x76, 0x77, 0xf4,
511
-
0x64, 0x1d, 0x43, 0xfb, 0x84, 0x7a, 0xea, 0x91,
512
-
0xae, 0xf9, 0x9e, 0xb7, 0xb4, 0xb0, 0x91, 0x5f,
513
-
0x16, 0x35, 0x9a, 0x11, 0xb8, 0xc7, 0xc1, 0x8c,
514
-
0xc6, 0x10, 0x8d, 0x2f, 0x63, 0x4a, 0xa7, 0x57,
515
-
0x3a, 0x51, 0xd6, 0x32, 0x2d, 0x64, 0x72, 0xd4,
516
-
0x66, 0xdc, 0x10, 0xa6, 0x67, 0xd6, 0x04, 0x23,
517
-
0x9d, 0x0a, 0x11, 0x77, 0xdd, 0x37, 0x94, 0x17,
518
-
0x3c, 0xbf, 0x8b, 0x65, 0xb0, 0x2e, 0x5e, 0x66,
519
-
0x47, 0x64, 0xac, 0xdd, 0xf0, 0x84, 0xfd, 0x39,
520
-
0xfa, 0x15, 0x5d, 0xef, 0xae, 0xca, 0xc1, 0x36,
521
-
0xa7, 0x5c, 0xbf, 0xc7, 0x08, 0xc2, 0x66, 0x00,
522
-
0x74, 0x74, 0x4e, 0x27, 0x3f, 0x55, 0x8a, 0xb7,
523
-
0x38, 0x66, 0x83, 0x6d, 0xcf, 0x99, 0x9e, 0x60,
524
-
0x8f, 0xdd, 0x2e, 0x62, 0x22, 0x0e, 0xef, 0x0c,
525
-
0x98, 0xa7, 0x85, 0x74, 0x3b, 0x9d, 0xec, 0x9e,
526
-
0xa9, 0x19, 0x72, 0xa5, 0x7f, 0x2c, 0x39, 0xb7,
527
-
0x7d, 0xb7, 0xf1, 0x12, 0x65, 0x27, 0x4b, 0x5a,
528
-
0xde, 0x17, 0xfe, 0xad, 0x44, 0xf3, 0x20, 0x4d,
529
-
0xfd, 0xe4, 0x1f, 0xb5, 0x81, 0xb0, 0x36, 0x37,
530
-
0x08, 0x6f, 0xc3, 0x0c, 0xe9, 0x85, 0x98, 0x82,
531
-
0xa9, 0x62, 0x0c, 0xc4, 0x97, 0xc0, 0x50, 0xc8,
532
-
0xa7, 0x3c, 0x50, 0x9f, 0x43, 0xb9, 0xcd, 0x5e,
533
-
0x4d, 0xfa, 0x1c, 0x4b, 0x0b, 0xa9, 0x98, 0x85,
534
-
0x38, 0x92, 0xac, 0x8d, 0xe4, 0xad, 0x9b, 0x98,
535
-
0xab, 0xd9, 0x38, 0xac, 0x62, 0x52, 0xa3, 0x22,
536
-
0x63, 0x0f, 0xbf, 0x95, 0x48, 0xdf, 0x69, 0xe7,
537
-
0x8b, 0x33, 0xd5, 0xb2, 0xbd, 0x05, 0x49, 0x49,
538
-
0x9d, 0x57, 0x73, 0x19, 0x33, 0xae, 0xfa, 0x33,
539
-
0xf1, 0x19, 0xa8, 0x80, 0xce, 0x04, 0x9f, 0xbc,
540
-
0x1d, 0x65, 0x82, 0x1b, 0xe5, 0x3a, 0x51, 0xc8,
541
-
0x1c, 0x21, 0xe3, 0x5d, 0xf3, 0x7d, 0x9b, 0x2f,
542
-
0x2c, 0x1d, 0x4a, 0x7f, 0x9b, 0x68, 0x35, 0xa3,
543
-
0xb2, 0x50, 0xf7, 0x62, 0x79, 0xcd, 0xf4, 0x98,
544
-
0x4f, 0xe5, 0x63, 0x7c, 0x3e, 0x45, 0x31, 0x8c,
545
-
0x16, 0xa0, 0x12, 0xc8, 0x58, 0xce, 0x39, 0xa6,
546
-
0xbc, 0x54, 0xdb, 0xc5, 0xe0, 0xd5, 0xba, 0xbc,
547
-
0xb9, 0x04, 0xf4, 0x8d, 0xe8, 0x2f, 0x15, 0x9d,
548
-
};
549
-
550
-
/* 100 test cases */
551
-
static struct crc_test {
552
-
u32 crc; /* random starting crc */
553
-
u32 start; /* random 6 bit offset in buf */
554
-
u32 length; /* random 11 bit length of test */
555
-
u32 crc_le; /* expected crc32_le result */
556
-
u32 crc_be; /* expected crc32_be result */
557
-
u32 crc32c_le; /* expected crc32c_le result */
558
-
} const test[] __initconst =
559
-
{
560
-
{0x674bf11d, 0x00000038, 0x00000542, 0x0af6d466, 0xd8b6e4c1, 0xf6e93d6c},
561
-
{0x35c672c6, 0x0000003a, 0x000001aa, 0xc6d3dfba, 0x28aaf3ad, 0x0fe92aca},
562
-
{0x496da28e, 0x00000039, 0x000005af, 0xd933660f, 0x5d57e81f, 0x52e1ebb8},
563
-
{0x09a9b90e, 0x00000027, 0x000001f8, 0xb45fe007, 0xf45fca9a, 0x0798af9a},
564
-
{0xdc97e5a9, 0x00000025, 0x000003b6, 0xf81a3562, 0xe0126ba2, 0x18eb3152},
565
-
{0x47c58900, 0x0000000a, 0x000000b9, 0x8e58eccf, 0xf3afc793, 0xd00d08c7},
566
-
{0x292561e8, 0x0000000c, 0x00000403, 0xa2ba8aaf, 0x0b797aed, 0x8ba966bc},
567
-
{0x415037f6, 0x00000003, 0x00000676, 0xa17d52e8, 0x7f0fdf35, 0x11d694a2},
568
-
{0x3466e707, 0x00000026, 0x00000042, 0x258319be, 0x75c484a2, 0x6ab3208d},
569
-
{0xafd1281b, 0x00000023, 0x000002ee, 0x4428eaf8, 0x06c7ad10, 0xba4603c5},
570
-
{0xd3857b18, 0x00000028, 0x000004a2, 0x5c430821, 0xb062b7cb, 0xe6071c6f},
571
-
{0x1d825a8f, 0x0000002b, 0x0000050b, 0xd2c45f0c, 0xd68634e0, 0x179ec30a},
572
-
{0x5033e3bc, 0x0000000b, 0x00000078, 0xa3ea4113, 0xac6d31fb, 0x0903beb8},
573
-
{0x94f1fb5e, 0x0000000f, 0x000003a2, 0xfbfc50b1, 0x3cfe50ed, 0x6a7cb4fa},
574
-
{0xc9a0fe14, 0x00000009, 0x00000473, 0x5fb61894, 0x87070591, 0xdb535801},
575
-
{0x88a034b1, 0x0000001c, 0x000005ad, 0xc1b16053, 0x46f95c67, 0x92bed597},
576
-
{0xf0f72239, 0x00000020, 0x0000026d, 0xa6fa58f3, 0xf8c2c1dd, 0x192a3f1b},
577
-
{0xcc20a5e3, 0x0000003b, 0x0000067a, 0x7740185a, 0x308b979a, 0xccbaec1a},
578
-
{0xce589c95, 0x0000002b, 0x00000641, 0xd055e987, 0x40aae25b, 0x7eabae4d},
579
-
{0x78edc885, 0x00000035, 0x000005be, 0xa39cb14b, 0x035b0d1f, 0x28c72982},
580
-
{0x9d40a377, 0x0000003b, 0x00000038, 0x1f47ccd2, 0x197fbc9d, 0xc3cd4d18},
581
-
{0x703d0e01, 0x0000003c, 0x000006f1, 0x88735e7c, 0xfed57c5a, 0xbca8f0e7},
582
-
{0x776bf505, 0x0000000f, 0x000005b2, 0x5cc4fc01, 0xf32efb97, 0x713f60b3},
583
-
{0x4a3e7854, 0x00000027, 0x000004b8, 0x8d923c82, 0x0cbfb4a2, 0xebd08fd5},
584
-
{0x209172dd, 0x0000003b, 0x00000356, 0xb89e9c2b, 0xd7868138, 0x64406c59},
585
-
{0x3ba4cc5b, 0x0000002f, 0x00000203, 0xe51601a9, 0x5b2a1032, 0x7421890e},
586
-
{0xfc62f297, 0x00000000, 0x00000079, 0x71a8e1a2, 0x5d88685f, 0xe9347603},
587
-
{0x64280b8b, 0x00000016, 0x000007ab, 0x0fa7a30c, 0xda3a455f, 0x1bef9060},
588
-
{0x97dd724b, 0x00000033, 0x000007ad, 0x5788b2f4, 0xd7326d32, 0x34720072},
589
-
{0x61394b52, 0x00000035, 0x00000571, 0xc66525f1, 0xcabe7fef, 0x48310f59},
590
-
{0x29b4faff, 0x00000024, 0x0000006e, 0xca13751e, 0x993648e0, 0x783a4213},
591
-
{0x29bfb1dc, 0x0000000b, 0x00000244, 0x436c43f7, 0x429f7a59, 0x9e8efd41},
592
-
{0x86ae934b, 0x00000035, 0x00000104, 0x0760ec93, 0x9cf7d0f4, 0xfc3d34a5},
593
-
{0xc4c1024e, 0x0000002e, 0x000006b1, 0x6516a3ec, 0x19321f9c, 0x17a52ae2},
594
-
{0x3287a80a, 0x00000026, 0x00000496, 0x0b257eb1, 0x754ebd51, 0x886d935a},
595
-
{0xa4db423e, 0x00000023, 0x0000045d, 0x9b3a66dc, 0x873e9f11, 0xeaaeaeb2},
596
-
{0x7a1078df, 0x00000015, 0x0000014a, 0x8c2484c5, 0x6a628659, 0x8e900a4b},
597
-
{0x6048bd5b, 0x00000006, 0x0000006a, 0x897e3559, 0xac9961af, 0xd74662b1},
598
-
{0xd8f9ea20, 0x0000003d, 0x00000277, 0x60eb905b, 0xed2aaf99, 0xd26752ba},
599
-
{0xea5ec3b4, 0x0000002a, 0x000004fe, 0x869965dc, 0x6c1f833b, 0x8b1fcd62},
600
-
{0x2dfb005d, 0x00000016, 0x00000345, 0x6a3b117e, 0xf05e8521, 0xf54342fe},
601
-
{0x5a214ade, 0x00000020, 0x000005b6, 0x467f70be, 0xcb22ccd3, 0x5b95b988},
602
-
{0xf0ab9cca, 0x00000032, 0x00000515, 0xed223df3, 0x7f3ef01d, 0x2e1176be},
603
-
{0x91b444f9, 0x0000002e, 0x000007f8, 0x84e9a983, 0x5676756f, 0x66120546},
604
-
{0x1b5d2ddb, 0x0000002e, 0x0000012c, 0xba638c4c, 0x3f42047b, 0xf256a5cc},
605
-
{0xd824d1bb, 0x0000003a, 0x000007b5, 0x6288653b, 0x3a3ebea0, 0x4af1dd69},
606
-
{0x0470180c, 0x00000034, 0x000001f0, 0x9d5b80d6, 0x3de08195, 0x56f0a04a},
607
-
{0xffaa3a3f, 0x00000036, 0x00000299, 0xf3a82ab8, 0x53e0c13d, 0x74f6b6b2},
608
-
{0x6406cfeb, 0x00000023, 0x00000600, 0xa920b8e8, 0xe4e2acf4, 0x085951fd},
609
-
{0xb24aaa38, 0x0000003e, 0x000004a1, 0x657cc328, 0x5077b2c3, 0xc65387eb},
610
-
{0x58b2ab7c, 0x00000039, 0x000002b4, 0x3a17ee7e, 0x9dcb3643, 0x1ca9257b},
611
-
{0x3db85970, 0x00000006, 0x000002b6, 0x95268b59, 0xb9812c10, 0xfd196d76},
612
-
{0x857830c5, 0x00000003, 0x00000590, 0x4ef439d5, 0xf042161d, 0x5ef88339},
613
-
{0xe1fcd978, 0x0000003e, 0x000007d8, 0xae8d8699, 0xce0a1ef5, 0x2c3714d9},
614
-
{0xb982a768, 0x00000016, 0x000006e0, 0x62fad3df, 0x5f8a067b, 0x58576548},
615
-
{0x1d581ce8, 0x0000001e, 0x0000058b, 0xf0f5da53, 0x26e39eee, 0xfd7c57de},
616
-
{0x2456719b, 0x00000025, 0x00000503, 0x4296ac64, 0xd50e4c14, 0xd5fedd59},
617
-
{0xfae6d8f2, 0x00000000, 0x0000055d, 0x057fdf2e, 0x2a31391a, 0x1cc3b17b},
618
-
{0xcba828e3, 0x00000039, 0x000002ce, 0xe3f22351, 0x8f00877b, 0x270eed73},
619
-
{0x13d25952, 0x0000000a, 0x0000072d, 0x76d4b4cc, 0x5eb67ec3, 0x91ecbb11},
620
-
{0x0342be3f, 0x00000015, 0x00000599, 0xec75d9f1, 0x9d4d2826, 0x05ed8d0c},
621
-
{0xeaa344e0, 0x00000014, 0x000004d8, 0x72a4c981, 0x2064ea06, 0x0b09ad5b},
622
-
{0xbbb52021, 0x0000003b, 0x00000272, 0x04af99fc, 0xaf042d35, 0xf8d511fb},
623
-
{0xb66384dc, 0x0000001d, 0x000007fc, 0xd7629116, 0x782bd801, 0x5ad832cc},
624
-
{0x616c01b6, 0x00000022, 0x000002c8, 0x5b1dab30, 0x783ce7d2, 0x1214d196},
625
-
{0xce2bdaad, 0x00000016, 0x0000062a, 0x932535c8, 0x3f02926d, 0x5747218a},
626
-
{0x00fe84d7, 0x00000005, 0x00000205, 0x850e50aa, 0x753d649c, 0xde8f14de},
627
-
{0xbebdcb4c, 0x00000006, 0x0000055d, 0xbeaa37a2, 0x2d8c9eba, 0x3563b7b9},
628
-
{0xd8b1a02a, 0x00000010, 0x00000387, 0x5017d2fc, 0x503541a5, 0x071475d0},
629
-
{0x3b96cad2, 0x00000036, 0x00000347, 0x1d2372ae, 0x926cd90b, 0x54c79d60},
630
-
{0xc94c1ed7, 0x00000005, 0x0000038b, 0x9e9fdb22, 0x144a9178, 0x4c53eee6},
631
-
{0x1aad454e, 0x00000025, 0x000002b2, 0xc3f6315c, 0x5c7a35b3, 0x10137a3c},
632
-
{0xa4fec9a6, 0x00000000, 0x000006d6, 0x90be5080, 0xa4107605, 0xaa9d6c73},
633
-
{0x1bbe71e2, 0x0000001f, 0x000002fd, 0x4e504c3b, 0x284ccaf1, 0xb63d23e7},
634
-
{0x4201c7e4, 0x00000002, 0x000002b7, 0x7822e3f9, 0x0cc912a9, 0x7f53e9cf},
635
-
{0x23fddc96, 0x00000003, 0x00000627, 0x8a385125, 0x07767e78, 0x13c1cd83},
636
-
{0xd82ba25c, 0x00000016, 0x0000063e, 0x98e4148a, 0x283330c9, 0x49ff5867},
637
-
{0x786f2032, 0x0000002d, 0x0000060f, 0xf201600a, 0xf561bfcd, 0x8467f211},
638
-
{0xfebe4e1f, 0x0000002a, 0x000004f2, 0x95e51961, 0xfd80dcab, 0x3f9683b2},
639
-
{0x1a6e0a39, 0x00000008, 0x00000672, 0x8af6c2a5, 0x78dd84cb, 0x76a3f874},
640
-
{0x56000ab8, 0x0000000e, 0x000000e5, 0x36bacb8f, 0x22ee1f77, 0x863b702f},
641
-
{0x4717fe0c, 0x00000000, 0x000006ec, 0x8439f342, 0x5c8e03da, 0xdc6c58ff},
642
-
{0xd5d5d68e, 0x0000003c, 0x000003a3, 0x46fff083, 0x177d1b39, 0x0622cc95},
643
-
{0xc25dd6c6, 0x00000024, 0x000006c0, 0x5ceb8eb4, 0x892b0d16, 0xe85605cd},
644
-
{0xe9b11300, 0x00000023, 0x00000683, 0x07a5d59a, 0x6c6a3208, 0x31da5f06},
645
-
{0x95cd285e, 0x00000001, 0x00000047, 0x7b3a4368, 0x0202c07e, 0xa1f2e784},
646
-
{0xd9245a25, 0x0000001e, 0x000003a6, 0xd33c1841, 0x1936c0d5, 0xb07cc616},
647
-
{0x103279db, 0x00000006, 0x0000039b, 0xca09b8a0, 0x77d62892, 0xbf943b6c},
648
-
{0x1cba3172, 0x00000027, 0x000001c8, 0xcb377194, 0xebe682db, 0x2c01af1c},
649
-
{0x8f613739, 0x0000000c, 0x000001df, 0xb4b0bc87, 0x7710bd43, 0x0fe5f56d},
650
-
{0x1c6aa90d, 0x0000001b, 0x0000053c, 0x70559245, 0xda7894ac, 0xf8943b2d},
651
-
{0xaabe5b93, 0x0000003d, 0x00000715, 0xcdbf42fa, 0x0c3b99e7, 0xe4d89272},
652
-
{0xf15dd038, 0x00000006, 0x000006db, 0x6e104aea, 0x8d5967f2, 0x7c2f6bbb},
653
-
{0x584dd49c, 0x00000020, 0x000007bc, 0x36b6cfd6, 0xad4e23b2, 0xabbf388b},
654
-
{0x5d8c9506, 0x00000020, 0x00000470, 0x4c62378e, 0x31d92640, 0x1dca1f4e},
655
-
{0xb80d17b0, 0x00000032, 0x00000346, 0x22a5bb88, 0x9a7ec89f, 0x5c170e23},
656
-
{0xdaf0592e, 0x00000023, 0x000007b0, 0x3cab3f99, 0x9b1fdd99, 0xc0e9d672},
657
-
{0x4793cc85, 0x0000000d, 0x00000706, 0xe82e04f6, 0xed3db6b7, 0xc18bdc86},
658
-
{0x82ebf64e, 0x00000009, 0x000007c3, 0x69d590a9, 0x9efa8499, 0xa874fcdd},
659
-
{0xb18a0319, 0x00000026, 0x000007db, 0x1cf98dcc, 0x8fa9ad6a, 0x9dc0bb48},
660
-
};
661
-
662
-
#include <linux/time.h>
663
-
664
-
static int __init crc32c_test(void)
665
-
{
666
-
int i;
667
-
int errors = 0;
668
-
int bytes = 0;
669
-
u64 nsec;
670
-
unsigned long flags;
671
-
672
-
/* keep static to prevent cache warming code from
673
-
* getting eliminated by the compiler */
674
-
static u32 crc;
675
-
676
-
/* pre-warm the cache */
677
-
for (i = 0; i < 100; i++) {
678
-
bytes += test[i].length;
679
-
680
-
crc ^= __crc32c_le(test[i].crc, test_buf +
681
-
test[i].start, test[i].length);
682
-
}
683
-
684
-
/* reduce OS noise */
685
-
local_irq_save(flags);
686
-
687
-
nsec = ktime_get_ns();
688
-
for (i = 0; i < 100; i++) {
689
-
if (test[i].crc32c_le != __crc32c_le(test[i].crc, test_buf +
690
-
test[i].start, test[i].length))
691
-
errors++;
692
-
}
693
-
nsec = ktime_get_ns() - nsec;
694
-
695
-
local_irq_restore(flags);
696
-
697
-
pr_info("crc32c: CRC_LE_BITS = %d\n", CRC_LE_BITS);
698
-
699
-
if (errors)
700
-
pr_warn("crc32c: %d self tests failed\n", errors);
701
-
else {
702
-
pr_info("crc32c: self tests passed, processed %d bytes in %lld nsec\n",
703
-
bytes, nsec);
704
-
}
705
-
706
-
return 0;
707
-
}
708
-
709
-
static int __init crc32c_combine_test(void)
710
-
{
711
-
int i, j;
712
-
int errors = 0, runs = 0;
713
-
714
-
for (i = 0; i < 10; i++) {
715
-
u32 crc_full;
716
-
717
-
crc_full = __crc32c_le(test[i].crc, test_buf + test[i].start,
718
-
test[i].length);
719
-
for (j = 0; j <= test[i].length; ++j) {
720
-
u32 crc1, crc2;
721
-
u32 len1 = j, len2 = test[i].length - j;
722
-
723
-
crc1 = __crc32c_le(test[i].crc, test_buf +
724
-
test[i].start, len1);
725
-
crc2 = __crc32c_le(0, test_buf + test[i].start +
726
-
len1, len2);
727
-
728
-
if (!(crc_full == __crc32c_le_combine(crc1, crc2, len2) &&
729
-
crc_full == test[i].crc32c_le))
730
-
errors++;
731
-
runs++;
732
-
cond_resched();
733
-
}
734
-
}
735
-
736
-
if (errors)
737
-
pr_warn("crc32c_combine: %d/%d self tests failed\n", errors, runs);
738
-
else
739
-
pr_info("crc32c_combine: %d self tests passed\n", runs);
740
-
741
-
return 0;
742
-
}
743
-
744
-
static int __init crc32_test(void)
745
-
{
746
-
int i;
747
-
int errors = 0;
748
-
int bytes = 0;
749
-
u64 nsec;
750
-
unsigned long flags;
751
-
752
-
/* keep static to prevent cache warming code from
753
-
* getting eliminated by the compiler */
754
-
static u32 crc;
755
-
756
-
/* pre-warm the cache */
757
-
for (i = 0; i < 100; i++) {
758
-
bytes += 2*test[i].length;
759
-
760
-
crc ^= crc32_le(test[i].crc, test_buf +
761
-
test[i].start, test[i].length);
762
-
763
-
crc ^= crc32_be(test[i].crc, test_buf +
764
-
test[i].start, test[i].length);
765
-
}
766
-
767
-
/* reduce OS noise */
768
-
local_irq_save(flags);
769
-
770
-
nsec = ktime_get_ns();
771
-
for (i = 0; i < 100; i++) {
772
-
if (test[i].crc_le != crc32_le(test[i].crc, test_buf +
773
-
test[i].start, test[i].length))
774
-
errors++;
775
-
776
-
if (test[i].crc_be != crc32_be(test[i].crc, test_buf +
777
-
test[i].start, test[i].length))
778
-
errors++;
779
-
}
780
-
nsec = ktime_get_ns() - nsec;
781
-
782
-
local_irq_restore(flags);
783
-
784
-
pr_info("crc32: CRC_LE_BITS = %d, CRC_BE BITS = %d\n",
785
-
CRC_LE_BITS, CRC_BE_BITS);
786
-
787
-
if (errors)
788
-
pr_warn("crc32: %d self tests failed\n", errors);
789
-
else {
790
-
pr_info("crc32: self tests passed, processed %d bytes in %lld nsec\n",
791
-
bytes, nsec);
792
-
}
793
-
794
-
return 0;
795
-
}
796
-
797
-
static int __init crc32_combine_test(void)
798
-
{
799
-
int i, j;
800
-
int errors = 0, runs = 0;
801
-
802
-
for (i = 0; i < 10; i++) {
803
-
u32 crc_full;
804
-
805
-
crc_full = crc32_le(test[i].crc, test_buf + test[i].start,
806
-
test[i].length);
807
-
for (j = 0; j <= test[i].length; ++j) {
808
-
u32 crc1, crc2;
809
-
u32 len1 = j, len2 = test[i].length - j;
810
-
811
-
crc1 = crc32_le(test[i].crc, test_buf +
812
-
test[i].start, len1);
813
-
crc2 = crc32_le(0, test_buf + test[i].start +
814
-
len1, len2);
815
-
816
-
if (!(crc_full == crc32_le_combine(crc1, crc2, len2) &&
817
-
crc_full == test[i].crc_le))
818
-
errors++;
819
-
runs++;
820
-
cond_resched();
821
-
}
822
-
}
823
-
824
-
if (errors)
825
-
pr_warn("crc32_combine: %d/%d self tests failed\n", errors, runs);
826
-
else
827
-
pr_info("crc32_combine: %d self tests passed\n", runs);
828
-
829
-
return 0;
830
-
}
831
-
832
-
static int __init crc32test_init(void)
833
-
{
834
-
crc32_test();
835
-
crc32c_test();
836
-
837
-
crc32_combine_test();
838
-
crc32c_combine_test();
839
-
840
-
return 0;
841
-
}
842
-
843
-
static void __exit crc32_exit(void)
844
-
{
845
-
}
846
-
847
-
module_init(crc32test_init);
848
-
module_exit(crc32_exit);
849
-
850
-
MODULE_AUTHOR("Matt Domsch <Matt_Domsch@dell.com>");
851
-
MODULE_DESCRIPTION("CRC32 selftest");
852
-
MODULE_LICENSE("GPL");
+435
lib/crc_kunit.c
+435
lib/crc_kunit.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-or-later
2
+
/*
3
+
* Unit tests and benchmarks for the CRC library functions
4
+
*
5
+
* Copyright 2024 Google LLC
6
+
*
7
+
* Author: Eric Biggers <ebiggers@google.com>
8
+
*/
9
+
#include <kunit/test.h>
10
+
#include <linux/crc16.h>
11
+
#include <linux/crc-t10dif.h>
12
+
#include <linux/crc32.h>
13
+
#include <linux/crc32c.h>
14
+
#include <linux/crc64.h>
15
+
#include <linux/prandom.h>
16
+
#include <linux/vmalloc.h>
17
+
18
+
#define CRC_KUNIT_SEED 42
19
+
#define CRC_KUNIT_MAX_LEN 16384
20
+
#define CRC_KUNIT_NUM_TEST_ITERS 1000
21
+
22
+
static struct rnd_state rng;
23
+
static u8 *test_buffer;
24
+
static size_t test_buflen;
25
+
26
+
/**
27
+
* struct crc_variant - describes a CRC variant
28
+
* @bits: Number of bits in the CRC, 1 <= @bits <= 64.
29
+
* @le: true if it's a "little endian" CRC (reversed mapping between bits and
30
+
* polynomial coefficients in each byte), false if it's a "big endian" CRC
31
+
* (natural mapping between bits and polynomial coefficients in each byte)
32
+
* @poly: The generator polynomial with the highest-order term omitted.
33
+
* Bit-reversed if @le is true.
34
+
* @func: The function to compute a CRC. The type signature uses u64 so that it
35
+
* can fit any CRC up to CRC-64.
36
+
* @combine_func: Optional function to combine two CRCs.
37
+
*/
38
+
struct crc_variant {
39
+
int bits;
40
+
bool le;
41
+
u64 poly;
42
+
u64 (*func)(u64 crc, const u8 *p, size_t len);
43
+
u64 (*combine_func)(u64 crc1, u64 crc2, size_t len2);
44
+
};
45
+
46
+
static u32 rand32(void)
47
+
{
48
+
return prandom_u32_state(&rng);
49
+
}
50
+
51
+
static u64 rand64(void)
52
+
{
53
+
u32 n = rand32();
54
+
55
+
return ((u64)n << 32) | rand32();
56
+
}
57
+
58
+
static u64 crc_mask(const struct crc_variant *v)
59
+
{
60
+
return (u64)-1 >> (64 - v->bits);
61
+
}
62
+
63
+
/* Reference implementation of any CRC variant */
64
+
static u64 crc_ref(const struct crc_variant *v,
65
+
u64 crc, const u8 *p, size_t len)
66
+
{
67
+
size_t i, j;
68
+
69
+
for (i = 0; i < len; i++) {
70
+
for (j = 0; j < 8; j++) {
71
+
if (v->le) {
72
+
crc ^= (p[i] >> j) & 1;
73
+
crc = (crc >> 1) ^ ((crc & 1) ? v->poly : 0);
74
+
} else {
75
+
crc ^= (u64)((p[i] >> (7 - j)) & 1) <<
76
+
(v->bits - 1);
77
+
if (crc & (1ULL << (v->bits - 1)))
78
+
crc = ((crc << 1) ^ v->poly) &
79
+
crc_mask(v);
80
+
else
81
+
crc <<= 1;
82
+
}
83
+
}
84
+
}
85
+
return crc;
86
+
}
87
+
88
+
static int crc_suite_init(struct kunit_suite *suite)
89
+
{
90
+
/*
91
+
* Allocate the test buffer using vmalloc() with a page-aligned length
92
+
* so that it is immediately followed by a guard page. This allows
93
+
* buffer overreads to be detected, even in assembly code.
94
+
*/
95
+
test_buflen = round_up(CRC_KUNIT_MAX_LEN, PAGE_SIZE);
96
+
test_buffer = vmalloc(test_buflen);
97
+
if (!test_buffer)
98
+
return -ENOMEM;
99
+
100
+
prandom_seed_state(&rng, CRC_KUNIT_SEED);
101
+
prandom_bytes_state(&rng, test_buffer, test_buflen);
102
+
return 0;
103
+
}
104
+
105
+
static void crc_suite_exit(struct kunit_suite *suite)
106
+
{
107
+
vfree(test_buffer);
108
+
test_buffer = NULL;
109
+
}
110
+
111
+
/* Generate a random initial CRC. */
112
+
static u64 generate_random_initial_crc(const struct crc_variant *v)
113
+
{
114
+
switch (rand32() % 4) {
115
+
case 0:
116
+
return 0;
117
+
case 1:
118
+
return crc_mask(v); /* All 1 bits */
119
+
default:
120
+
return rand64() & crc_mask(v);
121
+
}
122
+
}
123
+
124
+
/* Generate a random length, preferring small lengths. */
125
+
static size_t generate_random_length(size_t max_length)
126
+
{
127
+
size_t len;
128
+
129
+
switch (rand32() % 3) {
130
+
case 0:
131
+
len = rand32() % 128;
132
+
break;
133
+
case 1:
134
+
len = rand32() % 3072;
135
+
break;
136
+
default:
137
+
len = rand32();
138
+
break;
139
+
}
140
+
return len % (max_length + 1);
141
+
}
142
+
143
+
/* Test that v->func gives the same CRCs as a reference implementation. */
144
+
static void crc_main_test(struct kunit *test, const struct crc_variant *v)
145
+
{
146
+
size_t i;
147
+
148
+
for (i = 0; i < CRC_KUNIT_NUM_TEST_ITERS; i++) {
149
+
u64 init_crc, expected_crc, actual_crc;
150
+
size_t len, offset;
151
+
bool nosimd;
152
+
153
+
init_crc = generate_random_initial_crc(v);
154
+
len = generate_random_length(CRC_KUNIT_MAX_LEN);
155
+
156
+
/* Generate a random offset. */
157
+
if (rand32() % 2 == 0) {
158
+
/* Use a random alignment mod 64 */
159
+
offset = rand32() % 64;
160
+
offset = min(offset, CRC_KUNIT_MAX_LEN - len);
161
+
} else {
162
+
/* Go up to the guard page, to catch buffer overreads */
163
+
offset = test_buflen - len;
164
+
}
165
+
166
+
if (rand32() % 8 == 0)
167
+
/* Refresh the data occasionally. */
168
+
prandom_bytes_state(&rng, &test_buffer[offset], len);
169
+
170
+
nosimd = rand32() % 8 == 0;
171
+
172
+
/*
173
+
* Compute the CRC, and verify that it equals the CRC computed
174
+
* by a simple bit-at-a-time reference implementation.
175
+
*/
176
+
expected_crc = crc_ref(v, init_crc, &test_buffer[offset], len);
177
+
if (nosimd)
178
+
local_irq_disable();
179
+
actual_crc = v->func(init_crc, &test_buffer[offset], len);
180
+
if (nosimd)
181
+
local_irq_enable();
182
+
KUNIT_EXPECT_EQ_MSG(test, expected_crc, actual_crc,
183
+
"Wrong result with len=%zu offset=%zu nosimd=%d",
184
+
len, offset, nosimd);
185
+
}
186
+
}
187
+
188
+
/* Test that CRC(concat(A, B)) == combine_CRCs(CRC(A), CRC(B), len(B)). */
189
+
static void crc_combine_test(struct kunit *test, const struct crc_variant *v)
190
+
{
191
+
int i;
192
+
193
+
for (i = 0; i < 100; i++) {
194
+
u64 init_crc = generate_random_initial_crc(v);
195
+
size_t len1 = generate_random_length(CRC_KUNIT_MAX_LEN);
196
+
size_t len2 = generate_random_length(CRC_KUNIT_MAX_LEN - len1);
197
+
u64 crc1, crc2, expected_crc, actual_crc;
198
+
199
+
prandom_bytes_state(&rng, test_buffer, len1 + len2);
200
+
crc1 = v->func(init_crc, test_buffer, len1);
201
+
crc2 = v->func(0, &test_buffer[len1], len2);
202
+
expected_crc = v->func(init_crc, test_buffer, len1 + len2);
203
+
actual_crc = v->combine_func(crc1, crc2, len2);
204
+
KUNIT_EXPECT_EQ_MSG(test, expected_crc, actual_crc,
205
+
"CRC combination gave wrong result with len1=%zu len2=%zu\n",
206
+
len1, len2);
207
+
}
208
+
}
209
+
210
+
static void crc_test(struct kunit *test, const struct crc_variant *v)
211
+
{
212
+
crc_main_test(test, v);
213
+
if (v->combine_func)
214
+
crc_combine_test(test, v);
215
+
}
216
+
217
+
static __always_inline void
218
+
crc_benchmark(struct kunit *test,
219
+
u64 (*crc_func)(u64 crc, const u8 *p, size_t len))
220
+
{
221
+
static const size_t lens_to_test[] = {
222
+
1, 16, 64, 127, 128, 200, 256, 511, 512, 1024, 3173, 4096, 16384,
223
+
};
224
+
size_t len, i, j, num_iters;
225
+
/*
226
+
* Some of the CRC library functions are marked as __pure, so use
227
+
* volatile to ensure that all calls are really made as intended.
228
+
*/
229
+
volatile u64 crc = 0;
230
+
u64 t;
231
+
232
+
if (!IS_ENABLED(CONFIG_CRC_BENCHMARK))
233
+
kunit_skip(test, "not enabled");
234
+
235
+
/* warm-up */
236
+
for (i = 0; i < 10000000; i += CRC_KUNIT_MAX_LEN)
237
+
crc = crc_func(crc, test_buffer, CRC_KUNIT_MAX_LEN);
238
+
239
+
for (i = 0; i < ARRAY_SIZE(lens_to_test); i++) {
240
+
len = lens_to_test[i];
241
+
KUNIT_ASSERT_LE(test, len, CRC_KUNIT_MAX_LEN);
242
+
num_iters = 10000000 / (len + 128);
243
+
preempt_disable();
244
+
t = ktime_get_ns();
245
+
for (j = 0; j < num_iters; j++)
246
+
crc = crc_func(crc, test_buffer, len);
247
+
t = ktime_get_ns() - t;
248
+
preempt_enable();
249
+
kunit_info(test, "len=%zu: %llu MB/s\n",
250
+
len, div64_u64((u64)len * num_iters * 1000, t));
251
+
}
252
+
}
253
+
254
+
/* crc16 */
255
+
256
+
static u64 crc16_wrapper(u64 crc, const u8 *p, size_t len)
257
+
{
258
+
return crc16(crc, p, len);
259
+
}
260
+
261
+
static const struct crc_variant crc_variant_crc16 = {
262
+
.bits = 16,
263
+
.le = true,
264
+
.poly = 0xa001,
265
+
.func = crc16_wrapper,
266
+
};
267
+
268
+
static void crc16_test(struct kunit *test)
269
+
{
270
+
crc_test(test, &crc_variant_crc16);
271
+
}
272
+
273
+
static void crc16_benchmark(struct kunit *test)
274
+
{
275
+
crc_benchmark(test, crc16_wrapper);
276
+
}
277
+
278
+
/* crc_t10dif */
279
+
280
+
static u64 crc_t10dif_wrapper(u64 crc, const u8 *p, size_t len)
281
+
{
282
+
return crc_t10dif_update(crc, p, len);
283
+
}
284
+
285
+
static const struct crc_variant crc_variant_crc_t10dif = {
286
+
.bits = 16,
287
+
.le = false,
288
+
.poly = 0x8bb7,
289
+
.func = crc_t10dif_wrapper,
290
+
};
291
+
292
+
static void crc_t10dif_test(struct kunit *test)
293
+
{
294
+
crc_test(test, &crc_variant_crc_t10dif);
295
+
}
296
+
297
+
static void crc_t10dif_benchmark(struct kunit *test)
298
+
{
299
+
crc_benchmark(test, crc_t10dif_wrapper);
300
+
}
301
+
302
+
/* crc32_le */
303
+
304
+
static u64 crc32_le_wrapper(u64 crc, const u8 *p, size_t len)
305
+
{
306
+
return crc32_le(crc, p, len);
307
+
}
308
+
309
+
static u64 crc32_le_combine_wrapper(u64 crc1, u64 crc2, size_t len2)
310
+
{
311
+
return crc32_le_combine(crc1, crc2, len2);
312
+
}
313
+
314
+
static const struct crc_variant crc_variant_crc32_le = {
315
+
.bits = 32,
316
+
.le = true,
317
+
.poly = 0xedb88320,
318
+
.func = crc32_le_wrapper,
319
+
.combine_func = crc32_le_combine_wrapper,
320
+
};
321
+
322
+
static void crc32_le_test(struct kunit *test)
323
+
{
324
+
crc_test(test, &crc_variant_crc32_le);
325
+
}
326
+
327
+
static void crc32_le_benchmark(struct kunit *test)
328
+
{
329
+
crc_benchmark(test, crc32_le_wrapper);
330
+
}
331
+
332
+
/* crc32_be */
333
+
334
+
static u64 crc32_be_wrapper(u64 crc, const u8 *p, size_t len)
335
+
{
336
+
return crc32_be(crc, p, len);
337
+
}
338
+
339
+
static const struct crc_variant crc_variant_crc32_be = {
340
+
.bits = 32,
341
+
.le = false,
342
+
.poly = 0x04c11db7,
343
+
.func = crc32_be_wrapper,
344
+
};
345
+
346
+
static void crc32_be_test(struct kunit *test)
347
+
{
348
+
crc_test(test, &crc_variant_crc32_be);
349
+
}
350
+
351
+
static void crc32_be_benchmark(struct kunit *test)
352
+
{
353
+
crc_benchmark(test, crc32_be_wrapper);
354
+
}
355
+
356
+
/* crc32c */
357
+
358
+
static u64 crc32c_wrapper(u64 crc, const u8 *p, size_t len)
359
+
{
360
+
return crc32c(crc, p, len);
361
+
}
362
+
363
+
static u64 crc32c_combine_wrapper(u64 crc1, u64 crc2, size_t len2)
364
+
{
365
+
return __crc32c_le_combine(crc1, crc2, len2);
366
+
}
367
+
368
+
static const struct crc_variant crc_variant_crc32c = {
369
+
.bits = 32,
370
+
.le = true,
371
+
.poly = 0x82f63b78,
372
+
.func = crc32c_wrapper,
373
+
.combine_func = crc32c_combine_wrapper,
374
+
};
375
+
376
+
static void crc32c_test(struct kunit *test)
377
+
{
378
+
crc_test(test, &crc_variant_crc32c);
379
+
}
380
+
381
+
static void crc32c_benchmark(struct kunit *test)
382
+
{
383
+
crc_benchmark(test, crc32c_wrapper);
384
+
}
385
+
386
+
/* crc64_be */
387
+
388
+
static u64 crc64_be_wrapper(u64 crc, const u8 *p, size_t len)
389
+
{
390
+
return crc64_be(crc, p, len);
391
+
}
392
+
393
+
static const struct crc_variant crc_variant_crc64_be = {
394
+
.bits = 64,
395
+
.le = false,
396
+
.poly = 0x42f0e1eba9ea3693,
397
+
.func = crc64_be_wrapper,
398
+
};
399
+
400
+
static void crc64_be_test(struct kunit *test)
401
+
{
402
+
crc_test(test, &crc_variant_crc64_be);
403
+
}
404
+
405
+
static void crc64_be_benchmark(struct kunit *test)
406
+
{
407
+
crc_benchmark(test, crc64_be_wrapper);
408
+
}
409
+
410
+
static struct kunit_case crc_test_cases[] = {
411
+
KUNIT_CASE(crc16_test),
412
+
KUNIT_CASE(crc16_benchmark),
413
+
KUNIT_CASE(crc_t10dif_test),
414
+
KUNIT_CASE(crc_t10dif_benchmark),
415
+
KUNIT_CASE(crc32_le_test),
416
+
KUNIT_CASE(crc32_le_benchmark),
417
+
KUNIT_CASE(crc32_be_test),
418
+
KUNIT_CASE(crc32_be_benchmark),
419
+
KUNIT_CASE(crc32c_test),
420
+
KUNIT_CASE(crc32c_benchmark),
421
+
KUNIT_CASE(crc64_be_test),
422
+
KUNIT_CASE(crc64_be_benchmark),
423
+
{},
424
+
};
425
+
426
+
static struct kunit_suite crc_test_suite = {
427
+
.name = "crc",
428
+
.test_cases = crc_test_cases,
429
+
.suite_init = crc_suite_init,
430
+
.suite_exit = crc_suite_exit,
431
+
};
432
+
kunit_test_suite(crc_test_suite);
433
+
434
+
MODULE_DESCRIPTION("Unit tests and benchmarks for the CRC library functions");
435
+
MODULE_LICENSE("GPL");
-74
lib/libcrc32c.c
-74
lib/libcrc32c.c
···
1
-
// SPDX-License-Identifier: GPL-2.0-or-later
2
-
/*
3
-
* CRC32C
4
-
*@Article{castagnoli-crc,
5
-
* author = { Guy Castagnoli and Stefan Braeuer and Martin Herrman},
6
-
* title = {{Optimization of Cyclic Redundancy-Check Codes with 24
7
-
* and 32 Parity Bits}},
8
-
* journal = IEEE Transactions on Communication,
9
-
* year = {1993},
10
-
* volume = {41},
11
-
* number = {6},
12
-
* pages = {},
13
-
* month = {June},
14
-
*}
15
-
* Used by the iSCSI driver, possibly others, and derived from
16
-
* the iscsi-crc.c module of the linux-iscsi driver at
17
-
* http://linux-iscsi.sourceforge.net.
18
-
*
19
-
* Following the example of lib/crc32, this function is intended to be
20
-
* flexible and useful for all users. Modules that currently have their
21
-
* own crc32c, but hopefully may be able to use this one are:
22
-
* net/sctp (please add all your doco to here if you change to
23
-
* use this one!)
24
-
* <endoflist>
25
-
*
26
-
* Copyright (c) 2004 Cisco Systems, Inc.
27
-
*/
28
-
29
-
#include <crypto/hash.h>
30
-
#include <linux/err.h>
31
-
#include <linux/init.h>
32
-
#include <linux/kernel.h>
33
-
#include <linux/module.h>
34
-
#include <linux/crc32c.h>
35
-
36
-
static struct crypto_shash *tfm;
37
-
38
-
u32 crc32c(u32 crc, const void *address, unsigned int length)
39
-
{
40
-
SHASH_DESC_ON_STACK(shash, tfm);
41
-
u32 ret, *ctx = (u32 *)shash_desc_ctx(shash);
42
-
int err;
43
-
44
-
shash->tfm = tfm;
45
-
*ctx = crc;
46
-
47
-
err = crypto_shash_update(shash, address, length);
48
-
BUG_ON(err);
49
-
50
-
ret = *ctx;
51
-
barrier_data(ctx);
52
-
return ret;
53
-
}
54
-
55
-
EXPORT_SYMBOL(crc32c);
56
-
57
-
static int __init libcrc32c_mod_init(void)
58
-
{
59
-
tfm = crypto_alloc_shash("crc32c", 0, 0);
60
-
return PTR_ERR_OR_ZERO(tfm);
61
-
}
62
-
63
-
static void __exit libcrc32c_mod_fini(void)
64
-
{
65
-
crypto_free_shash(tfm);
66
-
}
67
-
68
-
module_init(libcrc32c_mod_init);
69
-
module_exit(libcrc32c_mod_fini);
70
-
71
-
MODULE_AUTHOR("Clay Haapala <chaapala@cisco.com>");
72
-
MODULE_DESCRIPTION("CRC32c (Castagnoli) calculations");
73
-
MODULE_LICENSE("GPL");
74
-
MODULE_SOFTDEP("pre: crc32c");