Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'crc-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux
Pull CRC updates from Eric Biggers:
- Several improvements related to crc_kunit, to align with the standard
KUnit conventions and make it easier for developers and CI systems to
run this test suite
- Add an arm64-optimized implementation of CRC64-NVME
- Remove unused code for big endian arm64
* tag 'crc-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux:
lib/crc: arm64: Simplify intrinsics implementation
lib/crc: arm64: Use existing macros for kernel-mode FPU cflags
lib/crc: arm64: Drop unnecessary chunking logic from crc64
lib/crc: arm64: Assume a little-endian kernel
lib/crc: arm64: add NEON accelerated CRC64-NVMe implementation
lib/crc: arm64: Drop check for CONFIG_KERNEL_MODE_NEON
crypto: crc32c - Remove another outdated comment
crypto: crc32c - Remove more outdated usage information
kunit: configs: Enable all CRC tests in all_tests.config
lib/crc: tests: Add a .kunitconfig file
lib/crc: tests: Add CRC_ENABLE_ALL_FOR_KUNIT
lib/crc: tests: Make crc_kunit test only the enabled CRC variants
+172
-65
+1
-18
crypto/crc32c.c
+1
-18
crypto/crc32c.c
···
1
1
// SPDX-License-Identifier: GPL-2.0-or-later
2
2
/*
3
-
* Cryptographic API.
4
-
*
5
-
* CRC32C chksum
3
+
* crypto_shash support for CRC-32C
6
4
*
7
5
*@Article{castagnoli-crc,
8
6
* author = { Guy Castagnoli and Stefan Braeuer and Martin Herrman},
···
13
15
* pages = {},
14
16
* month = {June},
15
17
*}
16
-
* Used by the iSCSI driver, possibly others, and derived from
17
-
* the iscsi-crc.c module of the linux-iscsi driver at
18
-
* http://linux-iscsi.sourceforge.net.
19
-
*
20
-
* Following the example of lib/crc32, this function is intended to be
21
-
* flexible and useful for all users. Modules that currently have their
22
-
* own crc32c, but hopefully may be able to use this one are:
23
-
* net/sctp (please add all your doco to here if you change to
24
-
* use this one!)
25
-
* <endoflist>
26
18
*
27
19
* Copyright (c) 2004 Cisco Systems, Inc.
28
20
* Copyright (c) 2008 Herbert Xu <herbert@gondor.apana.org.au>
···
36
48
struct chksum_desc_ctx {
37
49
u32 crc;
38
50
};
39
-
40
-
/*
41
-
* Steps through buffer one byte at a time, calculates reflected
42
-
* crc using table.
43
-
*/
44
51
45
52
static int chksum_init(struct shash_desc *desc)
46
53
{
+3
lib/crc/.kunitconfig
+3
lib/crc/.kunitconfig
+15
-5
lib/crc/Kconfig
+15
-5
lib/crc/Kconfig
···
48
48
bool
49
49
depends on CRC_T10DIF && CRC_OPTIMIZATIONS
50
50
default y if ARM && KERNEL_MODE_NEON
51
-
default y if ARM64 && KERNEL_MODE_NEON
51
+
default y if ARM64
52
52
default y if PPC64 && ALTIVEC
53
53
default y if RISCV && RISCV_ISA_ZBC
54
54
default y if X86
···
82
82
config CRC64_ARCH
83
83
bool
84
84
depends on CRC64 && CRC_OPTIMIZATIONS
85
+
default y if ARM64
85
86
default y if RISCV && RISCV_ISA_ZBC && 64BIT
86
87
default y if X86_64
87
88
···
100
99
101
100
config CRC_KUNIT_TEST
102
101
tristate "KUnit tests for CRC functions" if !KUNIT_ALL_TESTS
103
-
depends on KUNIT
102
+
depends on KUNIT && (CRC7 || CRC16 || CRC_T10DIF || CRC32 || CRC64)
104
103
default KUNIT_ALL_TESTS
104
+
help
105
+
Unit tests for the CRC library functions.
106
+
107
+
This is intended to help people writing architecture-specific
108
+
optimized versions. If unsure, say N.
109
+
110
+
config CRC_ENABLE_ALL_FOR_KUNIT
111
+
tristate "Enable all CRC functions for KUnit test"
112
+
depends on KUNIT
105
113
select CRC7
106
114
select CRC16
107
115
select CRC_T10DIF
108
116
select CRC32
109
117
select CRC64
110
118
help
111
-
Unit tests for the CRC library functions.
119
+
Enable all CRC functions that have test code in CRC_KUNIT_TEST.
112
120
113
-
This is intended to help people writing architecture-specific
114
-
optimized versions. If unsure, say N.
121
+
Enable this only if you'd like the CRC KUnit test suite to test all
122
+
the CRC variants, even ones that wouldn't otherwise need to be built.
115
123
116
124
config CRC_BENCHMARK
117
125
bool "Benchmark for the CRC functions"
+6
-1
lib/crc/Makefile
+6
-1
lib/crc/Makefile
···
38
38
crc64-y := crc64-main.o
39
39
ifeq ($(CONFIG_CRC64_ARCH),y)
40
40
CFLAGS_crc64-main.o += -I$(src)/$(SRCARCH)
41
+
42
+
CFLAGS_REMOVE_arm64/crc64-neon-inner.o += $(CC_FLAGS_NO_FPU)
43
+
CFLAGS_arm64/crc64-neon-inner.o += $(CC_FLAGS_FPU) -march=armv8-a+crypto
44
+
crc64-$(CONFIG_ARM64) += arm64/crc64-neon-inner.o
45
+
41
46
crc64-$(CONFIG_RISCV) += riscv/crc64_lsb.o riscv/crc64_msb.o
42
47
crc64-$(CONFIG_X86) += x86/crc64-pclmul.o
43
-
endif
48
+
endif # CONFIG_CRC64_ARCH
44
49
45
50
obj-y += tests/
46
51
+28
-28
lib/crc/arm64/crc-t10dif-core.S
+28
-28
lib/crc/arm64/crc-t10dif-core.S
···
181
181
182
182
pmull16x64_\p fold_consts, \reg1, v8
183
183
184
-
CPU_LE( rev64 v11.16b, v11.16b )
185
-
CPU_LE( rev64 v12.16b, v12.16b )
184
+
rev64 v11.16b, v11.16b
185
+
rev64 v12.16b, v12.16b
186
186
187
187
pmull16x64_\p fold_consts, \reg2, v9
188
188
189
-
CPU_LE( ext v11.16b, v11.16b, v11.16b, #8 )
190
-
CPU_LE( ext v12.16b, v12.16b, v12.16b, #8 )
189
+
ext v11.16b, v11.16b, v11.16b, #8
190
+
ext v12.16b, v12.16b, v12.16b, #8
191
191
192
192
eor \reg1\().16b, \reg1\().16b, v8.16b
193
193
eor \reg2\().16b, \reg2\().16b, v9.16b
···
220
220
ldp q4, q5, [buf, #0x40]
221
221
ldp q6, q7, [buf, #0x60]
222
222
add buf, buf, #0x80
223
-
CPU_LE( rev64 v0.16b, v0.16b )
224
-
CPU_LE( rev64 v1.16b, v1.16b )
225
-
CPU_LE( rev64 v2.16b, v2.16b )
226
-
CPU_LE( rev64 v3.16b, v3.16b )
227
-
CPU_LE( rev64 v4.16b, v4.16b )
228
-
CPU_LE( rev64 v5.16b, v5.16b )
229
-
CPU_LE( rev64 v6.16b, v6.16b )
230
-
CPU_LE( rev64 v7.16b, v7.16b )
231
-
CPU_LE( ext v0.16b, v0.16b, v0.16b, #8 )
232
-
CPU_LE( ext v1.16b, v1.16b, v1.16b, #8 )
233
-
CPU_LE( ext v2.16b, v2.16b, v2.16b, #8 )
234
-
CPU_LE( ext v3.16b, v3.16b, v3.16b, #8 )
235
-
CPU_LE( ext v4.16b, v4.16b, v4.16b, #8 )
236
-
CPU_LE( ext v5.16b, v5.16b, v5.16b, #8 )
237
-
CPU_LE( ext v6.16b, v6.16b, v6.16b, #8 )
238
-
CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 )
223
+
rev64 v0.16b, v0.16b
224
+
rev64 v1.16b, v1.16b
225
+
rev64 v2.16b, v2.16b
226
+
rev64 v3.16b, v3.16b
227
+
rev64 v4.16b, v4.16b
228
+
rev64 v5.16b, v5.16b
229
+
rev64 v6.16b, v6.16b
230
+
rev64 v7.16b, v7.16b
231
+
ext v0.16b, v0.16b, v0.16b, #8
232
+
ext v1.16b, v1.16b, v1.16b, #8
233
+
ext v2.16b, v2.16b, v2.16b, #8
234
+
ext v3.16b, v3.16b, v3.16b, #8
235
+
ext v4.16b, v4.16b, v4.16b, #8
236
+
ext v5.16b, v5.16b, v5.16b, #8
237
+
ext v6.16b, v6.16b, v6.16b, #8
238
+
ext v7.16b, v7.16b, v7.16b, #8
239
239
240
240
// XOR the first 16 data *bits* with the initial CRC value.
241
241
movi v8.16b, #0
···
288
288
pmull16x64_\p fold_consts, v7, v8
289
289
eor v7.16b, v7.16b, v8.16b
290
290
ldr q0, [buf], #16
291
-
CPU_LE( rev64 v0.16b, v0.16b )
292
-
CPU_LE( ext v0.16b, v0.16b, v0.16b, #8 )
291
+
rev64 v0.16b, v0.16b
292
+
ext v0.16b, v0.16b, v0.16b, #8
293
293
eor v7.16b, v7.16b, v0.16b
294
294
subs len, len, #16
295
295
b.ge .Lfold_16_bytes_loop_\@
···
310
310
// v0 = last 16 original data bytes
311
311
add buf, buf, len
312
312
ldr q0, [buf, #-16]
313
-
CPU_LE( rev64 v0.16b, v0.16b )
314
-
CPU_LE( ext v0.16b, v0.16b, v0.16b, #8 )
313
+
rev64 v0.16b, v0.16b
314
+
ext v0.16b, v0.16b, v0.16b, #8
315
315
316
316
// v1 = high order part of second chunk: v7 left-shifted by 'len' bytes.
317
317
adr_l x4, .Lbyteshift_table + 16
···
344
344
345
345
// Load the first 16 data bytes.
346
346
ldr q7, [buf], #0x10
347
-
CPU_LE( rev64 v7.16b, v7.16b )
348
-
CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 )
347
+
rev64 v7.16b, v7.16b
348
+
ext v7.16b, v7.16b, v7.16b, #8
349
349
350
350
// XOR the first 16 data *bits* with the initial CRC value.
351
351
movi v0.16b, #0
···
382
382
383
383
crc_t10dif_pmull p8
384
384
385
-
CPU_LE( rev64 v7.16b, v7.16b )
386
-
CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 )
385
+
rev64 v7.16b, v7.16b
386
+
ext v7.16b, v7.16b, v7.16b, #8
387
387
str q7, [x3]
388
388
389
389
frame_pop
+2
-7
lib/crc/arm64/crc32-core.S
+2
-7
lib/crc/arm64/crc32-core.S
···
29
29
.endm
30
30
31
31
.macro hwordle, reg
32
-
CPU_BE( rev16 \reg, \reg )
33
32
.endm
34
33
35
34
.macro hwordbe, reg
36
-
CPU_LE( rev \reg, \reg )
35
+
rev \reg, \reg
37
36
rbit \reg, \reg
38
-
CPU_BE( lsr \reg, \reg, #16 )
39
37
.endm
40
38
41
39
.macro le, regs:vararg
42
-
.irp r, \regs
43
-
CPU_BE( rev \r, \r )
44
-
.endr
45
40
.endm
46
41
47
42
.macro be, regs:vararg
48
43
.irp r, \regs
49
-
CPU_LE( rev \r, \r )
44
+
rev \r, \r
50
45
.endr
51
46
.irp r, \regs
52
47
rbit \r, \r
+65
lib/crc/arm64/crc64-neon-inner.c
+65
lib/crc/arm64/crc64-neon-inner.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/*
3
+
* Accelerated CRC64 (NVMe) using ARM NEON C intrinsics
4
+
*/
5
+
6
+
#include <linux/types.h>
7
+
#include <asm/neon-intrinsics.h>
8
+
9
+
u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len);
10
+
11
+
/* x^191 mod G, x^127 mod G */
12
+
static const u64 fold_consts_val[2] = { 0xeadc41fd2ba3d420ULL,
13
+
0x21e9761e252621acULL };
14
+
/* floor(x^127 / G), (G - x^64) / x */
15
+
static const u64 bconsts_val[2] = { 0x27ecfa329aef9f77ULL,
16
+
0x34d926535897936aULL };
17
+
18
+
static inline uint64x2_t pmull64(uint64x2_t a, uint64x2_t b)
19
+
{
20
+
return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 0),
21
+
vgetq_lane_u64(b, 0)));
22
+
}
23
+
24
+
static inline uint64x2_t pmull64_high(uint64x2_t a, uint64x2_t b)
25
+
{
26
+
poly64x2_t l = vreinterpretq_p64_u64(a);
27
+
poly64x2_t m = vreinterpretq_p64_u64(b);
28
+
29
+
return vreinterpretq_u64_p128(vmull_high_p64(l, m));
30
+
}
31
+
32
+
static inline uint64x2_t pmull64_hi_lo(uint64x2_t a, uint64x2_t b)
33
+
{
34
+
return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 1),
35
+
vgetq_lane_u64(b, 0)));
36
+
}
37
+
38
+
u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len)
39
+
{
40
+
uint64x2_t fold_consts = vld1q_u64(fold_consts_val);
41
+
uint64x2_t v0 = { crc, 0 };
42
+
uint64x2_t zero = { };
43
+
44
+
for (;;) {
45
+
v0 ^= vreinterpretq_u64_u8(vld1q_u8(p));
46
+
47
+
p += 16;
48
+
len -= 16;
49
+
if (len < 16)
50
+
break;
51
+
52
+
v0 = pmull64(fold_consts, v0) ^ pmull64_high(fold_consts, v0);
53
+
}
54
+
55
+
/* Multiply the 128-bit value by x^64 and reduce it back to 128 bits. */
56
+
v0 = vextq_u64(v0, zero, 1) ^ pmull64_hi_lo(fold_consts, v0);
57
+
58
+
/* Final Barrett reduction */
59
+
uint64x2_t bconsts = vld1q_u64(bconsts_val);
60
+
uint64x2_t final = pmull64(bconsts, v0);
61
+
62
+
v0 ^= vextq_u64(zero, final, 1) ^ pmull64_hi_lo(bconsts, final);
63
+
64
+
return vgetq_lane_u64(v0, 1);
65
+
}
+28
lib/crc/arm64/crc64.h
+28
lib/crc/arm64/crc64.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
/*
3
+
* CRC64 using ARM64 PMULL instructions
4
+
*/
5
+
6
+
#include <linux/cpufeature.h>
7
+
#include <asm/simd.h>
8
+
#include <linux/minmax.h>
9
+
#include <linux/sizes.h>
10
+
11
+
u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len);
12
+
13
+
#define crc64_be_arch crc64_be_generic
14
+
15
+
static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len)
16
+
{
17
+
if (len >= 128 && cpu_have_named_feature(PMULL) &&
18
+
likely(may_use_simd())) {
19
+
size_t chunk = len & ~15;
20
+
21
+
scoped_ksimd()
22
+
crc = crc64_nvme_arm64_c(crc, p, chunk);
23
+
24
+
p += chunk;
25
+
len &= 15;
26
+
}
27
+
return crc64_nvme_generic(crc, p, len);
28
+
}
+22
-6
lib/crc/tests/crc_kunit.c
+22
-6
lib/crc/tests/crc_kunit.c
···
268
268
}
269
269
}
270
270
271
-
/* crc7_be */
272
-
271
+
#if IS_REACHABLE(CONFIG_CRC7)
273
272
static u64 crc7_be_wrapper(u64 crc, const u8 *p, size_t len)
274
273
{
275
274
/*
···
293
294
{
294
295
crc_benchmark(test, crc7_be_wrapper);
295
296
}
297
+
#endif /* CONFIG_CRC7 */
296
298
297
-
/* crc16 */
298
-
299
+
#if IS_REACHABLE(CONFIG_CRC16)
299
300
static u64 crc16_wrapper(u64 crc, const u8 *p, size_t len)
300
301
{
301
302
return crc16(crc, p, len);
···
317
318
{
318
319
crc_benchmark(test, crc16_wrapper);
319
320
}
321
+
#endif /* CONFIG_CRC16 */
320
322
321
-
/* crc_t10dif */
322
-
323
+
#if IS_REACHABLE(CONFIG_CRC_T10DIF)
323
324
static u64 crc_t10dif_wrapper(u64 crc, const u8 *p, size_t len)
324
325
{
325
326
return crc_t10dif_update(crc, p, len);
···
341
342
{
342
343
crc_benchmark(test, crc_t10dif_wrapper);
343
344
}
345
+
#endif /* CONFIG_CRC_T10DIF */
346
+
347
+
#if IS_REACHABLE(CONFIG_CRC32)
344
348
345
349
/* crc32_le */
346
350
···
416
414
{
417
415
crc_benchmark(test, crc32c_wrapper);
418
416
}
417
+
#endif /* CONFIG_CRC32 */
418
+
419
+
#if IS_REACHABLE(CONFIG_CRC64)
419
420
420
421
/* crc64_be */
421
422
···
468
463
{
469
464
crc_benchmark(test, crc64_nvme_wrapper);
470
465
}
466
+
#endif /* CONFIG_CRC64 */
471
467
472
468
static struct kunit_case crc_test_cases[] = {
469
+
#if IS_REACHABLE(CONFIG_CRC7)
473
470
KUNIT_CASE(crc7_be_test),
474
471
KUNIT_CASE(crc7_be_benchmark),
472
+
#endif
473
+
#if IS_REACHABLE(CONFIG_CRC16)
475
474
KUNIT_CASE(crc16_test),
476
475
KUNIT_CASE(crc16_benchmark),
476
+
#endif
477
+
#if IS_REACHABLE(CONFIG_CRC_T10DIF)
477
478
KUNIT_CASE(crc_t10dif_test),
478
479
KUNIT_CASE(crc_t10dif_benchmark),
480
+
#endif
481
+
#if IS_REACHABLE(CONFIG_CRC32)
479
482
KUNIT_CASE(crc32_le_test),
480
483
KUNIT_CASE(crc32_le_benchmark),
481
484
KUNIT_CASE(crc32_be_test),
482
485
KUNIT_CASE(crc32_be_benchmark),
483
486
KUNIT_CASE(crc32c_test),
484
487
KUNIT_CASE(crc32c_benchmark),
488
+
#endif
489
+
#if IS_REACHABLE(CONFIG_CRC64)
485
490
KUNIT_CASE(crc64_be_test),
486
491
KUNIT_CASE(crc64_be_benchmark),
487
492
KUNIT_CASE(crc64_nvme_test),
488
493
KUNIT_CASE(crc64_nvme_benchmark),
494
+
#endif
489
495
{},
490
496
};
491
497