Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'crc-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux
Pull CRC updates from Eric Biggers:
"Update crc_kunit to test the CRC functions in softirq and hardirq
contexts, similar to what the lib/crypto/ KUnit tests do. Move the
helper function needed to do this into a common header.
This is useful mainly to test fallback code paths for when
FPU/SIMD/vector registers are unusable"
* tag 'crc-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux:
Documentation/staging: Fix typo and incorrect citation in crc32.rst
lib/crc: Drop inline from all *_mod_init_arch() functions
lib/crc: Use underlying functions instead of crypto_simd_usable()
lib/crc: crc_kunit: Test CRC computation in interrupt contexts
kunit, lib/crypto: Move run_irq_test() to common header
+219
-165
+2
-2
Documentation/staging/crc32.rst
+2
-2
Documentation/staging/crc32.rst
···
34
34
Just like with ordinary division, you proceed one digit (bit) at a time.
35
35
Each step of the division you take one more digit (bit) of the dividend
36
36
and append it to the current remainder. Then you figure out the
37
-
appropriate multiple of the divisor to subtract to being the remainder
37
+
appropriate multiple of the divisor to subtract to bring the remainder
38
38
back into range. In binary, this is easy - it has to be either 0 or 1,
39
39
and to make the XOR cancel, it's just a copy of bit 32 of the remainder.
40
40
···
116
116
To reduce the number of conditional branches, software commonly uses
117
117
the byte-at-a-time table method, popularized by Dilip V. Sarwate,
118
118
"Computation of Cyclic Redundancy Checks via Table Look-Up", Comm. ACM
119
-
v.31 no.8 (August 1998) p. 1008-1013.
119
+
v.31 no.8 (August 1988) p. 1008-1013.
120
120
121
121
Here, rather than just shifting one bit of the remainder to decide
122
122
in the correct multiple to subtract, we can shift a byte at a time.
+129
include/kunit/run-in-irq-context.h
+129
include/kunit/run-in-irq-context.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-or-later */
2
+
/*
3
+
* Helper function for testing code in interrupt contexts
4
+
*
5
+
* Copyright 2025 Google LLC
6
+
*/
7
+
#ifndef _KUNIT_RUN_IN_IRQ_CONTEXT_H
8
+
#define _KUNIT_RUN_IN_IRQ_CONTEXT_H
9
+
10
+
#include <kunit/test.h>
11
+
#include <linux/timekeeping.h>
12
+
#include <linux/hrtimer.h>
13
+
#include <linux/workqueue.h>
14
+
15
+
#define KUNIT_IRQ_TEST_HRTIMER_INTERVAL us_to_ktime(5)
16
+
17
+
struct kunit_irq_test_state {
18
+
bool (*func)(void *test_specific_state);
19
+
void *test_specific_state;
20
+
bool task_func_reported_failure;
21
+
bool hardirq_func_reported_failure;
22
+
bool softirq_func_reported_failure;
23
+
unsigned long hardirq_func_calls;
24
+
unsigned long softirq_func_calls;
25
+
struct hrtimer timer;
26
+
struct work_struct bh_work;
27
+
};
28
+
29
+
static enum hrtimer_restart kunit_irq_test_timer_func(struct hrtimer *timer)
30
+
{
31
+
struct kunit_irq_test_state *state =
32
+
container_of(timer, typeof(*state), timer);
33
+
34
+
WARN_ON_ONCE(!in_hardirq());
35
+
state->hardirq_func_calls++;
36
+
37
+
if (!state->func(state->test_specific_state))
38
+
state->hardirq_func_reported_failure = true;
39
+
40
+
hrtimer_forward_now(&state->timer, KUNIT_IRQ_TEST_HRTIMER_INTERVAL);
41
+
queue_work(system_bh_wq, &state->bh_work);
42
+
return HRTIMER_RESTART;
43
+
}
44
+
45
+
static void kunit_irq_test_bh_work_func(struct work_struct *work)
46
+
{
47
+
struct kunit_irq_test_state *state =
48
+
container_of(work, typeof(*state), bh_work);
49
+
50
+
WARN_ON_ONCE(!in_serving_softirq());
51
+
state->softirq_func_calls++;
52
+
53
+
if (!state->func(state->test_specific_state))
54
+
state->softirq_func_reported_failure = true;
55
+
}
56
+
57
+
/*
58
+
* Helper function which repeatedly runs the given @func in task, softirq, and
59
+
* hardirq context concurrently, and reports a failure to KUnit if any
60
+
* invocation of @func in any context returns false. @func is passed
61
+
* @test_specific_state as its argument. At most 3 invocations of @func will
62
+
* run concurrently: one in each of task, softirq, and hardirq context.
63
+
*
64
+
* The main purpose of this interrupt context testing is to validate fallback
65
+
* code paths that run in contexts where the normal code path cannot be used,
66
+
* typically due to the FPU or vector registers already being in-use in kernel
67
+
* mode. These code paths aren't covered when the test code is executed only by
68
+
* the KUnit test runner thread in task context. The reason for the concurrency
69
+
* is because merely using hardirq context is not sufficient to reach a fallback
70
+
* code path on some architectures; the hardirq actually has to occur while the
71
+
* FPU or vector unit was already in-use in kernel mode.
72
+
*
73
+
* Another purpose of this testing is to detect issues with the architecture's
74
+
* irq_fpu_usable() and kernel_fpu_begin/end() or equivalent functions,
75
+
* especially in softirq context when the softirq may have interrupted a task
76
+
* already using kernel-mode FPU or vector (if the arch didn't prevent that).
77
+
* Crypto functions are often executed in softirqs, so this is important.
78
+
*/
79
+
static inline void kunit_run_irq_test(struct kunit *test, bool (*func)(void *),
80
+
int max_iterations,
81
+
void *test_specific_state)
82
+
{
83
+
struct kunit_irq_test_state state = {
84
+
.func = func,
85
+
.test_specific_state = test_specific_state,
86
+
};
87
+
unsigned long end_jiffies;
88
+
89
+
/*
90
+
* Set up a hrtimer (the way we access hardirq context) and a work
91
+
* struct for the BH workqueue (the way we access softirq context).
92
+
*/
93
+
hrtimer_setup_on_stack(&state.timer, kunit_irq_test_timer_func,
94
+
CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
95
+
INIT_WORK_ONSTACK(&state.bh_work, kunit_irq_test_bh_work_func);
96
+
97
+
/* Run for up to max_iterations or 1 second, whichever comes first. */
98
+
end_jiffies = jiffies + HZ;
99
+
hrtimer_start(&state.timer, KUNIT_IRQ_TEST_HRTIMER_INTERVAL,
100
+
HRTIMER_MODE_REL_HARD);
101
+
for (int i = 0; i < max_iterations && !time_after(jiffies, end_jiffies);
102
+
i++) {
103
+
if (!func(test_specific_state))
104
+
state.task_func_reported_failure = true;
105
+
}
106
+
107
+
/* Cancel the timer and work. */
108
+
hrtimer_cancel(&state.timer);
109
+
flush_work(&state.bh_work);
110
+
111
+
/* Sanity check: the timer and BH functions should have been run. */
112
+
KUNIT_EXPECT_GT_MSG(test, state.hardirq_func_calls, 0,
113
+
"Timer function was not called");
114
+
KUNIT_EXPECT_GT_MSG(test, state.softirq_func_calls, 0,
115
+
"BH work function was not called");
116
+
117
+
/* Check for incorrect hash values reported from any context. */
118
+
KUNIT_EXPECT_FALSE_MSG(
119
+
test, state.task_func_reported_failure,
120
+
"Incorrect hash values reported from task context");
121
+
KUNIT_EXPECT_FALSE_MSG(
122
+
test, state.hardirq_func_reported_failure,
123
+
"Incorrect hash values reported from hardirq context");
124
+
KUNIT_EXPECT_FALSE_MSG(
125
+
test, state.softirq_func_reported_failure,
126
+
"Incorrect hash values reported from softirq context");
127
+
}
128
+
129
+
#endif /* _KUNIT_RUN_IN_IRQ_CONTEXT_H */
+3
-5
lib/crc/arm/crc-t10dif.h
+3
-5
lib/crc/arm/crc-t10dif.h
···
5
5
* Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
6
6
*/
7
7
8
-
#include <crypto/internal/simd.h>
9
-
10
8
#include <asm/neon.h>
11
9
#include <asm/simd.h>
12
10
···
21
23
{
22
24
if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) {
23
25
if (static_branch_likely(&have_pmull)) {
24
-
if (crypto_simd_usable()) {
26
+
if (likely(may_use_simd())) {
25
27
kernel_neon_begin();
26
28
crc = crc_t10dif_pmull64(crc, data, length);
27
29
kernel_neon_end();
···
29
31
}
30
32
} else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE &&
31
33
static_branch_likely(&have_neon) &&
32
-
crypto_simd_usable()) {
34
+
likely(may_use_simd())) {
33
35
u8 buf[16] __aligned(16);
34
36
35
37
kernel_neon_begin();
···
43
45
}
44
46
45
47
#define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch
46
-
static inline void crc_t10dif_mod_init_arch(void)
48
+
static void crc_t10dif_mod_init_arch(void)
47
49
{
48
50
if (elf_hwcap & HWCAP_NEON) {
49
51
static_branch_enable(&have_neon);
+3
-5
lib/crc/arm/crc32.h
+3
-5
lib/crc/arm/crc32.h
···
7
7
8
8
#include <linux/cpufeature.h>
9
9
10
-
#include <crypto/internal/simd.h>
11
-
12
10
#include <asm/hwcap.h>
13
11
#include <asm/neon.h>
14
12
#include <asm/simd.h>
···
32
34
static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
33
35
{
34
36
if (len >= PMULL_MIN_LEN + 15 &&
35
-
static_branch_likely(&have_pmull) && crypto_simd_usable()) {
37
+
static_branch_likely(&have_pmull) && likely(may_use_simd())) {
36
38
size_t n = -(uintptr_t)p & 15;
37
39
38
40
/* align p to 16-byte boundary */
···
61
63
static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
62
64
{
63
65
if (len >= PMULL_MIN_LEN + 15 &&
64
-
static_branch_likely(&have_pmull) && crypto_simd_usable()) {
66
+
static_branch_likely(&have_pmull) && likely(may_use_simd())) {
65
67
size_t n = -(uintptr_t)p & 15;
66
68
67
69
/* align p to 16-byte boundary */
···
83
85
#define crc32_be_arch crc32_be_base /* not implemented on this arch */
84
86
85
87
#define crc32_mod_init_arch crc32_mod_init_arch
86
-
static inline void crc32_mod_init_arch(void)
88
+
static void crc32_mod_init_arch(void)
87
89
{
88
90
if (elf_hwcap2 & HWCAP2_CRC32)
89
91
static_branch_enable(&have_crc32);
+3
-5
lib/crc/arm64/crc-t10dif.h
+3
-5
lib/crc/arm64/crc-t10dif.h
···
7
7
8
8
#include <linux/cpufeature.h>
9
9
10
-
#include <crypto/internal/simd.h>
11
-
12
10
#include <asm/neon.h>
13
11
#include <asm/simd.h>
14
12
···
23
25
{
24
26
if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) {
25
27
if (static_branch_likely(&have_pmull)) {
26
-
if (crypto_simd_usable()) {
28
+
if (likely(may_use_simd())) {
27
29
kernel_neon_begin();
28
30
crc = crc_t10dif_pmull_p64(crc, data, length);
29
31
kernel_neon_end();
···
31
33
}
32
34
} else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE &&
33
35
static_branch_likely(&have_asimd) &&
34
-
crypto_simd_usable()) {
36
+
likely(may_use_simd())) {
35
37
u8 buf[16];
36
38
37
39
kernel_neon_begin();
···
45
47
}
46
48
47
49
#define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch
48
-
static inline void crc_t10dif_mod_init_arch(void)
50
+
static void crc_t10dif_mod_init_arch(void)
49
51
{
50
52
if (cpu_have_named_feature(ASIMD)) {
51
53
static_branch_enable(&have_asimd);
+6
-5
lib/crc/arm64/crc32.h
+6
-5
lib/crc/arm64/crc32.h
···
5
5
#include <asm/neon.h>
6
6
#include <asm/simd.h>
7
7
8
-
#include <crypto/internal/simd.h>
9
-
10
8
// The minimum input length to consider the 4-way interleaved code path
11
9
static const size_t min_len = 1024;
12
10
···
21
23
if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
22
24
return crc32_le_base(crc, p, len);
23
25
24
-
if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) {
26
+
if (len >= min_len && cpu_have_named_feature(PMULL) &&
27
+
likely(may_use_simd())) {
25
28
kernel_neon_begin();
26
29
crc = crc32_le_arm64_4way(crc, p, len);
27
30
kernel_neon_end();
···
42
43
if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
43
44
return crc32c_base(crc, p, len);
44
45
45
-
if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) {
46
+
if (len >= min_len && cpu_have_named_feature(PMULL) &&
47
+
likely(may_use_simd())) {
46
48
kernel_neon_begin();
47
49
crc = crc32c_le_arm64_4way(crc, p, len);
48
50
kernel_neon_end();
···
63
63
if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
64
64
return crc32_be_base(crc, p, len);
65
65
66
-
if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) {
66
+
if (len >= min_len && cpu_have_named_feature(PMULL) &&
67
+
likely(may_use_simd())) {
67
68
kernel_neon_begin();
68
69
crc = crc32_be_arm64_4way(crc, p, len);
69
70
kernel_neon_end();
+1
-1
lib/crc/loongarch/crc32.h
+1
-1
lib/crc/loongarch/crc32.h
···
101
101
#define crc32_be_arch crc32_be_base /* not implemented on this arch */
102
102
103
103
#define crc32_mod_init_arch crc32_mod_init_arch
104
-
static inline void crc32_mod_init_arch(void)
104
+
static void crc32_mod_init_arch(void)
105
105
{
106
106
if (cpu_has_crc32)
107
107
static_branch_enable(&have_crc32);
+1
-1
lib/crc/mips/crc32.h
+1
-1
lib/crc/mips/crc32.h
···
148
148
#define crc32_be_arch crc32_be_base /* not implemented on this arch */
149
149
150
150
#define crc32_mod_init_arch crc32_mod_init_arch
151
-
static inline void crc32_mod_init_arch(void)
151
+
static void crc32_mod_init_arch(void)
152
152
{
153
153
if (cpu_have_feature(cpu_feature(MIPS_CRC32)))
154
154
static_branch_enable(&have_crc32);
+4
-3
lib/crc/powerpc/crc-t10dif.h
+4
-3
lib/crc/powerpc/crc-t10dif.h
···
6
6
* [based on crc32c-vpmsum_glue.c]
7
7
*/
8
8
9
+
#include <asm/simd.h>
9
10
#include <asm/switch_to.h>
10
-
#include <crypto/internal/simd.h>
11
11
#include <linux/cpufeature.h>
12
12
#include <linux/jump_label.h>
13
13
#include <linux/preempt.h>
···
29
29
u32 crc = crci;
30
30
31
31
if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) ||
32
-
!static_branch_likely(&have_vec_crypto) || !crypto_simd_usable())
32
+
!static_branch_likely(&have_vec_crypto) ||
33
+
unlikely(!may_use_simd()))
33
34
return crc_t10dif_generic(crc, p, len);
34
35
35
36
if ((unsigned long)p & VMX_ALIGN_MASK) {
···
62
61
}
63
62
64
63
#define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch
65
-
static inline void crc_t10dif_mod_init_arch(void)
64
+
static void crc_t10dif_mod_init_arch(void)
66
65
{
67
66
if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
68
67
(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
+4
-3
lib/crc/powerpc/crc32.h
+4
-3
lib/crc/powerpc/crc32.h
···
1
1
// SPDX-License-Identifier: GPL-2.0-only
2
+
#include <asm/simd.h>
2
3
#include <asm/switch_to.h>
3
-
#include <crypto/internal/simd.h>
4
4
#include <linux/cpufeature.h>
5
5
#include <linux/jump_label.h>
6
6
#include <linux/preempt.h>
···
24
24
unsigned int tail;
25
25
26
26
if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) ||
27
-
!static_branch_likely(&have_vec_crypto) || !crypto_simd_usable())
27
+
!static_branch_likely(&have_vec_crypto) ||
28
+
unlikely(!may_use_simd()))
28
29
return crc32c_base(crc, p, len);
29
30
30
31
if ((unsigned long)p & VMX_ALIGN_MASK) {
···
55
54
}
56
55
57
56
#define crc32_mod_init_arch crc32_mod_init_arch
58
-
static inline void crc32_mod_init_arch(void)
57
+
static void crc32_mod_init_arch(void)
59
58
{
60
59
if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
61
60
(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
+1
-1
lib/crc/sparc/crc32.h
+1
-1
lib/crc/sparc/crc32.h
+53
-9
lib/crc/tests/crc_kunit.c
+53
-9
lib/crc/tests/crc_kunit.c
···
6
6
*
7
7
* Author: Eric Biggers <ebiggers@google.com>
8
8
*/
9
+
#include <kunit/run-in-irq-context.h>
9
10
#include <kunit/test.h>
10
11
#include <linux/crc7.h>
11
12
#include <linux/crc16.h>
···
142
141
return len % (max_length + 1);
143
142
}
144
143
144
+
#define IRQ_TEST_DATA_LEN 512
145
+
#define IRQ_TEST_NUM_BUFFERS 3 /* matches max concurrency level */
146
+
147
+
struct crc_irq_test_state {
148
+
const struct crc_variant *v;
149
+
u64 initial_crc;
150
+
u64 expected_crcs[IRQ_TEST_NUM_BUFFERS];
151
+
atomic_t seqno;
152
+
};
153
+
154
+
/*
155
+
* Compute the CRC of one of the test messages and verify that it matches the
156
+
* expected CRC from @state->expected_crcs. To increase the chance of detecting
157
+
* problems, cycle through multiple messages.
158
+
*/
159
+
static bool crc_irq_test_func(void *state_)
160
+
{
161
+
struct crc_irq_test_state *state = state_;
162
+
const struct crc_variant *v = state->v;
163
+
u32 i = (u32)atomic_inc_return(&state->seqno) % IRQ_TEST_NUM_BUFFERS;
164
+
u64 actual_crc = v->func(state->initial_crc,
165
+
&test_buffer[i * IRQ_TEST_DATA_LEN],
166
+
IRQ_TEST_DATA_LEN);
167
+
168
+
return actual_crc == state->expected_crcs[i];
169
+
}
170
+
171
+
/*
172
+
* Test that if CRCs are computed in task, softirq, and hardirq context
173
+
* concurrently, then all results are as expected.
174
+
*/
175
+
static void crc_interrupt_context_test(struct kunit *test,
176
+
const struct crc_variant *v)
177
+
{
178
+
struct crc_irq_test_state state = {
179
+
.v = v,
180
+
.initial_crc = generate_random_initial_crc(v),
181
+
};
182
+
183
+
for (int i = 0; i < IRQ_TEST_NUM_BUFFERS; i++) {
184
+
state.expected_crcs[i] = crc_ref(
185
+
v, state.initial_crc,
186
+
&test_buffer[i * IRQ_TEST_DATA_LEN], IRQ_TEST_DATA_LEN);
187
+
}
188
+
189
+
kunit_run_irq_test(test, crc_irq_test_func, 100000, &state);
190
+
}
191
+
145
192
/* Test that v->func gives the same CRCs as a reference implementation. */
146
193
static void crc_test(struct kunit *test, const struct crc_variant *v)
147
194
{
···
198
149
for (i = 0; i < CRC_KUNIT_NUM_TEST_ITERS; i++) {
199
150
u64 init_crc, expected_crc, actual_crc;
200
151
size_t len, offset;
201
-
bool nosimd;
202
152
203
153
init_crc = generate_random_initial_crc(v);
204
154
len = generate_random_length(CRC_KUNIT_MAX_LEN);
···
216
168
/* Refresh the data occasionally. */
217
169
prandom_bytes_state(&rng, &test_buffer[offset], len);
218
170
219
-
nosimd = rand32() % 8 == 0;
220
-
221
171
/*
222
172
* Compute the CRC, and verify that it equals the CRC computed
223
173
* by a simple bit-at-a-time reference implementation.
224
174
*/
225
175
expected_crc = crc_ref(v, init_crc, &test_buffer[offset], len);
226
-
if (nosimd)
227
-
local_irq_disable();
228
176
actual_crc = v->func(init_crc, &test_buffer[offset], len);
229
-
if (nosimd)
230
-
local_irq_enable();
231
177
KUNIT_EXPECT_EQ_MSG(test, expected_crc, actual_crc,
232
-
"Wrong result with len=%zu offset=%zu nosimd=%d",
233
-
len, offset, nosimd);
178
+
"Wrong result with len=%zu offset=%zu",
179
+
len, offset);
234
180
}
181
+
182
+
crc_interrupt_context_test(test, v);
235
183
}
236
184
237
185
static __always_inline void
+1
-2
lib/crc/x86/crc-pclmul-template.h
+1
-2
lib/crc/x86/crc-pclmul-template.h
···
12
12
13
13
#include <asm/cpufeatures.h>
14
14
#include <asm/simd.h>
15
-
#include <crypto/internal/simd.h>
16
15
#include <linux/static_call.h>
17
16
#include "crc-pclmul-consts.h"
18
17
···
56
57
#define CRC_PCLMUL(crc, p, len, prefix, consts, have_pclmulqdq) \
57
58
do { \
58
59
if ((len) >= 16 && static_branch_likely(&(have_pclmulqdq)) && \
59
-
crypto_simd_usable()) { \
60
+
likely(irq_fpu_usable())) { \
60
61
const void *consts_ptr; \
61
62
\
62
63
consts_ptr = (consts).fold_across_128_bits_consts; \
+1
-1
lib/crc/x86/crc-t10dif.h
+1
-1
lib/crc/x86/crc-t10dif.h
+2
-2
lib/crc/x86/crc32.h
+2
-2
lib/crc/x86/crc32.h
···
44
44
return crc32c_base(crc, p, len);
45
45
46
46
if (IS_ENABLED(CONFIG_X86_64) && len >= CRC32C_PCLMUL_BREAKEVEN &&
47
-
static_branch_likely(&have_pclmulqdq) && crypto_simd_usable()) {
47
+
static_branch_likely(&have_pclmulqdq) && likely(irq_fpu_usable())) {
48
48
/*
49
49
* Long length, the vector registers are usable, and the CPU is
50
50
* 64-bit and supports both CRC32 and PCLMULQDQ instructions.
···
106
106
#define crc32_be_arch crc32_be_base /* not implemented on this arch */
107
107
108
108
#define crc32_mod_init_arch crc32_mod_init_arch
109
-
static inline void crc32_mod_init_arch(void)
109
+
static void crc32_mod_init_arch(void)
110
110
{
111
111
if (boot_cpu_has(X86_FEATURE_XMM4_2))
112
112
static_branch_enable(&have_crc32);
+1
-1
lib/crc/x86/crc64.h
+1
-1
lib/crc/x86/crc64.h
+4
-119
lib/crypto/tests/hash-test-template.h
+4
-119
lib/crypto/tests/hash-test-template.h
···
5
5
*
6
6
* Copyright 2025 Google LLC
7
7
*/
8
+
#include <kunit/run-in-irq-context.h>
8
9
#include <kunit/test.h>
9
-
#include <linux/hrtimer.h>
10
-
#include <linux/timekeeping.h>
11
10
#include <linux/vmalloc.h>
12
-
#include <linux/workqueue.h>
13
11
14
12
/* test_buf is a guarded buffer, i.e. &test_buf[TEST_BUF_LEN] is not mapped. */
15
13
#define TEST_BUF_LEN 16384
···
317
319
"Hash context was not zeroized by finalization");
318
320
}
319
321
320
-
#define IRQ_TEST_HRTIMER_INTERVAL us_to_ktime(5)
321
-
322
-
struct hash_irq_test_state {
323
-
bool (*func)(void *test_specific_state);
324
-
void *test_specific_state;
325
-
bool task_func_reported_failure;
326
-
bool hardirq_func_reported_failure;
327
-
bool softirq_func_reported_failure;
328
-
unsigned long hardirq_func_calls;
329
-
unsigned long softirq_func_calls;
330
-
struct hrtimer timer;
331
-
struct work_struct bh_work;
332
-
};
333
-
334
-
static enum hrtimer_restart hash_irq_test_timer_func(struct hrtimer *timer)
335
-
{
336
-
struct hash_irq_test_state *state =
337
-
container_of(timer, typeof(*state), timer);
338
-
339
-
WARN_ON_ONCE(!in_hardirq());
340
-
state->hardirq_func_calls++;
341
-
342
-
if (!state->func(state->test_specific_state))
343
-
state->hardirq_func_reported_failure = true;
344
-
345
-
hrtimer_forward_now(&state->timer, IRQ_TEST_HRTIMER_INTERVAL);
346
-
queue_work(system_bh_wq, &state->bh_work);
347
-
return HRTIMER_RESTART;
348
-
}
349
-
350
-
static void hash_irq_test_bh_work_func(struct work_struct *work)
351
-
{
352
-
struct hash_irq_test_state *state =
353
-
container_of(work, typeof(*state), bh_work);
354
-
355
-
WARN_ON_ONCE(!in_serving_softirq());
356
-
state->softirq_func_calls++;
357
-
358
-
if (!state->func(state->test_specific_state))
359
-
state->softirq_func_reported_failure = true;
360
-
}
361
-
362
-
/*
363
-
* Helper function which repeatedly runs the given @func in task, softirq, and
364
-
* hardirq context concurrently, and reports a failure to KUnit if any
365
-
* invocation of @func in any context returns false. @func is passed
366
-
* @test_specific_state as its argument. At most 3 invocations of @func will
367
-
* run concurrently: one in each of task, softirq, and hardirq context.
368
-
*
369
-
* The main purpose of this interrupt context testing is to validate fallback
370
-
* code paths that run in contexts where the normal code path cannot be used,
371
-
* typically due to the FPU or vector registers already being in-use in kernel
372
-
* mode. These code paths aren't covered when the test code is executed only by
373
-
* the KUnit test runner thread in task context. The reason for the concurrency
374
-
* is because merely using hardirq context is not sufficient to reach a fallback
375
-
* code path on some architectures; the hardirq actually has to occur while the
376
-
* FPU or vector unit was already in-use in kernel mode.
377
-
*
378
-
* Another purpose of this testing is to detect issues with the architecture's
379
-
* irq_fpu_usable() and kernel_fpu_begin/end() or equivalent functions,
380
-
* especially in softirq context when the softirq may have interrupted a task
381
-
* already using kernel-mode FPU or vector (if the arch didn't prevent that).
382
-
* Crypto functions are often executed in softirqs, so this is important.
383
-
*/
384
-
static void run_irq_test(struct kunit *test, bool (*func)(void *),
385
-
int max_iterations, void *test_specific_state)
386
-
{
387
-
struct hash_irq_test_state state = {
388
-
.func = func,
389
-
.test_specific_state = test_specific_state,
390
-
};
391
-
unsigned long end_jiffies;
392
-
393
-
/*
394
-
* Set up a hrtimer (the way we access hardirq context) and a work
395
-
* struct for the BH workqueue (the way we access softirq context).
396
-
*/
397
-
hrtimer_setup_on_stack(&state.timer, hash_irq_test_timer_func,
398
-
CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
399
-
INIT_WORK_ONSTACK(&state.bh_work, hash_irq_test_bh_work_func);
400
-
401
-
/* Run for up to max_iterations or 1 second, whichever comes first. */
402
-
end_jiffies = jiffies + HZ;
403
-
hrtimer_start(&state.timer, IRQ_TEST_HRTIMER_INTERVAL,
404
-
HRTIMER_MODE_REL_HARD);
405
-
for (int i = 0; i < max_iterations && !time_after(jiffies, end_jiffies);
406
-
i++) {
407
-
if (!func(test_specific_state))
408
-
state.task_func_reported_failure = true;
409
-
}
410
-
411
-
/* Cancel the timer and work. */
412
-
hrtimer_cancel(&state.timer);
413
-
flush_work(&state.bh_work);
414
-
415
-
/* Sanity check: the timer and BH functions should have been run. */
416
-
KUNIT_EXPECT_GT_MSG(test, state.hardirq_func_calls, 0,
417
-
"Timer function was not called");
418
-
KUNIT_EXPECT_GT_MSG(test, state.softirq_func_calls, 0,
419
-
"BH work function was not called");
420
-
421
-
/* Check for incorrect hash values reported from any context. */
422
-
KUNIT_EXPECT_FALSE_MSG(
423
-
test, state.task_func_reported_failure,
424
-
"Incorrect hash values reported from task context");
425
-
KUNIT_EXPECT_FALSE_MSG(
426
-
test, state.hardirq_func_reported_failure,
427
-
"Incorrect hash values reported from hardirq context");
428
-
KUNIT_EXPECT_FALSE_MSG(
429
-
test, state.softirq_func_reported_failure,
430
-
"Incorrect hash values reported from softirq context");
431
-
}
432
-
433
322
#define IRQ_TEST_DATA_LEN 256
434
323
#define IRQ_TEST_NUM_BUFFERS 3 /* matches max concurrency level */
435
324
···
354
469
HASH(&test_buf[i * IRQ_TEST_DATA_LEN], IRQ_TEST_DATA_LEN,
355
470
state.expected_hashes[i]);
356
471
357
-
run_irq_test(test, hash_irq_test1_func, 100000, &state);
472
+
kunit_run_irq_test(test, hash_irq_test1_func, 100000, &state);
358
473
}
359
474
360
475
struct hash_irq_test2_hash_ctx {
···
385
500
if (WARN_ON_ONCE(ctx == &state->ctxs[ARRAY_SIZE(state->ctxs)])) {
386
501
/*
387
502
* This should never happen, as the number of contexts is equal
388
-
* to the maximum concurrency level of run_irq_test().
503
+
* to the maximum concurrency level of kunit_run_irq_test().
389
504
*/
390
505
return false;
391
506
}
···
451
566
state->update_lens[state->num_steps++] = remaining;
452
567
state->num_steps += 2; /* for init and final */
453
568
454
-
run_irq_test(test, hash_irq_test2_func, 250000, state);
569
+
kunit_run_irq_test(test, hash_irq_test2_func, 250000, state);
455
570
}
456
571
457
572
#define UNKEYED_HASH_KUNIT_CASES \