Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
s390/crc32: expose CRC32 functions through lib
Move the s390 CRC32 assembly code into the lib directory and wire it up
to the library interface. This allows it to be used without going
through the crypto API. It remains usable via the crypto API too via
the shash algorithms that use the library interface. Thus all the
arch-specific "shash" code becomes unnecessary and is removed.
Note: to see the diff from arch/s390/crypto/crc32-vx.c to
arch/s390/lib/crc32-glue.c, view this commit with 'git show -M10'.
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20241202010844.144356-10-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@google.com>
+96
-322
+1
arch/s390/Kconfig
+1
arch/s390/Kconfig
···
72
72
select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM
73
73
select ARCH_ENABLE_MEMORY_HOTREMOVE
74
74
select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2
75
+
select ARCH_HAS_CRC32
75
76
select ARCH_HAS_CURRENT_STACK_POINTER
76
77
select ARCH_HAS_DEBUG_VIRTUAL
77
78
select ARCH_HAS_DEBUG_VM_PGTABLE
-1
arch/s390/configs/debug_defconfig
-1
arch/s390/configs/debug_defconfig
-1
arch/s390/configs/defconfig
-1
arch/s390/configs/defconfig
-12
arch/s390/crypto/Kconfig
-12
arch/s390/crypto/Kconfig
···
2
2
3
3
menu "Accelerated Cryptographic Algorithms for CPU (s390)"
4
4
5
-
config CRYPTO_CRC32_S390
6
-
tristate "CRC32c and CRC32"
7
-
depends on S390
8
-
select CRYPTO_HASH
9
-
select CRC32
10
-
help
11
-
CRC32c and CRC32 CRC algorithms
12
-
13
-
Architecture: s390
14
-
15
-
It is available with IBM z13 or later.
16
-
17
5
config CRYPTO_SHA512_S390
18
6
tristate "Hash functions: SHA-384 and SHA-512"
19
7
depends on S390
-2
arch/s390/crypto/Makefile
-2
arch/s390/crypto/Makefile
···
14
14
obj-$(CONFIG_CRYPTO_CHACHA_S390) += chacha_s390.o
15
15
obj-$(CONFIG_S390_PRNG) += prng.o
16
16
obj-$(CONFIG_CRYPTO_GHASH_S390) += ghash_s390.o
17
-
obj-$(CONFIG_CRYPTO_CRC32_S390) += crc32-vx_s390.o
18
17
obj-$(CONFIG_CRYPTO_HMAC_S390) += hmac_s390.o
19
18
obj-y += arch_random.o
20
19
21
-
crc32-vx_s390-y := crc32-vx.o crc32le-vx.o crc32be-vx.o
22
20
chacha_s390-y := chacha-glue.o chacha-s390.o
-306
arch/s390/crypto/crc32-vx.c
-306
arch/s390/crypto/crc32-vx.c
···
1
-
// SPDX-License-Identifier: GPL-2.0
2
-
/*
3
-
* Crypto-API module for CRC-32 algorithms implemented with the
4
-
* z/Architecture Vector Extension Facility.
5
-
*
6
-
* Copyright IBM Corp. 2015
7
-
* Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
8
-
*/
9
-
#define KMSG_COMPONENT "crc32-vx"
10
-
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
11
-
12
-
#include <linux/module.h>
13
-
#include <linux/cpufeature.h>
14
-
#include <linux/crc32.h>
15
-
#include <crypto/internal/hash.h>
16
-
#include <asm/fpu.h>
17
-
#include "crc32-vx.h"
18
-
19
-
#define CRC32_BLOCK_SIZE 1
20
-
#define CRC32_DIGEST_SIZE 4
21
-
22
-
#define VX_MIN_LEN 64
23
-
#define VX_ALIGNMENT 16L
24
-
#define VX_ALIGN_MASK (VX_ALIGNMENT - 1)
25
-
26
-
struct crc_ctx {
27
-
u32 key;
28
-
};
29
-
30
-
struct crc_desc_ctx {
31
-
u32 crc;
32
-
};
33
-
34
-
/*
35
-
* DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
36
-
*
37
-
* Creates a function to perform a particular CRC-32 computation. Depending
38
-
* on the message buffer, the hardware-accelerated or software implementation
39
-
* is used. Note that the message buffer is aligned to improve fetch
40
-
* operations of VECTOR LOAD MULTIPLE instructions.
41
-
*
42
-
*/
43
-
#define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \
44
-
static u32 __pure ___fname(u32 crc, \
45
-
unsigned char const *data, size_t datalen) \
46
-
{ \
47
-
unsigned long prealign, aligned, remaining; \
48
-
DECLARE_KERNEL_FPU_ONSTACK16(vxstate); \
49
-
\
50
-
if (datalen < VX_MIN_LEN + VX_ALIGN_MASK) \
51
-
return ___crc32_sw(crc, data, datalen); \
52
-
\
53
-
if ((unsigned long)data & VX_ALIGN_MASK) { \
54
-
prealign = VX_ALIGNMENT - \
55
-
((unsigned long)data & VX_ALIGN_MASK); \
56
-
datalen -= prealign; \
57
-
crc = ___crc32_sw(crc, data, prealign); \
58
-
data = (void *)((unsigned long)data + prealign); \
59
-
} \
60
-
\
61
-
aligned = datalen & ~VX_ALIGN_MASK; \
62
-
remaining = datalen & VX_ALIGN_MASK; \
63
-
\
64
-
kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \
65
-
crc = ___crc32_vx(crc, data, aligned); \
66
-
kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \
67
-
\
68
-
if (remaining) \
69
-
crc = ___crc32_sw(crc, data + aligned, remaining); \
70
-
\
71
-
return crc; \
72
-
}
73
-
74
-
DEFINE_CRC32_VX(crc32_le_vx, crc32_le_vgfm_16, crc32_le)
75
-
DEFINE_CRC32_VX(crc32_be_vx, crc32_be_vgfm_16, crc32_be)
76
-
DEFINE_CRC32_VX(crc32c_le_vx, crc32c_le_vgfm_16, __crc32c_le)
77
-
78
-
79
-
static int crc32_vx_cra_init_zero(struct crypto_tfm *tfm)
80
-
{
81
-
struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
82
-
83
-
mctx->key = 0;
84
-
return 0;
85
-
}
86
-
87
-
static int crc32_vx_cra_init_invert(struct crypto_tfm *tfm)
88
-
{
89
-
struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
90
-
91
-
mctx->key = ~0;
92
-
return 0;
93
-
}
94
-
95
-
static int crc32_vx_init(struct shash_desc *desc)
96
-
{
97
-
struct crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
98
-
struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
99
-
100
-
ctx->crc = mctx->key;
101
-
return 0;
102
-
}
103
-
104
-
static int crc32_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
105
-
unsigned int newkeylen)
106
-
{
107
-
struct crc_ctx *mctx = crypto_shash_ctx(tfm);
108
-
109
-
if (newkeylen != sizeof(mctx->key))
110
-
return -EINVAL;
111
-
mctx->key = le32_to_cpu(*(__le32 *)newkey);
112
-
return 0;
113
-
}
114
-
115
-
static int crc32be_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
116
-
unsigned int newkeylen)
117
-
{
118
-
struct crc_ctx *mctx = crypto_shash_ctx(tfm);
119
-
120
-
if (newkeylen != sizeof(mctx->key))
121
-
return -EINVAL;
122
-
mctx->key = be32_to_cpu(*(__be32 *)newkey);
123
-
return 0;
124
-
}
125
-
126
-
static int crc32le_vx_final(struct shash_desc *desc, u8 *out)
127
-
{
128
-
struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
129
-
130
-
*(__le32 *)out = cpu_to_le32p(&ctx->crc);
131
-
return 0;
132
-
}
133
-
134
-
static int crc32be_vx_final(struct shash_desc *desc, u8 *out)
135
-
{
136
-
struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
137
-
138
-
*(__be32 *)out = cpu_to_be32p(&ctx->crc);
139
-
return 0;
140
-
}
141
-
142
-
static int crc32c_vx_final(struct shash_desc *desc, u8 *out)
143
-
{
144
-
struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
145
-
146
-
/*
147
-
* Perform a final XOR with 0xFFFFFFFF to be in sync
148
-
* with the generic crc32c shash implementation.
149
-
*/
150
-
*(__le32 *)out = ~cpu_to_le32p(&ctx->crc);
151
-
return 0;
152
-
}
153
-
154
-
static int __crc32le_vx_finup(u32 *crc, const u8 *data, unsigned int len,
155
-
u8 *out)
156
-
{
157
-
*(__le32 *)out = cpu_to_le32(crc32_le_vx(*crc, data, len));
158
-
return 0;
159
-
}
160
-
161
-
static int __crc32be_vx_finup(u32 *crc, const u8 *data, unsigned int len,
162
-
u8 *out)
163
-
{
164
-
*(__be32 *)out = cpu_to_be32(crc32_be_vx(*crc, data, len));
165
-
return 0;
166
-
}
167
-
168
-
static int __crc32c_vx_finup(u32 *crc, const u8 *data, unsigned int len,
169
-
u8 *out)
170
-
{
171
-
/*
172
-
* Perform a final XOR with 0xFFFFFFFF to be in sync
173
-
* with the generic crc32c shash implementation.
174
-
*/
175
-
*(__le32 *)out = ~cpu_to_le32(crc32c_le_vx(*crc, data, len));
176
-
return 0;
177
-
}
178
-
179
-
180
-
#define CRC32_VX_FINUP(alg, func) \
181
-
static int alg ## _vx_finup(struct shash_desc *desc, const u8 *data, \
182
-
unsigned int datalen, u8 *out) \
183
-
{ \
184
-
return __ ## alg ## _vx_finup(shash_desc_ctx(desc), \
185
-
data, datalen, out); \
186
-
}
187
-
188
-
CRC32_VX_FINUP(crc32le, crc32_le_vx)
189
-
CRC32_VX_FINUP(crc32be, crc32_be_vx)
190
-
CRC32_VX_FINUP(crc32c, crc32c_le_vx)
191
-
192
-
#define CRC32_VX_DIGEST(alg, func) \
193
-
static int alg ## _vx_digest(struct shash_desc *desc, const u8 *data, \
194
-
unsigned int len, u8 *out) \
195
-
{ \
196
-
return __ ## alg ## _vx_finup(crypto_shash_ctx(desc->tfm), \
197
-
data, len, out); \
198
-
}
199
-
200
-
CRC32_VX_DIGEST(crc32le, crc32_le_vx)
201
-
CRC32_VX_DIGEST(crc32be, crc32_be_vx)
202
-
CRC32_VX_DIGEST(crc32c, crc32c_le_vx)
203
-
204
-
#define CRC32_VX_UPDATE(alg, func) \
205
-
static int alg ## _vx_update(struct shash_desc *desc, const u8 *data, \
206
-
unsigned int datalen) \
207
-
{ \
208
-
struct crc_desc_ctx *ctx = shash_desc_ctx(desc); \
209
-
ctx->crc = func(ctx->crc, data, datalen); \
210
-
return 0; \
211
-
}
212
-
213
-
CRC32_VX_UPDATE(crc32le, crc32_le_vx)
214
-
CRC32_VX_UPDATE(crc32be, crc32_be_vx)
215
-
CRC32_VX_UPDATE(crc32c, crc32c_le_vx)
216
-
217
-
218
-
static struct shash_alg crc32_vx_algs[] = {
219
-
/* CRC-32 LE */
220
-
{
221
-
.init = crc32_vx_init,
222
-
.setkey = crc32_vx_setkey,
223
-
.update = crc32le_vx_update,
224
-
.final = crc32le_vx_final,
225
-
.finup = crc32le_vx_finup,
226
-
.digest = crc32le_vx_digest,
227
-
.descsize = sizeof(struct crc_desc_ctx),
228
-
.digestsize = CRC32_DIGEST_SIZE,
229
-
.base = {
230
-
.cra_name = "crc32",
231
-
.cra_driver_name = "crc32-vx",
232
-
.cra_priority = 200,
233
-
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
234
-
.cra_blocksize = CRC32_BLOCK_SIZE,
235
-
.cra_ctxsize = sizeof(struct crc_ctx),
236
-
.cra_module = THIS_MODULE,
237
-
.cra_init = crc32_vx_cra_init_zero,
238
-
},
239
-
},
240
-
/* CRC-32 BE */
241
-
{
242
-
.init = crc32_vx_init,
243
-
.setkey = crc32be_vx_setkey,
244
-
.update = crc32be_vx_update,
245
-
.final = crc32be_vx_final,
246
-
.finup = crc32be_vx_finup,
247
-
.digest = crc32be_vx_digest,
248
-
.descsize = sizeof(struct crc_desc_ctx),
249
-
.digestsize = CRC32_DIGEST_SIZE,
250
-
.base = {
251
-
.cra_name = "crc32be",
252
-
.cra_driver_name = "crc32be-vx",
253
-
.cra_priority = 200,
254
-
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
255
-
.cra_blocksize = CRC32_BLOCK_SIZE,
256
-
.cra_ctxsize = sizeof(struct crc_ctx),
257
-
.cra_module = THIS_MODULE,
258
-
.cra_init = crc32_vx_cra_init_zero,
259
-
},
260
-
},
261
-
/* CRC-32C LE */
262
-
{
263
-
.init = crc32_vx_init,
264
-
.setkey = crc32_vx_setkey,
265
-
.update = crc32c_vx_update,
266
-
.final = crc32c_vx_final,
267
-
.finup = crc32c_vx_finup,
268
-
.digest = crc32c_vx_digest,
269
-
.descsize = sizeof(struct crc_desc_ctx),
270
-
.digestsize = CRC32_DIGEST_SIZE,
271
-
.base = {
272
-
.cra_name = "crc32c",
273
-
.cra_driver_name = "crc32c-vx",
274
-
.cra_priority = 200,
275
-
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
276
-
.cra_blocksize = CRC32_BLOCK_SIZE,
277
-
.cra_ctxsize = sizeof(struct crc_ctx),
278
-
.cra_module = THIS_MODULE,
279
-
.cra_init = crc32_vx_cra_init_invert,
280
-
},
281
-
},
282
-
};
283
-
284
-
285
-
static int __init crc_vx_mod_init(void)
286
-
{
287
-
return crypto_register_shashes(crc32_vx_algs,
288
-
ARRAY_SIZE(crc32_vx_algs));
289
-
}
290
-
291
-
static void __exit crc_vx_mod_exit(void)
292
-
{
293
-
crypto_unregister_shashes(crc32_vx_algs, ARRAY_SIZE(crc32_vx_algs));
294
-
}
295
-
296
-
module_cpu_feature_match(S390_CPU_FEATURE_VXRS, crc_vx_mod_init);
297
-
module_exit(crc_vx_mod_exit);
298
-
299
-
MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
300
-
MODULE_DESCRIPTION("CRC-32 algorithms using z/Architecture Vector Extension Facility");
301
-
MODULE_LICENSE("GPL");
302
-
303
-
MODULE_ALIAS_CRYPTO("crc32");
304
-
MODULE_ALIAS_CRYPTO("crc32-vx");
305
-
MODULE_ALIAS_CRYPTO("crc32c");
306
-
MODULE_ALIAS_CRYPTO("crc32c-vx");
arch/s390/crypto/crc32-vx.h
arch/s390/lib/crc32-vx.h
arch/s390/crypto/crc32-vx.h
arch/s390/lib/crc32-vx.h
arch/s390/crypto/crc32be-vx.c
arch/s390/lib/crc32be-vx.c
arch/s390/crypto/crc32be-vx.c
arch/s390/lib/crc32be-vx.c
arch/s390/crypto/crc32le-vx.c
arch/s390/lib/crc32le-vx.c
arch/s390/crypto/crc32le-vx.c
arch/s390/lib/crc32le-vx.c
+3
arch/s390/lib/Makefile
+3
arch/s390/lib/Makefile
+92
arch/s390/lib/crc32-glue.c
+92
arch/s390/lib/crc32-glue.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/*
3
+
* CRC-32 implemented with the z/Architecture Vector Extension Facility.
4
+
*
5
+
* Copyright IBM Corp. 2015
6
+
* Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
7
+
*/
8
+
#define KMSG_COMPONENT "crc32-vx"
9
+
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
10
+
11
+
#include <linux/module.h>
12
+
#include <linux/cpufeature.h>
13
+
#include <linux/crc32.h>
14
+
#include <asm/fpu.h>
15
+
#include "crc32-vx.h"
16
+
17
+
#define VX_MIN_LEN 64
18
+
#define VX_ALIGNMENT 16L
19
+
#define VX_ALIGN_MASK (VX_ALIGNMENT - 1)
20
+
21
+
static DEFINE_STATIC_KEY_FALSE(have_vxrs);
22
+
23
+
/*
24
+
* DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
25
+
*
26
+
* Creates a function to perform a particular CRC-32 computation. Depending
27
+
* on the message buffer, the hardware-accelerated or software implementation
28
+
* is used. Note that the message buffer is aligned to improve fetch
29
+
* operations of VECTOR LOAD MULTIPLE instructions.
30
+
*/
31
+
#define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \
32
+
u32 ___fname(u32 crc, const u8 *data, size_t datalen) \
33
+
{ \
34
+
unsigned long prealign, aligned, remaining; \
35
+
DECLARE_KERNEL_FPU_ONSTACK16(vxstate); \
36
+
\
37
+
if (datalen < VX_MIN_LEN + VX_ALIGN_MASK || \
38
+
!static_branch_likely(&have_vxrs)) \
39
+
return ___crc32_sw(crc, data, datalen); \
40
+
\
41
+
if ((unsigned long)data & VX_ALIGN_MASK) { \
42
+
prealign = VX_ALIGNMENT - \
43
+
((unsigned long)data & VX_ALIGN_MASK); \
44
+
datalen -= prealign; \
45
+
crc = ___crc32_sw(crc, data, prealign); \
46
+
data = (void *)((unsigned long)data + prealign); \
47
+
} \
48
+
\
49
+
aligned = datalen & ~VX_ALIGN_MASK; \
50
+
remaining = datalen & VX_ALIGN_MASK; \
51
+
\
52
+
kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \
53
+
crc = ___crc32_vx(crc, data, aligned); \
54
+
kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \
55
+
\
56
+
if (remaining) \
57
+
crc = ___crc32_sw(crc, data + aligned, remaining); \
58
+
\
59
+
return crc; \
60
+
} \
61
+
EXPORT_SYMBOL(___fname);
62
+
63
+
DEFINE_CRC32_VX(crc32_le_arch, crc32_le_vgfm_16, crc32_le_base)
64
+
DEFINE_CRC32_VX(crc32_be_arch, crc32_be_vgfm_16, crc32_be_base)
65
+
DEFINE_CRC32_VX(crc32c_le_arch, crc32c_le_vgfm_16, crc32c_le_base)
66
+
67
+
static int __init crc32_s390_init(void)
68
+
{
69
+
if (cpu_have_feature(S390_CPU_FEATURE_VXRS))
70
+
static_branch_enable(&have_vxrs);
71
+
return 0;
72
+
}
73
+
arch_initcall(crc32_s390_init);
74
+
75
+
static void __exit crc32_s390_exit(void)
76
+
{
77
+
}
78
+
module_exit(crc32_s390_exit);
79
+
80
+
u32 crc32_optimizations(void)
81
+
{
82
+
if (static_key_enabled(&have_vxrs))
83
+
return CRC32_LE_OPTIMIZATION |
84
+
CRC32_BE_OPTIMIZATION |
85
+
CRC32C_OPTIMIZATION;
86
+
return 0;
87
+
}
88
+
EXPORT_SYMBOL(crc32_optimizations);
89
+
90
+
MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
91
+
MODULE_DESCRIPTION("CRC-32 algorithms using z/Architecture Vector Extension Facility");
92
+
MODULE_LICENSE("GPL");