Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
crypto: x86/sha256 - implement library instead of shash
Instead of providing crypto_shash algorithms for the arch-optimized
SHA-256 code, instead implement the SHA-256 library. This is much
simpler, it makes the SHA-256 library functions be arch-optimized, and
it fixes the longstanding issue where the arch-optimized SHA-256 was
disabled by default. SHA-256 still remains available through
crypto_shash, but individual architectures no longer need to handle it.
To match sha256_blocks_arch(), change the type of the nblocks parameter
of the assembly functions from int to size_t. The assembly functions
actually already treated it as size_t.
Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
authored by
Eric Biggers
and committed by
Herbert Xu
11d7956d
699618d4
+118
-489
-14
arch/x86/crypto/Kconfig
-14
arch/x86/crypto/Kconfig
···
390
390
- AVX2 (Advanced Vector Extensions 2)
391
391
- SHA-NI (SHA Extensions New Instructions)
392
392
393
-
config CRYPTO_SHA256_SSSE3
394
-
tristate "Hash functions: SHA-224 and SHA-256 (SSSE3/AVX/AVX2/SHA-NI)"
395
-
depends on 64BIT
396
-
select CRYPTO_SHA256
397
-
select CRYPTO_HASH
398
-
help
399
-
SHA-224 and SHA-256 secure hash algorithms (FIPS 180)
400
-
401
-
Architecture: x86_64 using:
402
-
- SSSE3 (Supplemental SSE3)
403
-
- AVX (Advanced Vector Extensions)
404
-
- AVX2 (Advanced Vector Extensions 2)
405
-
- SHA-NI (SHA Extensions New Instructions)
406
-
407
393
config CRYPTO_SHA512_SSSE3
408
394
tristate "Hash functions: SHA-384 and SHA-512 (SSSE3/AVX/AVX2)"
409
395
depends on 64BIT
-3
arch/x86/crypto/Makefile
-3
arch/x86/crypto/Makefile
···
54
54
obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o
55
55
sha1-ssse3-y := sha1_avx2_x86_64_asm.o sha1_ssse3_asm.o sha1_ni_asm.o sha1_ssse3_glue.o
56
56
57
-
obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o
58
-
sha256-ssse3-y := sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256_ni_asm.o sha256_ssse3_glue.o
59
-
60
57
obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o
61
58
sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o
62
59
+6
-6
arch/x86/crypto/sha256-avx-asm.S
arch/x86/lib/crypto/sha256-avx-asm.S
+6
-6
arch/x86/crypto/sha256-avx-asm.S
arch/x86/lib/crypto/sha256-avx-asm.S
···
48
48
########################################################################
49
49
50
50
#include <linux/linkage.h>
51
-
#include <linux/cfi_types.h>
51
+
#include <linux/objtool.h>
52
52
53
53
## assume buffers not aligned
54
54
#define VMOVDQ vmovdqu
···
341
341
.endm
342
342
343
343
########################################################################
344
-
## void sha256_transform_avx(state sha256_state *state, const u8 *data, int blocks)
345
-
## arg 1 : pointer to state
346
-
## arg 2 : pointer to input data
347
-
## arg 3 : Num blocks
344
+
## void sha256_transform_avx(u32 state[SHA256_STATE_WORDS],
345
+
## const u8 *data, size_t nblocks);
348
346
########################################################################
349
347
.text
350
-
SYM_TYPED_FUNC_START(sha256_transform_avx)
348
+
SYM_FUNC_START(sha256_transform_avx)
349
+
ANNOTATE_NOENDBR # since this is called only via static_call
350
+
351
351
pushq %rbx
352
352
pushq %r12
353
353
pushq %r13
+6
-6
arch/x86/crypto/sha256-avx2-asm.S
arch/x86/lib/crypto/sha256-avx2-asm.S
+6
-6
arch/x86/crypto/sha256-avx2-asm.S
arch/x86/lib/crypto/sha256-avx2-asm.S
···
49
49
########################################################################
50
50
51
51
#include <linux/linkage.h>
52
-
#include <linux/cfi_types.h>
52
+
#include <linux/objtool.h>
53
53
54
54
## assume buffers not aligned
55
55
#define VMOVDQ vmovdqu
···
518
518
.endm
519
519
520
520
########################################################################
521
-
## void sha256_transform_rorx(struct sha256_state *state, const u8 *data, int blocks)
522
-
## arg 1 : pointer to state
523
-
## arg 2 : pointer to input data
524
-
## arg 3 : Num blocks
521
+
## void sha256_transform_rorx(u32 state[SHA256_STATE_WORDS],
522
+
## const u8 *data, size_t nblocks);
525
523
########################################################################
526
524
.text
527
-
SYM_TYPED_FUNC_START(sha256_transform_rorx)
525
+
SYM_FUNC_START(sha256_transform_rorx)
526
+
ANNOTATE_NOENDBR # since this is called only via static_call
527
+
528
528
pushq %rbx
529
529
pushq %r12
530
530
pushq %r13
+6
-8
arch/x86/crypto/sha256-ssse3-asm.S
arch/x86/lib/crypto/sha256-ssse3-asm.S
+6
-8
arch/x86/crypto/sha256-ssse3-asm.S
arch/x86/lib/crypto/sha256-ssse3-asm.S
···
47
47
########################################################################
48
48
49
49
#include <linux/linkage.h>
50
-
#include <linux/cfi_types.h>
50
+
#include <linux/objtool.h>
51
51
52
52
## assume buffers not aligned
53
53
#define MOVDQ movdqu
···
348
348
.endm
349
349
350
350
########################################################################
351
-
## void sha256_transform_ssse3(struct sha256_state *state, const u8 *data,
352
-
## int blocks);
353
-
## arg 1 : pointer to state
354
-
## (struct sha256_state is assumed to begin with u32 state[8])
355
-
## arg 2 : pointer to input data
356
-
## arg 3 : Num blocks
351
+
## void sha256_transform_ssse3(u32 state[SHA256_STATE_WORDS],
352
+
## const u8 *data, size_t nblocks);
357
353
########################################################################
358
354
.text
359
-
SYM_TYPED_FUNC_START(sha256_transform_ssse3)
355
+
SYM_FUNC_START(sha256_transform_ssse3)
356
+
ANNOTATE_NOENDBR # since this is called only via static_call
357
+
360
358
pushq %rbx
361
359
pushq %r12
362
360
pushq %r13
+16
-20
arch/x86/crypto/sha256_ni_asm.S
arch/x86/lib/crypto/sha256-ni-asm.S
+16
-20
arch/x86/crypto/sha256_ni_asm.S
arch/x86/lib/crypto/sha256-ni-asm.S
···
54
54
*/
55
55
56
56
#include <linux/linkage.h>
57
-
#include <linux/cfi_types.h>
57
+
#include <linux/objtool.h>
58
58
59
-
#define DIGEST_PTR %rdi /* 1st arg */
59
+
#define STATE_PTR %rdi /* 1st arg */
60
60
#define DATA_PTR %rsi /* 2nd arg */
61
61
#define NUM_BLKS %rdx /* 3rd arg */
62
62
···
98
98
.endm
99
99
100
100
/*
101
-
* Intel SHA Extensions optimized implementation of a SHA-256 update function
101
+
* Intel SHA Extensions optimized implementation of a SHA-256 block function
102
102
*
103
-
* The function takes a pointer to the current hash values, a pointer to the
104
-
* input data, and a number of 64 byte blocks to process. Once all blocks have
105
-
* been processed, the digest pointer is updated with the resulting hash value.
106
-
* The function only processes complete blocks, there is no functionality to
107
-
* store partial blocks. All message padding and hash value initialization must
108
-
* be done outside the update function.
103
+
* This function takes a pointer to the current SHA-256 state, a pointer to the
104
+
* input data, and the number of 64-byte blocks to process. Once all blocks
105
+
* have been processed, the state is updated with the new state. This function
106
+
* only processes complete blocks. State initialization, buffering of partial
107
+
* blocks, and digest finalization is expected to be handled elsewhere.
109
108
*
110
-
* void sha256_ni_transform(uint32_t *digest, const void *data,
111
-
uint32_t numBlocks);
112
-
* digest : pointer to digest
113
-
* data: pointer to input data
114
-
* numBlocks: Number of blocks to process
109
+
* void sha256_ni_transform(u32 state[SHA256_STATE_WORDS],
110
+
* const u8 *data, size_t nblocks);
115
111
*/
116
-
117
112
.text
118
-
SYM_TYPED_FUNC_START(sha256_ni_transform)
113
+
SYM_FUNC_START(sha256_ni_transform)
114
+
ANNOTATE_NOENDBR # since this is called only via static_call
119
115
120
116
shl $6, NUM_BLKS /* convert to bytes */
121
117
jz .Ldone_hash
···
122
126
* Need to reorder these appropriately
123
127
* DCBA, HGFE -> ABEF, CDGH
124
128
*/
125
-
movdqu 0*16(DIGEST_PTR), STATE0 /* DCBA */
126
-
movdqu 1*16(DIGEST_PTR), STATE1 /* HGFE */
129
+
movdqu 0*16(STATE_PTR), STATE0 /* DCBA */
130
+
movdqu 1*16(STATE_PTR), STATE1 /* HGFE */
127
131
128
132
movdqa STATE0, TMP
129
133
punpcklqdq STATE1, STATE0 /* FEBA */
···
162
166
pshufd $0xB1, STATE0, STATE0 /* HGFE */
163
167
pshufd $0x1B, STATE1, STATE1 /* DCBA */
164
168
165
-
movdqu STATE1, 0*16(DIGEST_PTR)
166
-
movdqu STATE0, 1*16(DIGEST_PTR)
169
+
movdqu STATE1, 0*16(STATE_PTR)
170
+
movdqu STATE0, 1*16(STATE_PTR)
167
171
168
172
.Ldone_hash:
169
173
-432
arch/x86/crypto/sha256_ssse3_glue.c
-432
arch/x86/crypto/sha256_ssse3_glue.c
···
1
-
/*
2
-
* Cryptographic API.
3
-
*
4
-
* Glue code for the SHA256 Secure Hash Algorithm assembler implementations
5
-
* using SSSE3, AVX, AVX2, and SHA-NI instructions.
6
-
*
7
-
* This file is based on sha256_generic.c
8
-
*
9
-
* Copyright (C) 2013 Intel Corporation.
10
-
*
11
-
* Author:
12
-
* Tim Chen <tim.c.chen@linux.intel.com>
13
-
*
14
-
* This program is free software; you can redistribute it and/or modify it
15
-
* under the terms of the GNU General Public License as published by the Free
16
-
* Software Foundation; either version 2 of the License, or (at your option)
17
-
* any later version.
18
-
*
19
-
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
20
-
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21
-
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
22
-
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
23
-
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
24
-
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
25
-
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26
-
* SOFTWARE.
27
-
*/
28
-
29
-
30
-
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
31
-
32
-
#include <asm/cpu_device_id.h>
33
-
#include <asm/fpu/api.h>
34
-
#include <crypto/internal/hash.h>
35
-
#include <crypto/sha2.h>
36
-
#include <crypto/sha256_base.h>
37
-
#include <linux/kernel.h>
38
-
#include <linux/module.h>
39
-
40
-
asmlinkage void sha256_transform_ssse3(struct crypto_sha256_state *state,
41
-
const u8 *data, int blocks);
42
-
43
-
static const struct x86_cpu_id module_cpu_ids[] = {
44
-
X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL),
45
-
X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL),
46
-
X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL),
47
-
X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL),
48
-
{}
49
-
};
50
-
MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids);
51
-
52
-
static int _sha256_update(struct shash_desc *desc, const u8 *data,
53
-
unsigned int len,
54
-
sha256_block_fn *sha256_xform)
55
-
{
56
-
int remain;
57
-
58
-
/*
59
-
* Make sure struct crypto_sha256_state begins directly with the SHA256
60
-
* 256-bit internal state, as this is what the asm functions expect.
61
-
*/
62
-
BUILD_BUG_ON(offsetof(struct crypto_sha256_state, state) != 0);
63
-
64
-
kernel_fpu_begin();
65
-
remain = sha256_base_do_update_blocks(desc, data, len, sha256_xform);
66
-
kernel_fpu_end();
67
-
68
-
return remain;
69
-
}
70
-
71
-
static int sha256_finup(struct shash_desc *desc, const u8 *data,
72
-
unsigned int len, u8 *out, sha256_block_fn *sha256_xform)
73
-
{
74
-
kernel_fpu_begin();
75
-
sha256_base_do_finup(desc, data, len, sha256_xform);
76
-
kernel_fpu_end();
77
-
78
-
return sha256_base_finish(desc, out);
79
-
}
80
-
81
-
static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data,
82
-
unsigned int len)
83
-
{
84
-
return _sha256_update(desc, data, len, sha256_transform_ssse3);
85
-
}
86
-
87
-
static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data,
88
-
unsigned int len, u8 *out)
89
-
{
90
-
return sha256_finup(desc, data, len, out, sha256_transform_ssse3);
91
-
}
92
-
93
-
static int sha256_ssse3_digest(struct shash_desc *desc, const u8 *data,
94
-
unsigned int len, u8 *out)
95
-
{
96
-
return sha256_base_init(desc) ?:
97
-
sha256_ssse3_finup(desc, data, len, out);
98
-
}
99
-
100
-
static struct shash_alg sha256_ssse3_algs[] = { {
101
-
.digestsize = SHA256_DIGEST_SIZE,
102
-
.init = sha256_base_init,
103
-
.update = sha256_ssse3_update,
104
-
.finup = sha256_ssse3_finup,
105
-
.digest = sha256_ssse3_digest,
106
-
.descsize = sizeof(struct crypto_sha256_state),
107
-
.base = {
108
-
.cra_name = "sha256",
109
-
.cra_driver_name = "sha256-ssse3",
110
-
.cra_priority = 150,
111
-
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
112
-
CRYPTO_AHASH_ALG_FINUP_MAX,
113
-
.cra_blocksize = SHA256_BLOCK_SIZE,
114
-
.cra_module = THIS_MODULE,
115
-
}
116
-
}, {
117
-
.digestsize = SHA224_DIGEST_SIZE,
118
-
.init = sha224_base_init,
119
-
.update = sha256_ssse3_update,
120
-
.finup = sha256_ssse3_finup,
121
-
.descsize = sizeof(struct crypto_sha256_state),
122
-
.base = {
123
-
.cra_name = "sha224",
124
-
.cra_driver_name = "sha224-ssse3",
125
-
.cra_priority = 150,
126
-
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
127
-
CRYPTO_AHASH_ALG_FINUP_MAX,
128
-
.cra_blocksize = SHA224_BLOCK_SIZE,
129
-
.cra_module = THIS_MODULE,
130
-
}
131
-
} };
132
-
133
-
static int register_sha256_ssse3(void)
134
-
{
135
-
if (boot_cpu_has(X86_FEATURE_SSSE3))
136
-
return crypto_register_shashes(sha256_ssse3_algs,
137
-
ARRAY_SIZE(sha256_ssse3_algs));
138
-
return 0;
139
-
}
140
-
141
-
static void unregister_sha256_ssse3(void)
142
-
{
143
-
if (boot_cpu_has(X86_FEATURE_SSSE3))
144
-
crypto_unregister_shashes(sha256_ssse3_algs,
145
-
ARRAY_SIZE(sha256_ssse3_algs));
146
-
}
147
-
148
-
asmlinkage void sha256_transform_avx(struct crypto_sha256_state *state,
149
-
const u8 *data, int blocks);
150
-
151
-
static int sha256_avx_update(struct shash_desc *desc, const u8 *data,
152
-
unsigned int len)
153
-
{
154
-
return _sha256_update(desc, data, len, sha256_transform_avx);
155
-
}
156
-
157
-
static int sha256_avx_finup(struct shash_desc *desc, const u8 *data,
158
-
unsigned int len, u8 *out)
159
-
{
160
-
return sha256_finup(desc, data, len, out, sha256_transform_avx);
161
-
}
162
-
163
-
static int sha256_avx_digest(struct shash_desc *desc, const u8 *data,
164
-
unsigned int len, u8 *out)
165
-
{
166
-
return sha256_base_init(desc) ?:
167
-
sha256_avx_finup(desc, data, len, out);
168
-
}
169
-
170
-
static struct shash_alg sha256_avx_algs[] = { {
171
-
.digestsize = SHA256_DIGEST_SIZE,
172
-
.init = sha256_base_init,
173
-
.update = sha256_avx_update,
174
-
.finup = sha256_avx_finup,
175
-
.digest = sha256_avx_digest,
176
-
.descsize = sizeof(struct crypto_sha256_state),
177
-
.base = {
178
-
.cra_name = "sha256",
179
-
.cra_driver_name = "sha256-avx",
180
-
.cra_priority = 160,
181
-
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
182
-
CRYPTO_AHASH_ALG_FINUP_MAX,
183
-
.cra_blocksize = SHA256_BLOCK_SIZE,
184
-
.cra_module = THIS_MODULE,
185
-
}
186
-
}, {
187
-
.digestsize = SHA224_DIGEST_SIZE,
188
-
.init = sha224_base_init,
189
-
.update = sha256_avx_update,
190
-
.finup = sha256_avx_finup,
191
-
.descsize = sizeof(struct crypto_sha256_state),
192
-
.base = {
193
-
.cra_name = "sha224",
194
-
.cra_driver_name = "sha224-avx",
195
-
.cra_priority = 160,
196
-
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
197
-
CRYPTO_AHASH_ALG_FINUP_MAX,
198
-
.cra_blocksize = SHA224_BLOCK_SIZE,
199
-
.cra_module = THIS_MODULE,
200
-
}
201
-
} };
202
-
203
-
static bool avx_usable(void)
204
-
{
205
-
if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
206
-
if (boot_cpu_has(X86_FEATURE_AVX))
207
-
pr_info("AVX detected but unusable.\n");
208
-
return false;
209
-
}
210
-
211
-
return true;
212
-
}
213
-
214
-
static int register_sha256_avx(void)
215
-
{
216
-
if (avx_usable())
217
-
return crypto_register_shashes(sha256_avx_algs,
218
-
ARRAY_SIZE(sha256_avx_algs));
219
-
return 0;
220
-
}
221
-
222
-
static void unregister_sha256_avx(void)
223
-
{
224
-
if (avx_usable())
225
-
crypto_unregister_shashes(sha256_avx_algs,
226
-
ARRAY_SIZE(sha256_avx_algs));
227
-
}
228
-
229
-
asmlinkage void sha256_transform_rorx(struct crypto_sha256_state *state,
230
-
const u8 *data, int blocks);
231
-
232
-
static int sha256_avx2_update(struct shash_desc *desc, const u8 *data,
233
-
unsigned int len)
234
-
{
235
-
return _sha256_update(desc, data, len, sha256_transform_rorx);
236
-
}
237
-
238
-
static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data,
239
-
unsigned int len, u8 *out)
240
-
{
241
-
return sha256_finup(desc, data, len, out, sha256_transform_rorx);
242
-
}
243
-
244
-
static int sha256_avx2_digest(struct shash_desc *desc, const u8 *data,
245
-
unsigned int len, u8 *out)
246
-
{
247
-
return sha256_base_init(desc) ?:
248
-
sha256_avx2_finup(desc, data, len, out);
249
-
}
250
-
251
-
static struct shash_alg sha256_avx2_algs[] = { {
252
-
.digestsize = SHA256_DIGEST_SIZE,
253
-
.init = sha256_base_init,
254
-
.update = sha256_avx2_update,
255
-
.finup = sha256_avx2_finup,
256
-
.digest = sha256_avx2_digest,
257
-
.descsize = sizeof(struct crypto_sha256_state),
258
-
.base = {
259
-
.cra_name = "sha256",
260
-
.cra_driver_name = "sha256-avx2",
261
-
.cra_priority = 170,
262
-
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
263
-
CRYPTO_AHASH_ALG_FINUP_MAX,
264
-
.cra_blocksize = SHA256_BLOCK_SIZE,
265
-
.cra_module = THIS_MODULE,
266
-
}
267
-
}, {
268
-
.digestsize = SHA224_DIGEST_SIZE,
269
-
.init = sha224_base_init,
270
-
.update = sha256_avx2_update,
271
-
.finup = sha256_avx2_finup,
272
-
.descsize = sizeof(struct crypto_sha256_state),
273
-
.base = {
274
-
.cra_name = "sha224",
275
-
.cra_driver_name = "sha224-avx2",
276
-
.cra_priority = 170,
277
-
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
278
-
CRYPTO_AHASH_ALG_FINUP_MAX,
279
-
.cra_blocksize = SHA224_BLOCK_SIZE,
280
-
.cra_module = THIS_MODULE,
281
-
}
282
-
} };
283
-
284
-
static bool avx2_usable(void)
285
-
{
286
-
if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
287
-
boot_cpu_has(X86_FEATURE_BMI2))
288
-
return true;
289
-
290
-
return false;
291
-
}
292
-
293
-
static int register_sha256_avx2(void)
294
-
{
295
-
if (avx2_usable())
296
-
return crypto_register_shashes(sha256_avx2_algs,
297
-
ARRAY_SIZE(sha256_avx2_algs));
298
-
return 0;
299
-
}
300
-
301
-
static void unregister_sha256_avx2(void)
302
-
{
303
-
if (avx2_usable())
304
-
crypto_unregister_shashes(sha256_avx2_algs,
305
-
ARRAY_SIZE(sha256_avx2_algs));
306
-
}
307
-
308
-
asmlinkage void sha256_ni_transform(struct crypto_sha256_state *digest,
309
-
const u8 *data, int rounds);
310
-
311
-
static int sha256_ni_update(struct shash_desc *desc, const u8 *data,
312
-
unsigned int len)
313
-
{
314
-
return _sha256_update(desc, data, len, sha256_ni_transform);
315
-
}
316
-
317
-
static int sha256_ni_finup(struct shash_desc *desc, const u8 *data,
318
-
unsigned int len, u8 *out)
319
-
{
320
-
return sha256_finup(desc, data, len, out, sha256_ni_transform);
321
-
}
322
-
323
-
static int sha256_ni_digest(struct shash_desc *desc, const u8 *data,
324
-
unsigned int len, u8 *out)
325
-
{
326
-
return sha256_base_init(desc) ?:
327
-
sha256_ni_finup(desc, data, len, out);
328
-
}
329
-
330
-
static struct shash_alg sha256_ni_algs[] = { {
331
-
.digestsize = SHA256_DIGEST_SIZE,
332
-
.init = sha256_base_init,
333
-
.update = sha256_ni_update,
334
-
.finup = sha256_ni_finup,
335
-
.digest = sha256_ni_digest,
336
-
.descsize = sizeof(struct crypto_sha256_state),
337
-
.base = {
338
-
.cra_name = "sha256",
339
-
.cra_driver_name = "sha256-ni",
340
-
.cra_priority = 250,
341
-
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
342
-
CRYPTO_AHASH_ALG_FINUP_MAX,
343
-
.cra_blocksize = SHA256_BLOCK_SIZE,
344
-
.cra_module = THIS_MODULE,
345
-
}
346
-
}, {
347
-
.digestsize = SHA224_DIGEST_SIZE,
348
-
.init = sha224_base_init,
349
-
.update = sha256_ni_update,
350
-
.finup = sha256_ni_finup,
351
-
.descsize = sizeof(struct crypto_sha256_state),
352
-
.base = {
353
-
.cra_name = "sha224",
354
-
.cra_driver_name = "sha224-ni",
355
-
.cra_priority = 250,
356
-
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
357
-
CRYPTO_AHASH_ALG_FINUP_MAX,
358
-
.cra_blocksize = SHA224_BLOCK_SIZE,
359
-
.cra_module = THIS_MODULE,
360
-
}
361
-
} };
362
-
363
-
static int register_sha256_ni(void)
364
-
{
365
-
if (boot_cpu_has(X86_FEATURE_SHA_NI))
366
-
return crypto_register_shashes(sha256_ni_algs,
367
-
ARRAY_SIZE(sha256_ni_algs));
368
-
return 0;
369
-
}
370
-
371
-
static void unregister_sha256_ni(void)
372
-
{
373
-
if (boot_cpu_has(X86_FEATURE_SHA_NI))
374
-
crypto_unregister_shashes(sha256_ni_algs,
375
-
ARRAY_SIZE(sha256_ni_algs));
376
-
}
377
-
378
-
static int __init sha256_ssse3_mod_init(void)
379
-
{
380
-
if (!x86_match_cpu(module_cpu_ids))
381
-
return -ENODEV;
382
-
383
-
if (register_sha256_ssse3())
384
-
goto fail;
385
-
386
-
if (register_sha256_avx()) {
387
-
unregister_sha256_ssse3();
388
-
goto fail;
389
-
}
390
-
391
-
if (register_sha256_avx2()) {
392
-
unregister_sha256_avx();
393
-
unregister_sha256_ssse3();
394
-
goto fail;
395
-
}
396
-
397
-
if (register_sha256_ni()) {
398
-
unregister_sha256_avx2();
399
-
unregister_sha256_avx();
400
-
unregister_sha256_ssse3();
401
-
goto fail;
402
-
}
403
-
404
-
return 0;
405
-
fail:
406
-
return -ENODEV;
407
-
}
408
-
409
-
static void __exit sha256_ssse3_mod_fini(void)
410
-
{
411
-
unregister_sha256_ni();
412
-
unregister_sha256_avx2();
413
-
unregister_sha256_avx();
414
-
unregister_sha256_ssse3();
415
-
}
416
-
417
-
module_init(sha256_ssse3_mod_init);
418
-
module_exit(sha256_ssse3_mod_fini);
419
-
420
-
MODULE_LICENSE("GPL");
421
-
MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated");
422
-
423
-
MODULE_ALIAS_CRYPTO("sha256");
424
-
MODULE_ALIAS_CRYPTO("sha256-ssse3");
425
-
MODULE_ALIAS_CRYPTO("sha256-avx");
426
-
MODULE_ALIAS_CRYPTO("sha256-avx2");
427
-
MODULE_ALIAS_CRYPTO("sha224");
428
-
MODULE_ALIAS_CRYPTO("sha224-ssse3");
429
-
MODULE_ALIAS_CRYPTO("sha224-avx");
430
-
MODULE_ALIAS_CRYPTO("sha224-avx2");
431
-
MODULE_ALIAS_CRYPTO("sha256-ni");
432
-
MODULE_ALIAS_CRYPTO("sha224-ni");
+7
arch/x86/lib/crypto/Kconfig
+7
arch/x86/lib/crypto/Kconfig
+3
arch/x86/lib/crypto/Makefile
+3
arch/x86/lib/crypto/Makefile
···
10
10
poly1305-x86_64-y := poly1305-x86_64-cryptogams.o poly1305_glue.o
11
11
targets += poly1305-x86_64-cryptogams.S
12
12
13
+
obj-$(CONFIG_CRYPTO_SHA256_X86_64) += sha256-x86_64.o
14
+
sha256-x86_64-y := sha256.o sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256-ni-asm.o
15
+
13
16
quiet_cmd_perlasm = PERLASM $@
14
17
cmd_perlasm = $(PERL) $< > $@
15
18
+74
arch/x86/lib/crypto/sha256.c
+74
arch/x86/lib/crypto/sha256.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-or-later
2
+
/*
3
+
* SHA-256 optimized for x86_64
4
+
*
5
+
* Copyright 2025 Google LLC
6
+
*/
7
+
#include <asm/fpu/api.h>
8
+
#include <crypto/internal/sha2.h>
9
+
#include <crypto/internal/simd.h>
10
+
#include <linux/kernel.h>
11
+
#include <linux/module.h>
12
+
#include <linux/static_call.h>
13
+
14
+
asmlinkage void sha256_transform_ssse3(u32 state[SHA256_STATE_WORDS],
15
+
const u8 *data, size_t nblocks);
16
+
asmlinkage void sha256_transform_avx(u32 state[SHA256_STATE_WORDS],
17
+
const u8 *data, size_t nblocks);
18
+
asmlinkage void sha256_transform_rorx(u32 state[SHA256_STATE_WORDS],
19
+
const u8 *data, size_t nblocks);
20
+
asmlinkage void sha256_ni_transform(u32 state[SHA256_STATE_WORDS],
21
+
const u8 *data, size_t nblocks);
22
+
23
+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_x86);
24
+
25
+
DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_transform_ssse3);
26
+
27
+
void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
28
+
const u8 *data, size_t nblocks)
29
+
{
30
+
if (static_branch_likely(&have_sha256_x86) && crypto_simd_usable()) {
31
+
kernel_fpu_begin();
32
+
static_call(sha256_blocks_x86)(state, data, nblocks);
33
+
kernel_fpu_end();
34
+
} else {
35
+
sha256_blocks_generic(state, data, nblocks);
36
+
}
37
+
}
38
+
EXPORT_SYMBOL(sha256_blocks_arch);
39
+
40
+
bool sha256_is_arch_optimized(void)
41
+
{
42
+
return static_key_enabled(&have_sha256_x86);
43
+
}
44
+
EXPORT_SYMBOL(sha256_is_arch_optimized);
45
+
46
+
static int __init sha256_x86_mod_init(void)
47
+
{
48
+
if (boot_cpu_has(X86_FEATURE_SHA_NI)) {
49
+
static_call_update(sha256_blocks_x86, sha256_ni_transform);
50
+
} else if (cpu_has_xfeatures(XFEATURE_MASK_SSE |
51
+
XFEATURE_MASK_YMM, NULL) &&
52
+
boot_cpu_has(X86_FEATURE_AVX)) {
53
+
if (boot_cpu_has(X86_FEATURE_AVX2) &&
54
+
boot_cpu_has(X86_FEATURE_BMI2))
55
+
static_call_update(sha256_blocks_x86,
56
+
sha256_transform_rorx);
57
+
else
58
+
static_call_update(sha256_blocks_x86,
59
+
sha256_transform_avx);
60
+
} else if (!boot_cpu_has(X86_FEATURE_SSSE3)) {
61
+
return 0;
62
+
}
63
+
static_branch_enable(&have_sha256_x86);
64
+
return 0;
65
+
}
66
+
arch_initcall(sha256_x86_mod_init);
67
+
68
+
static void __exit sha256_x86_mod_exit(void)
69
+
{
70
+
}
71
+
module_exit(sha256_x86_mod_exit);
72
+
73
+
MODULE_LICENSE("GPL");
74
+
MODULE_DESCRIPTION("SHA-256 optimized for x86_64");