Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
crypto: x86/sha512 - Use API partial block handling
Use the Crypto API partial block handling.
Also remove the unnecessary SIMD fallback path.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
+77
-57
+27
-52
arch/x86/crypto/sha512_ssse3_glue.c
+27
-52
arch/x86/crypto/sha512_ssse3_glue.c
···
27
27
28
28
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
29
29
30
-
#include <crypto/internal/hash.h>
31
-
#include <crypto/internal/simd.h>
32
-
#include <linux/init.h>
33
-
#include <linux/module.h>
34
-
#include <linux/mm.h>
35
-
#include <linux/string.h>
36
-
#include <linux/types.h>
37
-
#include <crypto/sha2.h>
38
-
#include <crypto/sha512_base.h>
39
30
#include <asm/cpu_device_id.h>
40
31
#include <asm/simd.h>
32
+
#include <crypto/internal/hash.h>
33
+
#include <linux/kernel.h>
34
+
#include <linux/module.h>
35
+
#include <crypto/sha2.h>
36
+
#include <crypto/sha512_base.h>
41
37
42
38
asmlinkage void sha512_transform_ssse3(struct sha512_state *state,
43
39
const u8 *data, int blocks);
···
41
45
static int sha512_update(struct shash_desc *desc, const u8 *data,
42
46
unsigned int len, sha512_block_fn *sha512_xform)
43
47
{
44
-
struct sha512_state *sctx = shash_desc_ctx(desc);
45
-
46
-
if (!crypto_simd_usable() ||
47
-
(sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE)
48
-
return crypto_sha512_update(desc, data, len);
48
+
int remain;
49
49
50
50
/*
51
51
* Make sure struct sha512_state begins directly with the SHA512
···
50
58
BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0);
51
59
52
60
kernel_fpu_begin();
53
-
sha512_base_do_update(desc, data, len, sha512_xform);
61
+
remain = sha512_base_do_update_blocks(desc, data, len, sha512_xform);
54
62
kernel_fpu_end();
55
63
56
-
return 0;
64
+
return remain;
57
65
}
58
66
59
67
static int sha512_finup(struct shash_desc *desc, const u8 *data,
60
68
unsigned int len, u8 *out, sha512_block_fn *sha512_xform)
61
69
{
62
-
if (!crypto_simd_usable())
63
-
return crypto_sha512_finup(desc, data, len, out);
64
-
65
70
kernel_fpu_begin();
66
-
if (len)
67
-
sha512_base_do_update(desc, data, len, sha512_xform);
68
-
sha512_base_do_finalize(desc, sha512_xform);
71
+
sha512_base_do_finup(desc, data, len, sha512_xform);
69
72
kernel_fpu_end();
70
73
71
74
return sha512_base_finish(desc, out);
···
78
91
return sha512_finup(desc, data, len, out, sha512_transform_ssse3);
79
92
}
80
93
81
-
/* Add padding and return the message digest. */
82
-
static int sha512_ssse3_final(struct shash_desc *desc, u8 *out)
83
-
{
84
-
return sha512_ssse3_finup(desc, NULL, 0, out);
85
-
}
86
-
87
94
static struct shash_alg sha512_ssse3_algs[] = { {
88
95
.digestsize = SHA512_DIGEST_SIZE,
89
96
.init = sha512_base_init,
90
97
.update = sha512_ssse3_update,
91
-
.final = sha512_ssse3_final,
92
98
.finup = sha512_ssse3_finup,
93
-
.descsize = sizeof(struct sha512_state),
99
+
.descsize = SHA512_STATE_SIZE,
94
100
.base = {
95
101
.cra_name = "sha512",
96
102
.cra_driver_name = "sha512-ssse3",
97
103
.cra_priority = 150,
104
+
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
105
+
CRYPTO_AHASH_ALG_FINUP_MAX,
98
106
.cra_blocksize = SHA512_BLOCK_SIZE,
99
107
.cra_module = THIS_MODULE,
100
108
}
···
97
115
.digestsize = SHA384_DIGEST_SIZE,
98
116
.init = sha384_base_init,
99
117
.update = sha512_ssse3_update,
100
-
.final = sha512_ssse3_final,
101
118
.finup = sha512_ssse3_finup,
102
-
.descsize = sizeof(struct sha512_state),
119
+
.descsize = SHA512_STATE_SIZE,
103
120
.base = {
104
121
.cra_name = "sha384",
105
122
.cra_driver_name = "sha384-ssse3",
106
123
.cra_priority = 150,
124
+
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
125
+
CRYPTO_AHASH_ALG_FINUP_MAX,
107
126
.cra_blocksize = SHA384_BLOCK_SIZE,
108
127
.cra_module = THIS_MODULE,
109
128
}
···
150
167
return sha512_finup(desc, data, len, out, sha512_transform_avx);
151
168
}
152
169
153
-
/* Add padding and return the message digest. */
154
-
static int sha512_avx_final(struct shash_desc *desc, u8 *out)
155
-
{
156
-
return sha512_avx_finup(desc, NULL, 0, out);
157
-
}
158
-
159
170
static struct shash_alg sha512_avx_algs[] = { {
160
171
.digestsize = SHA512_DIGEST_SIZE,
161
172
.init = sha512_base_init,
162
173
.update = sha512_avx_update,
163
-
.final = sha512_avx_final,
164
174
.finup = sha512_avx_finup,
165
-
.descsize = sizeof(struct sha512_state),
175
+
.descsize = SHA512_STATE_SIZE,
166
176
.base = {
167
177
.cra_name = "sha512",
168
178
.cra_driver_name = "sha512-avx",
169
179
.cra_priority = 160,
180
+
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
181
+
CRYPTO_AHASH_ALG_FINUP_MAX,
170
182
.cra_blocksize = SHA512_BLOCK_SIZE,
171
183
.cra_module = THIS_MODULE,
172
184
}
···
169
191
.digestsize = SHA384_DIGEST_SIZE,
170
192
.init = sha384_base_init,
171
193
.update = sha512_avx_update,
172
-
.final = sha512_avx_final,
173
194
.finup = sha512_avx_finup,
174
-
.descsize = sizeof(struct sha512_state),
195
+
.descsize = SHA512_STATE_SIZE,
175
196
.base = {
176
197
.cra_name = "sha384",
177
198
.cra_driver_name = "sha384-avx",
178
199
.cra_priority = 160,
200
+
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
201
+
CRYPTO_AHASH_ALG_FINUP_MAX,
179
202
.cra_blocksize = SHA384_BLOCK_SIZE,
180
203
.cra_module = THIS_MODULE,
181
204
}
···
212
233
return sha512_finup(desc, data, len, out, sha512_transform_rorx);
213
234
}
214
235
215
-
/* Add padding and return the message digest. */
216
-
static int sha512_avx2_final(struct shash_desc *desc, u8 *out)
217
-
{
218
-
return sha512_avx2_finup(desc, NULL, 0, out);
219
-
}
220
-
221
236
static struct shash_alg sha512_avx2_algs[] = { {
222
237
.digestsize = SHA512_DIGEST_SIZE,
223
238
.init = sha512_base_init,
224
239
.update = sha512_avx2_update,
225
-
.final = sha512_avx2_final,
226
240
.finup = sha512_avx2_finup,
227
-
.descsize = sizeof(struct sha512_state),
241
+
.descsize = SHA512_STATE_SIZE,
228
242
.base = {
229
243
.cra_name = "sha512",
230
244
.cra_driver_name = "sha512-avx2",
231
245
.cra_priority = 170,
246
+
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
247
+
CRYPTO_AHASH_ALG_FINUP_MAX,
232
248
.cra_blocksize = SHA512_BLOCK_SIZE,
233
249
.cra_module = THIS_MODULE,
234
250
}
···
231
257
.digestsize = SHA384_DIGEST_SIZE,
232
258
.init = sha384_base_init,
233
259
.update = sha512_avx2_update,
234
-
.final = sha512_avx2_final,
235
260
.finup = sha512_avx2_finup,
236
-
.descsize = sizeof(struct sha512_state),
261
+
.descsize = SHA512_STATE_SIZE,
237
262
.base = {
238
263
.cra_name = "sha384",
239
264
.cra_driver_name = "sha384-avx2",
240
265
.cra_priority = 170,
266
+
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
267
+
CRYPTO_AHASH_ALG_FINUP_MAX,
241
268
.cra_blocksize = SHA384_BLOCK_SIZE,
242
269
.cra_module = THIS_MODULE,
243
270
}
+1
include/crypto/sha2.h
+1
include/crypto/sha2.h
+49
-5
include/crypto/sha512_base.h
+49
-5
include/crypto/sha512_base.h
···
10
10
11
11
#include <crypto/internal/hash.h>
12
12
#include <crypto/sha2.h>
13
-
#include <linux/crypto.h>
14
-
#include <linux/module.h>
15
13
#include <linux/string.h>
16
-
17
14
#include <linux/unaligned.h>
18
15
19
16
typedef void (sha512_block_fn)(struct sha512_state *sst, u8 const *src,
···
90
93
return 0;
91
94
}
92
95
96
+
static inline int sha512_base_do_update_blocks(struct shash_desc *desc,
97
+
const u8 *data,
98
+
unsigned int len,
99
+
sha512_block_fn *block_fn)
100
+
{
101
+
unsigned int remain = len - round_down(len, SHA512_BLOCK_SIZE);
102
+
struct sha512_state *sctx = shash_desc_ctx(desc);
103
+
104
+
len -= remain;
105
+
sctx->count[0] += len;
106
+
if (sctx->count[0] < len)
107
+
sctx->count[1]++;
108
+
block_fn(sctx, data, len / SHA512_BLOCK_SIZE);
109
+
return remain;
110
+
}
111
+
112
+
static inline int sha512_base_do_finup(struct shash_desc *desc, const u8 *src,
113
+
unsigned int len,
114
+
sha512_block_fn *block_fn)
115
+
{
116
+
unsigned int bit_offset = SHA512_BLOCK_SIZE / 8 - 2;
117
+
struct sha512_state *sctx = shash_desc_ctx(desc);
118
+
union {
119
+
__be64 b64[SHA512_BLOCK_SIZE / 4];
120
+
u8 u8[SHA512_BLOCK_SIZE * 2];
121
+
} block = {};
122
+
123
+
if (len >= SHA512_BLOCK_SIZE) {
124
+
int remain;
125
+
126
+
remain = sha512_base_do_update_blocks(desc, src, len, block_fn);
127
+
src += len - remain;
128
+
len = remain;
129
+
}
130
+
131
+
if (len >= bit_offset * 8)
132
+
bit_offset += SHA512_BLOCK_SIZE / 8;
133
+
memcpy(&block, src, len);
134
+
block.u8[len] = 0x80;
135
+
sctx->count[0] += len;
136
+
block.b64[bit_offset] = cpu_to_be64(sctx->count[1] << 3 |
137
+
sctx->count[0] >> 61);
138
+
block.b64[bit_offset + 1] = cpu_to_be64(sctx->count[0] << 3);
139
+
block_fn(sctx, block.u8, (bit_offset + 2) * 8 / SHA512_BLOCK_SIZE);
140
+
memzero_explicit(&block, sizeof(block));
141
+
142
+
return 0;
143
+
}
144
+
93
145
static inline int sha512_base_do_finalize(struct shash_desc *desc,
94
146
sha512_block_fn *block_fn)
95
147
{
···
172
126
173
127
for (i = 0; digest_size > 0; i++, digest_size -= sizeof(__be64))
174
128
put_unaligned_be64(sctx->state[i], digest++);
175
-
176
-
memzero_explicit(sctx, sizeof(*sctx));
177
129
return 0;
178
130
}
179
131