Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
crypto: arm64/sha256 - Use API partial block handling
Use the Crypto API partial block handling.
Also remove the unnecessary SIMD fallback path.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
+37
-60
+37
-60
arch/arm64/crypto/sha256-glue.c
+37
-60
arch/arm64/crypto/sha256-glue.c
···
5
5
* Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org>
6
6
*/
7
7
8
-
#include <asm/hwcap.h>
9
8
#include <asm/neon.h>
10
-
#include <asm/simd.h>
11
9
#include <crypto/internal/hash.h>
12
-
#include <crypto/internal/simd.h>
13
10
#include <crypto/sha2.h>
14
11
#include <crypto/sha256_base.h>
12
+
#include <linux/cpufeature.h>
13
+
#include <linux/kernel.h>
15
14
#include <linux/module.h>
16
-
#include <linux/string.h>
17
-
#include <linux/types.h>
18
15
19
16
MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64");
20
17
MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>");
···
24
27
unsigned int num_blks);
25
28
EXPORT_SYMBOL(sha256_block_data_order);
26
29
27
-
static void sha256_arm64_transform(struct sha256_state *sst, u8 const *src,
28
-
int blocks)
30
+
static void sha256_arm64_transform(struct crypto_sha256_state *sst,
31
+
u8 const *src, int blocks)
29
32
{
30
33
sha256_block_data_order(sst->state, src, blocks);
31
34
}
···
33
36
asmlinkage void sha256_block_neon(u32 *digest, const void *data,
34
37
unsigned int num_blks);
35
38
36
-
static void sha256_neon_transform(struct sha256_state *sst, u8 const *src,
37
-
int blocks)
39
+
static void sha256_neon_transform(struct crypto_sha256_state *sst,
40
+
u8 const *src, int blocks)
38
41
{
42
+
kernel_neon_begin();
39
43
sha256_block_neon(sst->state, src, blocks);
44
+
kernel_neon_end();
40
45
}
41
46
42
47
static int crypto_sha256_arm64_update(struct shash_desc *desc, const u8 *data,
43
48
unsigned int len)
44
49
{
45
-
return sha256_base_do_update(desc, data, len, sha256_arm64_transform);
50
+
return sha256_base_do_update_blocks(desc, data, len,
51
+
sha256_arm64_transform);
46
52
}
47
53
48
54
static int crypto_sha256_arm64_finup(struct shash_desc *desc, const u8 *data,
49
55
unsigned int len, u8 *out)
50
56
{
51
-
if (len)
52
-
sha256_base_do_update(desc, data, len, sha256_arm64_transform);
53
-
sha256_base_do_finalize(desc, sha256_arm64_transform);
54
-
57
+
sha256_base_do_finup(desc, data, len, sha256_arm64_transform);
55
58
return sha256_base_finish(desc, out);
56
-
}
57
-
58
-
static int crypto_sha256_arm64_final(struct shash_desc *desc, u8 *out)
59
-
{
60
-
return crypto_sha256_arm64_finup(desc, NULL, 0, out);
61
59
}
62
60
63
61
static struct shash_alg algs[] = { {
64
62
.digestsize = SHA256_DIGEST_SIZE,
65
63
.init = sha256_base_init,
66
64
.update = crypto_sha256_arm64_update,
67
-
.final = crypto_sha256_arm64_final,
68
65
.finup = crypto_sha256_arm64_finup,
69
-
.descsize = sizeof(struct sha256_state),
66
+
.descsize = sizeof(struct crypto_sha256_state),
70
67
.base.cra_name = "sha256",
71
68
.base.cra_driver_name = "sha256-arm64",
72
69
.base.cra_priority = 125,
70
+
.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
71
+
CRYPTO_AHASH_ALG_FINUP_MAX,
73
72
.base.cra_blocksize = SHA256_BLOCK_SIZE,
74
73
.base.cra_module = THIS_MODULE,
75
74
}, {
76
75
.digestsize = SHA224_DIGEST_SIZE,
77
76
.init = sha224_base_init,
78
77
.update = crypto_sha256_arm64_update,
79
-
.final = crypto_sha256_arm64_final,
80
78
.finup = crypto_sha256_arm64_finup,
81
-
.descsize = sizeof(struct sha256_state),
79
+
.descsize = sizeof(struct crypto_sha256_state),
82
80
.base.cra_name = "sha224",
83
81
.base.cra_driver_name = "sha224-arm64",
84
82
.base.cra_priority = 125,
83
+
.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
84
+
CRYPTO_AHASH_ALG_FINUP_MAX,
85
85
.base.cra_blocksize = SHA224_BLOCK_SIZE,
86
86
.base.cra_module = THIS_MODULE,
87
87
} };
···
86
92
static int sha256_update_neon(struct shash_desc *desc, const u8 *data,
87
93
unsigned int len)
88
94
{
89
-
struct sha256_state *sctx = shash_desc_ctx(desc);
90
-
91
-
if (!crypto_simd_usable())
92
-
return sha256_base_do_update(desc, data, len,
93
-
sha256_arm64_transform);
94
-
95
-
while (len > 0) {
95
+
do {
96
96
unsigned int chunk = len;
97
97
98
98
/*
···
94
106
* input when running on a preemptible kernel, but process the
95
107
* data block by block instead.
96
108
*/
97
-
if (IS_ENABLED(CONFIG_PREEMPTION) &&
98
-
chunk + sctx->count % SHA256_BLOCK_SIZE > SHA256_BLOCK_SIZE)
99
-
chunk = SHA256_BLOCK_SIZE -
100
-
sctx->count % SHA256_BLOCK_SIZE;
109
+
if (IS_ENABLED(CONFIG_PREEMPTION))
110
+
chunk = SHA256_BLOCK_SIZE;
101
111
102
-
kernel_neon_begin();
103
-
sha256_base_do_update(desc, data, chunk, sha256_neon_transform);
104
-
kernel_neon_end();
112
+
chunk -= sha256_base_do_update_blocks(desc, data, chunk,
113
+
sha256_neon_transform);
105
114
data += chunk;
106
115
len -= chunk;
107
-
}
108
-
return 0;
116
+
} while (len >= SHA256_BLOCK_SIZE);
117
+
return len;
109
118
}
110
119
111
120
static int sha256_finup_neon(struct shash_desc *desc, const u8 *data,
112
121
unsigned int len, u8 *out)
113
122
{
114
-
if (!crypto_simd_usable()) {
115
-
if (len)
116
-
sha256_base_do_update(desc, data, len,
117
-
sha256_arm64_transform);
118
-
sha256_base_do_finalize(desc, sha256_arm64_transform);
119
-
} else {
120
-
if (len)
121
-
sha256_update_neon(desc, data, len);
122
-
kernel_neon_begin();
123
-
sha256_base_do_finalize(desc, sha256_neon_transform);
124
-
kernel_neon_end();
125
-
}
126
-
return sha256_base_finish(desc, out);
127
-
}
123
+
if (len >= SHA256_BLOCK_SIZE) {
124
+
int remain = sha256_update_neon(desc, data, len);
128
125
129
-
static int sha256_final_neon(struct shash_desc *desc, u8 *out)
130
-
{
131
-
return sha256_finup_neon(desc, NULL, 0, out);
126
+
data += len - remain;
127
+
len = remain;
128
+
}
129
+
sha256_base_do_finup(desc, data, len, sha256_neon_transform);
130
+
return sha256_base_finish(desc, out);
132
131
}
133
132
134
133
static struct shash_alg neon_algs[] = { {
135
134
.digestsize = SHA256_DIGEST_SIZE,
136
135
.init = sha256_base_init,
137
136
.update = sha256_update_neon,
138
-
.final = sha256_final_neon,
139
137
.finup = sha256_finup_neon,
140
-
.descsize = sizeof(struct sha256_state),
138
+
.descsize = sizeof(struct crypto_sha256_state),
141
139
.base.cra_name = "sha256",
142
140
.base.cra_driver_name = "sha256-arm64-neon",
143
141
.base.cra_priority = 150,
142
+
.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
143
+
CRYPTO_AHASH_ALG_FINUP_MAX,
144
144
.base.cra_blocksize = SHA256_BLOCK_SIZE,
145
145
.base.cra_module = THIS_MODULE,
146
146
}, {
147
147
.digestsize = SHA224_DIGEST_SIZE,
148
148
.init = sha224_base_init,
149
149
.update = sha256_update_neon,
150
-
.final = sha256_final_neon,
151
150
.finup = sha256_finup_neon,
152
-
.descsize = sizeof(struct sha256_state),
151
+
.descsize = sizeof(struct crypto_sha256_state),
153
152
.base.cra_name = "sha224",
154
153
.base.cra_driver_name = "sha224-arm64-neon",
155
154
.base.cra_priority = 150,
155
+
.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
156
+
CRYPTO_AHASH_ALG_FINUP_MAX,
156
157
.base.cra_blocksize = SHA224_BLOCK_SIZE,
157
158
.base.cra_module = THIS_MODULE,
158
159
} };