Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
crypto: arm64/ghash - Use API partial block handling
Use the Crypto API partial block handling.
Also remove the unnecessary SIMD fallback path.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
+60
-91
+60
-91
arch/arm64/crypto/ghash-ce-glue.c
+60
-91
arch/arm64/crypto/ghash-ce-glue.c
···
6
6
*/
7
7
8
8
#include <asm/neon.h>
9
-
#include <asm/simd.h>
10
-
#include <linux/unaligned.h>
11
9
#include <crypto/aes.h>
12
-
#include <crypto/gcm.h>
13
-
#include <crypto/algapi.h>
14
10
#include <crypto/b128ops.h>
11
+
#include <crypto/gcm.h>
12
+
#include <crypto/ghash.h>
15
13
#include <crypto/gf128mul.h>
16
14
#include <crypto/internal/aead.h>
17
15
#include <crypto/internal/hash.h>
18
-
#include <crypto/internal/simd.h>
19
16
#include <crypto/internal/skcipher.h>
20
17
#include <crypto/scatterwalk.h>
21
18
#include <linux/cpufeature.h>
22
-
#include <linux/crypto.h>
19
+
#include <linux/errno.h>
20
+
#include <linux/kernel.h>
23
21
#include <linux/module.h>
22
+
#include <linux/string.h>
23
+
#include <linux/unaligned.h>
24
24
25
25
MODULE_DESCRIPTION("GHASH and AES-GCM using ARMv8 Crypto Extensions");
26
26
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
27
27
MODULE_LICENSE("GPL v2");
28
28
MODULE_ALIAS_CRYPTO("ghash");
29
-
30
-
#define GHASH_BLOCK_SIZE 16
31
-
#define GHASH_DIGEST_SIZE 16
32
29
33
30
#define RFC4106_NONCE_SIZE 4
34
31
···
34
37
u64 h[][2];
35
38
};
36
39
37
-
struct ghash_desc_ctx {
40
+
struct arm_ghash_desc_ctx {
38
41
u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
39
-
u8 buf[GHASH_BLOCK_SIZE];
40
-
u32 count;
41
42
};
42
43
43
44
struct gcm_aes_ctx {
···
60
65
61
66
static int ghash_init(struct shash_desc *desc)
62
67
{
63
-
struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
68
+
struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
64
69
65
-
*ctx = (struct ghash_desc_ctx){};
70
+
*ctx = (struct arm_ghash_desc_ctx){};
66
71
return 0;
67
-
}
68
-
69
-
static void ghash_do_update(int blocks, u64 dg[], const char *src,
70
-
struct ghash_key *key, const char *head)
71
-
{
72
-
be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) };
73
-
74
-
do {
75
-
const u8 *in = src;
76
-
77
-
if (head) {
78
-
in = head;
79
-
blocks++;
80
-
head = NULL;
81
-
} else {
82
-
src += GHASH_BLOCK_SIZE;
83
-
}
84
-
85
-
crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE);
86
-
gf128mul_lle(&dst, &key->k);
87
-
} while (--blocks);
88
-
89
-
dg[0] = be64_to_cpu(dst.b);
90
-
dg[1] = be64_to_cpu(dst.a);
91
72
}
92
73
93
74
static __always_inline
···
74
103
u64 const h[][2],
75
104
const char *head))
76
105
{
77
-
if (likely(crypto_simd_usable())) {
78
-
kernel_neon_begin();
79
-
simd_update(blocks, dg, src, key->h, head);
80
-
kernel_neon_end();
81
-
} else {
82
-
ghash_do_update(blocks, dg, src, key, head);
83
-
}
106
+
kernel_neon_begin();
107
+
simd_update(blocks, dg, src, key->h, head);
108
+
kernel_neon_end();
84
109
}
85
110
86
111
/* avoid hogging the CPU for too long */
···
85
118
static int ghash_update(struct shash_desc *desc, const u8 *src,
86
119
unsigned int len)
87
120
{
88
-
struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
89
-
unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
121
+
struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
122
+
struct ghash_key *key = crypto_shash_ctx(desc->tfm);
123
+
int blocks;
90
124
91
-
ctx->count += len;
125
+
blocks = len / GHASH_BLOCK_SIZE;
126
+
len -= blocks * GHASH_BLOCK_SIZE;
92
127
93
-
if ((partial + len) >= GHASH_BLOCK_SIZE) {
94
-
struct ghash_key *key = crypto_shash_ctx(desc->tfm);
95
-
int blocks;
128
+
do {
129
+
int chunk = min(blocks, MAX_BLOCKS);
96
130
97
-
if (partial) {
98
-
int p = GHASH_BLOCK_SIZE - partial;
131
+
ghash_do_simd_update(chunk, ctx->digest, src, key, NULL,
132
+
pmull_ghash_update_p8);
133
+
blocks -= chunk;
134
+
src += chunk * GHASH_BLOCK_SIZE;
135
+
} while (unlikely(blocks > 0));
136
+
return len;
137
+
}
99
138
100
-
memcpy(ctx->buf + partial, src, p);
101
-
src += p;
102
-
len -= p;
103
-
}
139
+
static int ghash_export(struct shash_desc *desc, void *out)
140
+
{
141
+
struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
142
+
u8 *dst = out;
104
143
105
-
blocks = len / GHASH_BLOCK_SIZE;
106
-
len %= GHASH_BLOCK_SIZE;
107
-
108
-
do {
109
-
int chunk = min(blocks, MAX_BLOCKS);
110
-
111
-
ghash_do_simd_update(chunk, ctx->digest, src, key,
112
-
partial ? ctx->buf : NULL,
113
-
pmull_ghash_update_p8);
114
-
115
-
blocks -= chunk;
116
-
src += chunk * GHASH_BLOCK_SIZE;
117
-
partial = 0;
118
-
} while (unlikely(blocks > 0));
119
-
}
120
-
if (len)
121
-
memcpy(ctx->buf + partial, src, len);
144
+
put_unaligned_be64(ctx->digest[1], dst);
145
+
put_unaligned_be64(ctx->digest[0], dst + 8);
122
146
return 0;
123
147
}
124
148
125
-
static int ghash_final(struct shash_desc *desc, u8 *dst)
149
+
static int ghash_import(struct shash_desc *desc, const void *in)
126
150
{
127
-
struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
128
-
unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
151
+
struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
152
+
const u8 *src = in;
129
153
130
-
if (partial) {
131
-
struct ghash_key *key = crypto_shash_ctx(desc->tfm);
132
-
133
-
memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
134
-
135
-
ghash_do_simd_update(1, ctx->digest, ctx->buf, key, NULL,
136
-
pmull_ghash_update_p8);
137
-
}
138
-
put_unaligned_be64(ctx->digest[1], dst);
139
-
put_unaligned_be64(ctx->digest[0], dst + 8);
140
-
141
-
memzero_explicit(ctx, sizeof(*ctx));
154
+
ctx->digest[1] = get_unaligned_be64(src);
155
+
ctx->digest[0] = get_unaligned_be64(src + 8);
142
156
return 0;
157
+
}
158
+
159
+
static int ghash_finup(struct shash_desc *desc, const u8 *src,
160
+
unsigned int len, u8 *dst)
161
+
{
162
+
struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
163
+
struct ghash_key *key = crypto_shash_ctx(desc->tfm);
164
+
165
+
if (len) {
166
+
u8 buf[GHASH_BLOCK_SIZE] = {};
167
+
168
+
memcpy(buf, src, len);
169
+
ghash_do_simd_update(1, ctx->digest, src, key, NULL,
170
+
pmull_ghash_update_p8);
171
+
memzero_explicit(buf, sizeof(buf));
172
+
}
173
+
return ghash_export(desc, dst);
143
174
}
144
175
145
176
static void ghash_reflect(u64 h[], const be128 *k)
···
170
205
.base.cra_name = "ghash",
171
206
.base.cra_driver_name = "ghash-neon",
172
207
.base.cra_priority = 150,
208
+
.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
173
209
.base.cra_blocksize = GHASH_BLOCK_SIZE,
174
210
.base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]),
175
211
.base.cra_module = THIS_MODULE,
···
178
212
.digestsize = GHASH_DIGEST_SIZE,
179
213
.init = ghash_init,
180
214
.update = ghash_update,
181
-
.final = ghash_final,
215
+
.finup = ghash_finup,
182
216
.setkey = ghash_setkey,
183
-
.descsize = sizeof(struct ghash_desc_ctx),
217
+
.export = ghash_export,
218
+
.import = ghash_import,
219
+
.descsize = sizeof(struct arm_ghash_desc_ctx),
220
+
.statesize = sizeof(struct ghash_desc_ctx),
184
221
};
185
222
186
223
static int num_rounds(struct crypto_aes_ctx *ctx)