Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
crypto: s390 - New s390 specific protected key hash phmac
Add support for protected key hmac ("phmac") for s390 arch.
With the latest machine generation there is now support for
protected key (that is a key wrapped by a master key stored
in firmware) hmac for sha2 (sha224, sha256, sha384 and sha512)
for the s390 specific CPACF instruction kmac.
This patch adds support via 4 new ahashes registered as
phmac(sha224), phmac(sha256), phmac(sha384) and phmac(sha512).
Co-developed-by: Holger Dengler <dengler@linux.ibm.com>
Signed-off-by: Harald Freudenberger <freude@linux.ibm.com>
Reviewed-by: Holger Dengler <dengler@linux.ibm.com>
Signed-off-by: Holger Dengler <dengler@linux.ibm.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
authored by
Harald Freudenberger
and committed by
Herbert Xu
cbbc6755
86ca5cb8
+927
+1
arch/s390/configs/debug_defconfig
+1
arch/s390/configs/debug_defconfig
+1
arch/s390/configs/defconfig
+1
arch/s390/configs/defconfig
+1
arch/s390/crypto/Makefile
+1
arch/s390/crypto/Makefile
+911
arch/s390/crypto/phmac_s390.c
+911
arch/s390/crypto/phmac_s390.c
···
1
+
// SPDX-License-Identifier: GPL-2.0+
2
+
/*
3
+
* Copyright IBM Corp. 2025
4
+
*
5
+
* s390 specific HMAC support for protected keys.
6
+
*/
7
+
8
+
#define KMSG_COMPONENT "phmac_s390"
9
+
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
10
+
11
+
#include <asm/cpacf.h>
12
+
#include <asm/pkey.h>
13
+
#include <crypto/engine.h>
14
+
#include <crypto/hash.h>
15
+
#include <crypto/internal/hash.h>
16
+
#include <crypto/sha2.h>
17
+
#include <linux/atomic.h>
18
+
#include <linux/cpufeature.h>
19
+
#include <linux/delay.h>
20
+
#include <linux/miscdevice.h>
21
+
#include <linux/module.h>
22
+
#include <linux/spinlock.h>
23
+
24
+
static struct crypto_engine *phmac_crypto_engine;
25
+
#define MAX_QLEN 10
26
+
27
+
/*
28
+
* A simple hash walk helper
29
+
*/
30
+
31
+
struct hash_walk_helper {
32
+
struct crypto_hash_walk walk;
33
+
const u8 *walkaddr;
34
+
int walkbytes;
35
+
};
36
+
37
+
/*
38
+
* Prepare hash walk helper.
39
+
* Set up the base hash walk, fill walkaddr and walkbytes.
40
+
* Returns 0 on success or negative value on error.
41
+
*/
42
+
static inline int hwh_prepare(struct ahash_request *req,
43
+
struct hash_walk_helper *hwh)
44
+
{
45
+
hwh->walkbytes = crypto_hash_walk_first(req, &hwh->walk);
46
+
if (hwh->walkbytes < 0)
47
+
return hwh->walkbytes;
48
+
hwh->walkaddr = hwh->walk.data;
49
+
return 0;
50
+
}
51
+
52
+
/*
53
+
* Advance hash walk helper by n bytes.
54
+
* Progress the walkbytes and walkaddr fields by n bytes.
55
+
* If walkbytes is then 0, pull next hunk from hash walk
56
+
* and update walkbytes and walkaddr.
57
+
* If n is negative, unmap hash walk and return error.
58
+
* Returns 0 on success or negative value on error.
59
+
*/
60
+
static inline int hwh_advance(struct hash_walk_helper *hwh, int n)
61
+
{
62
+
if (n < 0)
63
+
return crypto_hash_walk_done(&hwh->walk, n);
64
+
65
+
hwh->walkbytes -= n;
66
+
hwh->walkaddr += n;
67
+
if (hwh->walkbytes > 0)
68
+
return 0;
69
+
70
+
hwh->walkbytes = crypto_hash_walk_done(&hwh->walk, 0);
71
+
if (hwh->walkbytes < 0)
72
+
return hwh->walkbytes;
73
+
74
+
hwh->walkaddr = hwh->walk.data;
75
+
return 0;
76
+
}
77
+
78
+
/*
79
+
* KMAC param block layout for sha2 function codes:
80
+
* The layout of the param block for the KMAC instruction depends on the
81
+
* blocksize of the used hashing sha2-algorithm function codes. The param block
82
+
* contains the hash chaining value (cv), the input message bit-length (imbl)
83
+
* and the hmac-secret (key). To prevent code duplication, the sizes of all
84
+
* these are calculated based on the blocksize.
85
+
*
86
+
* param-block:
87
+
* +-------+
88
+
* | cv |
89
+
* +-------+
90
+
* | imbl |
91
+
* +-------+
92
+
* | key |
93
+
* +-------+
94
+
*
95
+
* sizes:
96
+
* part | sh2-alg | calculation | size | type
97
+
* -----+---------+-------------+------+--------
98
+
* cv | 224/256 | blocksize/2 | 32 | u64[8]
99
+
* | 384/512 | | 64 | u128[8]
100
+
* imbl | 224/256 | blocksize/8 | 8 | u64
101
+
* | 384/512 | | 16 | u128
102
+
* key | 224/256 | blocksize | 96 | u8[96]
103
+
* | 384/512 | | 160 | u8[160]
104
+
*/
105
+
106
+
#define MAX_DIGEST_SIZE SHA512_DIGEST_SIZE
107
+
#define MAX_IMBL_SIZE sizeof(u128)
108
+
#define MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
109
+
110
+
#define SHA2_CV_SIZE(bs) ((bs) >> 1)
111
+
#define SHA2_IMBL_SIZE(bs) ((bs) >> 3)
112
+
113
+
#define SHA2_IMBL_OFFSET(bs) (SHA2_CV_SIZE(bs))
114
+
#define SHA2_KEY_OFFSET(bs) (SHA2_CV_SIZE(bs) + SHA2_IMBL_SIZE(bs))
115
+
116
+
#define PHMAC_MAX_KEYSIZE 256
117
+
#define PHMAC_SHA256_PK_SIZE (SHA256_BLOCK_SIZE + 32)
118
+
#define PHMAC_SHA512_PK_SIZE (SHA512_BLOCK_SIZE + 32)
119
+
#define PHMAC_MAX_PK_SIZE PHMAC_SHA512_PK_SIZE
120
+
121
+
/* phmac protected key struct */
122
+
struct phmac_protkey {
123
+
u32 type;
124
+
u32 len;
125
+
u8 protkey[PHMAC_MAX_PK_SIZE];
126
+
};
127
+
128
+
#define PK_STATE_NO_KEY 0
129
+
#define PK_STATE_CONVERT_IN_PROGRESS 1
130
+
#define PK_STATE_VALID 2
131
+
132
+
/* phmac tfm context */
133
+
struct phmac_tfm_ctx {
134
+
/* source key material used to derive a protected key from */
135
+
u8 keybuf[PHMAC_MAX_KEYSIZE];
136
+
unsigned int keylen;
137
+
138
+
/* cpacf function code to use with this protected key type */
139
+
long fc;
140
+
141
+
/* nr of requests enqueued via crypto engine which use this tfm ctx */
142
+
atomic_t via_engine_ctr;
143
+
144
+
/* spinlock to atomic read/update all the following fields */
145
+
spinlock_t pk_lock;
146
+
147
+
/* see PK_STATE* defines above, < 0 holds convert failure rc */
148
+
int pk_state;
149
+
/* if state is valid, pk holds the protected key */
150
+
struct phmac_protkey pk;
151
+
};
152
+
153
+
union kmac_gr0 {
154
+
unsigned long reg;
155
+
struct {
156
+
unsigned long : 48;
157
+
unsigned long ikp : 1;
158
+
unsigned long iimp : 1;
159
+
unsigned long ccup : 1;
160
+
unsigned long : 6;
161
+
unsigned long fc : 7;
162
+
};
163
+
};
164
+
165
+
struct kmac_sha2_ctx {
166
+
u8 param[MAX_DIGEST_SIZE + MAX_IMBL_SIZE + PHMAC_MAX_PK_SIZE];
167
+
union kmac_gr0 gr0;
168
+
u8 buf[MAX_BLOCK_SIZE];
169
+
u64 buflen[2];
170
+
};
171
+
172
+
/* phmac request context */
173
+
struct phmac_req_ctx {
174
+
struct hash_walk_helper hwh;
175
+
struct kmac_sha2_ctx kmac_ctx;
176
+
bool final;
177
+
};
178
+
179
+
/*
180
+
* phmac_tfm_ctx_setkey() - Set key value into tfm context, maybe construct
181
+
* a clear key token digestible by pkey from a clear key value.
182
+
*/
183
+
static inline int phmac_tfm_ctx_setkey(struct phmac_tfm_ctx *tfm_ctx,
184
+
const u8 *key, unsigned int keylen)
185
+
{
186
+
if (keylen > sizeof(tfm_ctx->keybuf))
187
+
return -EINVAL;
188
+
189
+
memcpy(tfm_ctx->keybuf, key, keylen);
190
+
tfm_ctx->keylen = keylen;
191
+
192
+
return 0;
193
+
}
194
+
195
+
/*
196
+
* Convert the raw key material into a protected key via PKEY api.
197
+
* This function may sleep - don't call in non-sleeping context.
198
+
*/
199
+
static inline int convert_key(const u8 *key, unsigned int keylen,
200
+
struct phmac_protkey *pk)
201
+
{
202
+
int rc, i;
203
+
204
+
pk->len = sizeof(pk->protkey);
205
+
206
+
/*
207
+
* In case of a busy card retry with increasing delay
208
+
* of 200, 400, 800 and 1600 ms - in total 3 s.
209
+
*/
210
+
for (rc = -EIO, i = 0; rc && i < 5; i++) {
211
+
if (rc == -EBUSY && msleep_interruptible((1 << i) * 100)) {
212
+
rc = -EINTR;
213
+
goto out;
214
+
}
215
+
rc = pkey_key2protkey(key, keylen,
216
+
pk->protkey, &pk->len, &pk->type,
217
+
PKEY_XFLAG_NOMEMALLOC);
218
+
}
219
+
220
+
out:
221
+
pr_debug("rc=%d\n", rc);
222
+
return rc;
223
+
}
224
+
225
+
/*
226
+
* (Re-)Convert the raw key material from the tfm ctx into a protected
227
+
* key via convert_key() function. Update the pk_state, pk_type, pk_len
228
+
* and the protected key in the tfm context.
229
+
* Please note this function may be invoked concurrently with the very
230
+
* same tfm context. The pk_lock spinlock in the context ensures an
231
+
* atomic update of the pk and the pk state but does not guarantee any
232
+
* order of update. So a fresh converted valid protected key may get
233
+
* updated with an 'old' expired key value. As the cpacf instructions
234
+
* detect this, refuse to operate with an invalid key and the calling
235
+
* code triggers a (re-)conversion this does no harm. This may lead to
236
+
* unnecessary additional conversion but never to invalid data on the
237
+
* hash operation.
238
+
*/
239
+
static int phmac_convert_key(struct phmac_tfm_ctx *tfm_ctx)
240
+
{
241
+
struct phmac_protkey pk;
242
+
int rc;
243
+
244
+
spin_lock_bh(&tfm_ctx->pk_lock);
245
+
tfm_ctx->pk_state = PK_STATE_CONVERT_IN_PROGRESS;
246
+
spin_unlock_bh(&tfm_ctx->pk_lock);
247
+
248
+
rc = convert_key(tfm_ctx->keybuf, tfm_ctx->keylen, &pk);
249
+
250
+
/* update context */
251
+
spin_lock_bh(&tfm_ctx->pk_lock);
252
+
if (rc) {
253
+
tfm_ctx->pk_state = rc;
254
+
} else {
255
+
tfm_ctx->pk_state = PK_STATE_VALID;
256
+
tfm_ctx->pk = pk;
257
+
}
258
+
spin_unlock_bh(&tfm_ctx->pk_lock);
259
+
260
+
memzero_explicit(&pk, sizeof(pk));
261
+
pr_debug("rc=%d\n", rc);
262
+
return rc;
263
+
}
264
+
265
+
/*
266
+
* kmac_sha2_set_imbl - sets the input message bit-length based on the blocksize
267
+
*/
268
+
static inline void kmac_sha2_set_imbl(u8 *param, u64 buflen_lo,
269
+
u64 buflen_hi, unsigned int blocksize)
270
+
{
271
+
u8 *imbl = param + SHA2_IMBL_OFFSET(blocksize);
272
+
273
+
switch (blocksize) {
274
+
case SHA256_BLOCK_SIZE:
275
+
*(u64 *)imbl = buflen_lo * BITS_PER_BYTE;
276
+
break;
277
+
case SHA512_BLOCK_SIZE:
278
+
*(u128 *)imbl = (((u128)buflen_hi << 64) + buflen_lo) << 3;
279
+
break;
280
+
default:
281
+
break;
282
+
}
283
+
}
284
+
285
+
static int phmac_kmac_update(struct ahash_request *req, bool maysleep)
286
+
{
287
+
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
288
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
289
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
290
+
struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx;
291
+
struct hash_walk_helper *hwh = &req_ctx->hwh;
292
+
unsigned int bs = crypto_ahash_blocksize(tfm);
293
+
unsigned int offset, k, n;
294
+
int rc = 0;
295
+
296
+
/*
297
+
* The walk is always mapped when this function is called.
298
+
* Note that in case of partial processing or failure the walk
299
+
* is NOT unmapped here. So a follow up task may reuse the walk
300
+
* or in case of unrecoverable failure needs to unmap it.
301
+
*/
302
+
303
+
while (hwh->walkbytes > 0) {
304
+
/* check sha2 context buffer */
305
+
offset = ctx->buflen[0] % bs;
306
+
if (offset + hwh->walkbytes < bs)
307
+
goto store;
308
+
309
+
if (offset) {
310
+
/* fill ctx buffer up to blocksize and process this block */
311
+
n = bs - offset;
312
+
memcpy(ctx->buf + offset, hwh->walkaddr, n);
313
+
ctx->gr0.iimp = 1;
314
+
for (;;) {
315
+
k = _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, bs);
316
+
if (likely(k == bs))
317
+
break;
318
+
if (unlikely(k > 0)) {
319
+
/*
320
+
* Can't deal with hunks smaller than blocksize.
321
+
* And kmac should always return the nr of
322
+
* processed bytes as 0 or a multiple of the
323
+
* blocksize.
324
+
*/
325
+
rc = -EIO;
326
+
goto out;
327
+
}
328
+
/* protected key is invalid and needs re-conversion */
329
+
if (!maysleep) {
330
+
rc = -EKEYEXPIRED;
331
+
goto out;
332
+
}
333
+
rc = phmac_convert_key(tfm_ctx);
334
+
if (rc)
335
+
goto out;
336
+
spin_lock_bh(&tfm_ctx->pk_lock);
337
+
memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
338
+
tfm_ctx->pk.protkey, tfm_ctx->pk.len);
339
+
spin_unlock_bh(&tfm_ctx->pk_lock);
340
+
}
341
+
ctx->buflen[0] += n;
342
+
if (ctx->buflen[0] < n)
343
+
ctx->buflen[1]++;
344
+
rc = hwh_advance(hwh, n);
345
+
if (unlikely(rc))
346
+
goto out;
347
+
offset = 0;
348
+
}
349
+
350
+
/* process as many blocks as possible from the walk */
351
+
while (hwh->walkbytes >= bs) {
352
+
n = (hwh->walkbytes / bs) * bs;
353
+
ctx->gr0.iimp = 1;
354
+
k = _cpacf_kmac(&ctx->gr0.reg, ctx->param, hwh->walkaddr, n);
355
+
if (likely(k > 0)) {
356
+
ctx->buflen[0] += k;
357
+
if (ctx->buflen[0] < k)
358
+
ctx->buflen[1]++;
359
+
rc = hwh_advance(hwh, k);
360
+
if (unlikely(rc))
361
+
goto out;
362
+
}
363
+
if (unlikely(k < n)) {
364
+
/* protected key is invalid and needs re-conversion */
365
+
if (!maysleep) {
366
+
rc = -EKEYEXPIRED;
367
+
goto out;
368
+
}
369
+
rc = phmac_convert_key(tfm_ctx);
370
+
if (rc)
371
+
goto out;
372
+
spin_lock_bh(&tfm_ctx->pk_lock);
373
+
memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
374
+
tfm_ctx->pk.protkey, tfm_ctx->pk.len);
375
+
spin_unlock_bh(&tfm_ctx->pk_lock);
376
+
}
377
+
}
378
+
379
+
store:
380
+
/* store incomplete block in context buffer */
381
+
if (hwh->walkbytes) {
382
+
memcpy(ctx->buf + offset, hwh->walkaddr, hwh->walkbytes);
383
+
ctx->buflen[0] += hwh->walkbytes;
384
+
if (ctx->buflen[0] < hwh->walkbytes)
385
+
ctx->buflen[1]++;
386
+
rc = hwh_advance(hwh, hwh->walkbytes);
387
+
if (unlikely(rc))
388
+
goto out;
389
+
}
390
+
391
+
} /* end of while (hwh->walkbytes > 0) */
392
+
393
+
out:
394
+
pr_debug("rc=%d\n", rc);
395
+
return rc;
396
+
}
397
+
398
+
static int phmac_kmac_final(struct ahash_request *req, bool maysleep)
399
+
{
400
+
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
401
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
402
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
403
+
struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx;
404
+
unsigned int ds = crypto_ahash_digestsize(tfm);
405
+
unsigned int bs = crypto_ahash_blocksize(tfm);
406
+
unsigned int k, n;
407
+
int rc = 0;
408
+
409
+
n = ctx->buflen[0] % bs;
410
+
ctx->gr0.iimp = 0;
411
+
kmac_sha2_set_imbl(ctx->param, ctx->buflen[0], ctx->buflen[1], bs);
412
+
for (;;) {
413
+
k = _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, n);
414
+
if (likely(k == n))
415
+
break;
416
+
if (unlikely(k > 0)) {
417
+
/* Can't deal with hunks smaller than blocksize. */
418
+
rc = -EIO;
419
+
goto out;
420
+
}
421
+
/* protected key is invalid and needs re-conversion */
422
+
if (!maysleep) {
423
+
rc = -EKEYEXPIRED;
424
+
goto out;
425
+
}
426
+
rc = phmac_convert_key(tfm_ctx);
427
+
if (rc)
428
+
goto out;
429
+
spin_lock_bh(&tfm_ctx->pk_lock);
430
+
memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
431
+
tfm_ctx->pk.protkey, tfm_ctx->pk.len);
432
+
spin_unlock_bh(&tfm_ctx->pk_lock);
433
+
}
434
+
435
+
memcpy(req->result, ctx->param, ds);
436
+
437
+
out:
438
+
pr_debug("rc=%d\n", rc);
439
+
return rc;
440
+
}
441
+
442
+
static int phmac_init(struct ahash_request *req)
443
+
{
444
+
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
445
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
446
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
447
+
struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
448
+
unsigned int bs = crypto_ahash_blocksize(tfm);
449
+
int rc = 0;
450
+
451
+
/* zero request context (includes the kmac sha2 context) */
452
+
memset(req_ctx, 0, sizeof(*req_ctx));
453
+
454
+
/*
455
+
* setkey() should have set a valid fc into the tfm context.
456
+
* Copy this function code into the gr0 field of the kmac context.
457
+
*/
458
+
if (!tfm_ctx->fc) {
459
+
rc = -ENOKEY;
460
+
goto out;
461
+
}
462
+
kmac_ctx->gr0.fc = tfm_ctx->fc;
463
+
464
+
/*
465
+
* Copy the pk from tfm ctx into kmac ctx. The protected key
466
+
* may be outdated but update() and final() will handle this.
467
+
*/
468
+
spin_lock_bh(&tfm_ctx->pk_lock);
469
+
memcpy(kmac_ctx->param + SHA2_KEY_OFFSET(bs),
470
+
tfm_ctx->pk.protkey, tfm_ctx->pk.len);
471
+
spin_unlock_bh(&tfm_ctx->pk_lock);
472
+
473
+
out:
474
+
pr_debug("rc=%d\n", rc);
475
+
return rc;
476
+
}
477
+
478
+
static int phmac_update(struct ahash_request *req)
479
+
{
480
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
481
+
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
482
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
483
+
struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
484
+
struct hash_walk_helper *hwh = &req_ctx->hwh;
485
+
int rc;
486
+
487
+
/* prep the walk in the request context */
488
+
rc = hwh_prepare(req, hwh);
489
+
if (rc)
490
+
goto out;
491
+
492
+
/* Try synchronous operation if no active engine usage */
493
+
if (!atomic_read(&tfm_ctx->via_engine_ctr)) {
494
+
rc = phmac_kmac_update(req, false);
495
+
if (rc == 0)
496
+
goto out;
497
+
}
498
+
499
+
/*
500
+
* If sync operation failed or key expired or there are already
501
+
* requests enqueued via engine, fallback to async. Mark tfm as
502
+
* using engine to serialize requests.
503
+
*/
504
+
if (rc == 0 || rc == -EKEYEXPIRED) {
505
+
atomic_inc(&tfm_ctx->via_engine_ctr);
506
+
rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req);
507
+
if (rc != -EINPROGRESS)
508
+
atomic_dec(&tfm_ctx->via_engine_ctr);
509
+
}
510
+
511
+
if (rc != -EINPROGRESS) {
512
+
hwh_advance(hwh, rc);
513
+
memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
514
+
}
515
+
516
+
out:
517
+
pr_debug("rc=%d\n", rc);
518
+
return rc;
519
+
}
520
+
521
+
static int phmac_final(struct ahash_request *req)
522
+
{
523
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
524
+
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
525
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
526
+
struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
527
+
int rc = 0;
528
+
529
+
/* Try synchronous operation if no active engine usage */
530
+
if (!atomic_read(&tfm_ctx->via_engine_ctr)) {
531
+
rc = phmac_kmac_final(req, false);
532
+
if (rc == 0)
533
+
goto out;
534
+
}
535
+
536
+
/*
537
+
* If sync operation failed or key expired or there are already
538
+
* requests enqueued via engine, fallback to async. Mark tfm as
539
+
* using engine to serialize requests.
540
+
*/
541
+
if (rc == 0 || rc == -EKEYEXPIRED) {
542
+
req->nbytes = 0;
543
+
req_ctx->final = true;
544
+
atomic_inc(&tfm_ctx->via_engine_ctr);
545
+
rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req);
546
+
if (rc != -EINPROGRESS)
547
+
atomic_dec(&tfm_ctx->via_engine_ctr);
548
+
}
549
+
550
+
out:
551
+
if (rc != -EINPROGRESS)
552
+
memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
553
+
pr_debug("rc=%d\n", rc);
554
+
return rc;
555
+
}
556
+
557
+
static int phmac_finup(struct ahash_request *req)
558
+
{
559
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
560
+
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
561
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
562
+
struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
563
+
struct hash_walk_helper *hwh = &req_ctx->hwh;
564
+
int rc;
565
+
566
+
/* prep the walk in the request context */
567
+
rc = hwh_prepare(req, hwh);
568
+
if (rc)
569
+
goto out;
570
+
571
+
/* Try synchronous operations if no active engine usage */
572
+
if (!atomic_read(&tfm_ctx->via_engine_ctr)) {
573
+
rc = phmac_kmac_update(req, false);
574
+
if (rc == 0)
575
+
req->nbytes = 0;
576
+
}
577
+
if (!rc && !req->nbytes && !atomic_read(&tfm_ctx->via_engine_ctr)) {
578
+
rc = phmac_kmac_final(req, false);
579
+
if (rc == 0)
580
+
goto out;
581
+
}
582
+
583
+
/*
584
+
* If sync operation failed or key expired or there are already
585
+
* requests enqueued via engine, fallback to async. Mark tfm as
586
+
* using engine to serialize requests.
587
+
*/
588
+
if (rc == 0 || rc == -EKEYEXPIRED) {
589
+
req_ctx->final = true;
590
+
atomic_inc(&tfm_ctx->via_engine_ctr);
591
+
rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req);
592
+
if (rc != -EINPROGRESS)
593
+
atomic_dec(&tfm_ctx->via_engine_ctr);
594
+
}
595
+
596
+
if (rc != -EINPROGRESS)
597
+
hwh_advance(hwh, rc);
598
+
599
+
out:
600
+
if (rc != -EINPROGRESS)
601
+
memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
602
+
pr_debug("rc=%d\n", rc);
603
+
return rc;
604
+
}
605
+
606
+
static int phmac_digest(struct ahash_request *req)
607
+
{
608
+
int rc;
609
+
610
+
rc = phmac_init(req);
611
+
if (rc)
612
+
goto out;
613
+
614
+
rc = phmac_finup(req);
615
+
616
+
out:
617
+
pr_debug("rc=%d\n", rc);
618
+
return rc;
619
+
}
620
+
621
+
static int phmac_setkey(struct crypto_ahash *tfm,
622
+
const u8 *key, unsigned int keylen)
623
+
{
624
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
625
+
unsigned int ds = crypto_ahash_digestsize(tfm);
626
+
int rc = 0;
627
+
628
+
/* copy raw key into tfm context */
629
+
rc = phmac_tfm_ctx_setkey(tfm_ctx, key, keylen);
630
+
if (rc)
631
+
goto out;
632
+
633
+
/* convert raw key into protected key */
634
+
rc = phmac_convert_key(tfm_ctx);
635
+
if (rc)
636
+
goto out;
637
+
638
+
/* set function code in tfm context, check for valid pk type */
639
+
switch (ds) {
640
+
case SHA224_DIGEST_SIZE:
641
+
if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_512)
642
+
rc = -EINVAL;
643
+
else
644
+
tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_224;
645
+
break;
646
+
case SHA256_DIGEST_SIZE:
647
+
if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_512)
648
+
rc = -EINVAL;
649
+
else
650
+
tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_256;
651
+
break;
652
+
case SHA384_DIGEST_SIZE:
653
+
if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_1024)
654
+
rc = -EINVAL;
655
+
else
656
+
tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_384;
657
+
break;
658
+
case SHA512_DIGEST_SIZE:
659
+
if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_1024)
660
+
rc = -EINVAL;
661
+
else
662
+
tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_512;
663
+
break;
664
+
default:
665
+
tfm_ctx->fc = 0;
666
+
rc = -EINVAL;
667
+
}
668
+
669
+
out:
670
+
pr_debug("rc=%d\n", rc);
671
+
return rc;
672
+
}
673
+
674
+
static int phmac_export(struct ahash_request *req, void *out)
675
+
{
676
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
677
+
struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx;
678
+
679
+
memcpy(out, ctx, sizeof(*ctx));
680
+
681
+
return 0;
682
+
}
683
+
684
+
static int phmac_import(struct ahash_request *req, const void *in)
685
+
{
686
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
687
+
struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx;
688
+
689
+
memset(req_ctx, 0, sizeof(*req_ctx));
690
+
memcpy(ctx, in, sizeof(*ctx));
691
+
692
+
return 0;
693
+
}
694
+
695
+
static int phmac_init_tfm(struct crypto_ahash *tfm)
696
+
{
697
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
698
+
699
+
memset(tfm_ctx, 0, sizeof(*tfm_ctx));
700
+
spin_lock_init(&tfm_ctx->pk_lock);
701
+
702
+
crypto_ahash_set_reqsize(tfm, sizeof(struct phmac_req_ctx));
703
+
704
+
return 0;
705
+
}
706
+
707
+
static void phmac_exit_tfm(struct crypto_ahash *tfm)
708
+
{
709
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
710
+
711
+
memzero_explicit(tfm_ctx->keybuf, sizeof(tfm_ctx->keybuf));
712
+
memzero_explicit(&tfm_ctx->pk, sizeof(tfm_ctx->pk));
713
+
}
714
+
715
+
static int phmac_do_one_request(struct crypto_engine *engine, void *areq)
716
+
{
717
+
struct ahash_request *req = ahash_request_cast(areq);
718
+
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
719
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
720
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
721
+
struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
722
+
struct hash_walk_helper *hwh = &req_ctx->hwh;
723
+
int rc = -EINVAL;
724
+
725
+
/*
726
+
* Three kinds of requests come in here:
727
+
* update when req->nbytes > 0 and req_ctx->final is false
728
+
* final when req->nbytes = 0 and req_ctx->final is true
729
+
* finup when req->nbytes > 0 and req_ctx->final is true
730
+
* For update and finup the hwh walk needs to be prepared and
731
+
* up to date but the actual nr of bytes in req->nbytes may be
732
+
* any non zero number. For final there is no hwh walk needed.
733
+
*/
734
+
735
+
if (req->nbytes) {
736
+
rc = phmac_kmac_update(req, true);
737
+
if (rc == -EKEYEXPIRED) {
738
+
/*
739
+
* Protected key expired, conversion is in process.
740
+
* Trigger a re-schedule of this request by returning
741
+
* -ENOSPC ("hardware queue full") to the crypto engine.
742
+
* To avoid immediately re-invocation of this callback,
743
+
* tell scheduler to voluntarily give up the CPU here.
744
+
*/
745
+
pr_debug("rescheduling request\n");
746
+
cond_resched();
747
+
return -ENOSPC;
748
+
} else if (rc) {
749
+
hwh_advance(hwh, rc);
750
+
goto out;
751
+
}
752
+
req->nbytes = 0;
753
+
}
754
+
755
+
if (req_ctx->final) {
756
+
rc = phmac_kmac_final(req, true);
757
+
if (rc == -EKEYEXPIRED) {
758
+
/*
759
+
* Protected key expired, conversion is in process.
760
+
* Trigger a re-schedule of this request by returning
761
+
* -ENOSPC ("hardware queue full") to the crypto engine.
762
+
* To avoid immediately re-invocation of this callback,
763
+
* tell scheduler to voluntarily give up the CPU here.
764
+
*/
765
+
pr_debug("rescheduling request\n");
766
+
cond_resched();
767
+
return -ENOSPC;
768
+
}
769
+
}
770
+
771
+
out:
772
+
if (rc || req_ctx->final)
773
+
memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
774
+
pr_debug("request complete with rc=%d\n", rc);
775
+
local_bh_disable();
776
+
atomic_dec(&tfm_ctx->via_engine_ctr);
777
+
crypto_finalize_hash_request(engine, req, rc);
778
+
local_bh_enable();
779
+
return rc;
780
+
}
781
+
782
+
#define S390_ASYNC_PHMAC_ALG(x) \
783
+
{ \
784
+
.base = { \
785
+
.init = phmac_init, \
786
+
.update = phmac_update, \
787
+
.final = phmac_final, \
788
+
.finup = phmac_finup, \
789
+
.digest = phmac_digest, \
790
+
.setkey = phmac_setkey, \
791
+
.import = phmac_import, \
792
+
.export = phmac_export, \
793
+
.init_tfm = phmac_init_tfm, \
794
+
.exit_tfm = phmac_exit_tfm, \
795
+
.halg = { \
796
+
.digestsize = SHA##x##_DIGEST_SIZE, \
797
+
.statesize = sizeof(struct kmac_sha2_ctx), \
798
+
.base = { \
799
+
.cra_name = "phmac(sha" #x ")", \
800
+
.cra_driver_name = "phmac_s390_sha" #x, \
801
+
.cra_blocksize = SHA##x##_BLOCK_SIZE, \
802
+
.cra_priority = 400, \
803
+
.cra_flags = CRYPTO_ALG_ASYNC | \
804
+
CRYPTO_ALG_NO_FALLBACK, \
805
+
.cra_ctxsize = sizeof(struct phmac_tfm_ctx), \
806
+
.cra_module = THIS_MODULE, \
807
+
}, \
808
+
}, \
809
+
}, \
810
+
.op = { \
811
+
.do_one_request = phmac_do_one_request, \
812
+
}, \
813
+
}
814
+
815
+
static struct phmac_alg {
816
+
unsigned int fc;
817
+
struct ahash_engine_alg alg;
818
+
bool registered;
819
+
} phmac_algs[] = {
820
+
{
821
+
.fc = CPACF_KMAC_PHMAC_SHA_224,
822
+
.alg = S390_ASYNC_PHMAC_ALG(224),
823
+
}, {
824
+
.fc = CPACF_KMAC_PHMAC_SHA_256,
825
+
.alg = S390_ASYNC_PHMAC_ALG(256),
826
+
}, {
827
+
.fc = CPACF_KMAC_PHMAC_SHA_384,
828
+
.alg = S390_ASYNC_PHMAC_ALG(384),
829
+
}, {
830
+
.fc = CPACF_KMAC_PHMAC_SHA_512,
831
+
.alg = S390_ASYNC_PHMAC_ALG(512),
832
+
}
833
+
};
834
+
835
+
static struct miscdevice phmac_dev = {
836
+
.name = "phmac",
837
+
.minor = MISC_DYNAMIC_MINOR,
838
+
};
839
+
840
+
static void s390_phmac_exit(void)
841
+
{
842
+
struct phmac_alg *phmac;
843
+
int i;
844
+
845
+
if (phmac_crypto_engine) {
846
+
crypto_engine_stop(phmac_crypto_engine);
847
+
crypto_engine_exit(phmac_crypto_engine);
848
+
}
849
+
850
+
for (i = ARRAY_SIZE(phmac_algs) - 1; i >= 0; i--) {
851
+
phmac = &phmac_algs[i];
852
+
if (phmac->registered)
853
+
crypto_engine_unregister_ahash(&phmac->alg);
854
+
}
855
+
856
+
misc_deregister(&phmac_dev);
857
+
}
858
+
859
+
static int __init s390_phmac_init(void)
860
+
{
861
+
struct phmac_alg *phmac;
862
+
int i, rc;
863
+
864
+
/* register a simple phmac pseudo misc device */
865
+
rc = misc_register(&phmac_dev);
866
+
if (rc)
867
+
return rc;
868
+
869
+
/* with this pseudo device alloc and start a crypto engine */
870
+
phmac_crypto_engine =
871
+
crypto_engine_alloc_init_and_set(phmac_dev.this_device,
872
+
true, NULL, false, MAX_QLEN);
873
+
if (!phmac_crypto_engine) {
874
+
rc = -ENOMEM;
875
+
goto out_err;
876
+
}
877
+
rc = crypto_engine_start(phmac_crypto_engine);
878
+
if (rc) {
879
+
crypto_engine_exit(phmac_crypto_engine);
880
+
phmac_crypto_engine = NULL;
881
+
goto out_err;
882
+
}
883
+
884
+
for (i = 0; i < ARRAY_SIZE(phmac_algs); i++) {
885
+
phmac = &phmac_algs[i];
886
+
if (!cpacf_query_func(CPACF_KMAC, phmac->fc))
887
+
continue;
888
+
rc = crypto_engine_register_ahash(&phmac->alg);
889
+
if (rc)
890
+
goto out_err;
891
+
phmac->registered = true;
892
+
pr_debug("%s registered\n", phmac->alg.base.halg.base.cra_name);
893
+
}
894
+
895
+
return 0;
896
+
897
+
out_err:
898
+
s390_phmac_exit();
899
+
return rc;
900
+
}
901
+
902
+
module_init(s390_phmac_init);
903
+
module_exit(s390_phmac_exit);
904
+
905
+
MODULE_ALIAS_CRYPTO("phmac(sha224)");
906
+
MODULE_ALIAS_CRYPTO("phmac(sha256)");
907
+
MODULE_ALIAS_CRYPTO("phmac(sha384)");
908
+
MODULE_ALIAS_CRYPTO("phmac(sha512)");
909
+
910
+
MODULE_DESCRIPTION("S390 HMAC driver for protected keys");
911
+
MODULE_LICENSE("GPL");
+13
drivers/crypto/Kconfig
+13
drivers/crypto/Kconfig
···
188
188
Select this option if you want to use the paes cipher
189
189
for example to use protected key encrypted devices.
190
190
191
+
config CRYPTO_PHMAC_S390
192
+
tristate "PHMAC cipher algorithms"
193
+
depends on S390
194
+
depends on PKEY
195
+
select CRYPTO_HASH
196
+
select CRYPTO_ENGINE
197
+
help
198
+
This is the s390 hardware accelerated implementation of the
199
+
protected key HMAC support for SHA224, SHA256, SHA384 and SHA512.
200
+
201
+
Select this option if you want to use the phmac digests
202
+
for example to use dm-integrity with secure/protected keys.
203
+
191
204
config S390_PRNG
192
205
tristate "Pseudo random number generator device driver"
193
206
depends on S390