tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

crypto: powerpc/p10-aes-gcm - Register modules as SIMD

This patch is to fix an issue when simd is not usable that data mismatch
may occur. The fix is to register algs as SIMD modules so that the
algorithm is excecuted when SIMD instructions is usable. Called
gcm_update() to generate the final digest if needed.

A new module rfc4106(gcm(aes)) is also added.

Fixes: cdcecfd9991f ("crypto: p10-aes-gcm - Glue code for AES/GCM stitched implementation")

Signed-off-by: Danny Tsen <dtsen@linux.ibm.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

authored by

Danny Tsen and committed by
Herbert Xu c954b252 7aa747ed

+118 -23
+118 -23
arch/powerpc/crypto/aes-gcm-p10-glue.c
··· 8 8 #include <asm/unaligned.h> 9 9 #include <asm/simd.h> 10 10 #include <asm/switch_to.h> 11 + #include <crypto/gcm.h> 11 12 #include <crypto/aes.h> 12 13 #include <crypto/algapi.h> 13 14 #include <crypto/b128ops.h> ··· 25 24 26 25 #define PPC_ALIGN 16 27 26 #define GCM_IV_SIZE 12 27 + #define RFC4106_NONCE_SIZE 4 28 28 29 29 MODULE_DESCRIPTION("PPC64le AES-GCM with Stitched implementation"); 30 30 MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com"); ··· 33 31 MODULE_ALIAS_CRYPTO("aes"); 34 32 35 33 asmlinkage int aes_p10_set_encrypt_key(const u8 *userKey, const int bits, 36 - void *key); 34 + void *key); 37 35 asmlinkage void aes_p10_encrypt(const u8 *in, u8 *out, const void *key); 38 36 asmlinkage void aes_p10_gcm_encrypt(u8 *in, u8 *out, size_t len, 39 37 void *rkey, u8 *iv, void *Xi); ··· 41 39 void *rkey, u8 *iv, void *Xi); 42 40 asmlinkage void gcm_init_htable(unsigned char htable[], unsigned char Xi[]); 43 41 asmlinkage void gcm_ghash_p10(unsigned char *Xi, unsigned char *Htable, 44 - unsigned char *aad, unsigned int alen); 42 + unsigned char *aad, unsigned int alen); 43 + asmlinkage void gcm_update(u8 *iv, void *Xi); 45 44 46 45 struct aes_key { 47 46 u8 key[AES_MAX_KEYLENGTH]; ··· 55 52 u8 aad_hash[16]; 56 53 u64 aadLen; 57 54 u64 Plen; /* offset 56 - used in aes_p10_gcm_{en/de}crypt */ 55 + u8 pblock[16]; 58 56 }; 59 57 struct Hash_ctx { 60 58 u8 H[16]; /* subkey */ ··· 64 60 65 61 struct p10_aes_gcm_ctx { 66 62 struct aes_key enc_key; 63 + u8 nonce[RFC4106_NONCE_SIZE]; 67 64 }; 68 65 69 66 static void vsx_begin(void) 70 67 { 71 68 preempt_disable(); 69 + pagefault_disable(); 72 70 enable_kernel_vsx(); 73 71 } 74 72 75 73 static void vsx_end(void) 76 74 { 77 75 disable_kernel_vsx(); 76 + pagefault_enable(); 78 77 preempt_enable(); 79 78 } 80 79 ··· 192 185 } 193 186 194 187 static int p10_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, 195 - unsigned int keylen) 188 + unsigned int keylen) 196 189 { 197 190 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 198 191 struct p10_aes_gcm_ctx *ctx = crypto_tfm_ctx(tfm); ··· 205 198 return ret ? -EINVAL : 0; 206 199 } 207 200 208 - static int p10_aes_gcm_crypt(struct aead_request *req, int enc) 201 + static int p10_aes_gcm_crypt(struct aead_request *req, u8 *riv, 202 + int assoclen, int enc) 209 203 { 210 204 struct crypto_tfm *tfm = req->base.tfm; 211 205 struct p10_aes_gcm_ctx *ctx = crypto_tfm_ctx(tfm); ··· 218 210 struct skcipher_walk walk; 219 211 u8 *assocmem = NULL; 220 212 u8 *assoc; 221 - unsigned int assoclen = req->assoclen; 222 213 unsigned int cryptlen = req->cryptlen; 223 214 unsigned char ivbuf[AES_BLOCK_SIZE+PPC_ALIGN]; 224 215 unsigned char *iv = PTR_ALIGN((void *)ivbuf, PPC_ALIGN); ··· 225 218 unsigned long auth_tag_len = crypto_aead_authsize(__crypto_aead_cast(tfm)); 226 219 u8 otag[16]; 227 220 int total_processed = 0; 221 + int nbytes; 228 222 229 223 memset(databuf, 0, sizeof(databuf)); 230 224 memset(hashbuf, 0, sizeof(hashbuf)); 231 225 memset(ivbuf, 0, sizeof(ivbuf)); 232 - memcpy(iv, req->iv, GCM_IV_SIZE); 226 + memcpy(iv, riv, GCM_IV_SIZE); 233 227 234 228 /* Linearize assoc, if not already linear */ 235 229 if (req->src->length >= assoclen && req->src->length) { ··· 265 257 if (ret) 266 258 return ret; 267 259 268 - while (walk.nbytes > 0 && ret == 0) { 260 + while ((nbytes = walk.nbytes) > 0 && ret == 0) { 261 + u8 *src = walk.src.virt.addr; 262 + u8 *dst = walk.dst.virt.addr; 263 + u8 buf[AES_BLOCK_SIZE]; 264 + 265 + if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) 266 + src = dst = memcpy(buf, src, nbytes); 269 267 270 268 vsx_begin(); 271 269 if (enc) 272 - aes_p10_gcm_encrypt(walk.src.virt.addr, 273 - walk.dst.virt.addr, 274 - walk.nbytes, 270 + aes_p10_gcm_encrypt(src, dst, nbytes, 275 271 &ctx->enc_key, gctx->iv, hash->Htable); 276 272 else 277 - aes_p10_gcm_decrypt(walk.src.virt.addr, 278 - walk.dst.virt.addr, 279 - walk.nbytes, 273 + aes_p10_gcm_decrypt(src, dst, nbytes, 280 274 &ctx->enc_key, gctx->iv, hash->Htable); 275 + 276 + if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) 277 + memcpy(walk.dst.virt.addr, buf, nbytes); 278 + 281 279 vsx_end(); 282 280 283 281 total_processed += walk.nbytes; ··· 295 281 296 282 /* Finalize hash */ 297 283 vsx_begin(); 284 + gcm_update(gctx->iv, hash->Htable); 298 285 finish_tag(gctx, hash, total_processed); 299 286 vsx_end(); 300 287 ··· 317 302 return 0; 318 303 } 319 304 305 + static int rfc4106_setkey(struct crypto_aead *tfm, const u8 *inkey, 306 + unsigned int keylen) 307 + { 308 + struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); 309 + int err; 310 + 311 + keylen -= RFC4106_NONCE_SIZE; 312 + err = p10_aes_gcm_setkey(tfm, inkey, keylen); 313 + if (err) 314 + return err; 315 + 316 + memcpy(ctx->nonce, inkey + keylen, RFC4106_NONCE_SIZE); 317 + return 0; 318 + } 319 + 320 + static int rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 321 + { 322 + return crypto_rfc4106_check_authsize(authsize); 323 + } 324 + 325 + static int rfc4106_encrypt(struct aead_request *req) 326 + { 327 + struct crypto_aead *aead = crypto_aead_reqtfm(req); 328 + struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(aead); 329 + u8 iv[AES_BLOCK_SIZE]; 330 + 331 + memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 332 + memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 333 + 334 + return crypto_ipsec_check_assoclen(req->assoclen) ?: 335 + p10_aes_gcm_crypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE, 1); 336 + } 337 + 338 + static int rfc4106_decrypt(struct aead_request *req) 339 + { 340 + struct crypto_aead *aead = crypto_aead_reqtfm(req); 341 + struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(aead); 342 + u8 iv[AES_BLOCK_SIZE]; 343 + 344 + memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 345 + memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 346 + 347 + return crypto_ipsec_check_assoclen(req->assoclen) ?: 348 + p10_aes_gcm_crypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE, 0); 349 + } 350 + 320 351 static int p10_aes_gcm_encrypt(struct aead_request *req) 321 352 { 322 - return p10_aes_gcm_crypt(req, 1); 353 + return p10_aes_gcm_crypt(req, req->iv, req->assoclen, 1); 323 354 } 324 355 325 356 static int p10_aes_gcm_decrypt(struct aead_request *req) 326 357 { 327 - return p10_aes_gcm_crypt(req, 0); 358 + return p10_aes_gcm_crypt(req, req->iv, req->assoclen, 0); 328 359 } 329 360 330 - static struct aead_alg gcm_aes_alg = { 361 + static struct aead_alg gcm_aes_algs[] = {{ 331 362 .ivsize = GCM_IV_SIZE, 332 363 .maxauthsize = 16, 333 364 ··· 382 321 .encrypt = p10_aes_gcm_encrypt, 383 322 .decrypt = p10_aes_gcm_decrypt, 384 323 385 - .base.cra_name = "gcm(aes)", 386 - .base.cra_driver_name = "aes_gcm_p10", 324 + .base.cra_name = "__gcm(aes)", 325 + .base.cra_driver_name = "__aes_gcm_p10", 387 326 .base.cra_priority = 2100, 388 327 .base.cra_blocksize = 1, 389 - .base.cra_ctxsize = sizeof(struct p10_aes_gcm_ctx), 328 + .base.cra_ctxsize = sizeof(struct p10_aes_gcm_ctx)+ 329 + 4 * sizeof(u64[2]), 390 330 .base.cra_module = THIS_MODULE, 391 - }; 331 + .base.cra_flags = CRYPTO_ALG_INTERNAL, 332 + }, { 333 + .ivsize = GCM_RFC4106_IV_SIZE, 334 + .maxauthsize = 16, 335 + .setkey = rfc4106_setkey, 336 + .setauthsize = rfc4106_setauthsize, 337 + .encrypt = rfc4106_encrypt, 338 + .decrypt = rfc4106_decrypt, 339 + 340 + .base.cra_name = "__rfc4106(gcm(aes))", 341 + .base.cra_driver_name = "__rfc4106_aes_gcm_p10", 342 + .base.cra_priority = 2100, 343 + .base.cra_blocksize = 1, 344 + .base.cra_ctxsize = sizeof(struct p10_aes_gcm_ctx) + 345 + 4 * sizeof(u64[2]), 346 + .base.cra_module = THIS_MODULE, 347 + .base.cra_flags = CRYPTO_ALG_INTERNAL, 348 + }}; 349 + 350 + static struct simd_aead_alg *p10_simd_aeads[ARRAY_SIZE(gcm_aes_algs)]; 392 351 393 352 static int __init p10_init(void) 394 353 { 395 - return crypto_register_aead(&gcm_aes_alg); 354 + int ret; 355 + 356 + if (!cpu_has_feature(PPC_FEATURE2_ARCH_3_1)) 357 + return 0; 358 + 359 + ret = simd_register_aeads_compat(gcm_aes_algs, 360 + ARRAY_SIZE(gcm_aes_algs), 361 + p10_simd_aeads); 362 + if (ret) { 363 + simd_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs), 364 + p10_simd_aeads); 365 + return ret; 366 + } 367 + return 0; 396 368 } 397 369 398 370 static void __exit p10_exit(void) 399 371 { 400 - crypto_unregister_aead(&gcm_aes_alg); 372 + simd_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs), 373 + p10_simd_aeads); 401 374 } 402 375 403 - module_cpu_feature_match(PPC_MODULE_FEATURE_P10, p10_init); 376 + module_init(p10_init); 404 377 module_exit(p10_exit);