tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

crypto: arm/ghash - Move NEON GHASH assembly into its own file

arch/arm/crypto/ghash-ce-core.S implements pmull_ghash_update_p8(),
which is used only by a crypto_shash implementation of GHASH. It also
implements other functions, including pmull_ghash_update_p64() and
others, which are used only by a crypto_aead implementation of AES-GCM.

While some code is shared between pmull_ghash_update_p8() and
pmull_ghash_update_p64(), it's not very much. Since
pmull_ghash_update_p8() will also need to be migrated into lib/crypto/
to achieve parity in the standalone GHASH support, let's move it into a
separate file ghash-neon-core.S.

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260319061723.1140720-7-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>

Eric Biggers ca5ff14c 39afaff9

+222 -158
+1 -1
arch/arm/crypto/Makefile
··· 10 10 11 11 aes-arm-bs-y := aes-neonbs-core.o aes-neonbs-glue.o 12 12 aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o 13 - ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o 13 + ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o ghash-neon-core.o
+14 -157
arch/arm/crypto/ghash-ce-core.S
··· 1 1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 2 /* 3 - * Accelerated GHASH implementation with NEON/ARMv8 vmull.p8/64 instructions. 3 + * Accelerated AES-GCM implementation with ARMv8 Crypto Extensions. 4 4 * 5 5 * Copyright (C) 2015 - 2017 Linaro Ltd. 6 6 * Copyright (C) 2023 Google LLC. <ardb@google.com> ··· 29 29 XM_H .req d7 30 30 XH_L .req d8 31 31 32 - t0l .req d10 33 - t0h .req d11 34 - t1l .req d12 35 - t1h .req d13 36 - t2l .req d14 37 - t2h .req d15 38 - t3l .req d16 39 - t3h .req d17 40 - t4l .req d18 41 - t4h .req d19 42 - 43 - t0q .req q5 44 - t1q .req q6 45 - t2q .req q7 46 - t3q .req q8 47 - t4q .req q9 48 32 XH2 .req q9 49 33 50 - s1l .req d20 51 - s1h .req d21 52 - s2l .req d22 53 - s2h .req d23 54 - s3l .req d24 55 - s3h .req d25 56 - s4l .req d26 57 - s4h .req d27 58 - 59 34 MASK .req d28 60 - SHASH2_p8 .req d28 61 35 62 - k16 .req d29 63 - k32 .req d30 64 - k48 .req d31 65 36 SHASH2_p64 .req d31 66 37 67 38 HH .req q10 ··· 64 93 65 94 .text 66 95 67 - .macro __pmull_p64, rd, rn, rm, b1, b2, b3, b4 68 - vmull.p64 \rd, \rn, \rm 69 - .endm 70 - 71 - /* 72 - * This implementation of 64x64 -> 128 bit polynomial multiplication 73 - * using vmull.p8 instructions (8x8 -> 16) is taken from the paper 74 - * "Fast Software Polynomial Multiplication on ARM Processors Using 75 - * the NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and 76 - * Ricardo Dahab (https://hal.inria.fr/hal-01506572) 77 - * 78 - * It has been slightly tweaked for in-order performance, and to allow 79 - * 'rq' to overlap with 'ad' or 'bd'. 80 - */ 81 - .macro __pmull_p8, rq, ad, bd, b1=t4l, b2=t3l, b3=t4l, b4=t3l 82 - vext.8 t0l, \ad, \ad, #1 @ A1 83 - .ifc \b1, t4l 84 - vext.8 t4l, \bd, \bd, #1 @ B1 85 - .endif 86 - vmull.p8 t0q, t0l, \bd @ F = A1*B 87 - vext.8 t1l, \ad, \ad, #2 @ A2 88 - vmull.p8 t4q, \ad, \b1 @ E = A*B1 89 - .ifc \b2, t3l 90 - vext.8 t3l, \bd, \bd, #2 @ B2 91 - .endif 92 - vmull.p8 t1q, t1l, \bd @ H = A2*B 93 - vext.8 t2l, \ad, \ad, #3 @ A3 94 - vmull.p8 t3q, \ad, \b2 @ G = A*B2 95 - veor t0q, t0q, t4q @ L = E + F 96 - .ifc \b3, t4l 97 - vext.8 t4l, \bd, \bd, #3 @ B3 98 - .endif 99 - vmull.p8 t2q, t2l, \bd @ J = A3*B 100 - veor t0l, t0l, t0h @ t0 = (L) (P0 + P1) << 8 101 - veor t1q, t1q, t3q @ M = G + H 102 - .ifc \b4, t3l 103 - vext.8 t3l, \bd, \bd, #4 @ B4 104 - .endif 105 - vmull.p8 t4q, \ad, \b3 @ I = A*B3 106 - veor t1l, t1l, t1h @ t1 = (M) (P2 + P3) << 16 107 - vmull.p8 t3q, \ad, \b4 @ K = A*B4 108 - vand t0h, t0h, k48 109 - vand t1h, t1h, k32 110 - veor t2q, t2q, t4q @ N = I + J 111 - veor t0l, t0l, t0h 112 - veor t1l, t1l, t1h 113 - veor t2l, t2l, t2h @ t2 = (N) (P4 + P5) << 24 114 - vand t2h, t2h, k16 115 - veor t3l, t3l, t3h @ t3 = (K) (P6 + P7) << 32 116 - vmov.i64 t3h, #0 117 - vext.8 t0q, t0q, t0q, #15 118 - veor t2l, t2l, t2h 119 - vext.8 t1q, t1q, t1q, #14 120 - vmull.p8 \rq, \ad, \bd @ D = A*B 121 - vext.8 t2q, t2q, t2q, #13 122 - vext.8 t3q, t3q, t3q, #12 123 - veor t0q, t0q, t1q 124 - veor t2q, t2q, t3q 125 - veor \rq, \rq, t0q 126 - veor \rq, \rq, t2q 127 - .endm 128 - 129 - // 130 - // PMULL (64x64->128) based reduction for CPUs that can do 131 - // it in a single instruction. 132 - // 133 96 .macro __pmull_reduce_p64 134 97 vmull.p64 T1, XL_L, MASK 135 98 ··· 75 170 vmull.p64 XL, T1_H, MASK 76 171 .endm 77 172 78 - // 79 - // Alternative reduction for CPUs that lack support for the 80 - // 64x64->128 PMULL instruction 81 - // 82 - .macro __pmull_reduce_p8 83 - veor XL_H, XL_H, XM_L 84 - veor XH_L, XH_L, XM_H 85 - 86 - vshl.i64 T1, XL, #57 87 - vshl.i64 T2, XL, #62 88 - veor T1, T1, T2 89 - vshl.i64 T2, XL, #63 90 - veor T1, T1, T2 91 - veor XL_H, XL_H, T1_L 92 - veor XH_L, XH_L, T1_H 93 - 94 - vshr.u64 T1, XL, #1 95 - veor XH, XH, XL 96 - veor XL, XL, T1 97 - vshr.u64 T1, T1, #6 98 - vshr.u64 XL, XL, #1 99 - .endm 100 - 101 - .macro ghash_update, pn, enc, aggregate=1, head=1 173 + .macro ghash_update, enc, aggregate=1, head=1 102 174 vld1.64 {XL}, [r1] 103 175 104 176 .if \head ··· 88 206 b 3f 89 207 .endif 90 208 91 - 0: .ifc \pn, p64 92 - .if \aggregate 209 + 0: .if \aggregate 93 210 tst r0, #3 // skip until #blocks is a 94 211 bne 2f // round multiple of 4 95 212 ··· 169 288 170 289 b 1b 171 290 .endif 172 - .endif 173 291 174 292 2: vld1.8 {T1}, [r2]! 175 293 ··· 188 308 veor T1_L, T1_L, XL_H 189 309 veor XL, XL, IN1 190 310 191 - __pmull_\pn XH, XL_H, SHASH_H, s1h, s2h, s3h, s4h @ a1 * b1 311 + vmull.p64 XH, XL_H, SHASH_H @ a1 * b1 192 312 veor T1, T1, XL 193 - __pmull_\pn XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l @ a0 * b0 194 - __pmull_\pn XM, T1_L, SHASH2_\pn @ (a1+a0)(b1+b0) 313 + vmull.p64 XL, XL_L, SHASH_L @ a0 * b0 314 + vmull.p64 XM, T1_L, SHASH2_p64 @ (a1+a0)(b1+b0) 195 315 196 316 4: veor T1, XL, XH 197 317 veor XM, XM, T1 198 318 199 - __pmull_reduce_\pn 319 + __pmull_reduce_p64 200 320 201 321 veor T1, T1, XH 202 322 veor XL, XL, T1 ··· 205 325 .endm 206 326 207 327 /* 208 - * void pmull_ghash_update(int blocks, u64 dg[], const char *src, 209 - * struct ghash_key const *k, const char *head) 328 + * void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, 329 + * u64 const h[4][2], const char *head) 210 330 */ 211 331 ENTRY(pmull_ghash_update_p64) 212 332 vld1.64 {SHASH}, [r3]! ··· 221 341 vmov.i8 MASK, #0xe1 222 342 vshl.u64 MASK, MASK, #57 223 343 224 - ghash_update p64 344 + ghash_update 225 345 vst1.64 {XL}, [r1] 226 346 227 347 bx lr 228 348 ENDPROC(pmull_ghash_update_p64) 229 - 230 - ENTRY(pmull_ghash_update_p8) 231 - vld1.64 {SHASH}, [r3] 232 - veor SHASH2_p8, SHASH_L, SHASH_H 233 - 234 - vext.8 s1l, SHASH_L, SHASH_L, #1 235 - vext.8 s2l, SHASH_L, SHASH_L, #2 236 - vext.8 s3l, SHASH_L, SHASH_L, #3 237 - vext.8 s4l, SHASH_L, SHASH_L, #4 238 - vext.8 s1h, SHASH_H, SHASH_H, #1 239 - vext.8 s2h, SHASH_H, SHASH_H, #2 240 - vext.8 s3h, SHASH_H, SHASH_H, #3 241 - vext.8 s4h, SHASH_H, SHASH_H, #4 242 - 243 - vmov.i64 k16, #0xffff 244 - vmov.i64 k32, #0xffffffff 245 - vmov.i64 k48, #0xffffffffffff 246 - 247 - ghash_update p8 248 - vst1.64 {XL}, [r1] 249 - 250 - bx lr 251 - ENDPROC(pmull_ghash_update_p8) 252 349 253 350 e0 .req q9 254 351 e1 .req q10 ··· 393 536 394 537 vld1.64 {SHASH}, [r3] 395 538 396 - ghash_update p64, enc, head=0 539 + ghash_update enc, head=0 397 540 vst1.64 {XL}, [r1] 398 541 399 542 pop {r4-r8, pc} ··· 411 554 412 555 vld1.64 {SHASH}, [r3] 413 556 414 - ghash_update p64, dec, head=0 557 + ghash_update dec, head=0 415 558 vst1.64 {XL}, [r1] 416 559 417 560 pop {r4-r8, pc} ··· 460 603 vshl.u64 MASK, MASK, #57 461 604 mov r0, #1 462 605 bne 3f // process head block first 463 - ghash_update p64, aggregate=0, head=0 606 + ghash_update aggregate=0, head=0 464 607 465 608 vrev64.8 XL, XL 466 609 vext.8 XL, XL, XL, #8 ··· 517 660 vshl.u64 MASK, MASK, #57 518 661 mov r0, #1 519 662 bne 3f // process head block first 520 - ghash_update p64, aggregate=0, head=0 663 + ghash_update aggregate=0, head=0 521 664 522 665 vrev64.8 XL, XL 523 666 vext.8 XL, XL, XL, #8
+207
arch/arm/crypto/ghash-neon-core.S
··· 1 + /* SPDX-License-Identifier: GPL-2.0-only */ 2 + /* 3 + * Accelerated GHASH implementation with NEON vmull.p8 instructions. 4 + * 5 + * Copyright (C) 2015 - 2017 Linaro Ltd. 6 + * Copyright (C) 2023 Google LLC. <ardb@google.com> 7 + */ 8 + 9 + #include <linux/linkage.h> 10 + #include <asm/assembler.h> 11 + 12 + .fpu neon 13 + 14 + SHASH .req q0 15 + T1 .req q1 16 + XL .req q2 17 + XM .req q3 18 + XH .req q4 19 + IN1 .req q4 20 + 21 + SHASH_L .req d0 22 + SHASH_H .req d1 23 + T1_L .req d2 24 + T1_H .req d3 25 + XL_L .req d4 26 + XL_H .req d5 27 + XM_L .req d6 28 + XM_H .req d7 29 + XH_L .req d8 30 + 31 + t0l .req d10 32 + t0h .req d11 33 + t1l .req d12 34 + t1h .req d13 35 + t2l .req d14 36 + t2h .req d15 37 + t3l .req d16 38 + t3h .req d17 39 + t4l .req d18 40 + t4h .req d19 41 + 42 + t0q .req q5 43 + t1q .req q6 44 + t2q .req q7 45 + t3q .req q8 46 + t4q .req q9 47 + 48 + s1l .req d20 49 + s1h .req d21 50 + s2l .req d22 51 + s2h .req d23 52 + s3l .req d24 53 + s3h .req d25 54 + s4l .req d26 55 + s4h .req d27 56 + 57 + SHASH2_p8 .req d28 58 + 59 + k16 .req d29 60 + k32 .req d30 61 + k48 .req d31 62 + 63 + T2 .req q7 64 + 65 + .text 66 + 67 + /* 68 + * This implementation of 64x64 -> 128 bit polynomial multiplication 69 + * using vmull.p8 instructions (8x8 -> 16) is taken from the paper 70 + * "Fast Software Polynomial Multiplication on ARM Processors Using 71 + * the NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and 72 + * Ricardo Dahab (https://hal.inria.fr/hal-01506572) 73 + * 74 + * It has been slightly tweaked for in-order performance, and to allow 75 + * 'rq' to overlap with 'ad' or 'bd'. 76 + */ 77 + .macro __pmull_p8, rq, ad, bd, b1=t4l, b2=t3l, b3=t4l, b4=t3l 78 + vext.8 t0l, \ad, \ad, #1 @ A1 79 + .ifc \b1, t4l 80 + vext.8 t4l, \bd, \bd, #1 @ B1 81 + .endif 82 + vmull.p8 t0q, t0l, \bd @ F = A1*B 83 + vext.8 t1l, \ad, \ad, #2 @ A2 84 + vmull.p8 t4q, \ad, \b1 @ E = A*B1 85 + .ifc \b2, t3l 86 + vext.8 t3l, \bd, \bd, #2 @ B2 87 + .endif 88 + vmull.p8 t1q, t1l, \bd @ H = A2*B 89 + vext.8 t2l, \ad, \ad, #3 @ A3 90 + vmull.p8 t3q, \ad, \b2 @ G = A*B2 91 + veor t0q, t0q, t4q @ L = E + F 92 + .ifc \b3, t4l 93 + vext.8 t4l, \bd, \bd, #3 @ B3 94 + .endif 95 + vmull.p8 t2q, t2l, \bd @ J = A3*B 96 + veor t0l, t0l, t0h @ t0 = (L) (P0 + P1) << 8 97 + veor t1q, t1q, t3q @ M = G + H 98 + .ifc \b4, t3l 99 + vext.8 t3l, \bd, \bd, #4 @ B4 100 + .endif 101 + vmull.p8 t4q, \ad, \b3 @ I = A*B3 102 + veor t1l, t1l, t1h @ t1 = (M) (P2 + P3) << 16 103 + vmull.p8 t3q, \ad, \b4 @ K = A*B4 104 + vand t0h, t0h, k48 105 + vand t1h, t1h, k32 106 + veor t2q, t2q, t4q @ N = I + J 107 + veor t0l, t0l, t0h 108 + veor t1l, t1l, t1h 109 + veor t2l, t2l, t2h @ t2 = (N) (P4 + P5) << 24 110 + vand t2h, t2h, k16 111 + veor t3l, t3l, t3h @ t3 = (K) (P6 + P7) << 32 112 + vmov.i64 t3h, #0 113 + vext.8 t0q, t0q, t0q, #15 114 + veor t2l, t2l, t2h 115 + vext.8 t1q, t1q, t1q, #14 116 + vmull.p8 \rq, \ad, \bd @ D = A*B 117 + vext.8 t2q, t2q, t2q, #13 118 + vext.8 t3q, t3q, t3q, #12 119 + veor t0q, t0q, t1q 120 + veor t2q, t2q, t3q 121 + veor \rq, \rq, t0q 122 + veor \rq, \rq, t2q 123 + .endm 124 + 125 + .macro __pmull_reduce_p8 126 + veor XL_H, XL_H, XM_L 127 + veor XH_L, XH_L, XM_H 128 + 129 + vshl.i64 T1, XL, #57 130 + vshl.i64 T2, XL, #62 131 + veor T1, T1, T2 132 + vshl.i64 T2, XL, #63 133 + veor T1, T1, T2 134 + veor XL_H, XL_H, T1_L 135 + veor XH_L, XH_L, T1_H 136 + 137 + vshr.u64 T1, XL, #1 138 + veor XH, XH, XL 139 + veor XL, XL, T1 140 + vshr.u64 T1, T1, #6 141 + vshr.u64 XL, XL, #1 142 + .endm 143 + 144 + .macro ghash_update 145 + vld1.64 {XL}, [r1] 146 + 147 + /* do the head block first, if supplied */ 148 + ldr ip, [sp] 149 + teq ip, #0 150 + beq 0f 151 + vld1.64 {T1}, [ip] 152 + teq r0, #0 153 + b 3f 154 + 155 + 0: 156 + vld1.8 {T1}, [r2]! 157 + subs r0, r0, #1 158 + 159 + 3: /* multiply XL by SHASH in GF(2^128) */ 160 + vrev64.8 T1, T1 161 + 162 + vext.8 IN1, T1, T1, #8 163 + veor T1_L, T1_L, XL_H 164 + veor XL, XL, IN1 165 + 166 + __pmull_p8 XH, XL_H, SHASH_H, s1h, s2h, s3h, s4h @ a1 * b1 167 + veor T1, T1, XL 168 + __pmull_p8 XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l @ a0 * b0 169 + __pmull_p8 XM, T1_L, SHASH2_p8 @ (a1+a0)(b1+b0) 170 + 171 + veor T1, XL, XH 172 + veor XM, XM, T1 173 + 174 + __pmull_reduce_p8 175 + 176 + veor T1, T1, XH 177 + veor XL, XL, T1 178 + 179 + bne 0b 180 + .endm 181 + 182 + /* 183 + * void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, 184 + * u64 const h[1][2], const char *head) 185 + */ 186 + ENTRY(pmull_ghash_update_p8) 187 + vld1.64 {SHASH}, [r3] 188 + veor SHASH2_p8, SHASH_L, SHASH_H 189 + 190 + vext.8 s1l, SHASH_L, SHASH_L, #1 191 + vext.8 s2l, SHASH_L, SHASH_L, #2 192 + vext.8 s3l, SHASH_L, SHASH_L, #3 193 + vext.8 s4l, SHASH_L, SHASH_L, #4 194 + vext.8 s1h, SHASH_H, SHASH_H, #1 195 + vext.8 s2h, SHASH_H, SHASH_H, #2 196 + vext.8 s3h, SHASH_H, SHASH_H, #3 197 + vext.8 s4h, SHASH_H, SHASH_H, #4 198 + 199 + vmov.i64 k16, #0xffff 200 + vmov.i64 k32, #0xffffffff 201 + vmov.i64 k48, #0xffffffffffff 202 + 203 + ghash_update 204 + vst1.64 {XL}, [r1] 205 + 206 + bx lr 207 + ENDPROC(pmull_ghash_update_p8)