crypto: arm64/ghash - Move NEON GHASH assembly into its own file

+1 -1

arch/arm64/crypto/Makefile

··· 27 27 sm4-neon-y := sm4-neon-glue.o sm4-neon-core.o 28 28 29 29 obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o 30 - ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o 30 + ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o ghash-neon-core.o 31 31 32 32 obj-$(CONFIG_CRYPTO_AES_ARM64_CE_CCM) += aes-ce-ccm.o 33 33 aes-ce-ccm-y := aes-ce-ccm-glue.o aes-ce-ccm-core.o

+12 -195

arch/arm64/crypto/ghash-ce-core.S

··· 1 1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 2 /* 3 - * Accelerated GHASH implementation with ARMv8 PMULL instructions. 3 + * Accelerated AES-GCM implementation with ARMv8 Crypto Extensions. 4 4 * 5 5 * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> 6 6 */ ··· 19 19 XH .req v7 20 20 IN1 .req v7 21 21 22 - k00_16 .req v8 23 - k32_48 .req v9 24 - 25 - t3 .req v10 26 - t4 .req v11 27 - t5 .req v12 28 - t6 .req v13 29 - t7 .req v14 30 - t8 .req v15 31 - t9 .req v16 32 - 33 - perm1 .req v17 34 - perm2 .req v18 35 - perm3 .req v19 36 - 37 - sh1 .req v20 38 - sh2 .req v21 39 - sh3 .req v22 40 - sh4 .req v23 41 - 42 - ss1 .req v24 43 - ss2 .req v25 44 - ss3 .req v26 45 - ss4 .req v27 46 - 47 22 XL2 .req v8 48 23 XM2 .req v9 49 24 XH2 .req v10 ··· 34 59 35 60 .text 36 61 .arch armv8-a+crypto 37 - 38 - .macro __pmull_p64, rd, rn, rm 39 - pmull \rd\().1q, \rn\().1d, \rm\().1d 40 - .endm 41 - 42 - .macro __pmull2_p64, rd, rn, rm 43 - pmull2 \rd\().1q, \rn\().2d, \rm\().2d 44 - .endm 45 - 46 - .macro __pmull_p8, rq, ad, bd 47 - ext t3.8b, \ad\().8b, \ad\().8b, #1 // A1 48 - ext t5.8b, \ad\().8b, \ad\().8b, #2 // A2 49 - ext t7.8b, \ad\().8b, \ad\().8b, #3 // A3 50 - 51 - __pmull_p8_\bd \rq, \ad 52 - .endm 53 - 54 - .macro __pmull2_p8, rq, ad, bd 55 - tbl t3.16b, {\ad\().16b}, perm1.16b // A1 56 - tbl t5.16b, {\ad\().16b}, perm2.16b // A2 57 - tbl t7.16b, {\ad\().16b}, perm3.16b // A3 58 - 59 - __pmull2_p8_\bd \rq, \ad 60 - .endm 61 - 62 - .macro __pmull_p8_SHASH, rq, ad 63 - __pmull_p8_tail \rq, \ad\().8b, SHASH.8b, 8b,, sh1, sh2, sh3, sh4 64 - .endm 65 - 66 - .macro __pmull_p8_SHASH2, rq, ad 67 - __pmull_p8_tail \rq, \ad\().8b, SHASH2.8b, 8b,, ss1, ss2, ss3, ss4 68 - .endm 69 - 70 - .macro __pmull2_p8_SHASH, rq, ad 71 - __pmull_p8_tail \rq, \ad\().16b, SHASH.16b, 16b, 2, sh1, sh2, sh3, sh4 72 - .endm 73 - 74 - .macro __pmull_p8_tail, rq, ad, bd, nb, t, b1, b2, b3, b4 75 - pmull\t t3.8h, t3.\nb, \bd // F = A1*B 76 - pmull\t t4.8h, \ad, \b1\().\nb // E = A*B1 77 - pmull\t t5.8h, t5.\nb, \bd // H = A2*B 78 - pmull\t t6.8h, \ad, \b2\().\nb // G = A*B2 79 - pmull\t t7.8h, t7.\nb, \bd // J = A3*B 80 - pmull\t t8.8h, \ad, \b3\().\nb // I = A*B3 81 - pmull\t t9.8h, \ad, \b4\().\nb // K = A*B4 82 - pmull\t \rq\().8h, \ad, \bd // D = A*B 83 - 84 - eor t3.16b, t3.16b, t4.16b // L = E + F 85 - eor t5.16b, t5.16b, t6.16b // M = G + H 86 - eor t7.16b, t7.16b, t8.16b // N = I + J 87 - 88 - uzp1 t4.2d, t3.2d, t5.2d 89 - uzp2 t3.2d, t3.2d, t5.2d 90 - uzp1 t6.2d, t7.2d, t9.2d 91 - uzp2 t7.2d, t7.2d, t9.2d 92 - 93 - // t3 = (L) (P0 + P1) << 8 94 - // t5 = (M) (P2 + P3) << 16 95 - eor t4.16b, t4.16b, t3.16b 96 - and t3.16b, t3.16b, k32_48.16b 97 - 98 - // t7 = (N) (P4 + P5) << 24 99 - // t9 = (K) (P6 + P7) << 32 100 - eor t6.16b, t6.16b, t7.16b 101 - and t7.16b, t7.16b, k00_16.16b 102 - 103 - eor t4.16b, t4.16b, t3.16b 104 - eor t6.16b, t6.16b, t7.16b 105 - 106 - zip2 t5.2d, t4.2d, t3.2d 107 - zip1 t3.2d, t4.2d, t3.2d 108 - zip2 t9.2d, t6.2d, t7.2d 109 - zip1 t7.2d, t6.2d, t7.2d 110 - 111 - ext t3.16b, t3.16b, t3.16b, #15 112 - ext t5.16b, t5.16b, t5.16b, #14 113 - ext t7.16b, t7.16b, t7.16b, #13 114 - ext t9.16b, t9.16b, t9.16b, #12 115 - 116 - eor t3.16b, t3.16b, t5.16b 117 - eor t7.16b, t7.16b, t9.16b 118 - eor \rq\().16b, \rq\().16b, t3.16b 119 - eor \rq\().16b, \rq\().16b, t7.16b 120 - .endm 121 62 122 63 .macro __pmull_pre_p64 123 64 add x8, x3, #16 ··· 51 160 shl MASK.2d, MASK.2d, #57 52 161 .endm 53 162 54 - .macro __pmull_pre_p8 55 - ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 56 - eor SHASH2.16b, SHASH2.16b, SHASH.16b 57 - 58 - // k00_16 := 0x0000000000000000_000000000000ffff 59 - // k32_48 := 0x00000000ffffffff_0000ffffffffffff 60 - movi k32_48.2d, #0xffffffff 61 - mov k32_48.h[2], k32_48.h[0] 62 - ushr k00_16.2d, k32_48.2d, #32 63 - 64 - // prepare the permutation vectors 65 - mov_q x5, 0x080f0e0d0c0b0a09 66 - movi T1.8b, #8 67 - dup perm1.2d, x5 68 - eor perm1.16b, perm1.16b, T1.16b 69 - ushr perm2.2d, perm1.2d, #8 70 - ushr perm3.2d, perm1.2d, #16 71 - ushr T1.2d, perm1.2d, #24 72 - sli perm2.2d, perm1.2d, #56 73 - sli perm3.2d, perm1.2d, #48 74 - sli T1.2d, perm1.2d, #40 75 - 76 - // precompute loop invariants 77 - tbl sh1.16b, {SHASH.16b}, perm1.16b 78 - tbl sh2.16b, {SHASH.16b}, perm2.16b 79 - tbl sh3.16b, {SHASH.16b}, perm3.16b 80 - tbl sh4.16b, {SHASH.16b}, T1.16b 81 - ext ss1.8b, SHASH2.8b, SHASH2.8b, #1 82 - ext ss2.8b, SHASH2.8b, SHASH2.8b, #2 83 - ext ss3.8b, SHASH2.8b, SHASH2.8b, #3 84 - ext ss4.8b, SHASH2.8b, SHASH2.8b, #4 85 - .endm 86 - 87 - // 88 - // PMULL (64x64->128) based reduction for CPUs that can do 89 - // it in a single instruction. 90 - // 91 163 .macro __pmull_reduce_p64 92 164 pmull T2.1q, XL.1d, MASK.1d 93 165 eor XM.16b, XM.16b, T1.16b ··· 63 209 pmull XL.1q, XL.1d, MASK.1d 64 210 .endm 65 211 66 - // 67 - // Alternative reduction for CPUs that lack support for the 68 - // 64x64->128 PMULL instruction 69 - // 70 - .macro __pmull_reduce_p8 71 - eor XM.16b, XM.16b, T1.16b 72 - 73 - mov XL.d[1], XM.d[0] 74 - mov XH.d[0], XM.d[1] 75 - 76 - shl T1.2d, XL.2d, #57 77 - shl T2.2d, XL.2d, #62 78 - eor T2.16b, T2.16b, T1.16b 79 - shl T1.2d, XL.2d, #63 80 - eor T2.16b, T2.16b, T1.16b 81 - ext T1.16b, XL.16b, XH.16b, #8 82 - eor T2.16b, T2.16b, T1.16b 83 - 84 - mov XL.d[1], T2.d[0] 85 - mov XH.d[0], T2.d[1] 86 - 87 - ushr T2.2d, XL.2d, #1 88 - eor XH.16b, XH.16b, XL.16b 89 - eor XL.16b, XL.16b, T2.16b 90 - ushr T2.2d, T2.2d, #6 91 - ushr XL.2d, XL.2d, #1 92 - .endm 93 - 94 - .macro __pmull_ghash, pn 212 + /* 213 + * void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, 214 + * u64 const h[][2], const char *head) 215 + */ 216 + SYM_TYPED_FUNC_START(pmull_ghash_update_p64) 95 217 ld1 {SHASH.2d}, [x3] 96 218 ld1 {XL.2d}, [x1] 97 219 98 - __pmull_pre_\pn 220 + __pmull_pre_p64 99 221 100 222 /* do the head block first, if supplied */ 101 223 cbz x4, 0f ··· 79 249 mov x4, xzr 80 250 b 3f 81 251 82 - 0: .ifc \pn, p64 252 + 0: 83 253 tbnz w0, #0, 2f // skip until #blocks is a 84 254 tbnz w0, #1, 2f // round multiple of 4 85 255 ··· 144 314 145 315 cbz w0, 5f 146 316 b 1b 147 - .endif 148 317 149 318 2: ld1 {T1.2d}, [x2], #16 150 319 sub w0, w0, #1 ··· 156 327 eor T1.16b, T1.16b, T2.16b 157 328 eor XL.16b, XL.16b, IN1.16b 158 329 159 - __pmull2_\pn XH, XL, SHASH // a1 * b1 330 + pmull2 XH.1q, XL.2d, SHASH.2d // a1 * b1 160 331 eor T1.16b, T1.16b, XL.16b 161 - __pmull_\pn XL, XL, SHASH // a0 * b0 162 - __pmull_\pn XM, T1, SHASH2 // (a1 + a0)(b1 + b0) 332 + pmull XL.1q, XL.1d, SHASH.1d // a0 * b0 333 + pmull XM.1q, T1.1d, SHASH2.1d // (a1 + a0)(b1 + b0) 163 334 164 335 4: eor T2.16b, XL.16b, XH.16b 165 336 ext T1.16b, XL.16b, XH.16b, #8 166 337 eor XM.16b, XM.16b, T2.16b 167 338 168 - __pmull_reduce_\pn 339 + __pmull_reduce_p64 169 340 170 341 eor T2.16b, T2.16b, XH.16b 171 342 eor XL.16b, XL.16b, T2.16b ··· 174 345 175 346 5: st1 {XL.2d}, [x1] 176 347 ret 177 - .endm 178 - 179 - /* 180 - * void pmull_ghash_update(int blocks, u64 dg[], const char *src, 181 - * struct ghash_key const *k, const char *head) 182 - */ 183 - SYM_TYPED_FUNC_START(pmull_ghash_update_p64) 184 - __pmull_ghash p64 185 348 SYM_FUNC_END(pmull_ghash_update_p64) 186 - 187 - SYM_TYPED_FUNC_START(pmull_ghash_update_p8) 188 - __pmull_ghash p8 189 - SYM_FUNC_END(pmull_ghash_update_p8) 190 349 191 350 KS0 .req v8 192 351 KS1 .req v9

+226

arch/arm64/crypto/ghash-neon-core.S

··· 1 + /* SPDX-License-Identifier: GPL-2.0-only */ 2 + /* 3 + * Accelerated GHASH implementation with ARMv8 ASIMD instructions. 4 + * 5 + * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> 6 + */ 7 + 8 + #include <linux/linkage.h> 9 + #include <linux/cfi_types.h> 10 + #include <asm/assembler.h> 11 + 12 + SHASH .req v0 13 + SHASH2 .req v1 14 + T1 .req v2 15 + T2 .req v3 16 + XM .req v5 17 + XL .req v6 18 + XH .req v7 19 + IN1 .req v7 20 + 21 + k00_16 .req v8 22 + k32_48 .req v9 23 + 24 + t3 .req v10 25 + t4 .req v11 26 + t5 .req v12 27 + t6 .req v13 28 + t7 .req v14 29 + t8 .req v15 30 + t9 .req v16 31 + 32 + perm1 .req v17 33 + perm2 .req v18 34 + perm3 .req v19 35 + 36 + sh1 .req v20 37 + sh2 .req v21 38 + sh3 .req v22 39 + sh4 .req v23 40 + 41 + ss1 .req v24 42 + ss2 .req v25 43 + ss3 .req v26 44 + ss4 .req v27 45 + 46 + .text 47 + 48 + .macro __pmull_p8, rq, ad, bd 49 + ext t3.8b, \ad\().8b, \ad\().8b, #1 // A1 50 + ext t5.8b, \ad\().8b, \ad\().8b, #2 // A2 51 + ext t7.8b, \ad\().8b, \ad\().8b, #3 // A3 52 + 53 + __pmull_p8_\bd \rq, \ad 54 + .endm 55 + 56 + .macro __pmull2_p8, rq, ad, bd 57 + tbl t3.16b, {\ad\().16b}, perm1.16b // A1 58 + tbl t5.16b, {\ad\().16b}, perm2.16b // A2 59 + tbl t7.16b, {\ad\().16b}, perm3.16b // A3 60 + 61 + __pmull2_p8_\bd \rq, \ad 62 + .endm 63 + 64 + .macro __pmull_p8_SHASH, rq, ad 65 + __pmull_p8_tail \rq, \ad\().8b, SHASH.8b, 8b,, sh1, sh2, sh3, sh4 66 + .endm 67 + 68 + .macro __pmull_p8_SHASH2, rq, ad 69 + __pmull_p8_tail \rq, \ad\().8b, SHASH2.8b, 8b,, ss1, ss2, ss3, ss4 70 + .endm 71 + 72 + .macro __pmull2_p8_SHASH, rq, ad 73 + __pmull_p8_tail \rq, \ad\().16b, SHASH.16b, 16b, 2, sh1, sh2, sh3, sh4 74 + .endm 75 + 76 + .macro __pmull_p8_tail, rq, ad, bd, nb, t, b1, b2, b3, b4 77 + pmull\t t3.8h, t3.\nb, \bd // F = A1*B 78 + pmull\t t4.8h, \ad, \b1\().\nb // E = A*B1 79 + pmull\t t5.8h, t5.\nb, \bd // H = A2*B 80 + pmull\t t6.8h, \ad, \b2\().\nb // G = A*B2 81 + pmull\t t7.8h, t7.\nb, \bd // J = A3*B 82 + pmull\t t8.8h, \ad, \b3\().\nb // I = A*B3 83 + pmull\t t9.8h, \ad, \b4\().\nb // K = A*B4 84 + pmull\t \rq\().8h, \ad, \bd // D = A*B 85 + 86 + eor t3.16b, t3.16b, t4.16b // L = E + F 87 + eor t5.16b, t5.16b, t6.16b // M = G + H 88 + eor t7.16b, t7.16b, t8.16b // N = I + J 89 + 90 + uzp1 t4.2d, t3.2d, t5.2d 91 + uzp2 t3.2d, t3.2d, t5.2d 92 + uzp1 t6.2d, t7.2d, t9.2d 93 + uzp2 t7.2d, t7.2d, t9.2d 94 + 95 + // t3 = (L) (P0 + P1) << 8 96 + // t5 = (M) (P2 + P3) << 16 97 + eor t4.16b, t4.16b, t3.16b 98 + and t3.16b, t3.16b, k32_48.16b 99 + 100 + // t7 = (N) (P4 + P5) << 24 101 + // t9 = (K) (P6 + P7) << 32 102 + eor t6.16b, t6.16b, t7.16b 103 + and t7.16b, t7.16b, k00_16.16b 104 + 105 + eor t4.16b, t4.16b, t3.16b 106 + eor t6.16b, t6.16b, t7.16b 107 + 108 + zip2 t5.2d, t4.2d, t3.2d 109 + zip1 t3.2d, t4.2d, t3.2d 110 + zip2 t9.2d, t6.2d, t7.2d 111 + zip1 t7.2d, t6.2d, t7.2d 112 + 113 + ext t3.16b, t3.16b, t3.16b, #15 114 + ext t5.16b, t5.16b, t5.16b, #14 115 + ext t7.16b, t7.16b, t7.16b, #13 116 + ext t9.16b, t9.16b, t9.16b, #12 117 + 118 + eor t3.16b, t3.16b, t5.16b 119 + eor t7.16b, t7.16b, t9.16b 120 + eor \rq\().16b, \rq\().16b, t3.16b 121 + eor \rq\().16b, \rq\().16b, t7.16b 122 + .endm 123 + 124 + .macro __pmull_pre_p8 125 + ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 126 + eor SHASH2.16b, SHASH2.16b, SHASH.16b 127 + 128 + // k00_16 := 0x0000000000000000_000000000000ffff 129 + // k32_48 := 0x00000000ffffffff_0000ffffffffffff 130 + movi k32_48.2d, #0xffffffff 131 + mov k32_48.h[2], k32_48.h[0] 132 + ushr k00_16.2d, k32_48.2d, #32 133 + 134 + // prepare the permutation vectors 135 + mov_q x5, 0x080f0e0d0c0b0a09 136 + movi T1.8b, #8 137 + dup perm1.2d, x5 138 + eor perm1.16b, perm1.16b, T1.16b 139 + ushr perm2.2d, perm1.2d, #8 140 + ushr perm3.2d, perm1.2d, #16 141 + ushr T1.2d, perm1.2d, #24 142 + sli perm2.2d, perm1.2d, #56 143 + sli perm3.2d, perm1.2d, #48 144 + sli T1.2d, perm1.2d, #40 145 + 146 + // precompute loop invariants 147 + tbl sh1.16b, {SHASH.16b}, perm1.16b 148 + tbl sh2.16b, {SHASH.16b}, perm2.16b 149 + tbl sh3.16b, {SHASH.16b}, perm3.16b 150 + tbl sh4.16b, {SHASH.16b}, T1.16b 151 + ext ss1.8b, SHASH2.8b, SHASH2.8b, #1 152 + ext ss2.8b, SHASH2.8b, SHASH2.8b, #2 153 + ext ss3.8b, SHASH2.8b, SHASH2.8b, #3 154 + ext ss4.8b, SHASH2.8b, SHASH2.8b, #4 155 + .endm 156 + 157 + .macro __pmull_reduce_p8 158 + eor XM.16b, XM.16b, T1.16b 159 + 160 + mov XL.d[1], XM.d[0] 161 + mov XH.d[0], XM.d[1] 162 + 163 + shl T1.2d, XL.2d, #57 164 + shl T2.2d, XL.2d, #62 165 + eor T2.16b, T2.16b, T1.16b 166 + shl T1.2d, XL.2d, #63 167 + eor T2.16b, T2.16b, T1.16b 168 + ext T1.16b, XL.16b, XH.16b, #8 169 + eor T2.16b, T2.16b, T1.16b 170 + 171 + mov XL.d[1], T2.d[0] 172 + mov XH.d[0], T2.d[1] 173 + 174 + ushr T2.2d, XL.2d, #1 175 + eor XH.16b, XH.16b, XL.16b 176 + eor XL.16b, XL.16b, T2.16b 177 + ushr T2.2d, T2.2d, #6 178 + ushr XL.2d, XL.2d, #1 179 + .endm 180 + 181 + /* 182 + * void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, 183 + * u64 const h[][2], const char *head) 184 + */ 185 + SYM_TYPED_FUNC_START(pmull_ghash_update_p8) 186 + ld1 {SHASH.2d}, [x3] 187 + ld1 {XL.2d}, [x1] 188 + 189 + __pmull_pre_p8 190 + 191 + /* do the head block first, if supplied */ 192 + cbz x4, 0f 193 + ld1 {T1.2d}, [x4] 194 + mov x4, xzr 195 + b 3f 196 + 197 + 0: ld1 {T1.2d}, [x2], #16 198 + sub w0, w0, #1 199 + 200 + 3: /* multiply XL by SHASH in GF(2^128) */ 201 + CPU_LE( rev64 T1.16b, T1.16b ) 202 + 203 + ext T2.16b, XL.16b, XL.16b, #8 204 + ext IN1.16b, T1.16b, T1.16b, #8 205 + eor T1.16b, T1.16b, T2.16b 206 + eor XL.16b, XL.16b, IN1.16b 207 + 208 + __pmull2_p8 XH, XL, SHASH // a1 * b1 209 + eor T1.16b, T1.16b, XL.16b 210 + __pmull_p8 XL, XL, SHASH // a0 * b0 211 + __pmull_p8 XM, T1, SHASH2 // (a1 + a0)(b1 + b0) 212 + 213 + eor T2.16b, XL.16b, XH.16b 214 + ext T1.16b, XL.16b, XH.16b, #8 215 + eor XM.16b, XM.16b, T2.16b 216 + 217 + __pmull_reduce_p8 218 + 219 + eor T2.16b, T2.16b, XH.16b 220 + eor XL.16b, XL.16b, T2.16b 221 + 222 + cbnz w0, 0b 223 + 224 + st1 {XL.2d}, [x1] 225 + ret 226 + SYM_FUNC_END(pmull_ghash_update_p8)

Configure Feed

Configure Feed