tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

lib/crypto: x86/blake2s: Improve readability

Various cleanups for readability. No change to the generated code:

- Add some comments
- Add #defines for arguments
- Rename some labels
- Use decimal constants instead of hex where it makes sense.
(The pshufd immediates intentionally remain as hex.)
- Add blank lines when there's a logical break

The round loop still could use some work, but this is at least a start.

Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20251102234209.62133-5-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>

Eric Biggers a7acd77e 83c1a867

+134 -97
+134 -97
lib/crypto/x86/blake2s-core.S
··· 50 50 .byte 15, 5, 4, 13, 10, 7, 3, 11, 12, 2, 0, 6, 9, 8, 1, 14 51 51 .byte 8, 7, 14, 11, 13, 15, 0, 12, 10, 4, 5, 6, 3, 2, 1, 9 52 52 53 + #define CTX %rdi 54 + #define DATA %rsi 55 + #define NBLOCKS %rdx 56 + #define INC %ecx 57 + 53 58 .text 59 + // 60 + // void blake2s_compress_ssse3(struct blake2s_ctx *ctx, 61 + // const u8 *data, size_t nblocks, u32 inc); 62 + // 63 + // Only the first three fields of struct blake2s_ctx are used: 64 + // u32 h[8]; (inout) 65 + // u32 t[2]; (inout) 66 + // u32 f[2]; (in) 67 + // 54 68 SYM_FUNC_START(blake2s_compress_ssse3) 55 - movdqu (%rdi),%xmm0 56 - movdqu 0x10(%rdi),%xmm1 69 + movdqu (CTX),%xmm0 // Load h[0..3] 70 + movdqu 16(CTX),%xmm1 // Load h[4..7] 57 71 movdqa .Lror16(%rip),%xmm12 58 72 movdqa .Lror8(%rip),%xmm13 59 - movdqu 0x20(%rdi),%xmm14 60 - movd %ecx,%xmm15 61 - leaq .Lsigma+0xa0(%rip),%r8 62 - jmp .Lbeginofloop 73 + movdqu 32(CTX),%xmm14 // Load t and f 74 + movd INC,%xmm15 // Load inc 75 + leaq .Lsigma+160(%rip),%r8 76 + jmp .Lssse3_mainloop 77 + 63 78 .align 32 64 - .Lbeginofloop: 65 - movdqa %xmm0,%xmm10 66 - movdqa %xmm1,%xmm11 67 - paddq %xmm15,%xmm14 68 - movdqa .Liv(%rip),%xmm2 79 + .Lssse3_mainloop: 80 + // Main loop: each iteration processes one 64-byte block. 81 + movdqa %xmm0,%xmm10 // Save h[0..3] and let v[0..3] = h[0..3] 82 + movdqa %xmm1,%xmm11 // Save h[4..7] and let v[4..7] = h[4..7] 83 + paddq %xmm15,%xmm14 // t += inc (64-bit addition) 84 + movdqa .Liv(%rip),%xmm2 // v[8..11] = iv[0..3] 69 85 movdqa %xmm14,%xmm3 70 - pxor .Liv+0x10(%rip),%xmm3 86 + pxor .Liv+16(%rip),%xmm3 // v[12..15] = iv[4..7] ^ [t, f] 71 87 leaq .Lsigma(%rip),%rcx 72 - .Lroundloop: 88 + 89 + .Lssse3_roundloop: 90 + // Round loop: each iteration does 1 round (of 10 rounds total). 73 91 movzbl (%rcx),%eax 74 - movd (%rsi,%rax,4),%xmm4 75 - movzbl 0x1(%rcx),%eax 76 - movd (%rsi,%rax,4),%xmm5 77 - movzbl 0x2(%rcx),%eax 78 - movd (%rsi,%rax,4),%xmm6 79 - movzbl 0x3(%rcx),%eax 80 - movd (%rsi,%rax,4),%xmm7 92 + movd (DATA,%rax,4),%xmm4 93 + movzbl 1(%rcx),%eax 94 + movd (DATA,%rax,4),%xmm5 95 + movzbl 2(%rcx),%eax 96 + movd (DATA,%rax,4),%xmm6 97 + movzbl 3(%rcx),%eax 98 + movd (DATA,%rax,4),%xmm7 81 99 punpckldq %xmm5,%xmm4 82 100 punpckldq %xmm7,%xmm6 83 101 punpcklqdq %xmm6,%xmm4 ··· 106 88 paddd %xmm3,%xmm2 107 89 pxor %xmm2,%xmm1 108 90 movdqa %xmm1,%xmm8 109 - psrld $0xc,%xmm1 110 - pslld $0x14,%xmm8 91 + psrld $12,%xmm1 92 + pslld $20,%xmm8 111 93 por %xmm8,%xmm1 112 - movzbl 0x4(%rcx),%eax 113 - movd (%rsi,%rax,4),%xmm5 114 - movzbl 0x5(%rcx),%eax 115 - movd (%rsi,%rax,4),%xmm6 116 - movzbl 0x6(%rcx),%eax 117 - movd (%rsi,%rax,4),%xmm7 118 - movzbl 0x7(%rcx),%eax 119 - movd (%rsi,%rax,4),%xmm4 94 + movzbl 4(%rcx),%eax 95 + movd (DATA,%rax,4),%xmm5 96 + movzbl 5(%rcx),%eax 97 + movd (DATA,%rax,4),%xmm6 98 + movzbl 6(%rcx),%eax 99 + movd (DATA,%rax,4),%xmm7 100 + movzbl 7(%rcx),%eax 101 + movd (DATA,%rax,4),%xmm4 120 102 punpckldq %xmm6,%xmm5 121 103 punpckldq %xmm4,%xmm7 122 104 punpcklqdq %xmm7,%xmm5 ··· 127 109 paddd %xmm3,%xmm2 128 110 pxor %xmm2,%xmm1 129 111 movdqa %xmm1,%xmm8 130 - psrld $0x7,%xmm1 131 - pslld $0x19,%xmm8 112 + psrld $7,%xmm1 113 + pslld $25,%xmm8 132 114 por %xmm8,%xmm1 133 115 pshufd $0x93,%xmm0,%xmm0 134 116 pshufd $0x4e,%xmm3,%xmm3 135 117 pshufd $0x39,%xmm2,%xmm2 136 - movzbl 0x8(%rcx),%eax 137 - movd (%rsi,%rax,4),%xmm6 138 - movzbl 0x9(%rcx),%eax 139 - movd (%rsi,%rax,4),%xmm7 140 - movzbl 0xa(%rcx),%eax 141 - movd (%rsi,%rax,4),%xmm4 142 - movzbl 0xb(%rcx),%eax 143 - movd (%rsi,%rax,4),%xmm5 118 + movzbl 8(%rcx),%eax 119 + movd (DATA,%rax,4),%xmm6 120 + movzbl 9(%rcx),%eax 121 + movd (DATA,%rax,4),%xmm7 122 + movzbl 10(%rcx),%eax 123 + movd (DATA,%rax,4),%xmm4 124 + movzbl 11(%rcx),%eax 125 + movd (DATA,%rax,4),%xmm5 144 126 punpckldq %xmm7,%xmm6 145 127 punpckldq %xmm5,%xmm4 146 128 punpcklqdq %xmm4,%xmm6 ··· 151 133 paddd %xmm3,%xmm2 152 134 pxor %xmm2,%xmm1 153 135 movdqa %xmm1,%xmm8 154 - psrld $0xc,%xmm1 155 - pslld $0x14,%xmm8 136 + psrld $12,%xmm1 137 + pslld $20,%xmm8 156 138 por %xmm8,%xmm1 157 - movzbl 0xc(%rcx),%eax 158 - movd (%rsi,%rax,4),%xmm7 159 - movzbl 0xd(%rcx),%eax 160 - movd (%rsi,%rax,4),%xmm4 161 - movzbl 0xe(%rcx),%eax 162 - movd (%rsi,%rax,4),%xmm5 163 - movzbl 0xf(%rcx),%eax 164 - movd (%rsi,%rax,4),%xmm6 139 + movzbl 12(%rcx),%eax 140 + movd (DATA,%rax,4),%xmm7 141 + movzbl 13(%rcx),%eax 142 + movd (DATA,%rax,4),%xmm4 143 + movzbl 14(%rcx),%eax 144 + movd (DATA,%rax,4),%xmm5 145 + movzbl 15(%rcx),%eax 146 + movd (DATA,%rax,4),%xmm6 165 147 punpckldq %xmm4,%xmm7 166 148 punpckldq %xmm6,%xmm5 167 149 punpcklqdq %xmm5,%xmm7 ··· 172 154 paddd %xmm3,%xmm2 173 155 pxor %xmm2,%xmm1 174 156 movdqa %xmm1,%xmm8 175 - psrld $0x7,%xmm1 176 - pslld $0x19,%xmm8 157 + psrld $7,%xmm1 158 + pslld $25,%xmm8 177 159 por %xmm8,%xmm1 178 160 pshufd $0x39,%xmm0,%xmm0 179 161 pshufd $0x4e,%xmm3,%xmm3 180 162 pshufd $0x93,%xmm2,%xmm2 181 - addq $0x10,%rcx 163 + addq $16,%rcx 182 164 cmpq %r8,%rcx 183 - jnz .Lroundloop 165 + jnz .Lssse3_roundloop 166 + 167 + // Compute the new h: h[0..7] ^= v[0..7] ^ v[8..15] 184 168 pxor %xmm2,%xmm0 185 169 pxor %xmm3,%xmm1 186 170 pxor %xmm10,%xmm0 187 171 pxor %xmm11,%xmm1 188 - addq $0x40,%rsi 189 - decq %rdx 190 - jnz .Lbeginofloop 191 - movdqu %xmm0,(%rdi) 192 - movdqu %xmm1,0x10(%rdi) 193 - movdqu %xmm14,0x20(%rdi) 172 + addq $64,DATA 173 + decq NBLOCKS 174 + jnz .Lssse3_mainloop 175 + 176 + movdqu %xmm0,(CTX) // Store new h[0..3] 177 + movdqu %xmm1,16(CTX) // Store new h[4..7] 178 + movdqu %xmm14,32(CTX) // Store new t and f 194 179 RET 195 180 SYM_FUNC_END(blake2s_compress_ssse3) 196 181 182 + // 183 + // void blake2s_compress_avx512(struct blake2s_ctx *ctx, 184 + // const u8 *data, size_t nblocks, u32 inc); 185 + // 186 + // Only the first three fields of struct blake2s_ctx are used: 187 + // u32 h[8]; (inout) 188 + // u32 t[2]; (inout) 189 + // u32 f[2]; (in) 190 + // 197 191 SYM_FUNC_START(blake2s_compress_avx512) 198 - vmovdqu (%rdi),%xmm0 199 - vmovdqu 0x10(%rdi),%xmm1 200 - vmovdqu 0x20(%rdi),%xmm4 201 - vmovd %ecx,%xmm5 202 - vmovdqa .Liv(%rip),%xmm14 203 - vmovdqa .Liv+16(%rip),%xmm15 204 - jmp .Lblake2s_compress_avx512_mainloop 205 - .align 32 206 - .Lblake2s_compress_avx512_mainloop: 207 - vmovdqa %xmm0,%xmm10 208 - vmovdqa %xmm1,%xmm11 209 - vpaddq %xmm5,%xmm4,%xmm4 210 - vmovdqa %xmm14,%xmm2 211 - vpxor %xmm15,%xmm4,%xmm3 212 - vmovdqu (%rsi),%ymm6 213 - vmovdqu 0x20(%rsi),%ymm7 214 - addq $0x40,%rsi 192 + vmovdqu (CTX),%xmm0 // Load h[0..3] 193 + vmovdqu 16(CTX),%xmm1 // Load h[4..7] 194 + vmovdqu 32(CTX),%xmm4 // Load t and f 195 + vmovd INC,%xmm5 // Load inc 196 + vmovdqa .Liv(%rip),%xmm14 // Load iv[0..3] 197 + vmovdqa .Liv+16(%rip),%xmm15 // Load iv[4..7] 198 + jmp .Lavx512_mainloop 199 + 200 + .align 32 201 + .Lavx512_mainloop: 202 + // Main loop: each iteration processes one 64-byte block. 203 + vmovdqa %xmm0,%xmm10 // Save h[0..3] and let v[0..3] = h[0..3] 204 + vmovdqa %xmm1,%xmm11 // Save h[4..7] and let v[4..7] = h[4..7] 205 + vpaddq %xmm5,%xmm4,%xmm4 // t += inc (64-bit addition) 206 + vmovdqa %xmm14,%xmm2 // v[8..11] = iv[0..3] 207 + vpxor %xmm15,%xmm4,%xmm3 // v[12..15] = iv[4..7] ^ [t, f] 208 + vmovdqu (DATA),%ymm6 // Load first 8 data words 209 + vmovdqu 32(DATA),%ymm7 // Load second 8 data words 210 + addq $64,DATA 215 211 leaq .Lsigma2(%rip),%rax 216 - movb $0xa,%cl 217 - .Lblake2s_compress_avx512_roundloop: 212 + movb $10,%cl // Set num rounds remaining 213 + 214 + .Lavx512_roundloop: 215 + // Round loop: each iteration does 1 round (of 10 rounds total). 218 216 vpmovzxbd (%rax),%ymm8 219 - vpmovzxbd 0x8(%rax),%ymm9 220 - addq $0x10,%rax 217 + vpmovzxbd 8(%rax),%ymm9 218 + addq $16,%rax 221 219 vpermi2d %ymm7,%ymm6,%ymm8 222 220 vpermi2d %ymm7,%ymm6,%ymm9 223 221 vmovdqa %ymm8,%ymm6 ··· 241 207 vpaddd %xmm8,%xmm0,%xmm0 242 208 vpaddd %xmm1,%xmm0,%xmm0 243 209 vpxor %xmm0,%xmm3,%xmm3 244 - vprord $0x10,%xmm3,%xmm3 210 + vprord $16,%xmm3,%xmm3 245 211 vpaddd %xmm3,%xmm2,%xmm2 246 212 vpxor %xmm2,%xmm1,%xmm1 247 - vprord $0xc,%xmm1,%xmm1 248 - vextracti128 $0x1,%ymm8,%xmm8 213 + vprord $12,%xmm1,%xmm1 214 + vextracti128 $1,%ymm8,%xmm8 249 215 vpaddd %xmm8,%xmm0,%xmm0 250 216 vpaddd %xmm1,%xmm0,%xmm0 251 217 vpxor %xmm0,%xmm3,%xmm3 252 - vprord $0x8,%xmm3,%xmm3 218 + vprord $8,%xmm3,%xmm3 253 219 vpaddd %xmm3,%xmm2,%xmm2 254 220 vpxor %xmm2,%xmm1,%xmm1 255 - vprord $0x7,%xmm1,%xmm1 221 + vprord $7,%xmm1,%xmm1 256 222 vpshufd $0x93,%xmm0,%xmm0 257 223 vpshufd $0x4e,%xmm3,%xmm3 258 224 vpshufd $0x39,%xmm2,%xmm2 259 225 vpaddd %xmm9,%xmm0,%xmm0 260 226 vpaddd %xmm1,%xmm0,%xmm0 261 227 vpxor %xmm0,%xmm3,%xmm3 262 - vprord $0x10,%xmm3,%xmm3 228 + vprord $16,%xmm3,%xmm3 263 229 vpaddd %xmm3,%xmm2,%xmm2 264 230 vpxor %xmm2,%xmm1,%xmm1 265 - vprord $0xc,%xmm1,%xmm1 266 - vextracti128 $0x1,%ymm9,%xmm9 231 + vprord $12,%xmm1,%xmm1 232 + vextracti128 $1,%ymm9,%xmm9 267 233 vpaddd %xmm9,%xmm0,%xmm0 268 234 vpaddd %xmm1,%xmm0,%xmm0 269 235 vpxor %xmm0,%xmm3,%xmm3 270 - vprord $0x8,%xmm3,%xmm3 236 + vprord $8,%xmm3,%xmm3 271 237 vpaddd %xmm3,%xmm2,%xmm2 272 238 vpxor %xmm2,%xmm1,%xmm1 273 - vprord $0x7,%xmm1,%xmm1 239 + vprord $7,%xmm1,%xmm1 274 240 vpshufd $0x39,%xmm0,%xmm0 275 241 vpshufd $0x4e,%xmm3,%xmm3 276 242 vpshufd $0x93,%xmm2,%xmm2 277 243 decb %cl 278 - jne .Lblake2s_compress_avx512_roundloop 244 + jne .Lavx512_roundloop 245 + 246 + // Compute the new h: h[0..7] ^= v[0..7] ^ v[8..15] 279 247 vpxor %xmm10,%xmm0,%xmm0 280 248 vpxor %xmm11,%xmm1,%xmm1 281 249 vpxor %xmm2,%xmm0,%xmm0 282 250 vpxor %xmm3,%xmm1,%xmm1 283 - decq %rdx 284 - jne .Lblake2s_compress_avx512_mainloop 285 - vmovdqu %xmm0,(%rdi) 286 - vmovdqu %xmm1,0x10(%rdi) 287 - vmovdqu %xmm4,0x20(%rdi) 251 + decq NBLOCKS 252 + jne .Lavx512_mainloop 253 + 254 + vmovdqu %xmm0,(CTX) // Store new h[0..3] 255 + vmovdqu %xmm1,16(CTX) // Store new h[4..7] 256 + vmovdqu %xmm4,32(CTX) // Store new t and f 288 257 vzeroupper 289 258 RET 290 259 SYM_FUNC_END(blake2s_compress_avx512)