-14

arch/x86/crypto/Kconfig

reviewed

··· 99 99 100 100 Processes eight blocks in parallel. 101 101 102 102 - config CRYPTO_DES3_EDE_X86_64 103 103 - tristate "Ciphers: Triple DES EDE with modes: ECB, CBC" 104 104 - depends on 64BIT 105 105 - select CRYPTO_SKCIPHER 106 106 - select CRYPTO_LIB_DES 107 107 - imply CRYPTO_CTR 108 108 - help 109 109 - Block cipher: Triple DES EDE (FIPS 46-3) cipher algorithm 110 110 - Length-preserving ciphers: Triple DES EDE with ECB and CBC modes 111 111 - 112 112 - Architecture: x86_64 113 113 - 114 114 - Processes one or three blocks in parallel. 115 115 - 116 102 config CRYPTO_SERPENT_SSE2_X86_64 117 103 tristate "Ciphers: Serpent with modes: ECB, CBC (SSE2)" 118 104 depends on 64BIT

-3

arch/x86/crypto/Makefile

reviewed

··· 20 20 obj-$(CONFIG_CRYPTO_SERPENT_AVX2_X86_64) += serpent-avx2.o 21 21 serpent-avx2-y := serpent-avx2-asm_64.o serpent_avx2_glue.o 22 22 23 23 - obj-$(CONFIG_CRYPTO_DES3_EDE_X86_64) += des3_ede-x86_64.o 24 24 - des3_ede-x86_64-y := des3_ede-asm_64.o des3_ede_glue.o 25 25 - 26 23 obj-$(CONFIG_CRYPTO_CAMELLIA_X86_64) += camellia-x86_64.o 27 24 camellia-x86_64-y := camellia-x86_64-asm_64.o camellia_glue.o 28 25 obj-$(CONFIG_CRYPTO_CAMELLIA_AESNI_AVX_X86_64) += camellia-aesni-avx-x86_64.o

-831

arch/x86/crypto/des3_ede-asm_64.S

reviewed

··· 1 1 - /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 2 - /* 3 3 - * des3_ede-asm_64.S - x86-64 assembly implementation of 3DES cipher 4 4 - * 5 5 - * Copyright © 2014 Jussi Kivilinna <jussi.kivilinna@iki.fi> 6 6 - */ 7 7 - 8 8 - #include <linux/linkage.h> 9 9 - 10 10 - .file "des3_ede-asm_64.S" 11 11 - .text 12 12 - 13 13 - #define s1 .L_s1 14 14 - #define s2 ((s1) + (64*8)) 15 15 - #define s3 ((s2) + (64*8)) 16 16 - #define s4 ((s3) + (64*8)) 17 17 - #define s5 ((s4) + (64*8)) 18 18 - #define s6 ((s5) + (64*8)) 19 19 - #define s7 ((s6) + (64*8)) 20 20 - #define s8 ((s7) + (64*8)) 21 21 - 22 22 - /* register macros */ 23 23 - #define CTX %rdi 24 24 - 25 25 - #define RL0 %r8 26 26 - #define RL1 %r9 27 27 - #define RL2 %r10 28 28 - 29 29 - #define RL0d %r8d 30 30 - #define RL1d %r9d 31 31 - #define RL2d %r10d 32 32 - 33 33 - #define RR0 %r11 34 34 - #define RR1 %r12 35 35 - #define RR2 %r13 36 36 - 37 37 - #define RR0d %r11d 38 38 - #define RR1d %r12d 39 39 - #define RR2d %r13d 40 40 - 41 41 - #define RW0 %rax 42 42 - #define RW1 %rbx 43 43 - #define RW2 %rcx 44 44 - 45 45 - #define RW0d %eax 46 46 - #define RW1d %ebx 47 47 - #define RW2d %ecx 48 48 - 49 49 - #define RW0bl %al 50 50 - #define RW1bl %bl 51 51 - #define RW2bl %cl 52 52 - 53 53 - #define RW0bh %ah 54 54 - #define RW1bh %bh 55 55 - #define RW2bh %ch 56 56 - 57 57 - #define RT0 %r15 58 58 - #define RT1 %rsi 59 59 - #define RT2 %r14 60 60 - #define RT3 %rdx 61 61 - 62 62 - #define RT0d %r15d 63 63 - #define RT1d %esi 64 64 - #define RT2d %r14d 65 65 - #define RT3d %edx 66 66 - 67 67 - /*********************************************************************** 68 68 - * 1-way 3DES 69 69 - ***********************************************************************/ 70 70 - #define do_permutation(a, b, offset, mask) \ 71 71 - movl a, RT0d; \ 72 72 - shrl $(offset), RT0d; \ 73 73 - xorl b, RT0d; \ 74 74 - andl $(mask), RT0d; \ 75 75 - xorl RT0d, b; \ 76 76 - shll $(offset), RT0d; \ 77 77 - xorl RT0d, a; 78 78 - 79 79 - #define expand_to_64bits(val, mask) \ 80 80 - movl val##d, RT0d; \ 81 81 - rorl $4, RT0d; \ 82 82 - shlq $32, RT0; \ 83 83 - orq RT0, val; \ 84 84 - andq mask, val; 85 85 - 86 86 - #define compress_to_64bits(val) \ 87 87 - movq val, RT0; \ 88 88 - shrq $32, RT0; \ 89 89 - roll $4, RT0d; \ 90 90 - orl RT0d, val##d; 91 91 - 92 92 - #define initial_permutation(left, right) \ 93 93 - do_permutation(left##d, right##d, 4, 0x0f0f0f0f); \ 94 94 - do_permutation(left##d, right##d, 16, 0x0000ffff); \ 95 95 - do_permutation(right##d, left##d, 2, 0x33333333); \ 96 96 - do_permutation(right##d, left##d, 8, 0x00ff00ff); \ 97 97 - movabs $0x3f3f3f3f3f3f3f3f, RT3; \ 98 98 - movl left##d, RW0d; \ 99 99 - roll $1, right##d; \ 100 100 - xorl right##d, RW0d; \ 101 101 - andl $0xaaaaaaaa, RW0d; \ 102 102 - xorl RW0d, left##d; \ 103 103 - xorl RW0d, right##d; \ 104 104 - roll $1, left##d; \ 105 105 - expand_to_64bits(right, RT3); \ 106 106 - expand_to_64bits(left, RT3); 107 107 - 108 108 - #define final_permutation(left, right) \ 109 109 - compress_to_64bits(right); \ 110 110 - compress_to_64bits(left); \ 111 111 - movl right##d, RW0d; \ 112 112 - rorl $1, left##d; \ 113 113 - xorl left##d, RW0d; \ 114 114 - andl $0xaaaaaaaa, RW0d; \ 115 115 - xorl RW0d, right##d; \ 116 116 - xorl RW0d, left##d; \ 117 117 - rorl $1, right##d; \ 118 118 - do_permutation(right##d, left##d, 8, 0x00ff00ff); \ 119 119 - do_permutation(right##d, left##d, 2, 0x33333333); \ 120 120 - do_permutation(left##d, right##d, 16, 0x0000ffff); \ 121 121 - do_permutation(left##d, right##d, 4, 0x0f0f0f0f); 122 122 - 123 123 - #define round1(n, from, to, load_next_key) \ 124 124 - xorq from, RW0; \ 125 125 - \ 126 126 - movzbl RW0bl, RT0d; \ 127 127 - movzbl RW0bh, RT1d; \ 128 128 - shrq $16, RW0; \ 129 129 - movzbl RW0bl, RT2d; \ 130 130 - movzbl RW0bh, RT3d; \ 131 131 - shrq $16, RW0; \ 132 132 - leaq s8(%rip), RW1; \ 133 133 - movq (RW1, RT0, 8), RT0; \ 134 134 - leaq s6(%rip), RW1; \ 135 135 - xorq (RW1, RT1, 8), to; \ 136 136 - movzbl RW0bl, RL1d; \ 137 137 - movzbl RW0bh, RT1d; \ 138 138 - shrl $16, RW0d; \ 139 139 - leaq s4(%rip), RW1; \ 140 140 - xorq (RW1, RT2, 8), RT0; \ 141 141 - leaq s2(%rip), RW1; \ 142 142 - xorq (RW1, RT3, 8), to; \ 143 143 - movzbl RW0bl, RT2d; \ 144 144 - movzbl RW0bh, RT3d; \ 145 145 - leaq s7(%rip), RW1; \ 146 146 - xorq (RW1, RL1, 8), RT0; \ 147 147 - leaq s5(%rip), RW1; \ 148 148 - xorq (RW1, RT1, 8), to; \ 149 149 - leaq s3(%rip), RW1; \ 150 150 - xorq (RW1, RT2, 8), RT0; \ 151 151 - load_next_key(n, RW0); \ 152 152 - xorq RT0, to; \ 153 153 - leaq s1(%rip), RW1; \ 154 154 - xorq (RW1, RT3, 8), to; \ 155 155 - 156 156 - #define load_next_key(n, RWx) \ 157 157 - movq (((n) + 1) * 8)(CTX), RWx; 158 158 - 159 159 - #define dummy2(a, b) /*_*/ 160 160 - 161 161 - #define read_block(io, left, right) \ 162 162 - movl (io), left##d; \ 163 163 - movl 4(io), right##d; \ 164 164 - bswapl left##d; \ 165 165 - bswapl right##d; 166 166 - 167 167 - #define write_block(io, left, right) \ 168 168 - bswapl left##d; \ 169 169 - bswapl right##d; \ 170 170 - movl left##d, (io); \ 171 171 - movl right##d, 4(io); 172 172 - 173 173 - SYM_FUNC_START(des3_ede_x86_64_crypt_blk) 174 174 - /* input: 175 175 - * %rdi: round keys, CTX 176 176 - * %rsi: dst 177 177 - * %rdx: src 178 178 - */ 179 179 - pushq %rbx; 180 180 - pushq %r12; 181 181 - pushq %r13; 182 182 - pushq %r14; 183 183 - pushq %r15; 184 184 - 185 185 - pushq %rsi; /* dst */ 186 186 - 187 187 - read_block(%rdx, RL0, RR0); 188 188 - initial_permutation(RL0, RR0); 189 189 - 190 190 - movq (CTX), RW0; 191 191 - 192 192 - round1(0, RR0, RL0, load_next_key); 193 193 - round1(1, RL0, RR0, load_next_key); 194 194 - round1(2, RR0, RL0, load_next_key); 195 195 - round1(3, RL0, RR0, load_next_key); 196 196 - round1(4, RR0, RL0, load_next_key); 197 197 - round1(5, RL0, RR0, load_next_key); 198 198 - round1(6, RR0, RL0, load_next_key); 199 199 - round1(7, RL0, RR0, load_next_key); 200 200 - round1(8, RR0, RL0, load_next_key); 201 201 - round1(9, RL0, RR0, load_next_key); 202 202 - round1(10, RR0, RL0, load_next_key); 203 203 - round1(11, RL0, RR0, load_next_key); 204 204 - round1(12, RR0, RL0, load_next_key); 205 205 - round1(13, RL0, RR0, load_next_key); 206 206 - round1(14, RR0, RL0, load_next_key); 207 207 - round1(15, RL0, RR0, load_next_key); 208 208 - 209 209 - round1(16+0, RL0, RR0, load_next_key); 210 210 - round1(16+1, RR0, RL0, load_next_key); 211 211 - round1(16+2, RL0, RR0, load_next_key); 212 212 - round1(16+3, RR0, RL0, load_next_key); 213 213 - round1(16+4, RL0, RR0, load_next_key); 214 214 - round1(16+5, RR0, RL0, load_next_key); 215 215 - round1(16+6, RL0, RR0, load_next_key); 216 216 - round1(16+7, RR0, RL0, load_next_key); 217 217 - round1(16+8, RL0, RR0, load_next_key); 218 218 - round1(16+9, RR0, RL0, load_next_key); 219 219 - round1(16+10, RL0, RR0, load_next_key); 220 220 - round1(16+11, RR0, RL0, load_next_key); 221 221 - round1(16+12, RL0, RR0, load_next_key); 222 222 - round1(16+13, RR0, RL0, load_next_key); 223 223 - round1(16+14, RL0, RR0, load_next_key); 224 224 - round1(16+15, RR0, RL0, load_next_key); 225 225 - 226 226 - round1(32+0, RR0, RL0, load_next_key); 227 227 - round1(32+1, RL0, RR0, load_next_key); 228 228 - round1(32+2, RR0, RL0, load_next_key); 229 229 - round1(32+3, RL0, RR0, load_next_key); 230 230 - round1(32+4, RR0, RL0, load_next_key); 231 231 - round1(32+5, RL0, RR0, load_next_key); 232 232 - round1(32+6, RR0, RL0, load_next_key); 233 233 - round1(32+7, RL0, RR0, load_next_key); 234 234 - round1(32+8, RR0, RL0, load_next_key); 235 235 - round1(32+9, RL0, RR0, load_next_key); 236 236 - round1(32+10, RR0, RL0, load_next_key); 237 237 - round1(32+11, RL0, RR0, load_next_key); 238 238 - round1(32+12, RR0, RL0, load_next_key); 239 239 - round1(32+13, RL0, RR0, load_next_key); 240 240 - round1(32+14, RR0, RL0, load_next_key); 241 241 - round1(32+15, RL0, RR0, dummy2); 242 242 - 243 243 - final_permutation(RR0, RL0); 244 244 - 245 245 - popq %rsi /* dst */ 246 246 - write_block(%rsi, RR0, RL0); 247 247 - 248 248 - popq %r15; 249 249 - popq %r14; 250 250 - popq %r13; 251 251 - popq %r12; 252 252 - popq %rbx; 253 253 - 254 254 - RET; 255 255 - SYM_FUNC_END(des3_ede_x86_64_crypt_blk) 256 256 - 257 257 - /*********************************************************************** 258 258 - * 3-way 3DES 259 259 - ***********************************************************************/ 260 260 - #define expand_to_64bits(val, mask) \ 261 261 - movl val##d, RT0d; \ 262 262 - rorl $4, RT0d; \ 263 263 - shlq $32, RT0; \ 264 264 - orq RT0, val; \ 265 265 - andq mask, val; 266 266 - 267 267 - #define compress_to_64bits(val) \ 268 268 - movq val, RT0; \ 269 269 - shrq $32, RT0; \ 270 270 - roll $4, RT0d; \ 271 271 - orl RT0d, val##d; 272 272 - 273 273 - #define initial_permutation3(left, right) \ 274 274 - do_permutation(left##0d, right##0d, 4, 0x0f0f0f0f); \ 275 275 - do_permutation(left##0d, right##0d, 16, 0x0000ffff); \ 276 276 - do_permutation(left##1d, right##1d, 4, 0x0f0f0f0f); \ 277 277 - do_permutation(left##1d, right##1d, 16, 0x0000ffff); \ 278 278 - do_permutation(left##2d, right##2d, 4, 0x0f0f0f0f); \ 279 279 - do_permutation(left##2d, right##2d, 16, 0x0000ffff); \ 280 280 - \ 281 281 - do_permutation(right##0d, left##0d, 2, 0x33333333); \ 282 282 - do_permutation(right##0d, left##0d, 8, 0x00ff00ff); \ 283 283 - do_permutation(right##1d, left##1d, 2, 0x33333333); \ 284 284 - do_permutation(right##1d, left##1d, 8, 0x00ff00ff); \ 285 285 - do_permutation(right##2d, left##2d, 2, 0x33333333); \ 286 286 - do_permutation(right##2d, left##2d, 8, 0x00ff00ff); \ 287 287 - \ 288 288 - movabs $0x3f3f3f3f3f3f3f3f, RT3; \ 289 289 - \ 290 290 - movl left##0d, RW0d; \ 291 291 - roll $1, right##0d; \ 292 292 - xorl right##0d, RW0d; \ 293 293 - andl $0xaaaaaaaa, RW0d; \ 294 294 - xorl RW0d, left##0d; \ 295 295 - xorl RW0d, right##0d; \ 296 296 - roll $1, left##0d; \ 297 297 - expand_to_64bits(right##0, RT3); \ 298 298 - expand_to_64bits(left##0, RT3); \ 299 299 - movl left##1d, RW1d; \ 300 300 - roll $1, right##1d; \ 301 301 - xorl right##1d, RW1d; \ 302 302 - andl $0xaaaaaaaa, RW1d; \ 303 303 - xorl RW1d, left##1d; \ 304 304 - xorl RW1d, right##1d; \ 305 305 - roll $1, left##1d; \ 306 306 - expand_to_64bits(right##1, RT3); \ 307 307 - expand_to_64bits(left##1, RT3); \ 308 308 - movl left##2d, RW2d; \ 309 309 - roll $1, right##2d; \ 310 310 - xorl right##2d, RW2d; \ 311 311 - andl $0xaaaaaaaa, RW2d; \ 312 312 - xorl RW2d, left##2d; \ 313 313 - xorl RW2d, right##2d; \ 314 314 - roll $1, left##2d; \ 315 315 - expand_to_64bits(right##2, RT3); \ 316 316 - expand_to_64bits(left##2, RT3); 317 317 - 318 318 - #define final_permutation3(left, right) \ 319 319 - compress_to_64bits(right##0); \ 320 320 - compress_to_64bits(left##0); \ 321 321 - movl right##0d, RW0d; \ 322 322 - rorl $1, left##0d; \ 323 323 - xorl left##0d, RW0d; \ 324 324 - andl $0xaaaaaaaa, RW0d; \ 325 325 - xorl RW0d, right##0d; \ 326 326 - xorl RW0d, left##0d; \ 327 327 - rorl $1, right##0d; \ 328 328 - compress_to_64bits(right##1); \ 329 329 - compress_to_64bits(left##1); \ 330 330 - movl right##1d, RW1d; \ 331 331 - rorl $1, left##1d; \ 332 332 - xorl left##1d, RW1d; \ 333 333 - andl $0xaaaaaaaa, RW1d; \ 334 334 - xorl RW1d, right##1d; \ 335 335 - xorl RW1d, left##1d; \ 336 336 - rorl $1, right##1d; \ 337 337 - compress_to_64bits(right##2); \ 338 338 - compress_to_64bits(left##2); \ 339 339 - movl right##2d, RW2d; \ 340 340 - rorl $1, left##2d; \ 341 341 - xorl left##2d, RW2d; \ 342 342 - andl $0xaaaaaaaa, RW2d; \ 343 343 - xorl RW2d, right##2d; \ 344 344 - xorl RW2d, left##2d; \ 345 345 - rorl $1, right##2d; \ 346 346 - \ 347 347 - do_permutation(right##0d, left##0d, 8, 0x00ff00ff); \ 348 348 - do_permutation(right##0d, left##0d, 2, 0x33333333); \ 349 349 - do_permutation(right##1d, left##1d, 8, 0x00ff00ff); \ 350 350 - do_permutation(right##1d, left##1d, 2, 0x33333333); \ 351 351 - do_permutation(right##2d, left##2d, 8, 0x00ff00ff); \ 352 352 - do_permutation(right##2d, left##2d, 2, 0x33333333); \ 353 353 - \ 354 354 - do_permutation(left##0d, right##0d, 16, 0x0000ffff); \ 355 355 - do_permutation(left##0d, right##0d, 4, 0x0f0f0f0f); \ 356 356 - do_permutation(left##1d, right##1d, 16, 0x0000ffff); \ 357 357 - do_permutation(left##1d, right##1d, 4, 0x0f0f0f0f); \ 358 358 - do_permutation(left##2d, right##2d, 16, 0x0000ffff); \ 359 359 - do_permutation(left##2d, right##2d, 4, 0x0f0f0f0f); 360 360 - 361 361 - #define round3(n, from, to, load_next_key, do_movq) \ 362 362 - xorq from##0, RW0; \ 363 363 - movzbl RW0bl, RT3d; \ 364 364 - movzbl RW0bh, RT1d; \ 365 365 - shrq $16, RW0; \ 366 366 - leaq s8(%rip), RT2; \ 367 367 - xorq (RT2, RT3, 8), to##0; \ 368 368 - leaq s6(%rip), RT2; \ 369 369 - xorq (RT2, RT1, 8), to##0; \ 370 370 - movzbl RW0bl, RT3d; \ 371 371 - movzbl RW0bh, RT1d; \ 372 372 - shrq $16, RW0; \ 373 373 - leaq s4(%rip), RT2; \ 374 374 - xorq (RT2, RT3, 8), to##0; \ 375 375 - leaq s2(%rip), RT2; \ 376 376 - xorq (RT2, RT1, 8), to##0; \ 377 377 - movzbl RW0bl, RT3d; \ 378 378 - movzbl RW0bh, RT1d; \ 379 379 - shrl $16, RW0d; \ 380 380 - leaq s7(%rip), RT2; \ 381 381 - xorq (RT2, RT3, 8), to##0; \ 382 382 - leaq s5(%rip), RT2; \ 383 383 - xorq (RT2, RT1, 8), to##0; \ 384 384 - movzbl RW0bl, RT3d; \ 385 385 - movzbl RW0bh, RT1d; \ 386 386 - load_next_key(n, RW0); \ 387 387 - leaq s3(%rip), RT2; \ 388 388 - xorq (RT2, RT3, 8), to##0; \ 389 389 - leaq s1(%rip), RT2; \ 390 390 - xorq (RT2, RT1, 8), to##0; \ 391 391 - xorq from##1, RW1; \ 392 392 - movzbl RW1bl, RT3d; \ 393 393 - movzbl RW1bh, RT1d; \ 394 394 - shrq $16, RW1; \ 395 395 - leaq s8(%rip), RT2; \ 396 396 - xorq (RT2, RT3, 8), to##1; \ 397 397 - leaq s6(%rip), RT2; \ 398 398 - xorq (RT2, RT1, 8), to##1; \ 399 399 - movzbl RW1bl, RT3d; \ 400 400 - movzbl RW1bh, RT1d; \ 401 401 - shrq $16, RW1; \ 402 402 - leaq s4(%rip), RT2; \ 403 403 - xorq (RT2, RT3, 8), to##1; \ 404 404 - leaq s2(%rip), RT2; \ 405 405 - xorq (RT2, RT1, 8), to##1; \ 406 406 - movzbl RW1bl, RT3d; \ 407 407 - movzbl RW1bh, RT1d; \ 408 408 - shrl $16, RW1d; \ 409 409 - leaq s7(%rip), RT2; \ 410 410 - xorq (RT2, RT3, 8), to##1; \ 411 411 - leaq s5(%rip), RT2; \ 412 412 - xorq (RT2, RT1, 8), to##1; \ 413 413 - movzbl RW1bl, RT3d; \ 414 414 - movzbl RW1bh, RT1d; \ 415 415 - do_movq(RW0, RW1); \ 416 416 - leaq s3(%rip), RT2; \ 417 417 - xorq (RT2, RT3, 8), to##1; \ 418 418 - leaq s1(%rip), RT2; \ 419 419 - xorq (RT2, RT1, 8), to##1; \ 420 420 - xorq from##2, RW2; \ 421 421 - movzbl RW2bl, RT3d; \ 422 422 - movzbl RW2bh, RT1d; \ 423 423 - shrq $16, RW2; \ 424 424 - leaq s8(%rip), RT2; \ 425 425 - xorq (RT2, RT3, 8), to##2; \ 426 426 - leaq s6(%rip), RT2; \ 427 427 - xorq (RT2, RT1, 8), to##2; \ 428 428 - movzbl RW2bl, RT3d; \ 429 429 - movzbl RW2bh, RT1d; \ 430 430 - shrq $16, RW2; \ 431 431 - leaq s4(%rip), RT2; \ 432 432 - xorq (RT2, RT3, 8), to##2; \ 433 433 - leaq s2(%rip), RT2; \ 434 434 - xorq (RT2, RT1, 8), to##2; \ 435 435 - movzbl RW2bl, RT3d; \ 436 436 - movzbl RW2bh, RT1d; \ 437 437 - shrl $16, RW2d; \ 438 438 - leaq s7(%rip), RT2; \ 439 439 - xorq (RT2, RT3, 8), to##2; \ 440 440 - leaq s5(%rip), RT2; \ 441 441 - xorq (RT2, RT1, 8), to##2; \ 442 442 - movzbl RW2bl, RT3d; \ 443 443 - movzbl RW2bh, RT1d; \ 444 444 - do_movq(RW0, RW2); \ 445 445 - leaq s3(%rip), RT2; \ 446 446 - xorq (RT2, RT3, 8), to##2; \ 447 447 - leaq s1(%rip), RT2; \ 448 448 - xorq (RT2, RT1, 8), to##2; 449 449 - 450 450 - #define __movq(src, dst) \ 451 451 - movq src, dst; 452 452 - 453 453 - SYM_FUNC_START(des3_ede_x86_64_crypt_blk_3way) 454 454 - /* input: 455 455 - * %rdi: ctx, round keys 456 456 - * %rsi: dst (3 blocks) 457 457 - * %rdx: src (3 blocks) 458 458 - */ 459 459 - 460 460 - pushq %rbx; 461 461 - pushq %r12; 462 462 - pushq %r13; 463 463 - pushq %r14; 464 464 - pushq %r15; 465 465 - 466 466 - pushq %rsi /* dst */ 467 467 - 468 468 - /* load input */ 469 469 - movl 0 * 4(%rdx), RL0d; 470 470 - movl 1 * 4(%rdx), RR0d; 471 471 - movl 2 * 4(%rdx), RL1d; 472 472 - movl 3 * 4(%rdx), RR1d; 473 473 - movl 4 * 4(%rdx), RL2d; 474 474 - movl 5 * 4(%rdx), RR2d; 475 475 - 476 476 - bswapl RL0d; 477 477 - bswapl RR0d; 478 478 - bswapl RL1d; 479 479 - bswapl RR1d; 480 480 - bswapl RL2d; 481 481 - bswapl RR2d; 482 482 - 483 483 - initial_permutation3(RL, RR); 484 484 - 485 485 - movq 0(CTX), RW0; 486 486 - movq RW0, RW1; 487 487 - movq RW0, RW2; 488 488 - 489 489 - round3(0, RR, RL, load_next_key, __movq); 490 490 - round3(1, RL, RR, load_next_key, __movq); 491 491 - round3(2, RR, RL, load_next_key, __movq); 492 492 - round3(3, RL, RR, load_next_key, __movq); 493 493 - round3(4, RR, RL, load_next_key, __movq); 494 494 - round3(5, RL, RR, load_next_key, __movq); 495 495 - round3(6, RR, RL, load_next_key, __movq); 496 496 - round3(7, RL, RR, load_next_key, __movq); 497 497 - round3(8, RR, RL, load_next_key, __movq); 498 498 - round3(9, RL, RR, load_next_key, __movq); 499 499 - round3(10, RR, RL, load_next_key, __movq); 500 500 - round3(11, RL, RR, load_next_key, __movq); 501 501 - round3(12, RR, RL, load_next_key, __movq); 502 502 - round3(13, RL, RR, load_next_key, __movq); 503 503 - round3(14, RR, RL, load_next_key, __movq); 504 504 - round3(15, RL, RR, load_next_key, __movq); 505 505 - 506 506 - round3(16+0, RL, RR, load_next_key, __movq); 507 507 - round3(16+1, RR, RL, load_next_key, __movq); 508 508 - round3(16+2, RL, RR, load_next_key, __movq); 509 509 - round3(16+3, RR, RL, load_next_key, __movq); 510 510 - round3(16+4, RL, RR, load_next_key, __movq); 511 511 - round3(16+5, RR, RL, load_next_key, __movq); 512 512 - round3(16+6, RL, RR, load_next_key, __movq); 513 513 - round3(16+7, RR, RL, load_next_key, __movq); 514 514 - round3(16+8, RL, RR, load_next_key, __movq); 515 515 - round3(16+9, RR, RL, load_next_key, __movq); 516 516 - round3(16+10, RL, RR, load_next_key, __movq); 517 517 - round3(16+11, RR, RL, load_next_key, __movq); 518 518 - round3(16+12, RL, RR, load_next_key, __movq); 519 519 - round3(16+13, RR, RL, load_next_key, __movq); 520 520 - round3(16+14, RL, RR, load_next_key, __movq); 521 521 - round3(16+15, RR, RL, load_next_key, __movq); 522 522 - 523 523 - round3(32+0, RR, RL, load_next_key, __movq); 524 524 - round3(32+1, RL, RR, load_next_key, __movq); 525 525 - round3(32+2, RR, RL, load_next_key, __movq); 526 526 - round3(32+3, RL, RR, load_next_key, __movq); 527 527 - round3(32+4, RR, RL, load_next_key, __movq); 528 528 - round3(32+5, RL, RR, load_next_key, __movq); 529 529 - round3(32+6, RR, RL, load_next_key, __movq); 530 530 - round3(32+7, RL, RR, load_next_key, __movq); 531 531 - round3(32+8, RR, RL, load_next_key, __movq); 532 532 - round3(32+9, RL, RR, load_next_key, __movq); 533 533 - round3(32+10, RR, RL, load_next_key, __movq); 534 534 - round3(32+11, RL, RR, load_next_key, __movq); 535 535 - round3(32+12, RR, RL, load_next_key, __movq); 536 536 - round3(32+13, RL, RR, load_next_key, __movq); 537 537 - round3(32+14, RR, RL, load_next_key, __movq); 538 538 - round3(32+15, RL, RR, dummy2, dummy2); 539 539 - 540 540 - final_permutation3(RR, RL); 541 541 - 542 542 - bswapl RR0d; 543 543 - bswapl RL0d; 544 544 - bswapl RR1d; 545 545 - bswapl RL1d; 546 546 - bswapl RR2d; 547 547 - bswapl RL2d; 548 548 - 549 549 - popq %rsi /* dst */ 550 550 - movl RR0d, 0 * 4(%rsi); 551 551 - movl RL0d, 1 * 4(%rsi); 552 552 - movl RR1d, 2 * 4(%rsi); 553 553 - movl RL1d, 3 * 4(%rsi); 554 554 - movl RR2d, 4 * 4(%rsi); 555 555 - movl RL2d, 5 * 4(%rsi); 556 556 - 557 557 - popq %r15; 558 558 - popq %r14; 559 559 - popq %r13; 560 560 - popq %r12; 561 561 - popq %rbx; 562 562 - 563 563 - RET; 564 564 - SYM_FUNC_END(des3_ede_x86_64_crypt_blk_3way) 565 565 - 566 566 - .section .rodata, "a", @progbits 567 567 - .align 16 568 568 - .L_s1: 569 569 - .quad 0x0010100001010400, 0x0000000000000000 570 570 - .quad 0x0000100000010000, 0x0010100001010404 571 571 - .quad 0x0010100001010004, 0x0000100000010404 572 572 - .quad 0x0000000000000004, 0x0000100000010000 573 573 - .quad 0x0000000000000400, 0x0010100001010400 574 574 - .quad 0x0010100001010404, 0x0000000000000400 575 575 - .quad 0x0010000001000404, 0x0010100001010004 576 576 - .quad 0x0010000001000000, 0x0000000000000004 577 577 - .quad 0x0000000000000404, 0x0010000001000400 578 578 - .quad 0x0010000001000400, 0x0000100000010400 579 579 - .quad 0x0000100000010400, 0x0010100001010000 580 580 - .quad 0x0010100001010000, 0x0010000001000404 581 581 - .quad 0x0000100000010004, 0x0010000001000004 582 582 - .quad 0x0010000001000004, 0x0000100000010004 583 583 - .quad 0x0000000000000000, 0x0000000000000404 584 584 - .quad 0x0000100000010404, 0x0010000001000000 585 585 - .quad 0x0000100000010000, 0x0010100001010404 586 586 - .quad 0x0000000000000004, 0x0010100001010000 587 587 - .quad 0x0010100001010400, 0x0010000001000000 588 588 - .quad 0x0010000001000000, 0x0000000000000400 589 589 - .quad 0x0010100001010004, 0x0000100000010000 590 590 - .quad 0x0000100000010400, 0x0010000001000004 591 591 - .quad 0x0000000000000400, 0x0000000000000004 592 592 - .quad 0x0010000001000404, 0x0000100000010404 593 593 - .quad 0x0010100001010404, 0x0000100000010004 594 594 - .quad 0x0010100001010000, 0x0010000001000404 595 595 - .quad 0x0010000001000004, 0x0000000000000404 596 596 - .quad 0x0000100000010404, 0x0010100001010400 597 597 - .quad 0x0000000000000404, 0x0010000001000400 598 598 - .quad 0x0010000001000400, 0x0000000000000000 599 599 - .quad 0x0000100000010004, 0x0000100000010400 600 600 - .quad 0x0000000000000000, 0x0010100001010004 601 601 - .L_s2: 602 602 - .quad 0x0801080200100020, 0x0800080000000000 603 603 - .quad 0x0000080000000000, 0x0001080200100020 604 604 - .quad 0x0001000000100000, 0x0000000200000020 605 605 - .quad 0x0801000200100020, 0x0800080200000020 606 606 - .quad 0x0800000200000020, 0x0801080200100020 607 607 - .quad 0x0801080000100000, 0x0800000000000000 608 608 - .quad 0x0800080000000000, 0x0001000000100000 609 609 - .quad 0x0000000200000020, 0x0801000200100020 610 610 - .quad 0x0001080000100000, 0x0001000200100020 611 611 - .quad 0x0800080200000020, 0x0000000000000000 612 612 - .quad 0x0800000000000000, 0x0000080000000000 613 613 - .quad 0x0001080200100020, 0x0801000000100000 614 614 - .quad 0x0001000200100020, 0x0800000200000020 615 615 - .quad 0x0000000000000000, 0x0001080000100000 616 616 - .quad 0x0000080200000020, 0x0801080000100000 617 617 - .quad 0x0801000000100000, 0x0000080200000020 618 618 - .quad 0x0000000000000000, 0x0001080200100020 619 619 - .quad 0x0801000200100020, 0x0001000000100000 620 620 - .quad 0x0800080200000020, 0x0801000000100000 621 621 - .quad 0x0801080000100000, 0x0000080000000000 622 622 - .quad 0x0801000000100000, 0x0800080000000000 623 623 - .quad 0x0000000200000020, 0x0801080200100020 624 624 - .quad 0x0001080200100020, 0x0000000200000020 625 625 - .quad 0x0000080000000000, 0x0800000000000000 626 626 - .quad 0x0000080200000020, 0x0801080000100000 627 627 - .quad 0x0001000000100000, 0x0800000200000020 628 628 - .quad 0x0001000200100020, 0x0800080200000020 629 629 - .quad 0x0800000200000020, 0x0001000200100020 630 630 - .quad 0x0001080000100000, 0x0000000000000000 631 631 - .quad 0x0800080000000000, 0x0000080200000020 632 632 - .quad 0x0800000000000000, 0x0801000200100020 633 633 - .quad 0x0801080200100020, 0x0001080000100000 634 634 - .L_s3: 635 635 - .quad 0x0000002000000208, 0x0000202008020200 636 636 - .quad 0x0000000000000000, 0x0000200008020008 637 637 - .quad 0x0000002008000200, 0x0000000000000000 638 638 - .quad 0x0000202000020208, 0x0000002008000200 639 639 - .quad 0x0000200000020008, 0x0000000008000008 640 640 - .quad 0x0000000008000008, 0x0000200000020000 641 641 - .quad 0x0000202008020208, 0x0000200000020008 642 642 - .quad 0x0000200008020000, 0x0000002000000208 643 643 - .quad 0x0000000008000000, 0x0000000000000008 644 644 - .quad 0x0000202008020200, 0x0000002000000200 645 645 - .quad 0x0000202000020200, 0x0000200008020000 646 646 - .quad 0x0000200008020008, 0x0000202000020208 647 647 - .quad 0x0000002008000208, 0x0000202000020200 648 648 - .quad 0x0000200000020000, 0x0000002008000208 649 649 - .quad 0x0000000000000008, 0x0000202008020208 650 650 - .quad 0x0000002000000200, 0x0000000008000000 651 651 - .quad 0x0000202008020200, 0x0000000008000000 652 652 - .quad 0x0000200000020008, 0x0000002000000208 653 653 - .quad 0x0000200000020000, 0x0000202008020200 654 654 - .quad 0x0000002008000200, 0x0000000000000000 655 655 - .quad 0x0000002000000200, 0x0000200000020008 656 656 - .quad 0x0000202008020208, 0x0000002008000200 657 657 - .quad 0x0000000008000008, 0x0000002000000200 658 658 - .quad 0x0000000000000000, 0x0000200008020008 659 659 - .quad 0x0000002008000208, 0x0000200000020000 660 660 - .quad 0x0000000008000000, 0x0000202008020208 661 661 - .quad 0x0000000000000008, 0x0000202000020208 662 662 - .quad 0x0000202000020200, 0x0000000008000008 663 663 - .quad 0x0000200008020000, 0x0000002008000208 664 664 - .quad 0x0000002000000208, 0x0000200008020000 665 665 - .quad 0x0000202000020208, 0x0000000000000008 666 666 - .quad 0x0000200008020008, 0x0000202000020200 667 667 - .L_s4: 668 668 - .quad 0x1008020000002001, 0x1000020800002001 669 669 - .quad 0x1000020800002001, 0x0000000800000000 670 670 - .quad 0x0008020800002000, 0x1008000800000001 671 671 - .quad 0x1008000000000001, 0x1000020000002001 672 672 - .quad 0x0000000000000000, 0x0008020000002000 673 673 - .quad 0x0008020000002000, 0x1008020800002001 674 674 - .quad 0x1000000800000001, 0x0000000000000000 675 675 - .quad 0x0008000800000000, 0x1008000000000001 676 676 - .quad 0x1000000000000001, 0x0000020000002000 677 677 - .quad 0x0008000000000000, 0x1008020000002001 678 678 - .quad 0x0000000800000000, 0x0008000000000000 679 679 - .quad 0x1000020000002001, 0x0000020800002000 680 680 - .quad 0x1008000800000001, 0x1000000000000001 681 681 - .quad 0x0000020800002000, 0x0008000800000000 682 682 - .quad 0x0000020000002000, 0x0008020800002000 683 683 - .quad 0x1008020800002001, 0x1000000800000001 684 684 - .quad 0x0008000800000000, 0x1008000000000001 685 685 - .quad 0x0008020000002000, 0x1008020800002001 686 686 - .quad 0x1000000800000001, 0x0000000000000000 687 687 - .quad 0x0000000000000000, 0x0008020000002000 688 688 - .quad 0x0000020800002000, 0x0008000800000000 689 689 - .quad 0x1008000800000001, 0x1000000000000001 690 690 - .quad 0x1008020000002001, 0x1000020800002001 691 691 - .quad 0x1000020800002001, 0x0000000800000000 692 692 - .quad 0x1008020800002001, 0x1000000800000001 693 693 - .quad 0x1000000000000001, 0x0000020000002000 694 694 - .quad 0x1008000000000001, 0x1000020000002001 695 695 - .quad 0x0008020800002000, 0x1008000800000001 696 696 - .quad 0x1000020000002001, 0x0000020800002000 697 697 - .quad 0x0008000000000000, 0x1008020000002001 698 698 - .quad 0x0000000800000000, 0x0008000000000000 699 699 - .quad 0x0000020000002000, 0x0008020800002000 700 700 - .L_s5: 701 701 - .quad 0x0000001000000100, 0x0020001002080100 702 702 - .quad 0x0020000002080000, 0x0420001002000100 703 703 - .quad 0x0000000000080000, 0x0000001000000100 704 704 - .quad 0x0400000000000000, 0x0020000002080000 705 705 - .quad 0x0400001000080100, 0x0000000000080000 706 706 - .quad 0x0020001002000100, 0x0400001000080100 707 707 - .quad 0x0420001002000100, 0x0420000002080000 708 708 - .quad 0x0000001000080100, 0x0400000000000000 709 709 - .quad 0x0020000002000000, 0x0400000000080000 710 710 - .quad 0x0400000000080000, 0x0000000000000000 711 711 - .quad 0x0400001000000100, 0x0420001002080100 712 712 - .quad 0x0420001002080100, 0x0020001002000100 713 713 - .quad 0x0420000002080000, 0x0400001000000100 714 714 - .quad 0x0000000000000000, 0x0420000002000000 715 715 - .quad 0x0020001002080100, 0x0020000002000000 716 716 - .quad 0x0420000002000000, 0x0000001000080100 717 717 - .quad 0x0000000000080000, 0x0420001002000100 718 718 - .quad 0x0000001000000100, 0x0020000002000000 719 719 - .quad 0x0400000000000000, 0x0020000002080000 720 720 - .quad 0x0420001002000100, 0x0400001000080100 721 721 - .quad 0x0020001002000100, 0x0400000000000000 722 722 - .quad 0x0420000002080000, 0x0020001002080100 723 723 - .quad 0x0400001000080100, 0x0000001000000100 724 724 - .quad 0x0020000002000000, 0x0420000002080000 725 725 - .quad 0x0420001002080100, 0x0000001000080100 726 726 - .quad 0x0420000002000000, 0x0420001002080100 727 727 - .quad 0x0020000002080000, 0x0000000000000000 728 728 - .quad 0x0400000000080000, 0x0420000002000000 729 729 - .quad 0x0000001000080100, 0x0020001002000100 730 730 - .quad 0x0400001000000100, 0x0000000000080000 731 731 - .quad 0x0000000000000000, 0x0400000000080000 732 732 - .quad 0x0020001002080100, 0x0400001000000100 733 733 - .L_s6: 734 734 - .quad 0x0200000120000010, 0x0204000020000000 735 735 - .quad 0x0000040000000000, 0x0204040120000010 736 736 - .quad 0x0204000020000000, 0x0000000100000010 737 737 - .quad 0x0204040120000010, 0x0004000000000000 738 738 - .quad 0x0200040020000000, 0x0004040100000010 739 739 - .quad 0x0004000000000000, 0x0200000120000010 740 740 - .quad 0x0004000100000010, 0x0200040020000000 741 741 - .quad 0x0200000020000000, 0x0000040100000010 742 742 - .quad 0x0000000000000000, 0x0004000100000010 743 743 - .quad 0x0200040120000010, 0x0000040000000000 744 744 - .quad 0x0004040000000000, 0x0200040120000010 745 745 - .quad 0x0000000100000010, 0x0204000120000010 746 746 - .quad 0x0204000120000010, 0x0000000000000000 747 747 - .quad 0x0004040100000010, 0x0204040020000000 748 748 - .quad 0x0000040100000010, 0x0004040000000000 749 749 - .quad 0x0204040020000000, 0x0200000020000000 750 750 - .quad 0x0200040020000000, 0x0000000100000010 751 751 - .quad 0x0204000120000010, 0x0004040000000000 752 752 - .quad 0x0204040120000010, 0x0004000000000000 753 753 - .quad 0x0000040100000010, 0x0200000120000010 754 754 - .quad 0x0004000000000000, 0x0200040020000000 755 755 - .quad 0x0200000020000000, 0x0000040100000010 756 756 - .quad 0x0200000120000010, 0x0204040120000010 757 757 - .quad 0x0004040000000000, 0x0204000020000000 758 758 - .quad 0x0004040100000010, 0x0204040020000000 759 759 - .quad 0x0000000000000000, 0x0204000120000010 760 760 - .quad 0x0000000100000010, 0x0000040000000000 761 761 - .quad 0x0204000020000000, 0x0004040100000010 762 762 - .quad 0x0000040000000000, 0x0004000100000010 763 763 - .quad 0x0200040120000010, 0x0000000000000000 764 764 - .quad 0x0204040020000000, 0x0200000020000000 765 765 - .quad 0x0004000100000010, 0x0200040120000010 766 766 - .L_s7: 767 767 - .quad 0x0002000000200000, 0x2002000004200002 768 768 - .quad 0x2000000004000802, 0x0000000000000000 769 769 - .quad 0x0000000000000800, 0x2000000004000802 770 770 - .quad 0x2002000000200802, 0x0002000004200800 771 771 - .quad 0x2002000004200802, 0x0002000000200000 772 772 - .quad 0x0000000000000000, 0x2000000004000002 773 773 - .quad 0x2000000000000002, 0x0000000004000000 774 774 - .quad 0x2002000004200002, 0x2000000000000802 775 775 - .quad 0x0000000004000800, 0x2002000000200802 776 776 - .quad 0x2002000000200002, 0x0000000004000800 777 777 - .quad 0x2000000004000002, 0x0002000004200000 778 778 - .quad 0x0002000004200800, 0x2002000000200002 779 779 - .quad 0x0002000004200000, 0x0000000000000800 780 780 - .quad 0x2000000000000802, 0x2002000004200802 781 781 - .quad 0x0002000000200800, 0x2000000000000002 782 782 - .quad 0x0000000004000000, 0x0002000000200800 783 783 - .quad 0x0000000004000000, 0x0002000000200800 784 784 - .quad 0x0002000000200000, 0x2000000004000802 785 785 - .quad 0x2000000004000802, 0x2002000004200002 786 786 - .quad 0x2002000004200002, 0x2000000000000002 787 787 - .quad 0x2002000000200002, 0x0000000004000000 788 788 - .quad 0x0000000004000800, 0x0002000000200000 789 789 - .quad 0x0002000004200800, 0x2000000000000802 790 790 - .quad 0x2002000000200802, 0x0002000004200800 791 791 - .quad 0x2000000000000802, 0x2000000004000002 792 792 - .quad 0x2002000004200802, 0x0002000004200000 793 793 - .quad 0x0002000000200800, 0x0000000000000000 794 794 - .quad 0x2000000000000002, 0x2002000004200802 795 795 - .quad 0x0000000000000000, 0x2002000000200802 796 796 - .quad 0x0002000004200000, 0x0000000000000800 797 797 - .quad 0x2000000004000002, 0x0000000004000800 798 798 - .quad 0x0000000000000800, 0x2002000000200002 799 799 - .L_s8: 800 800 - .quad 0x0100010410001000, 0x0000010000001000 801 801 - .quad 0x0000000000040000, 0x0100010410041000 802 802 - .quad 0x0100000010000000, 0x0100010410001000 803 803 - .quad 0x0000000400000000, 0x0100000010000000 804 804 - .quad 0x0000000400040000, 0x0100000010040000 805 805 - .quad 0x0100010410041000, 0x0000010000041000 806 806 - .quad 0x0100010010041000, 0x0000010400041000 807 807 - .quad 0x0000010000001000, 0x0000000400000000 808 808 - .quad 0x0100000010040000, 0x0100000410000000 809 809 - .quad 0x0100010010001000, 0x0000010400001000 810 810 - .quad 0x0000010000041000, 0x0000000400040000 811 811 - .quad 0x0100000410040000, 0x0100010010041000 812 812 - .quad 0x0000010400001000, 0x0000000000000000 813 813 - .quad 0x0000000000000000, 0x0100000410040000 814 814 - .quad 0x0100000410000000, 0x0100010010001000 815 815 - .quad 0x0000010400041000, 0x0000000000040000 816 816 - .quad 0x0000010400041000, 0x0000000000040000 817 817 - .quad 0x0100010010041000, 0x0000010000001000 818 818 - .quad 0x0000000400000000, 0x0100000410040000 819 819 - .quad 0x0000010000001000, 0x0000010400041000 820 820 - .quad 0x0100010010001000, 0x0000000400000000 821 821 - .quad 0x0100000410000000, 0x0100000010040000 822 822 - .quad 0x0100000410040000, 0x0100000010000000 823 823 - .quad 0x0000000000040000, 0x0100010410001000 824 824 - .quad 0x0000000000000000, 0x0100010410041000 825 825 - .quad 0x0000000400040000, 0x0100000410000000 826 826 - .quad 0x0100000010040000, 0x0100010010001000 827 827 - .quad 0x0100010410001000, 0x0000000000000000 828 828 - .quad 0x0100010410041000, 0x0000010000041000 829 829 - .quad 0x0000010000041000, 0x0000010400001000 830 830 - .quad 0x0000010400001000, 0x0000000400040000 831 831 - .quad 0x0100000010000000, 0x0100010010041000

-391

arch/x86/crypto/des3_ede_glue.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-or-later 2 2 - /* 3 3 - * Glue Code for assembler optimized version of 3DES 4 4 - * 5 5 - * Copyright © 2014 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 6 6 - * 7 7 - * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: 8 8 - * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 9 9 - */ 10 10 - 11 11 - #include <crypto/algapi.h> 12 12 - #include <crypto/des.h> 13 13 - #include <crypto/internal/skcipher.h> 14 14 - #include <linux/crypto.h> 15 15 - #include <linux/init.h> 16 16 - #include <linux/module.h> 17 17 - #include <linux/types.h> 18 18 - 19 19 - struct des3_ede_x86_ctx { 20 20 - struct des3_ede_ctx enc; 21 21 - struct des3_ede_ctx dec; 22 22 - }; 23 23 - 24 24 - /* regular block cipher functions */ 25 25 - asmlinkage void des3_ede_x86_64_crypt_blk(const u32 *expkey, u8 *dst, 26 26 - const u8 *src); 27 27 - 28 28 - /* 3-way parallel cipher functions */ 29 29 - asmlinkage void des3_ede_x86_64_crypt_blk_3way(const u32 *expkey, u8 *dst, 30 30 - const u8 *src); 31 31 - 32 32 - static inline void des3_ede_enc_blk(struct des3_ede_x86_ctx *ctx, u8 *dst, 33 33 - const u8 *src) 34 34 - { 35 35 - u32 *enc_ctx = ctx->enc.expkey; 36 36 - 37 37 - des3_ede_x86_64_crypt_blk(enc_ctx, dst, src); 38 38 - } 39 39 - 40 40 - static inline void des3_ede_dec_blk(struct des3_ede_x86_ctx *ctx, u8 *dst, 41 41 - const u8 *src) 42 42 - { 43 43 - u32 *dec_ctx = ctx->dec.expkey; 44 44 - 45 45 - des3_ede_x86_64_crypt_blk(dec_ctx, dst, src); 46 46 - } 47 47 - 48 48 - static inline void des3_ede_dec_blk_3way(struct des3_ede_x86_ctx *ctx, u8 *dst, 49 49 - const u8 *src) 50 50 - { 51 51 - u32 *dec_ctx = ctx->dec.expkey; 52 52 - 53 53 - des3_ede_x86_64_crypt_blk_3way(dec_ctx, dst, src); 54 54 - } 55 55 - 56 56 - static void des3_ede_x86_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 57 57 - { 58 58 - des3_ede_enc_blk(crypto_tfm_ctx(tfm), dst, src); 59 59 - } 60 60 - 61 61 - static void des3_ede_x86_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 62 62 - { 63 63 - des3_ede_dec_blk(crypto_tfm_ctx(tfm), dst, src); 64 64 - } 65 65 - 66 66 - static int ecb_crypt(struct skcipher_request *req, const u32 *expkey) 67 67 - { 68 68 - const unsigned int bsize = DES3_EDE_BLOCK_SIZE; 69 69 - struct skcipher_walk walk; 70 70 - unsigned int nbytes; 71 71 - int err; 72 72 - 73 73 - err = skcipher_walk_virt(&walk, req, false); 74 74 - 75 75 - while ((nbytes = walk.nbytes)) { 76 76 - const u8 *wsrc = walk.src.virt.addr; 77 77 - u8 *wdst = walk.dst.virt.addr; 78 78 - 79 79 - /* Process four block batch */ 80 80 - if (nbytes >= bsize * 3) { 81 81 - do { 82 82 - des3_ede_x86_64_crypt_blk_3way(expkey, wdst, 83 83 - wsrc); 84 84 - 85 85 - wsrc += bsize * 3; 86 86 - wdst += bsize * 3; 87 87 - nbytes -= bsize * 3; 88 88 - } while (nbytes >= bsize * 3); 89 89 - 90 90 - if (nbytes < bsize) 91 91 - goto done; 92 92 - } 93 93 - 94 94 - /* Handle leftovers */ 95 95 - do { 96 96 - des3_ede_x86_64_crypt_blk(expkey, wdst, wsrc); 97 97 - 98 98 - wsrc += bsize; 99 99 - wdst += bsize; 100 100 - nbytes -= bsize; 101 101 - } while (nbytes >= bsize); 102 102 - 103 103 - done: 104 104 - err = skcipher_walk_done(&walk, nbytes); 105 105 - } 106 106 - 107 107 - return err; 108 108 - } 109 109 - 110 110 - static int ecb_encrypt(struct skcipher_request *req) 111 111 - { 112 112 - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 113 113 - struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); 114 114 - 115 115 - return ecb_crypt(req, ctx->enc.expkey); 116 116 - } 117 117 - 118 118 - static int ecb_decrypt(struct skcipher_request *req) 119 119 - { 120 120 - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 121 121 - struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); 122 122 - 123 123 - return ecb_crypt(req, ctx->dec.expkey); 124 124 - } 125 125 - 126 126 - static unsigned int __cbc_encrypt(struct des3_ede_x86_ctx *ctx, 127 127 - struct skcipher_walk *walk) 128 128 - { 129 129 - unsigned int bsize = DES3_EDE_BLOCK_SIZE; 130 130 - unsigned int nbytes = walk->nbytes; 131 131 - u64 *src = (u64 *)walk->src.virt.addr; 132 132 - u64 *dst = (u64 *)walk->dst.virt.addr; 133 133 - u64 *iv = (u64 *)walk->iv; 134 134 - 135 135 - do { 136 136 - *dst = *src ^ *iv; 137 137 - des3_ede_enc_blk(ctx, (u8 *)dst, (u8 *)dst); 138 138 - iv = dst; 139 139 - 140 140 - src += 1; 141 141 - dst += 1; 142 142 - nbytes -= bsize; 143 143 - } while (nbytes >= bsize); 144 144 - 145 145 - *(u64 *)walk->iv = *iv; 146 146 - return nbytes; 147 147 - } 148 148 - 149 149 - static int cbc_encrypt(struct skcipher_request *req) 150 150 - { 151 151 - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 152 152 - struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); 153 153 - struct skcipher_walk walk; 154 154 - unsigned int nbytes; 155 155 - int err; 156 156 - 157 157 - err = skcipher_walk_virt(&walk, req, false); 158 158 - 159 159 - while (walk.nbytes) { 160 160 - nbytes = __cbc_encrypt(ctx, &walk); 161 161 - err = skcipher_walk_done(&walk, nbytes); 162 162 - } 163 163 - 164 164 - return err; 165 165 - } 166 166 - 167 167 - static unsigned int __cbc_decrypt(struct des3_ede_x86_ctx *ctx, 168 168 - struct skcipher_walk *walk) 169 169 - { 170 170 - unsigned int bsize = DES3_EDE_BLOCK_SIZE; 171 171 - unsigned int nbytes = walk->nbytes; 172 172 - u64 *src = (u64 *)walk->src.virt.addr; 173 173 - u64 *dst = (u64 *)walk->dst.virt.addr; 174 174 - u64 ivs[3 - 1]; 175 175 - u64 last_iv; 176 176 - 177 177 - /* Start of the last block. */ 178 178 - src += nbytes / bsize - 1; 179 179 - dst += nbytes / bsize - 1; 180 180 - 181 181 - last_iv = *src; 182 182 - 183 183 - /* Process four block batch */ 184 184 - if (nbytes >= bsize * 3) { 185 185 - do { 186 186 - nbytes -= bsize * 3 - bsize; 187 187 - src -= 3 - 1; 188 188 - dst -= 3 - 1; 189 189 - 190 190 - ivs[0] = src[0]; 191 191 - ivs[1] = src[1]; 192 192 - 193 193 - des3_ede_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src); 194 194 - 195 195 - dst[1] ^= ivs[0]; 196 196 - dst[2] ^= ivs[1]; 197 197 - 198 198 - nbytes -= bsize; 199 199 - if (nbytes < bsize) 200 200 - goto done; 201 201 - 202 202 - *dst ^= *(src - 1); 203 203 - src -= 1; 204 204 - dst -= 1; 205 205 - } while (nbytes >= bsize * 3); 206 206 - } 207 207 - 208 208 - /* Handle leftovers */ 209 209 - for (;;) { 210 210 - des3_ede_dec_blk(ctx, (u8 *)dst, (u8 *)src); 211 211 - 212 212 - nbytes -= bsize; 213 213 - if (nbytes < bsize) 214 214 - break; 215 215 - 216 216 - *dst ^= *(src - 1); 217 217 - src -= 1; 218 218 - dst -= 1; 219 219 - } 220 220 - 221 221 - done: 222 222 - *dst ^= *(u64 *)walk->iv; 223 223 - *(u64 *)walk->iv = last_iv; 224 224 - 225 225 - return nbytes; 226 226 - } 227 227 - 228 228 - static int cbc_decrypt(struct skcipher_request *req) 229 229 - { 230 230 - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 231 231 - struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); 232 232 - struct skcipher_walk walk; 233 233 - unsigned int nbytes; 234 234 - int err; 235 235 - 236 236 - err = skcipher_walk_virt(&walk, req, false); 237 237 - 238 238 - while (walk.nbytes) { 239 239 - nbytes = __cbc_decrypt(ctx, &walk); 240 240 - err = skcipher_walk_done(&walk, nbytes); 241 241 - } 242 242 - 243 243 - return err; 244 244 - } 245 245 - 246 246 - static int des3_ede_x86_setkey(struct crypto_tfm *tfm, const u8 *key, 247 247 - unsigned int keylen) 248 248 - { 249 249 - struct des3_ede_x86_ctx *ctx = crypto_tfm_ctx(tfm); 250 250 - u32 i, j, tmp; 251 251 - int err; 252 252 - 253 253 - err = des3_ede_expand_key(&ctx->enc, key, keylen); 254 254 - if (err == -ENOKEY) { 255 255 - if (crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) 256 256 - err = -EINVAL; 257 257 - else 258 258 - err = 0; 259 259 - } 260 260 - 261 261 - if (err) { 262 262 - memset(ctx, 0, sizeof(*ctx)); 263 263 - return err; 264 264 - } 265 265 - 266 266 - /* Fix encryption context for this implementation and form decryption 267 267 - * context. */ 268 268 - j = DES3_EDE_EXPKEY_WORDS - 2; 269 269 - for (i = 0; i < DES3_EDE_EXPKEY_WORDS; i += 2, j -= 2) { 270 270 - tmp = ror32(ctx->enc.expkey[i + 1], 4); 271 271 - ctx->enc.expkey[i + 1] = tmp; 272 272 - 273 273 - ctx->dec.expkey[j + 0] = ctx->enc.expkey[i + 0]; 274 274 - ctx->dec.expkey[j + 1] = tmp; 275 275 - } 276 276 - 277 277 - return 0; 278 278 - } 279 279 - 280 280 - static int des3_ede_x86_setkey_skcipher(struct crypto_skcipher *tfm, 281 281 - const u8 *key, 282 282 - unsigned int keylen) 283 283 - { 284 284 - return des3_ede_x86_setkey(&tfm->base, key, keylen); 285 285 - } 286 286 - 287 287 - static struct crypto_alg des3_ede_cipher = { 288 288 - .cra_name = "des3_ede", 289 289 - .cra_driver_name = "des3_ede-asm", 290 290 - .cra_priority = 200, 291 291 - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 292 292 - .cra_blocksize = DES3_EDE_BLOCK_SIZE, 293 293 - .cra_ctxsize = sizeof(struct des3_ede_x86_ctx), 294 294 - .cra_module = THIS_MODULE, 295 295 - .cra_u = { 296 296 - .cipher = { 297 297 - .cia_min_keysize = DES3_EDE_KEY_SIZE, 298 298 - .cia_max_keysize = DES3_EDE_KEY_SIZE, 299 299 - .cia_setkey = des3_ede_x86_setkey, 300 300 - .cia_encrypt = des3_ede_x86_encrypt, 301 301 - .cia_decrypt = des3_ede_x86_decrypt, 302 302 - } 303 303 - } 304 304 - }; 305 305 - 306 306 - static struct skcipher_alg des3_ede_skciphers[] = { 307 307 - { 308 308 - .base.cra_name = "ecb(des3_ede)", 309 309 - .base.cra_driver_name = "ecb-des3_ede-asm", 310 310 - .base.cra_priority = 300, 311 311 - .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 312 312 - .base.cra_ctxsize = sizeof(struct des3_ede_x86_ctx), 313 313 - .base.cra_module = THIS_MODULE, 314 314 - .min_keysize = DES3_EDE_KEY_SIZE, 315 315 - .max_keysize = DES3_EDE_KEY_SIZE, 316 316 - .setkey = des3_ede_x86_setkey_skcipher, 317 317 - .encrypt = ecb_encrypt, 318 318 - .decrypt = ecb_decrypt, 319 319 - }, { 320 320 - .base.cra_name = "cbc(des3_ede)", 321 321 - .base.cra_driver_name = "cbc-des3_ede-asm", 322 322 - .base.cra_priority = 300, 323 323 - .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 324 324 - .base.cra_ctxsize = sizeof(struct des3_ede_x86_ctx), 325 325 - .base.cra_module = THIS_MODULE, 326 326 - .min_keysize = DES3_EDE_KEY_SIZE, 327 327 - .max_keysize = DES3_EDE_KEY_SIZE, 328 328 - .ivsize = DES3_EDE_BLOCK_SIZE, 329 329 - .setkey = des3_ede_x86_setkey_skcipher, 330 330 - .encrypt = cbc_encrypt, 331 331 - .decrypt = cbc_decrypt, 332 332 - } 333 333 - }; 334 334 - 335 335 - static bool is_blacklisted_cpu(void) 336 336 - { 337 337 - if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) 338 338 - return false; 339 339 - 340 340 - if (boot_cpu_data.x86 == 0x0f) { 341 341 - /* 342 342 - * On Pentium 4, des3_ede-x86_64 is slower than generic C 343 343 - * implementation because use of 64bit rotates (which are really 344 344 - * slow on P4). Therefore blacklist P4s. 345 345 - */ 346 346 - return true; 347 347 - } 348 348 - 349 349 - return false; 350 350 - } 351 351 - 352 352 - static int force; 353 353 - module_param(force, int, 0); 354 354 - MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist"); 355 355 - 356 356 - static int __init des3_ede_x86_init(void) 357 357 - { 358 358 - int err; 359 359 - 360 360 - if (!force && is_blacklisted_cpu()) { 361 361 - pr_info("des3_ede-x86_64: performance on this CPU would be suboptimal: disabling des3_ede-x86_64.\n"); 362 362 - return -ENODEV; 363 363 - } 364 364 - 365 365 - err = crypto_register_alg(&des3_ede_cipher); 366 366 - if (err) 367 367 - return err; 368 368 - 369 369 - err = crypto_register_skciphers(des3_ede_skciphers, 370 370 - ARRAY_SIZE(des3_ede_skciphers)); 371 371 - if (err) 372 372 - crypto_unregister_alg(&des3_ede_cipher); 373 373 - 374 374 - return err; 375 375 - } 376 376 - 377 377 - static void __exit des3_ede_x86_fini(void) 378 378 - { 379 379 - crypto_unregister_alg(&des3_ede_cipher); 380 380 - crypto_unregister_skciphers(des3_ede_skciphers, 381 381 - ARRAY_SIZE(des3_ede_skciphers)); 382 382 - } 383 383 - 384 384 - module_init(des3_ede_x86_init); 385 385 - module_exit(des3_ede_x86_fini); 386 386 - 387 387 - MODULE_LICENSE("GPL"); 388 388 - MODULE_DESCRIPTION("Triple DES EDE Cipher Algorithm, asm optimized"); 389 389 - MODULE_ALIAS_CRYPTO("des3_ede"); 390 390 - MODULE_ALIAS_CRYPTO("des3_ede-asm"); 391 391 - MODULE_AUTHOR("Jussi Kivilinna <jussi.kivilinna@iki.fi>");