crypto: algif_aead - Revert to operating out-of-place

+10 -39

crypto/af_alg.c

··· 637 637 /** 638 638 * af_alg_count_tsgl - Count number of TX SG entries 639 639 * 640 - * The counting starts from the beginning of the SGL to @bytes. If 641 - * an @offset is provided, the counting of the SG entries starts at the @offset. 640 + * The counting starts from the beginning of the SGL to @bytes. 642 641 * 643 642 * @sk: socket of connection to user space 644 643 * @bytes: Count the number of SG entries holding given number of bytes. 645 - * @offset: Start the counting of SG entries from the given offset. 646 644 * Return: Number of TX SG entries found given the constraints 647 645 */ 648 - unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset) 646 + unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes) 649 647 { 650 648 const struct alg_sock *ask = alg_sk(sk); 651 649 const struct af_alg_ctx *ctx = ask->private; ··· 658 660 const struct scatterlist *sg = sgl->sg; 659 661 660 662 for (i = 0; i < sgl->cur; i++) { 661 - size_t bytes_count; 662 - 663 - /* Skip offset */ 664 - if (offset >= sg[i].length) { 665 - offset -= sg[i].length; 666 - bytes -= sg[i].length; 667 - continue; 668 - } 669 - 670 - bytes_count = sg[i].length - offset; 671 - 672 - offset = 0; 673 663 sgl_count++; 674 - 675 - /* If we have seen requested number of bytes, stop */ 676 - if (bytes_count >= bytes) 664 + if (sg[i].length >= bytes) 677 665 return sgl_count; 678 666 679 - bytes -= bytes_count; 667 + bytes -= sg[i].length; 680 668 } 681 669 } 682 670 ··· 674 690 * af_alg_pull_tsgl - Release the specified buffers from TX SGL 675 691 * 676 692 * If @dst is non-null, reassign the pages to @dst. The caller must release 677 - * the pages. If @dst_offset is given only reassign the pages to @dst starting 678 - * at the @dst_offset (byte). The caller must ensure that @dst is large 679 - * enough (e.g. by using af_alg_count_tsgl with the same offset). 693 + * the pages. 680 694 * 681 695 * @sk: socket of connection to user space 682 696 * @used: Number of bytes to pull from TX SGL 683 697 * @dst: If non-NULL, buffer is reassigned to dst SGL instead of releasing. The 684 698 * caller must release the buffers in dst. 685 - * @dst_offset: Reassign the TX SGL from given offset. All buffers before 686 - * reaching the offset is released. 687 699 */ 688 - void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst, 689 - size_t dst_offset) 700 + void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst) 690 701 { 691 702 struct alg_sock *ask = alg_sk(sk); 692 703 struct af_alg_ctx *ctx = ask->private; ··· 706 727 * SG entries in dst. 707 728 */ 708 729 if (dst) { 709 - if (dst_offset >= plen) { 710 - /* discard page before offset */ 711 - dst_offset -= plen; 712 - } else { 713 - /* reassign page to dst after offset */ 714 - get_page(page); 715 - sg_set_page(dst + j, page, 716 - plen - dst_offset, 717 - sg[i].offset + dst_offset); 718 - dst_offset = 0; 719 - j++; 720 - } 730 + /* reassign page to dst after offset */ 731 + get_page(page); 732 + sg_set_page(dst + j, page, plen, sg[i].offset); 733 + j++; 721 734 } 722 735 723 736 sg[i].length -= plen;

+19 -81

crypto/algif_aead.c

··· 26 26 #include <crypto/internal/aead.h> 27 27 #include <crypto/scatterwalk.h> 28 28 #include <crypto/if_alg.h> 29 - #include <crypto/skcipher.h> 30 29 #include <linux/init.h> 31 30 #include <linux/list.h> 32 31 #include <linux/kernel.h> ··· 71 72 struct alg_sock *pask = alg_sk(psk); 72 73 struct af_alg_ctx *ctx = ask->private; 73 74 struct crypto_aead *tfm = pask->private; 74 - unsigned int i, as = crypto_aead_authsize(tfm); 75 + unsigned int as = crypto_aead_authsize(tfm); 75 76 struct af_alg_async_req *areq; 76 - struct af_alg_tsgl *tsgl, *tmp; 77 77 struct scatterlist *rsgl_src, *tsgl_src = NULL; 78 78 int err = 0; 79 79 size_t used = 0; /* [in] TX bufs to be en/decrypted */ ··· 152 154 outlen -= less; 153 155 } 154 156 157 + /* 158 + * Create a per request TX SGL for this request which tracks the 159 + * SG entries from the global TX SGL. 160 + */ 155 161 processed = used + ctx->aead_assoclen; 156 - list_for_each_entry_safe(tsgl, tmp, &ctx->tsgl_list, list) { 157 - for (i = 0; i < tsgl->cur; i++) { 158 - struct scatterlist *process_sg = tsgl->sg + i; 159 - 160 - if (!(process_sg->length) || !sg_page(process_sg)) 161 - continue; 162 - tsgl_src = process_sg; 163 - break; 164 - } 165 - if (tsgl_src) 166 - break; 167 - } 168 - if (processed && !tsgl_src) { 169 - err = -EFAULT; 162 + areq->tsgl_entries = af_alg_count_tsgl(sk, processed); 163 + if (!areq->tsgl_entries) 164 + areq->tsgl_entries = 1; 165 + areq->tsgl = sock_kmalloc(sk, array_size(sizeof(*areq->tsgl), 166 + areq->tsgl_entries), 167 + GFP_KERNEL); 168 + if (!areq->tsgl) { 169 + err = -ENOMEM; 170 170 goto free; 171 171 } 172 + sg_init_table(areq->tsgl, areq->tsgl_entries); 173 + af_alg_pull_tsgl(sk, processed, areq->tsgl); 174 + tsgl_src = areq->tsgl; 172 175 173 176 /* 174 177 * Copy of AAD from source to destination ··· 178 179 * when user space uses an in-place cipher operation, the kernel 179 180 * will copy the data as it does not see whether such in-place operation 180 181 * is initiated. 181 - * 182 - * To ensure efficiency, the following implementation ensure that the 183 - * ciphers are invoked to perform a crypto operation in-place. This 184 - * is achieved by memory management specified as follows. 185 182 */ 186 183 187 184 /* Use the RX SGL as source (and destination) for crypto op. */ 188 185 rsgl_src = areq->first_rsgl.sgl.sgt.sgl; 189 186 190 - if (ctx->enc) { 191 - /* 192 - * Encryption operation - The in-place cipher operation is 193 - * achieved by the following operation: 194 - * 195 - * TX SGL: AAD || PT 196 - * | | 197 - * | copy | 198 - * v v 199 - * RX SGL: AAD || PT || Tag 200 - */ 201 - memcpy_sglist(areq->first_rsgl.sgl.sgt.sgl, tsgl_src, 202 - processed); 203 - af_alg_pull_tsgl(sk, processed, NULL, 0); 204 - } else { 205 - /* 206 - * Decryption operation - To achieve an in-place cipher 207 - * operation, the following SGL structure is used: 208 - * 209 - * TX SGL: AAD || CT || Tag 210 - * | | ^ 211 - * | copy | | Create SGL link. 212 - * v v | 213 - * RX SGL: AAD || CT ----+ 214 - */ 215 - 216 - /* Copy AAD || CT to RX SGL buffer for in-place operation. */ 217 - memcpy_sglist(areq->first_rsgl.sgl.sgt.sgl, tsgl_src, outlen); 218 - 219 - /* Create TX SGL for tag and chain it to RX SGL. */ 220 - areq->tsgl_entries = af_alg_count_tsgl(sk, processed, 221 - processed - as); 222 - if (!areq->tsgl_entries) 223 - areq->tsgl_entries = 1; 224 - areq->tsgl = sock_kmalloc(sk, array_size(sizeof(*areq->tsgl), 225 - areq->tsgl_entries), 226 - GFP_KERNEL); 227 - if (!areq->tsgl) { 228 - err = -ENOMEM; 229 - goto free; 230 - } 231 - sg_init_table(areq->tsgl, areq->tsgl_entries); 232 - 233 - /* Release TX SGL, except for tag data and reassign tag data. */ 234 - af_alg_pull_tsgl(sk, processed, areq->tsgl, processed - as); 235 - 236 - /* chain the areq TX SGL holding the tag with RX SGL */ 237 - if (usedpages) { 238 - /* RX SGL present */ 239 - struct af_alg_sgl *sgl_prev = &areq->last_rsgl->sgl; 240 - struct scatterlist *sg = sgl_prev->sgt.sgl; 241 - 242 - sg_unmark_end(sg + sgl_prev->sgt.nents - 1); 243 - sg_chain(sg, sgl_prev->sgt.nents + 1, areq->tsgl); 244 - } else 245 - /* no RX SGL present (e.g. authentication only) */ 246 - rsgl_src = areq->tsgl; 247 - } 187 + memcpy_sglist(rsgl_src, tsgl_src, ctx->aead_assoclen); 248 188 249 189 /* Initialize the crypto operation */ 250 - aead_request_set_crypt(&areq->cra_u.aead_req, rsgl_src, 190 + aead_request_set_crypt(&areq->cra_u.aead_req, tsgl_src, 251 191 areq->first_rsgl.sgl.sgt.sgl, used, ctx->iv); 252 192 aead_request_set_ad(&areq->cra_u.aead_req, ctx->aead_assoclen); 253 193 aead_request_set_tfm(&areq->cra_u.aead_req, tfm); ··· 388 450 struct crypto_aead *tfm = pask->private; 389 451 unsigned int ivlen = crypto_aead_ivsize(tfm); 390 452 391 - af_alg_pull_tsgl(sk, ctx->used, NULL, 0); 453 + af_alg_pull_tsgl(sk, ctx->used, NULL); 392 454 sock_kzfree_s(sk, ctx->iv, ivlen); 393 455 sock_kfree_s(sk, ctx, ctx->len); 394 456 af_alg_release_parent(sk);

+3 -3

crypto/algif_skcipher.c

··· 138 138 * Create a per request TX SGL for this request which tracks the 139 139 * SG entries from the global TX SGL. 140 140 */ 141 - areq->tsgl_entries = af_alg_count_tsgl(sk, len, 0); 141 + areq->tsgl_entries = af_alg_count_tsgl(sk, len); 142 142 if (!areq->tsgl_entries) 143 143 areq->tsgl_entries = 1; 144 144 areq->tsgl = sock_kmalloc(sk, array_size(sizeof(*areq->tsgl), ··· 149 149 goto free; 150 150 } 151 151 sg_init_table(areq->tsgl, areq->tsgl_entries); 152 - af_alg_pull_tsgl(sk, len, areq->tsgl, 0); 152 + af_alg_pull_tsgl(sk, len, areq->tsgl); 153 153 154 154 /* Initialize the crypto operation */ 155 155 skcipher_request_set_tfm(&areq->cra_u.skcipher_req, tfm); ··· 363 363 struct alg_sock *pask = alg_sk(psk); 364 364 struct crypto_skcipher *tfm = pask->private; 365 365 366 - af_alg_pull_tsgl(sk, ctx->used, NULL, 0); 366 + af_alg_pull_tsgl(sk, ctx->used, NULL); 367 367 sock_kzfree_s(sk, ctx->iv, crypto_skcipher_ivsize(tfm)); 368 368 if (ctx->state) 369 369 sock_kzfree_s(sk, ctx->state, crypto_skcipher_statesize(tfm));

+2 -3

include/crypto/if_alg.h

··· 230 230 return PAGE_SIZE <= af_alg_rcvbuf(sk); 231 231 } 232 232 233 - unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset); 234 - void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst, 235 - size_t dst_offset); 233 + unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes); 234 + void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst); 236 235 void af_alg_wmem_wakeup(struct sock *sk); 237 236 int af_alg_wait_for_data(struct sock *sk, unsigned flags, unsigned min); 238 237 int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size,

Configure Feed

Configure Feed