Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

crypto: pcrypt - Avoid deadlock by using per-instance padata queues

If the pcrypt template is used multiple times in an algorithm, then a
deadlock occurs because all pcrypt instances share the same
padata_instance, which completes requests in the order submitted. That
is, the inner pcrypt request waits for the outer pcrypt request while
the outer request is already waiting for the inner.

This patch fixes this by allocating a set of queues for each pcrypt
instance instead of using two global queues. In order to maintain
the existing user-space interface, the pinst structure remains global
so any sysfs modifications will apply to every pcrypt instance.

Note that when an update occurs we have to allocate memory for
every pcrypt instance. Should one of the allocations fail we
will abort the update without rolling back changes already made.

The new per-instance data structure is called padata_shell and is
essentially a wrapper around parallel_data.

Reproducer:

#include <linux/if_alg.h>
#include <sys/socket.h>
#include <unistd.h>

int main()
{
struct sockaddr_alg addr = {
.salg_type = "aead",
.salg_name = "pcrypt(pcrypt(rfc4106-gcm-aesni))"
};
int algfd, reqfd;
char buf[32] = { 0 };

algfd = socket(AF_ALG, SOCK_SEQPACKET, 0);
bind(algfd, (void *)&addr, sizeof(addr));
setsockopt(algfd, SOL_ALG, ALG_SET_KEY, buf, 20);
reqfd = accept(algfd, 0, 0);
write(reqfd, buf, 32);
read(reqfd, buf, 16);
}

Reported-by: syzbot+56c7151cad94eec37c521f0e47d2eee53f9361c4@syzkaller.appspotmail.com
Fixes: 5068c7a883d1 ("crypto: pcrypt - Add pcrypt crypto parallelization wrapper")
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
Tested-by: Eric Biggers <ebiggers@kernel.org>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

+227 -79
+33 -3
crypto/pcrypt.c
··· 24 24 25 25 struct pcrypt_instance_ctx { 26 26 struct crypto_aead_spawn spawn; 27 + struct padata_shell *psenc; 28 + struct padata_shell *psdec; 27 29 atomic_t tfm_count; 28 30 }; 29 31 ··· 33 31 struct crypto_aead *child; 34 32 unsigned int cb_cpu; 35 33 }; 34 + 35 + static inline struct pcrypt_instance_ctx *pcrypt_tfm_ictx( 36 + struct crypto_aead *tfm) 37 + { 38 + return aead_instance_ctx(aead_alg_instance(tfm)); 39 + } 36 40 37 41 static int pcrypt_aead_setkey(struct crypto_aead *parent, 38 42 const u8 *key, unsigned int keylen) ··· 98 90 struct crypto_aead *aead = crypto_aead_reqtfm(req); 99 91 struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead); 100 92 u32 flags = aead_request_flags(req); 93 + struct pcrypt_instance_ctx *ictx; 94 + 95 + ictx = pcrypt_tfm_ictx(aead); 101 96 102 97 memset(padata, 0, sizeof(struct padata_priv)); 103 98 ··· 114 103 req->cryptlen, req->iv); 115 104 aead_request_set_ad(creq, req->assoclen); 116 105 117 - err = padata_do_parallel(pencrypt, padata, &ctx->cb_cpu); 106 + err = padata_do_parallel(ictx->psenc, padata, &ctx->cb_cpu); 118 107 if (!err) 119 108 return -EINPROGRESS; 120 109 ··· 143 132 struct crypto_aead *aead = crypto_aead_reqtfm(req); 144 133 struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead); 145 134 u32 flags = aead_request_flags(req); 135 + struct pcrypt_instance_ctx *ictx; 136 + 137 + ictx = pcrypt_tfm_ictx(aead); 146 138 147 139 memset(padata, 0, sizeof(struct padata_priv)); 148 140 ··· 159 145 req->cryptlen, req->iv); 160 146 aead_request_set_ad(creq, req->assoclen); 161 147 162 - err = padata_do_parallel(pdecrypt, padata, &ctx->cb_cpu); 148 + err = padata_do_parallel(ictx->psdec, padata, &ctx->cb_cpu); 163 149 if (!err) 164 150 return -EINPROGRESS; 165 151 ··· 206 192 struct pcrypt_instance_ctx *ctx = aead_instance_ctx(inst); 207 193 208 194 crypto_drop_aead(&ctx->spawn); 195 + padata_free_shell(ctx->psdec); 196 + padata_free_shell(ctx->psenc); 209 197 kfree(inst); 210 198 } 211 199 ··· 249 233 if (!inst) 250 234 return -ENOMEM; 251 235 236 + err = -ENOMEM; 237 + 252 238 ctx = aead_instance_ctx(inst); 239 + ctx->psenc = padata_alloc_shell(pencrypt); 240 + if (!ctx->psenc) 241 + goto out_free_inst; 242 + 243 + ctx->psdec = padata_alloc_shell(pdecrypt); 244 + if (!ctx->psdec) 245 + goto out_free_psenc; 246 + 253 247 crypto_set_aead_spawn(&ctx->spawn, aead_crypto_instance(inst)); 254 248 255 249 err = crypto_grab_aead(&ctx->spawn, name, 0, 0); 256 250 if (err) 257 - goto out_free_inst; 251 + goto out_free_psdec; 258 252 259 253 alg = crypto_spawn_aead_alg(&ctx->spawn); 260 254 err = pcrypt_init_instance(aead_crypto_instance(inst), &alg->base); ··· 297 271 298 272 out_drop_aead: 299 273 crypto_drop_aead(&ctx->spawn); 274 + out_free_psdec: 275 + padata_free_shell(ctx->psdec); 276 + out_free_psenc: 277 + padata_free_shell(ctx->psenc); 300 278 out_free_inst: 301 279 kfree(inst); 302 280 goto out;
+29 -5
include/linux/padata.h
··· 9 9 #ifndef PADATA_H 10 10 #define PADATA_H 11 11 12 + #include <linux/compiler_types.h> 12 13 #include <linux/workqueue.h> 13 14 #include <linux/spinlock.h> 14 15 #include <linux/list.h> ··· 99 98 * struct parallel_data - Internal control structure, covers everything 100 99 * that depends on the cpumask in use. 101 100 * 102 - * @pinst: padata instance. 101 + * @sh: padata_shell object. 103 102 * @pqueue: percpu padata queues used for parallelization. 104 103 * @squeue: percpu padata queues used for serialuzation. 105 104 * @reorder_objects: Number of objects waiting in the reorder queues. ··· 112 111 * @lock: Reorder lock. 113 112 */ 114 113 struct parallel_data { 115 - struct padata_instance *pinst; 114 + struct padata_shell *ps; 116 115 struct padata_parallel_queue __percpu *pqueue; 117 116 struct padata_serial_queue __percpu *squeue; 118 117 atomic_t reorder_objects; ··· 126 125 }; 127 126 128 127 /** 128 + * struct padata_shell - Wrapper around struct parallel_data, its 129 + * purpose is to allow the underlying control structure to be replaced 130 + * on the fly using RCU. 131 + * 132 + * @pinst: padat instance. 133 + * @pd: Actual parallel_data structure which may be substituted on the fly. 134 + * @opd: Pointer to old pd to be freed by padata_replace. 135 + * @list: List entry in padata_instance list. 136 + */ 137 + struct padata_shell { 138 + struct padata_instance *pinst; 139 + struct parallel_data __rcu *pd; 140 + struct parallel_data *opd; 141 + struct list_head list; 142 + }; 143 + 144 + /** 129 145 * struct padata_instance - The overall control structure. 130 146 * 131 147 * @cpu_notifier: cpu hotplug notifier. 132 148 * @parallel_wq: The workqueue used for parallel work. 133 149 * @serial_wq: The workqueue used for serial work. 134 - * @pd: The internal control structure. 150 + * @pslist: List of padata_shell objects attached to this instance. 135 151 * @cpumask: User supplied cpumasks for parallel and serial works. 152 + * @rcpumask: Actual cpumasks based on user cpumask and cpu_online_mask. 153 + * @omask: Temporary storage used to compute the notification mask. 136 154 * @cpumask_change_notifier: Notifiers chain for user-defined notify 137 155 * callbacks that will be called when either @pcpu or @cbcpu 138 156 * or both cpumasks change. ··· 163 143 struct hlist_node node; 164 144 struct workqueue_struct *parallel_wq; 165 145 struct workqueue_struct *serial_wq; 166 - struct parallel_data *pd; 146 + struct list_head pslist; 167 147 struct padata_cpumask cpumask; 148 + struct padata_cpumask rcpumask; 149 + cpumask_var_t omask; 168 150 struct blocking_notifier_head cpumask_change_notifier; 169 151 struct kobject kobj; 170 152 struct mutex lock; ··· 178 156 179 157 extern struct padata_instance *padata_alloc_possible(const char *name); 180 158 extern void padata_free(struct padata_instance *pinst); 181 - extern int padata_do_parallel(struct padata_instance *pinst, 159 + extern struct padata_shell *padata_alloc_shell(struct padata_instance *pinst); 160 + extern void padata_free_shell(struct padata_shell *ps); 161 + extern int padata_do_parallel(struct padata_shell *ps, 182 162 struct padata_priv *padata, int *cb_cpu); 183 163 extern void padata_do_serial(struct padata_priv *padata); 184 164 extern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type,
+165 -71
kernel/padata.c
··· 89 89 /** 90 90 * padata_do_parallel - padata parallelization function 91 91 * 92 - * @pinst: padata instance 92 + * @ps: padatashell 93 93 * @padata: object to be parallelized 94 94 * @cb_cpu: pointer to the CPU that the serialization callback function should 95 95 * run on. If it's not in the serial cpumask of @pinst ··· 100 100 * Note: Every object which is parallelized by padata_do_parallel 101 101 * must be seen by padata_do_serial. 102 102 */ 103 - int padata_do_parallel(struct padata_instance *pinst, 103 + int padata_do_parallel(struct padata_shell *ps, 104 104 struct padata_priv *padata, int *cb_cpu) 105 105 { 106 + struct padata_instance *pinst = ps->pinst; 106 107 int i, cpu, cpu_index, target_cpu, err; 107 108 struct padata_parallel_queue *queue; 108 109 struct parallel_data *pd; 109 110 110 111 rcu_read_lock_bh(); 111 112 112 - pd = rcu_dereference_bh(pinst->pd); 113 + pd = rcu_dereference_bh(ps->pd); 113 114 114 115 err = -EINVAL; 115 116 if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID) ··· 213 212 214 213 static void padata_reorder(struct parallel_data *pd) 215 214 { 215 + struct padata_instance *pinst = pd->ps->pinst; 216 216 int cb_cpu; 217 217 struct padata_priv *padata; 218 218 struct padata_serial_queue *squeue; 219 - struct padata_instance *pinst = pd->pinst; 220 219 struct padata_parallel_queue *next_queue; 221 220 222 221 /* ··· 350 349 } 351 350 EXPORT_SYMBOL(padata_do_serial); 352 351 353 - static int padata_setup_cpumasks(struct parallel_data *pd, 354 - const struct cpumask *pcpumask, 355 - const struct cpumask *cbcpumask) 352 + static int padata_setup_cpumasks(struct padata_instance *pinst) 356 353 { 357 354 struct workqueue_attrs *attrs; 355 + int err; 356 + 357 + attrs = alloc_workqueue_attrs(); 358 + if (!attrs) 359 + return -ENOMEM; 360 + 361 + /* Restrict parallel_wq workers to pd->cpumask.pcpu. */ 362 + cpumask_copy(attrs->cpumask, pinst->cpumask.pcpu); 363 + err = apply_workqueue_attrs(pinst->parallel_wq, attrs); 364 + free_workqueue_attrs(attrs); 365 + 366 + return err; 367 + } 368 + 369 + static int pd_setup_cpumasks(struct parallel_data *pd, 370 + const struct cpumask *pcpumask, 371 + const struct cpumask *cbcpumask) 372 + { 358 373 int err = -ENOMEM; 359 374 360 375 if (!alloc_cpumask_var(&pd->cpumask.pcpu, GFP_KERNEL)) 361 376 goto out; 362 - cpumask_and(pd->cpumask.pcpu, pcpumask, cpu_online_mask); 363 - 364 377 if (!alloc_cpumask_var(&pd->cpumask.cbcpu, GFP_KERNEL)) 365 378 goto free_pcpu_mask; 366 - cpumask_and(pd->cpumask.cbcpu, cbcpumask, cpu_online_mask); 367 379 368 - attrs = alloc_workqueue_attrs(); 369 - if (!attrs) 370 - goto free_cbcpu_mask; 371 - 372 - /* Restrict parallel_wq workers to pd->cpumask.pcpu. */ 373 - cpumask_copy(attrs->cpumask, pd->cpumask.pcpu); 374 - err = apply_workqueue_attrs(pd->pinst->parallel_wq, attrs); 375 - free_workqueue_attrs(attrs); 376 - if (err < 0) 377 - goto free_cbcpu_mask; 380 + cpumask_copy(pd->cpumask.pcpu, pcpumask); 381 + cpumask_copy(pd->cpumask.cbcpu, cbcpumask); 378 382 379 383 return 0; 380 384 381 - free_cbcpu_mask: 382 - free_cpumask_var(pd->cpumask.cbcpu); 383 385 free_pcpu_mask: 384 386 free_cpumask_var(pd->cpumask.pcpu); 385 387 out: ··· 426 422 } 427 423 428 424 /* Allocate and initialize the internal cpumask dependend resources. */ 429 - static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst, 430 - const struct cpumask *pcpumask, 431 - const struct cpumask *cbcpumask) 425 + static struct parallel_data *padata_alloc_pd(struct padata_shell *ps) 432 426 { 427 + struct padata_instance *pinst = ps->pinst; 428 + const struct cpumask *cbcpumask; 429 + const struct cpumask *pcpumask; 433 430 struct parallel_data *pd; 431 + 432 + cbcpumask = pinst->rcpumask.cbcpu; 433 + pcpumask = pinst->rcpumask.pcpu; 434 434 435 435 pd = kzalloc(sizeof(struct parallel_data), GFP_KERNEL); 436 436 if (!pd) ··· 448 440 if (!pd->squeue) 449 441 goto err_free_pqueue; 450 442 451 - pd->pinst = pinst; 452 - if (padata_setup_cpumasks(pd, pcpumask, cbcpumask) < 0) 443 + pd->ps = ps; 444 + if (pd_setup_cpumasks(pd, pcpumask, cbcpumask)) 453 445 goto err_free_squeue; 454 446 455 447 padata_init_pqueues(pd); ··· 498 490 } 499 491 500 492 /* Replace the internal control structure with a new one. */ 501 - static void padata_replace(struct padata_instance *pinst, 502 - struct parallel_data *pd_new) 493 + static int padata_replace_one(struct padata_shell *ps) 503 494 { 504 - struct parallel_data *pd_old = pinst->pd; 495 + struct parallel_data *pd_new; 496 + 497 + pd_new = padata_alloc_pd(ps); 498 + if (!pd_new) 499 + return -ENOMEM; 500 + 501 + ps->opd = rcu_dereference_protected(ps->pd, 1); 502 + rcu_assign_pointer(ps->pd, pd_new); 503 + 504 + return 0; 505 + } 506 + 507 + static int padata_replace(struct padata_instance *pinst, int cpu) 508 + { 505 509 int notification_mask = 0; 510 + struct padata_shell *ps; 511 + int err; 506 512 507 513 pinst->flags |= PADATA_RESET; 508 514 509 - rcu_assign_pointer(pinst->pd, pd_new); 515 + cpumask_copy(pinst->omask, pinst->rcpumask.pcpu); 516 + cpumask_and(pinst->rcpumask.pcpu, pinst->cpumask.pcpu, 517 + cpu_online_mask); 518 + if (cpu >= 0) 519 + cpumask_clear_cpu(cpu, pinst->rcpumask.pcpu); 520 + if (!cpumask_equal(pinst->omask, pinst->rcpumask.pcpu)) 521 + notification_mask |= PADATA_CPU_PARALLEL; 522 + 523 + cpumask_copy(pinst->omask, pinst->rcpumask.cbcpu); 524 + cpumask_and(pinst->rcpumask.cbcpu, pinst->cpumask.cbcpu, 525 + cpu_online_mask); 526 + if (cpu >= 0) 527 + cpumask_clear_cpu(cpu, pinst->rcpumask.cbcpu); 528 + if (!cpumask_equal(pinst->omask, pinst->rcpumask.cbcpu)) 529 + notification_mask |= PADATA_CPU_SERIAL; 530 + 531 + list_for_each_entry(ps, &pinst->pslist, list) { 532 + err = padata_replace_one(ps); 533 + if (err) 534 + break; 535 + } 510 536 511 537 synchronize_rcu(); 512 538 513 - if (!cpumask_equal(pd_old->cpumask.pcpu, pd_new->cpumask.pcpu)) 514 - notification_mask |= PADATA_CPU_PARALLEL; 515 - if (!cpumask_equal(pd_old->cpumask.cbcpu, pd_new->cpumask.cbcpu)) 516 - notification_mask |= PADATA_CPU_SERIAL; 517 - 518 - if (atomic_dec_and_test(&pd_old->refcnt)) 519 - padata_free_pd(pd_old); 539 + list_for_each_entry_continue_reverse(ps, &pinst->pslist, list) 540 + if (atomic_dec_and_test(&ps->opd->refcnt)) 541 + padata_free_pd(ps->opd); 520 542 521 543 if (notification_mask) 522 544 blocking_notifier_call_chain(&pinst->cpumask_change_notifier, 523 545 notification_mask, 524 - &pd_new->cpumask); 546 + &pinst->cpumask); 525 547 526 548 pinst->flags &= ~PADATA_RESET; 549 + 550 + return err; 527 551 } 528 552 529 553 /** ··· 608 568 cpumask_var_t cbcpumask) 609 569 { 610 570 int valid; 611 - struct parallel_data *pd; 571 + int err; 612 572 613 573 valid = padata_validate_cpumask(pinst, pcpumask); 614 574 if (!valid) { ··· 621 581 __padata_stop(pinst); 622 582 623 583 out_replace: 624 - pd = padata_alloc_pd(pinst, pcpumask, cbcpumask); 625 - if (!pd) 626 - return -ENOMEM; 627 - 628 584 cpumask_copy(pinst->cpumask.pcpu, pcpumask); 629 585 cpumask_copy(pinst->cpumask.cbcpu, cbcpumask); 630 586 631 - padata_replace(pinst, pd); 587 + err = padata_setup_cpumasks(pinst) ?: padata_replace(pinst, -1); 632 588 633 589 if (valid) 634 590 __padata_start(pinst); 635 591 636 - return 0; 592 + return err; 637 593 } 638 594 639 595 /** ··· 712 676 713 677 static int __padata_add_cpu(struct padata_instance *pinst, int cpu) 714 678 { 715 - struct parallel_data *pd; 679 + int err = 0; 716 680 717 681 if (cpumask_test_cpu(cpu, cpu_online_mask)) { 718 - pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu, 719 - pinst->cpumask.cbcpu); 720 - if (!pd) 721 - return -ENOMEM; 722 - 723 - padata_replace(pinst, pd); 682 + err = padata_replace(pinst, -1); 724 683 725 684 if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) && 726 685 padata_validate_cpumask(pinst, pinst->cpumask.cbcpu)) 727 686 __padata_start(pinst); 728 687 } 729 688 730 - return 0; 689 + return err; 731 690 } 732 691 733 692 static int __padata_remove_cpu(struct padata_instance *pinst, int cpu) 734 693 { 735 - struct parallel_data *pd = NULL; 694 + int err = 0; 736 695 737 696 if (cpumask_test_cpu(cpu, cpu_online_mask)) { 738 - 739 697 if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) || 740 698 !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu)) 741 699 __padata_stop(pinst); 742 700 743 - pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu, 744 - pinst->cpumask.cbcpu); 745 - if (!pd) 746 - return -ENOMEM; 747 - 748 - padata_replace(pinst, pd); 749 - 750 - cpumask_clear_cpu(cpu, pd->cpumask.cbcpu); 751 - cpumask_clear_cpu(cpu, pd->cpumask.pcpu); 701 + err = padata_replace(pinst, cpu); 752 702 } 753 703 754 - return 0; 704 + return err; 755 705 } 756 706 757 707 static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu) ··· 785 763 cpuhp_state_remove_instance_nocalls(hp_online, &pinst->node); 786 764 #endif 787 765 766 + WARN_ON(!list_empty(&pinst->pslist)); 767 + 788 768 padata_stop(pinst); 789 - padata_free_pd(pinst->pd); 769 + free_cpumask_var(pinst->omask); 770 + free_cpumask_var(pinst->rcpumask.cbcpu); 771 + free_cpumask_var(pinst->rcpumask.pcpu); 790 772 free_cpumask_var(pinst->cpumask.pcpu); 791 773 free_cpumask_var(pinst->cpumask.cbcpu); 792 774 destroy_workqueue(pinst->serial_wq); ··· 937 911 const struct cpumask *cbcpumask) 938 912 { 939 913 struct padata_instance *pinst; 940 - struct parallel_data *pd = NULL; 941 914 942 915 pinst = kzalloc(sizeof(struct padata_instance), GFP_KERNEL); 943 916 if (!pinst) ··· 964 939 !padata_validate_cpumask(pinst, cbcpumask)) 965 940 goto err_free_masks; 966 941 967 - pd = padata_alloc_pd(pinst, pcpumask, cbcpumask); 968 - if (!pd) 942 + if (!alloc_cpumask_var(&pinst->rcpumask.pcpu, GFP_KERNEL)) 969 943 goto err_free_masks; 944 + if (!alloc_cpumask_var(&pinst->rcpumask.cbcpu, GFP_KERNEL)) 945 + goto err_free_rcpumask_pcpu; 946 + if (!alloc_cpumask_var(&pinst->omask, GFP_KERNEL)) 947 + goto err_free_rcpumask_cbcpu; 970 948 971 - rcu_assign_pointer(pinst->pd, pd); 949 + INIT_LIST_HEAD(&pinst->pslist); 972 950 973 951 cpumask_copy(pinst->cpumask.pcpu, pcpumask); 974 952 cpumask_copy(pinst->cpumask.cbcpu, cbcpumask); 953 + cpumask_and(pinst->rcpumask.pcpu, pcpumask, cpu_online_mask); 954 + cpumask_and(pinst->rcpumask.cbcpu, cbcpumask, cpu_online_mask); 955 + 956 + if (padata_setup_cpumasks(pinst)) 957 + goto err_free_omask; 975 958 976 959 pinst->flags = 0; 977 960 ··· 995 962 996 963 return pinst; 997 964 965 + err_free_omask: 966 + free_cpumask_var(pinst->omask); 967 + err_free_rcpumask_cbcpu: 968 + free_cpumask_var(pinst->rcpumask.cbcpu); 969 + err_free_rcpumask_pcpu: 970 + free_cpumask_var(pinst->rcpumask.pcpu); 998 971 err_free_masks: 999 972 free_cpumask_var(pinst->cpumask.pcpu); 1000 973 free_cpumask_var(pinst->cpumask.cbcpu); ··· 1038 999 kobject_put(&pinst->kobj); 1039 1000 } 1040 1001 EXPORT_SYMBOL(padata_free); 1002 + 1003 + /** 1004 + * padata_alloc_shell - Allocate and initialize padata shell. 1005 + * 1006 + * @pinst: Parent padata_instance object. 1007 + */ 1008 + struct padata_shell *padata_alloc_shell(struct padata_instance *pinst) 1009 + { 1010 + struct parallel_data *pd; 1011 + struct padata_shell *ps; 1012 + 1013 + ps = kzalloc(sizeof(*ps), GFP_KERNEL); 1014 + if (!ps) 1015 + goto out; 1016 + 1017 + ps->pinst = pinst; 1018 + 1019 + get_online_cpus(); 1020 + pd = padata_alloc_pd(ps); 1021 + put_online_cpus(); 1022 + 1023 + if (!pd) 1024 + goto out_free_ps; 1025 + 1026 + mutex_lock(&pinst->lock); 1027 + RCU_INIT_POINTER(ps->pd, pd); 1028 + list_add(&ps->list, &pinst->pslist); 1029 + mutex_unlock(&pinst->lock); 1030 + 1031 + return ps; 1032 + 1033 + out_free_ps: 1034 + kfree(ps); 1035 + out: 1036 + return NULL; 1037 + } 1038 + EXPORT_SYMBOL(padata_alloc_shell); 1039 + 1040 + /** 1041 + * padata_free_shell - free a padata shell 1042 + * 1043 + * @ps: padata shell to free 1044 + */ 1045 + void padata_free_shell(struct padata_shell *ps) 1046 + { 1047 + struct padata_instance *pinst = ps->pinst; 1048 + 1049 + mutex_lock(&pinst->lock); 1050 + list_del(&ps->list); 1051 + padata_free_pd(rcu_dereference_protected(ps->pd, 1)); 1052 + mutex_unlock(&pinst->lock); 1053 + 1054 + kfree(ps); 1055 + } 1056 + EXPORT_SYMBOL(padata_free_shell); 1041 1057 1042 1058 #ifdef CONFIG_HOTPLUG_CPU 1043 1059