Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge git://git.kvack.org/~bcrl/aio-next
Pull aio fixes from Benjamin LaHaise.
* git://git.kvack.org/~bcrl/aio-next:
aio: nullify aio->ring_pages after freeing it
aio: prevent double free in ioctx_alloc
aio: Fix a trinity splat
+51
-83
+51
-83
fs/aio.c
+51
-83
fs/aio.c
···
80
80
struct percpu_ref users;
81
81
atomic_t dead;
82
82
83
+
struct percpu_ref reqs;
84
+
83
85
unsigned long user_id;
84
86
85
87
struct __percpu kioctx_cpu *cpu;
···
109
107
struct page **ring_pages;
110
108
long nr_pages;
111
109
112
-
struct rcu_head rcu_head;
113
110
struct work_struct free_work;
114
111
115
112
struct {
···
251
250
252
251
put_aio_ring_file(ctx);
253
252
254
-
if (ctx->ring_pages && ctx->ring_pages != ctx->internal_pages)
253
+
if (ctx->ring_pages && ctx->ring_pages != ctx->internal_pages) {
255
254
kfree(ctx->ring_pages);
255
+
ctx->ring_pages = NULL;
256
+
}
256
257
}
257
258
258
259
static int aio_ring_mmap(struct file *file, struct vm_area_struct *vma)
···
466
463
return cancel(kiocb);
467
464
}
468
465
469
-
static void free_ioctx_rcu(struct rcu_head *head)
466
+
static void free_ioctx(struct work_struct *work)
470
467
{
471
-
struct kioctx *ctx = container_of(head, struct kioctx, rcu_head);
468
+
struct kioctx *ctx = container_of(work, struct kioctx, free_work);
472
469
470
+
pr_debug("freeing %p\n", ctx);
471
+
472
+
aio_free_ring(ctx);
473
473
free_percpu(ctx->cpu);
474
474
kmem_cache_free(kioctx_cachep, ctx);
475
+
}
476
+
477
+
static void free_ioctx_reqs(struct percpu_ref *ref)
478
+
{
479
+
struct kioctx *ctx = container_of(ref, struct kioctx, reqs);
480
+
481
+
INIT_WORK(&ctx->free_work, free_ioctx);
482
+
schedule_work(&ctx->free_work);
475
483
}
476
484
477
485
/*
···
490
476
* and ctx->users has dropped to 0, so we know no more kiocbs can be submitted -
491
477
* now it's safe to cancel any that need to be.
492
478
*/
493
-
static void free_ioctx(struct work_struct *work)
479
+
static void free_ioctx_users(struct percpu_ref *ref)
494
480
{
495
-
struct kioctx *ctx = container_of(work, struct kioctx, free_work);
496
-
struct aio_ring *ring;
481
+
struct kioctx *ctx = container_of(ref, struct kioctx, users);
497
482
struct kiocb *req;
498
-
unsigned cpu, avail;
499
-
DEFINE_WAIT(wait);
500
483
501
484
spin_lock_irq(&ctx->ctx_lock);
502
485
···
507
496
508
497
spin_unlock_irq(&ctx->ctx_lock);
509
498
510
-
for_each_possible_cpu(cpu) {
511
-
struct kioctx_cpu *kcpu = per_cpu_ptr(ctx->cpu, cpu);
512
-
513
-
atomic_add(kcpu->reqs_available, &ctx->reqs_available);
514
-
kcpu->reqs_available = 0;
515
-
}
516
-
517
-
while (1) {
518
-
prepare_to_wait(&ctx->wait, &wait, TASK_UNINTERRUPTIBLE);
519
-
520
-
ring = kmap_atomic(ctx->ring_pages[0]);
521
-
avail = (ring->head <= ring->tail)
522
-
? ring->tail - ring->head
523
-
: ctx->nr_events - ring->head + ring->tail;
524
-
525
-
atomic_add(avail, &ctx->reqs_available);
526
-
ring->head = ring->tail;
527
-
kunmap_atomic(ring);
528
-
529
-
if (atomic_read(&ctx->reqs_available) >= ctx->nr_events - 1)
530
-
break;
531
-
532
-
schedule();
533
-
}
534
-
finish_wait(&ctx->wait, &wait);
535
-
536
-
WARN_ON(atomic_read(&ctx->reqs_available) > ctx->nr_events - 1);
537
-
538
-
aio_free_ring(ctx);
539
-
540
-
pr_debug("freeing %p\n", ctx);
541
-
542
-
/*
543
-
* Here the call_rcu() is between the wait_event() for reqs_active to
544
-
* hit 0, and freeing the ioctx.
545
-
*
546
-
* aio_complete() decrements reqs_active, but it has to touch the ioctx
547
-
* after to issue a wakeup so we use rcu.
548
-
*/
549
-
call_rcu(&ctx->rcu_head, free_ioctx_rcu);
550
-
}
551
-
552
-
static void free_ioctx_ref(struct percpu_ref *ref)
553
-
{
554
-
struct kioctx *ctx = container_of(ref, struct kioctx, users);
555
-
556
-
INIT_WORK(&ctx->free_work, free_ioctx);
557
-
schedule_work(&ctx->free_work);
499
+
percpu_ref_kill(&ctx->reqs);
500
+
percpu_ref_put(&ctx->reqs);
558
501
}
559
502
560
503
static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm)
···
567
602
}
568
603
}
569
604
605
+
static void aio_nr_sub(unsigned nr)
606
+
{
607
+
spin_lock(&aio_nr_lock);
608
+
if (WARN_ON(aio_nr - nr > aio_nr))
609
+
aio_nr = 0;
610
+
else
611
+
aio_nr -= nr;
612
+
spin_unlock(&aio_nr_lock);
613
+
}
614
+
570
615
/* ioctx_alloc
571
616
* Allocates and initializes an ioctx. Returns an ERR_PTR if it failed.
572
617
*/
···
614
639
615
640
ctx->max_reqs = nr_events;
616
641
617
-
if (percpu_ref_init(&ctx->users, free_ioctx_ref))
618
-
goto out_freectx;
642
+
if (percpu_ref_init(&ctx->users, free_ioctx_users))
643
+
goto err;
644
+
645
+
if (percpu_ref_init(&ctx->reqs, free_ioctx_reqs))
646
+
goto err;
619
647
620
648
spin_lock_init(&ctx->ctx_lock);
621
649
spin_lock_init(&ctx->completion_lock);
···
629
651
630
652
ctx->cpu = alloc_percpu(struct kioctx_cpu);
631
653
if (!ctx->cpu)
632
-
goto out_freeref;
654
+
goto err;
633
655
634
656
if (aio_setup_ring(ctx) < 0)
635
-
goto out_freepcpu;
657
+
goto err;
636
658
637
659
atomic_set(&ctx->reqs_available, ctx->nr_events - 1);
638
660
ctx->req_batch = (ctx->nr_events - 1) / (num_possible_cpus() * 4);
···
644
666
if (aio_nr + nr_events > (aio_max_nr * 2UL) ||
645
667
aio_nr + nr_events < aio_nr) {
646
668
spin_unlock(&aio_nr_lock);
647
-
goto out_cleanup;
669
+
err = -EAGAIN;
670
+
goto err;
648
671
}
649
672
aio_nr += ctx->max_reqs;
650
673
spin_unlock(&aio_nr_lock);
···
654
675
655
676
err = ioctx_add_table(ctx, mm);
656
677
if (err)
657
-
goto out_cleanup_put;
678
+
goto err_cleanup;
658
679
659
680
pr_debug("allocated ioctx %p[%ld]: mm=%p mask=0x%x\n",
660
681
ctx, ctx->user_id, mm, ctx->nr_events);
661
682
return ctx;
662
683
663
-
out_cleanup_put:
664
-
percpu_ref_put(&ctx->users);
665
-
out_cleanup:
666
-
err = -EAGAIN;
667
-
aio_free_ring(ctx);
668
-
out_freepcpu:
684
+
err_cleanup:
685
+
aio_nr_sub(ctx->max_reqs);
686
+
err:
669
687
free_percpu(ctx->cpu);
670
-
out_freeref:
688
+
free_percpu(ctx->reqs.pcpu_count);
671
689
free_percpu(ctx->users.pcpu_count);
672
-
out_freectx:
673
-
put_aio_ring_file(ctx);
674
690
kmem_cache_free(kioctx_cachep, ctx);
675
691
pr_debug("error allocating ioctx %d\n", err);
676
692
return ERR_PTR(err);
···
700
726
* -EAGAIN with no ioctxs actually in use (as far as userspace
701
727
* could tell).
702
728
*/
703
-
spin_lock(&aio_nr_lock);
704
-
BUG_ON(aio_nr - ctx->max_reqs > aio_nr);
705
-
aio_nr -= ctx->max_reqs;
706
-
spin_unlock(&aio_nr_lock);
729
+
aio_nr_sub(ctx->max_reqs);
707
730
708
731
if (ctx->mmap_size)
709
732
vm_munmap(ctx->mmap_base, ctx->mmap_size);
···
832
861
if (unlikely(!req))
833
862
goto out_put;
834
863
864
+
percpu_ref_get(&ctx->reqs);
865
+
835
866
req->ki_ctx = ctx;
836
867
return req;
837
868
out_put:
···
902
929
wake_up_process(iocb->ki_obj.tsk);
903
930
return;
904
931
}
905
-
906
-
/*
907
-
* Take rcu_read_lock() in case the kioctx is being destroyed, as we
908
-
* need to issue a wakeup after incrementing reqs_available.
909
-
*/
910
-
rcu_read_lock();
911
932
912
933
if (iocb->ki_list.next) {
913
934
unsigned long flags;
···
977
1010
if (waitqueue_active(&ctx->wait))
978
1011
wake_up(&ctx->wait);
979
1012
980
-
rcu_read_unlock();
1013
+
percpu_ref_put(&ctx->reqs);
981
1014
}
982
1015
EXPORT_SYMBOL(aio_complete);
983
1016
···
1388
1421
return 0;
1389
1422
out_put_req:
1390
1423
put_reqs_available(ctx, 1);
1424
+
percpu_ref_put(&ctx->reqs);
1391
1425
kiocb_free(req);
1392
1426
return ret;
1393
1427
}