kernfs: make ->attr.open RCU protected.

+102 -47

2 changed files

expand all

kernfs

file.c

include

linux

kernfs.h

+101 -46

fs/kernfs/file.c

··· 23 23 * for each kernfs_node with one or more open files. 24 24 * 25 25 * kernfs_node->attr.open points to kernfs_open_node. attr.open is 26 - * protected by kernfs_open_node_lock. 26 + * RCU protected. 27 27 * 28 28 * filp->private_data points to seq_file whose ->private points to 29 29 * kernfs_open_file. kernfs_open_files are chained at 30 30 * kernfs_open_node->files, which is protected by kernfs_open_file_mutex. 31 31 */ 32 - static DEFINE_SPINLOCK(kernfs_open_node_lock); 33 32 static DEFINE_MUTEX(kernfs_open_file_mutex); 34 33 35 34 struct kernfs_open_node { 35 + struct rcu_head rcu_head; 36 36 atomic_t event; 37 37 wait_queue_head_t poll; 38 38 struct list_head files; /* goes through kernfs_open_file.list */ ··· 50 50 51 51 static DEFINE_SPINLOCK(kernfs_notify_lock); 52 52 static struct kernfs_node *kernfs_notify_list = KERNFS_NOTIFY_EOL; 53 + 54 + /** 55 + * kernfs_deref_open_node - Get kernfs_open_node corresponding to @kn. 56 + * 57 + * @of: associated kernfs_open_file instance. 58 + * @kn: target kernfs_node. 59 + * 60 + * Fetch and return ->attr.open of @kn if @of->list is non empty. 61 + * If @of->list is not empty we can safely assume that @of is on 62 + * @kn->attr.open->files list and this guarantees that @kn->attr.open 63 + * will not vanish i.e. dereferencing outside RCU read-side critical 64 + * section is safe here. 65 + * 66 + * The caller needs to make sure that @of->list is not empty. 67 + */ 68 + static struct kernfs_open_node * 69 + kernfs_deref_open_node(struct kernfs_open_file *of, struct kernfs_node *kn) 70 + { 71 + struct kernfs_open_node *on; 72 + 73 + on = rcu_dereference_check(kn->attr.open, !list_empty(&of->list)); 74 + 75 + return on; 76 + } 77 + 78 + /** 79 + * kernfs_deref_open_node_protected - Get kernfs_open_node corresponding to @kn 80 + * 81 + * @kn: target kernfs_node. 82 + * 83 + * Fetch and return ->attr.open of @kn when caller holds the 84 + * kernfs_open_file_mutex. 85 + * 86 + * Update of ->attr.open happens under kernfs_open_file_mutex. So when 87 + * the caller guarantees that this mutex is being held, other updaters can't 88 + * change ->attr.open and this means that we can safely deref ->attr.open 89 + * outside RCU read-side critical section. 90 + * 91 + * The caller needs to make sure that kernfs_open_file_mutex is held. 92 + */ 93 + static struct kernfs_open_node * 94 + kernfs_deref_open_node_protected(struct kernfs_node *kn) 95 + { 96 + return rcu_dereference_protected(kn->attr.open, 97 + lockdep_is_held(&kernfs_open_file_mutex)); 98 + } 53 99 54 100 static struct kernfs_open_file *kernfs_of(struct file *file) 55 101 { ··· 202 156 static int kernfs_seq_show(struct seq_file *sf, void *v) 203 157 { 204 158 struct kernfs_open_file *of = sf->private; 159 + struct kernfs_open_node *on = kernfs_deref_open_node(of, of->kn); 205 160 206 - of->event = atomic_read(&of->kn->attr.open->event); 161 + if (!on) 162 + return -EINVAL; 163 + 164 + of->event = atomic_read(&on->event); 207 165 208 166 return of->kn->attr.ops->seq_show(sf, v); 209 167 } ··· 230 180 struct kernfs_open_file *of = kernfs_of(iocb->ki_filp); 231 181 ssize_t len = min_t(size_t, iov_iter_count(iter), PAGE_SIZE); 232 182 const struct kernfs_ops *ops; 183 + struct kernfs_open_node *on; 233 184 char *buf; 234 185 235 186 buf = of->prealloc_buf; ··· 252 201 goto out_free; 253 202 } 254 203 255 - of->event = atomic_read(&of->kn->attr.open->event); 204 + on = kernfs_deref_open_node(of, of->kn); 205 + if (!on) { 206 + len = -EINVAL; 207 + mutex_unlock(&of->mutex); 208 + goto out_free; 209 + } 210 + 211 + of->event = atomic_read(&on->event); 212 + 256 213 ops = kernfs_ops(of->kn); 257 214 if (ops->read) 258 215 len = ops->read(of, buf, len, iocb->ki_pos); ··· 577 518 { 578 519 struct kernfs_open_node *on, *new_on = NULL; 579 520 580 - retry: 581 521 mutex_lock(&kernfs_open_file_mutex); 582 - spin_lock_irq(&kernfs_open_node_lock); 583 - 584 - if (!kn->attr.open && new_on) { 585 - kn->attr.open = new_on; 586 - new_on = NULL; 587 - } 588 - 589 - on = kn->attr.open; 590 - if (on) 591 - list_add_tail(&of->list, &on->files); 592 - 593 - spin_unlock_irq(&kernfs_open_node_lock); 594 - mutex_unlock(&kernfs_open_file_mutex); 522 + on = kernfs_deref_open_node_protected(kn); 595 523 596 524 if (on) { 597 - kfree(new_on); 525 + list_add_tail(&of->list, &on->files); 526 + mutex_unlock(&kernfs_open_file_mutex); 598 527 return 0; 528 + } else { 529 + /* not there, initialize a new one */ 530 + new_on = kmalloc(sizeof(*new_on), GFP_KERNEL); 531 + if (!new_on) { 532 + mutex_unlock(&kernfs_open_file_mutex); 533 + return -ENOMEM; 534 + } 535 + atomic_set(&new_on->event, 1); 536 + init_waitqueue_head(&new_on->poll); 537 + INIT_LIST_HEAD(&new_on->files); 538 + list_add_tail(&of->list, &new_on->files); 539 + rcu_assign_pointer(kn->attr.open, new_on); 599 540 } 541 + mutex_unlock(&kernfs_open_file_mutex); 600 542 601 - /* not there, initialize a new one and retry */ 602 - new_on = kmalloc(sizeof(*new_on), GFP_KERNEL); 603 - if (!new_on) 604 - return -ENOMEM; 605 - 606 - atomic_set(&new_on->event, 1); 607 - init_waitqueue_head(&new_on->poll); 608 - INIT_LIST_HEAD(&new_on->files); 609 - goto retry; 543 + return 0; 610 544 } 611 545 612 546 /** ··· 618 566 static void kernfs_unlink_open_file(struct kernfs_node *kn, 619 567 struct kernfs_open_file *of) 620 568 { 621 - struct kernfs_open_node *on = kn->attr.open; 622 - unsigned long flags; 569 + struct kernfs_open_node *on; 623 570 624 571 mutex_lock(&kernfs_open_file_mutex); 625 - spin_lock_irqsave(&kernfs_open_node_lock, flags); 572 + 573 + on = kernfs_deref_open_node_protected(kn); 574 + if (!on) { 575 + mutex_unlock(&kernfs_open_file_mutex); 576 + return; 577 + } 626 578 627 579 if (of) 628 580 list_del(&of->list); 629 581 630 - if (list_empty(&on->files)) 631 - kn->attr.open = NULL; 632 - else 633 - on = NULL; 582 + if (list_empty(&on->files)) { 583 + rcu_assign_pointer(kn->attr.open, NULL); 584 + kfree_rcu(on, rcu_head); 585 + } 634 586 635 - spin_unlock_irqrestore(&kernfs_open_node_lock, flags); 636 587 mutex_unlock(&kernfs_open_file_mutex); 637 - 638 - kfree(on); 639 588 } 640 589 641 590 static int kernfs_fop_open(struct inode *inode, struct file *file) ··· 826 773 * check under kernfs_open_file_mutex will ensure bailing out if 827 774 * ->attr.open became NULL while waiting for the mutex. 828 775 */ 829 - if (!kn->attr.open) 776 + if (!rcu_access_pointer(kn->attr.open)) 830 777 return; 831 778 832 779 mutex_lock(&kernfs_open_file_mutex); 833 - if (!kn->attr.open) { 780 + on = kernfs_deref_open_node_protected(kn); 781 + if (!on) { 834 782 mutex_unlock(&kernfs_open_file_mutex); 835 783 return; 836 784 } 837 - 838 - on = kn->attr.open; 839 785 840 786 list_for_each_entry(of, &on->files, list) { 841 787 struct inode *inode = file_inode(of->file); ··· 866 814 __poll_t kernfs_generic_poll(struct kernfs_open_file *of, poll_table *wait) 867 815 { 868 816 struct kernfs_node *kn = kernfs_dentry_node(of->file->f_path.dentry); 869 - struct kernfs_open_node *on = kn->attr.open; 817 + struct kernfs_open_node *on = kernfs_deref_open_node(of, kn); 818 + 819 + if (!on) 820 + return EPOLLERR; 870 821 871 822 poll_wait(of->file, &on->poll, wait); 872 823 ··· 976 921 return; 977 922 978 923 /* kick poll immediately */ 979 - spin_lock_irqsave(&kernfs_open_node_lock, flags); 980 - on = kn->attr.open; 924 + rcu_read_lock(); 925 + on = rcu_dereference(kn->attr.open); 981 926 if (on) { 982 927 atomic_inc(&on->event); 983 928 wake_up_interruptible(&on->poll); 984 929 } 985 - spin_unlock_irqrestore(&kernfs_open_node_lock, flags); 930 + rcu_read_unlock(); 986 931 987 932 /* schedule work to kick fsnotify */ 988 933 spin_lock_irqsave(&kernfs_notify_lock, flags);

+1 -1

include/linux/kernfs.h

··· 114 114 115 115 struct kernfs_elem_attr { 116 116 const struct kernfs_ops *ops; 117 - struct kernfs_open_node *open; 117 + struct kernfs_open_node __rcu *open; 118 118 loff_t size; 119 119 struct kernfs_node *notify_next; /* for kernfs_notify() */ 120 120 };

Configure Feed

Configure Feed