tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

kernfs: Replace global kernfs_open_file_mutex with hashed mutexes.

In current kernfs design a single mutex, kernfs_open_file_mutex, protects
the list of kernfs_open_file instances corresponding to a sysfs attribute.
So even if different tasks are opening or closing different sysfs files
they can contend on osq_lock of this mutex. The contention is more apparent
in large scale systems with few hundred CPUs where most of the CPUs have
running tasks that are opening, accessing or closing sysfs files at any
point of time.

Using hashed mutexes in place of a single global mutex, can significantly
reduce contention around global mutex and hence can provide better
scalability. Moreover as these hashed mutexes are not part of kernfs_node
objects we will not see any singnificant change in memory utilization of
kernfs based file systems like sysfs, cgroupfs etc.

Modify interface introduced in previous patch to make use of hashed
mutexes. Use kernfs_node address as hashing key.

Acked-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Imran Khan <imran.f.khan@oracle.com>
Link: https://lore.kernel.org/r/20220615021059.862643-5-imran.f.khan@oracle.com
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

authored by

Imran Khan and committed by
Greg Kroah-Hartman 1d25b84e 41448c61

+83 -14
+3 -14
fs/kernfs/file.c
··· 18 18 19 19 #include "kernfs-internal.h" 20 20 21 - /* 22 - * There's one kernfs_open_file for each open file and one kernfs_open_node 23 - * for each kernfs_node with one or more open files. 24 - * 25 - * kernfs_node->attr.open points to kernfs_open_node. attr.open is 26 - * RCU protected. 27 - * 28 - * filp->private_data points to seq_file whose ->private points to 29 - * kernfs_open_file. kernfs_open_files are chained at 30 - * kernfs_open_node->files, which is protected by kernfs_open_file_mutex. 31 - */ 32 - static DEFINE_MUTEX(kernfs_open_file_mutex); 33 - 34 21 struct kernfs_open_node { 35 22 struct rcu_head rcu_head; 36 23 atomic_t event; ··· 38 51 39 52 static inline struct mutex *kernfs_open_file_mutex_ptr(struct kernfs_node *kn) 40 53 { 41 - return &kernfs_open_file_mutex; 54 + int idx = hash_ptr(kn, NR_KERNFS_LOCK_BITS); 55 + 56 + return &kernfs_locks->open_file_mutex[idx]; 42 57 } 43 58 44 59 static inline struct mutex *kernfs_open_file_mutex_lock(struct kernfs_node *kn)
+4
fs/kernfs/kernfs-internal.h
··· 164 164 */ 165 165 extern const struct inode_operations kernfs_symlink_iops; 166 166 167 + /* 168 + * kernfs locks 169 + */ 170 + extern struct kernfs_global_locks *kernfs_locks; 167 171 #endif /* __KERNFS_INTERNAL_H */
+19
fs/kernfs/mount.c
··· 20 20 #include "kernfs-internal.h" 21 21 22 22 struct kmem_cache *kernfs_node_cache, *kernfs_iattrs_cache; 23 + struct kernfs_global_locks *kernfs_locks; 23 24 24 25 static int kernfs_sop_show_options(struct seq_file *sf, struct dentry *dentry) 25 26 { ··· 388 387 kfree(info); 389 388 } 390 389 390 + static void __init kernfs_mutex_init(void) 391 + { 392 + int count; 393 + 394 + for (count = 0; count < NR_KERNFS_LOCKS; count++) 395 + mutex_init(&kernfs_locks->open_file_mutex[count]); 396 + } 397 + 398 + static void __init kernfs_lock_init(void) 399 + { 400 + kernfs_locks = kmalloc(sizeof(struct kernfs_global_locks), GFP_KERNEL); 401 + WARN_ON(!kernfs_locks); 402 + 403 + kernfs_mutex_init(); 404 + } 405 + 391 406 void __init kernfs_init(void) 392 407 { 393 408 kernfs_node_cache = kmem_cache_create("kernfs_node_cache", ··· 414 397 kernfs_iattrs_cache = kmem_cache_create("kernfs_iattrs_cache", 415 398 sizeof(struct kernfs_iattrs), 416 399 0, SLAB_PANIC, NULL); 400 + 401 + kernfs_lock_init(); 417 402 }
+57
include/linux/kernfs.h
··· 18 18 #include <linux/uidgid.h> 19 19 #include <linux/wait.h> 20 20 #include <linux/rwsem.h> 21 + #include <linux/cache.h> 21 22 22 23 struct file; 23 24 struct dentry; ··· 34 33 struct kernfs_fs_context; 35 34 struct kernfs_open_node; 36 35 struct kernfs_iattrs; 36 + 37 + /* 38 + * NR_KERNFS_LOCK_BITS determines size (NR_KERNFS_LOCKS) of hash 39 + * table of locks. 40 + * Having a small hash table would impact scalability, since 41 + * more and more kernfs_node objects will end up using same lock 42 + * and having a very large hash table would waste memory. 43 + * 44 + * At the moment size of hash table of locks is being set based on 45 + * the number of CPUs as follows: 46 + * 47 + * NR_CPU NR_KERNFS_LOCK_BITS NR_KERNFS_LOCKS 48 + * 1 1 2 49 + * 2-3 2 4 50 + * 4-7 4 16 51 + * 8-15 6 64 52 + * 16-31 8 256 53 + * 32 and more 10 1024 54 + * 55 + * The above relation between NR_CPU and number of locks is based 56 + * on some internal experimentation which involved booting qemu 57 + * with different values of smp, performing some sysfs operations 58 + * on all CPUs and observing how increase in number of locks impacts 59 + * completion time of these sysfs operations on each CPU. 60 + */ 61 + #ifdef CONFIG_SMP 62 + #define NR_KERNFS_LOCK_BITS (2 * (ilog2(NR_CPUS < 32 ? NR_CPUS : 32))) 63 + #else 64 + #define NR_KERNFS_LOCK_BITS 1 65 + #endif 66 + 67 + #define NR_KERNFS_LOCKS (1 << NR_KERNFS_LOCK_BITS) 68 + 69 + /* 70 + * There's one kernfs_open_file for each open file and one kernfs_open_node 71 + * for each kernfs_node with one or more open files. 72 + * 73 + * filp->private_data points to seq_file whose ->private points to 74 + * kernfs_open_file. 75 + * 76 + * kernfs_open_files are chained at kernfs_open_node->files, which is 77 + * protected by kernfs_global_locks.open_file_mutex[i]. 78 + * 79 + * To reduce possible contention in sysfs access, arising due to single 80 + * locks, use an array of locks (e.g. open_file_mutex) and use kernfs_node 81 + * object address as hash keys to get the index of these locks. 82 + * 83 + * Hashed mutexes are safe to use here because operations using these don't 84 + * rely on global exclusion. 85 + * 86 + * In future we intend to replace other global locks with hashed ones as well. 87 + * kernfs_global_locks acts as a holder for all such hash tables. 88 + */ 89 + struct kernfs_global_locks { 90 + struct mutex open_file_mutex[NR_KERNFS_LOCKS]; 91 + }; 37 92 38 93 enum kernfs_node_type { 39 94 KERNFS_DIR = 0x0001,