Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

mm: move lesser used vma_area_struct members into the last cacheline

Move several vma_area_struct members which are rarely or never used during
page fault handling into the last cacheline to better pack vm_area_struct.
As a result vm_area_struct will fit into 3 as opposed to 4 cachelines.
New typical vm_area_struct layout:

struct vm_area_struct {
union {
struct {
long unsigned int vm_start; /* 0 8 */
long unsigned int vm_end; /* 8 8 */
}; /* 0 16 */
freeptr_t vm_freeptr; /* 0 8 */
}; /* 0 16 */
struct mm_struct * vm_mm; /* 16 8 */
pgprot_t vm_page_prot; /* 24 8 */
union {
const vm_flags_t vm_flags; /* 32 8 */
vm_flags_t __vm_flags; /* 32 8 */
}; /* 32 8 */
unsigned int vm_lock_seq; /* 40 4 */

/* XXX 4 bytes hole, try to pack */

struct list_head anon_vma_chain; /* 48 16 */
/* --- cacheline 1 boundary (64 bytes) --- */
struct anon_vma * anon_vma; /* 64 8 */
const struct vm_operations_struct * vm_ops; /* 72 8 */
long unsigned int vm_pgoff; /* 80 8 */
struct file * vm_file; /* 88 8 */
void * vm_private_data; /* 96 8 */
atomic_long_t swap_readahead_info; /* 104 8 */
struct mempolicy * vm_policy; /* 112 8 */
struct vma_numab_state * numab_state; /* 120 8 */
/* --- cacheline 2 boundary (128 bytes) --- */
refcount_t vm_refcnt (__aligned__(64)); /* 128 4 */

/* XXX 4 bytes hole, try to pack */

struct {
struct rb_node rb (__aligned__(8)); /* 136 24 */
long unsigned int rb_subtree_last; /* 160 8 */
} __attribute__((__aligned__(8))) shared; /* 136 32 */
struct anon_vma_name * anon_name; /* 168 8 */
struct vm_userfaultfd_ctx vm_userfaultfd_ctx; /* 176 8 */

/* size: 192, cachelines: 3, members: 18 */
/* sum members: 176, holes: 2, sum holes: 8 */
/* padding: 8 */
/* forced alignments: 2, forced holes: 1, sum forced holes: 4 */
} __attribute__((__aligned__(64)));

Memory consumption per 1000 VMAs becomes 48 pages:

slabinfo after vm_area_struct changes:
<name> ... <objsize> <objperslab> <pagesperslab> : ...
vm_area_struct ... 192 42 2 : ...

Link: https://lkml.kernel.org/r/20250213224655.1680278-14-surenb@google.com
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Reviewed-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Tested-by: Shivank Garg <shivankg@amd.com>
Link: https://lkml.kernel.org/r/5e19ec93-8307-47c2-bb13-3ddf7150624e@amd.com
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Christian Brauner <brauner@kernel.org>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jann Horn <jannh@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Klara Modin <klarasmodin@gmail.com>
Cc: Liam R. Howlett <Liam.Howlett@Oracle.com>
Cc: Lokesh Gidra <lokeshgidra@google.com>
Cc: Mateusz Guzik <mjguzik@gmail.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Minchan Kim <minchan@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: "Paul E . McKenney" <paulmck@kernel.org>
Cc: Peter Xu <peterx@redhat.com>
Cc: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Sourav Panda <souravpanda@google.com>
Cc: Wei Yang <richard.weiyang@gmail.com>
Cc: Will Deacon <will@kernel.org>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Suren Baghdasaryan and committed by
Andrew Morton
6bef4c2f f35ab95c

+36 -39
+18 -20
include/linux/mm_types.h
··· 725 725 */ 726 726 unsigned int vm_lock_seq; 727 727 #endif 728 - 729 - /* 730 - * For areas with an address space and backing store, 731 - * linkage into the address_space->i_mmap interval tree. 732 - * 733 - */ 734 - struct { 735 - struct rb_node rb; 736 - unsigned long rb_subtree_last; 737 - } shared; 738 - 739 728 /* 740 729 * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma 741 730 * list, after a COW of one of the file pages. A MAP_SHARED vma ··· 744 755 struct file * vm_file; /* File we map to (can be NULL). */ 745 756 void * vm_private_data; /* was vm_pte (shared mem) */ 746 757 747 - #ifdef CONFIG_ANON_VMA_NAME 748 - /* 749 - * For private and shared anonymous mappings, a pointer to a null 750 - * terminated string containing the name given to the vma, or NULL if 751 - * unnamed. Serialized by mmap_lock. Use anon_vma_name to access. 752 - */ 753 - struct anon_vma_name *anon_name; 754 - #endif 755 758 #ifdef CONFIG_SWAP 756 759 atomic_long_t swap_readahead_info; 757 760 #endif ··· 756 775 #ifdef CONFIG_NUMA_BALANCING 757 776 struct vma_numab_state *numab_state; /* NUMA Balancing state */ 758 777 #endif 759 - struct vm_userfaultfd_ctx vm_userfaultfd_ctx; 760 778 #ifdef CONFIG_PER_VMA_LOCK 761 779 /* Unstable RCU readers are allowed to read this. */ 762 780 refcount_t vm_refcnt ____cacheline_aligned_in_smp; ··· 763 783 struct lockdep_map vmlock_dep_map; 764 784 #endif 765 785 #endif 786 + /* 787 + * For areas with an address space and backing store, 788 + * linkage into the address_space->i_mmap interval tree. 789 + * 790 + */ 791 + struct { 792 + struct rb_node rb; 793 + unsigned long rb_subtree_last; 794 + } shared; 795 + #ifdef CONFIG_ANON_VMA_NAME 796 + /* 797 + * For private and shared anonymous mappings, a pointer to a null 798 + * terminated string containing the name given to the vma, or NULL if 799 + * unnamed. Serialized by mmap_lock. Use anon_vma_name to access. 800 + */ 801 + struct anon_vma_name *anon_name; 802 + #endif 803 + struct vm_userfaultfd_ctx vm_userfaultfd_ctx; 766 804 } __randomize_layout; 767 805 768 806 #ifdef CONFIG_NUMA
+18 -19
tools/testing/vma/vma_internal.h
··· 280 280 #endif 281 281 282 282 /* 283 - * For areas with an address space and backing store, 284 - * linkage into the address_space->i_mmap interval tree. 285 - * 286 - */ 287 - struct { 288 - struct rb_node rb; 289 - unsigned long rb_subtree_last; 290 - } shared; 291 - 292 - /* 293 283 * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma 294 284 * list, after a COW of one of the file pages. A MAP_SHARED vma 295 285 * can only be in the i_mmap tree. An anonymous MAP_PRIVATE, stack ··· 298 308 struct file * vm_file; /* File we map to (can be NULL). */ 299 309 void * vm_private_data; /* was vm_pte (shared mem) */ 300 310 301 - #ifdef CONFIG_ANON_VMA_NAME 302 - /* 303 - * For private and shared anonymous mappings, a pointer to a null 304 - * terminated string containing the name given to the vma, or NULL if 305 - * unnamed. Serialized by mmap_lock. Use anon_vma_name to access. 306 - */ 307 - struct anon_vma_name *anon_name; 308 - #endif 309 311 #ifdef CONFIG_SWAP 310 312 atomic_long_t swap_readahead_info; 311 313 #endif ··· 310 328 #ifdef CONFIG_NUMA_BALANCING 311 329 struct vma_numab_state *numab_state; /* NUMA Balancing state */ 312 330 #endif 313 - struct vm_userfaultfd_ctx vm_userfaultfd_ctx; 314 331 #ifdef CONFIG_PER_VMA_LOCK 315 332 /* Unstable RCU readers are allowed to read this. */ 316 333 refcount_t vm_refcnt; 317 334 #endif 335 + /* 336 + * For areas with an address space and backing store, 337 + * linkage into the address_space->i_mmap interval tree. 338 + * 339 + */ 340 + struct { 341 + struct rb_node rb; 342 + unsigned long rb_subtree_last; 343 + } shared; 344 + #ifdef CONFIG_ANON_VMA_NAME 345 + /* 346 + * For private and shared anonymous mappings, a pointer to a null 347 + * terminated string containing the name given to the vma, or NULL if 348 + * unnamed. Serialized by mmap_lock. Use anon_vma_name to access. 349 + */ 350 + struct anon_vma_name *anon_name; 351 + #endif 352 + struct vm_userfaultfd_ctx vm_userfaultfd_ctx; 318 353 } __randomize_layout; 319 354 320 355 struct vm_fault {};