tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge branch 'net-introduce-net_aligned_data'

Eric Dumazet says:

====================
net: introduce net_aligned_data

____cacheline_aligned_in_smp on small fields like
tcp_memory_allocated and udp_memory_allocated is not good enough.

It makes sure to put these fields at the start of a cache line,
but does not prevent the linker from using the cache line for other
fields, with potential performance impact.

nm -v vmlinux|egrep -5 "tcp_memory_allocated|udp_memory_allocated"

...
ffffffff83e35cc0 B tcp_memory_allocated
ffffffff83e35cc8 b __key.0
ffffffff83e35cc8 b __tcp_tx_delay_enabled.1
ffffffff83e35ce0 b tcp_orphan_timer
...
ffffffff849dddc0 B udp_memory_allocated
ffffffff849dddc8 B udp_encap_needed_key
ffffffff849dddd8 B udpv6_encap_needed_key
ffffffff849dddf0 b inetsw_lock

One solution is to move these sensitive fields to a structure,
so that the compiler is forced to add empty holes between each member.

nm -v vmlinux|egrep -2 "tcp_memory_allocated|udp_memory_allocated|net_aligned_data"

ffffffff885af970 b mem_id_init
ffffffff885af980 b __key.0
ffffffff885af9c0 B net_aligned_data
ffffffff885afa80 B page_pool_mem_providers
ffffffff885afa90 b __key.2

v1: https://lore.kernel.org/netdev/20250627200551.348096-1-edumazet@google.com/
====================

Link: https://patch.msgid.link/20250630093540.3052835-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

Jakub Kicinski 8077b9a9 a8065af3

+41 -19
+22
include/net/aligned_data.h
··· 1 + /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 + #ifndef _NET_ALIGNED_DATA_H 3 + #define _NET_ALIGNED_DATA_H 4 + 5 + #include <linux/atomic.h> 6 + #include <linux/types.h> 7 + 8 + /* Structure holding cacheline aligned fields on SMP builds. 9 + * Each field or group should have an ____cacheline_aligned_in_smp 10 + * attribute to ensure no accidental false sharing can happen. 11 + */ 12 + struct net_aligned_data { 13 + atomic64_t net_cookie ____cacheline_aligned_in_smp; 14 + #if defined(CONFIG_INET) 15 + atomic_long_t tcp_memory_allocated ____cacheline_aligned_in_smp; 16 + atomic_long_t udp_memory_allocated ____cacheline_aligned_in_smp; 17 + #endif 18 + }; 19 + 20 + extern struct net_aligned_data net_aligned_data; 21 + 22 + #endif /* _NET_ALIGNED_DATA_H */
-1
include/net/tcp.h
··· 267 267 #define TCP_RACK_STATIC_REO_WND 0x2 /* Use static RACK reo wnd */ 268 268 #define TCP_RACK_NO_DUPTHRESH 0x4 /* Do not use DUPACK threshold in RACK */ 269 269 270 - extern atomic_long_t tcp_memory_allocated; 271 270 DECLARE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc); 272 271 273 272 extern struct percpu_counter tcp_sockets_allocated;
-1
include/net/udp.h
··· 205 205 206 206 extern struct proto udp_prot; 207 207 208 - extern atomic_long_t udp_memory_allocated; 209 208 DECLARE_PER_CPU(int, udp_memory_per_cpu_fw_alloc); 210 209 211 210 /* sysctl variables for udp */
+5
net/core/hotdata.c
··· 2 2 #include <linux/cache.h> 3 3 #include <linux/jiffies.h> 4 4 #include <linux/list.h> 5 + #include <net/aligned_data.h> 5 6 #include <net/hotdata.h> 7 + #include <net/ip.h> 6 8 #include <net/proto_memory.h> 7 9 8 10 struct net_hotdata net_hotdata __cacheline_aligned = { ··· 24 22 .sysctl_mem_pcpu_rsv = SK_MEMORY_PCPU_RESERVE 25 23 }; 26 24 EXPORT_SYMBOL(net_hotdata); 25 + 26 + struct net_aligned_data net_aligned_data; 27 + EXPORT_IPV6_MOD(net_aligned_data);
+2 -6
net/core/net_namespace.c
··· 19 19 #include <linux/net_namespace.h> 20 20 #include <linux/sched/task.h> 21 21 #include <linux/uidgid.h> 22 - #include <linux/cookie.h> 23 22 #include <linux/proc_fs.h> 24 23 24 + #include <net/aligned_data.h> 25 25 #include <net/sock.h> 26 26 #include <net/netlink.h> 27 27 #include <net/net_namespace.h> ··· 63 63 #define INITIAL_NET_GEN_PTRS 13 /* +1 for len +2 for rcu_head */ 64 64 65 65 static unsigned int max_gen_ptrs = INITIAL_NET_GEN_PTRS; 66 - 67 - DEFINE_COOKIE(net_cookie); 68 66 69 67 static struct net_generic *net_alloc_generic(void) 70 68 { ··· 432 434 LIST_HEAD(net_exit_list); 433 435 int error = 0; 434 436 435 - preempt_disable(); 436 - net->net_cookie = gen_cookie_next(&net_cookie); 437 - preempt_enable(); 437 + net->net_cookie = atomic64_inc_return(&net_aligned_data.net_cookie); 438 438 439 439 list_for_each_entry(ops, &pernet_list, list) { 440 440 error = ops_init(ops, net);
-2
net/ipv4/tcp.c
··· 302 302 long sysctl_tcp_mem[3] __read_mostly; 303 303 EXPORT_IPV6_MOD(sysctl_tcp_mem); 304 304 305 - atomic_long_t tcp_memory_allocated ____cacheline_aligned_in_smp; /* Current allocated memory. */ 306 - EXPORT_IPV6_MOD(tcp_memory_allocated); 307 305 DEFINE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc); 308 306 EXPORT_PER_CPU_SYMBOL_GPL(tcp_memory_per_cpu_fw_alloc); 309 307
+2 -1
net/ipv4/tcp_ipv4.c
··· 59 59 #include <linux/slab.h> 60 60 #include <linux/sched.h> 61 61 62 + #include <net/aligned_data.h> 62 63 #include <net/net_namespace.h> 63 64 #include <net/icmp.h> 64 65 #include <net/inet_hashtables.h> ··· 3391 3390 .sockets_allocated = &tcp_sockets_allocated, 3392 3391 .orphan_count = &tcp_orphan_count, 3393 3392 3394 - .memory_allocated = &tcp_memory_allocated, 3393 + .memory_allocated = &net_aligned_data.tcp_memory_allocated, 3395 3394 .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc, 3396 3395 3397 3396 .memory_pressure = &tcp_memory_pressure,
+1 -3
net/ipv4/udp.c
··· 127 127 long sysctl_udp_mem[3] __read_mostly; 128 128 EXPORT_IPV6_MOD(sysctl_udp_mem); 129 129 130 - atomic_long_t udp_memory_allocated ____cacheline_aligned_in_smp; 131 - EXPORT_IPV6_MOD(udp_memory_allocated); 132 130 DEFINE_PER_CPU(int, udp_memory_per_cpu_fw_alloc); 133 131 EXPORT_PER_CPU_SYMBOL_GPL(udp_memory_per_cpu_fw_alloc); 134 132 ··· 3233 3235 #ifdef CONFIG_BPF_SYSCALL 3234 3236 .psock_update_sk_prot = udp_bpf_update_proto, 3235 3237 #endif 3236 - .memory_allocated = &udp_memory_allocated, 3238 + .memory_allocated = &net_aligned_data.udp_memory_allocated, 3237 3239 .per_cpu_fw_alloc = &udp_memory_per_cpu_fw_alloc, 3238 3240 3239 3241 .sysctl_mem = sysctl_udp_mem,
+1
net/ipv4/udp_impl.h
··· 1 1 /* SPDX-License-Identifier: GPL-2.0 */ 2 2 #ifndef _UDP4_IMPL_H 3 3 #define _UDP4_IMPL_H 4 + #include <net/aligned_data.h> 4 5 #include <net/udp.h> 5 6 #include <net/udplite.h> 6 7 #include <net/protocol.h>
+1 -1
net/ipv4/udplite.c
··· 60 60 .rehash = udp_v4_rehash, 61 61 .get_port = udp_v4_get_port, 62 62 63 - .memory_allocated = &udp_memory_allocated, 63 + .memory_allocated = &net_aligned_data.udp_memory_allocated, 64 64 .per_cpu_fw_alloc = &udp_memory_per_cpu_fw_alloc, 65 65 66 66 .sysctl_mem = sysctl_udp_mem,
+2 -1
net/ipv6/tcp_ipv6.c
··· 41 41 #include <linux/random.h> 42 42 #include <linux/indirect_call_wrapper.h> 43 43 44 + #include <net/aligned_data.h> 44 45 #include <net/tcp.h> 45 46 #include <net/ndisc.h> 46 47 #include <net/inet6_hashtables.h> ··· 2357 2356 .stream_memory_free = tcp_stream_memory_free, 2358 2357 .sockets_allocated = &tcp_sockets_allocated, 2359 2358 2360 - .memory_allocated = &tcp_memory_allocated, 2359 + .memory_allocated = &net_aligned_data.tcp_memory_allocated, 2361 2360 .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc, 2362 2361 2363 2362 .memory_pressure = &tcp_memory_pressure,
+1 -1
net/ipv6/udp.c
··· 1925 1925 .psock_update_sk_prot = udp_bpf_update_proto, 1926 1926 #endif 1927 1927 1928 - .memory_allocated = &udp_memory_allocated, 1928 + .memory_allocated = &net_aligned_data.udp_memory_allocated, 1929 1929 .per_cpu_fw_alloc = &udp_memory_per_cpu_fw_alloc, 1930 1930 1931 1931 .sysctl_mem = sysctl_udp_mem,
+1
net/ipv6/udp_impl.h
··· 1 1 /* SPDX-License-Identifier: GPL-2.0 */ 2 2 #ifndef _UDP6_IMPL_H 3 3 #define _UDP6_IMPL_H 4 + #include <net/aligned_data.h> 4 5 #include <net/udp.h> 5 6 #include <net/udplite.h> 6 7 #include <net/protocol.h>
+1 -1
net/ipv6/udplite.c
··· 59 59 .rehash = udp_v6_rehash, 60 60 .get_port = udp_v6_get_port, 61 61 62 - .memory_allocated = &udp_memory_allocated, 62 + .memory_allocated = &net_aligned_data.udp_memory_allocated, 63 63 .per_cpu_fw_alloc = &udp_memory_per_cpu_fw_alloc, 64 64 65 65 .sysctl_mem = sysctl_udp_mem,
+2 -1
net/mptcp/protocol.c
··· 11 11 #include <linux/netdevice.h> 12 12 #include <linux/sched/signal.h> 13 13 #include <linux/atomic.h> 14 + #include <net/aligned_data.h> 14 15 #include <net/sock.h> 15 16 #include <net/inet_common.h> 16 17 #include <net/inet_hashtables.h> ··· 3730 3729 .stream_memory_free = mptcp_stream_memory_free, 3731 3730 .sockets_allocated = &mptcp_sockets_allocated, 3732 3731 3733 - .memory_allocated = &tcp_memory_allocated, 3732 + .memory_allocated = &net_aligned_data.tcp_memory_allocated, 3734 3733 .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc, 3735 3734 3736 3735 .memory_pressure = &tcp_memory_pressure,