Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'net-three-additions-to-net_hotdata'
Eric Dumazet says:
====================
net: three additions to net_hotdata
This series moves three fast path sysctls to net_hotdata.
To avoid <net/hotdata.h> inclusion from <net/sock.h>,
create <net/proto_memory.h> to hold proto memory definitions.
====================
Link: https://lore.kernel.org/r/20240429134025.1233626-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+117
-105
-2
include/linux/skbuff.h
-2
include/linux/skbuff.h
+3
include/net/hotdata.h
+3
include/net/hotdata.h
+83
include/net/proto_memory.h
+83
include/net/proto_memory.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-or-later */
2
+
#ifndef _PROTO_MEMORY_H
3
+
#define _PROTO_MEMORY_H
4
+
5
+
#include <net/sock.h>
6
+
#include <net/hotdata.h>
7
+
8
+
/* 1 MB per cpu, in page units */
9
+
#define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT))
10
+
11
+
static inline bool sk_has_memory_pressure(const struct sock *sk)
12
+
{
13
+
return sk->sk_prot->memory_pressure != NULL;
14
+
}
15
+
16
+
static inline bool
17
+
proto_memory_pressure(const struct proto *prot)
18
+
{
19
+
if (!prot->memory_pressure)
20
+
return false;
21
+
return !!READ_ONCE(*prot->memory_pressure);
22
+
}
23
+
24
+
static inline bool sk_under_global_memory_pressure(const struct sock *sk)
25
+
{
26
+
return proto_memory_pressure(sk->sk_prot);
27
+
}
28
+
29
+
static inline bool sk_under_memory_pressure(const struct sock *sk)
30
+
{
31
+
if (!sk->sk_prot->memory_pressure)
32
+
return false;
33
+
34
+
if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
35
+
mem_cgroup_under_socket_pressure(sk->sk_memcg))
36
+
return true;
37
+
38
+
return !!READ_ONCE(*sk->sk_prot->memory_pressure);
39
+
}
40
+
41
+
static inline long
42
+
proto_memory_allocated(const struct proto *prot)
43
+
{
44
+
return max(0L, atomic_long_read(prot->memory_allocated));
45
+
}
46
+
47
+
static inline long
48
+
sk_memory_allocated(const struct sock *sk)
49
+
{
50
+
return proto_memory_allocated(sk->sk_prot);
51
+
}
52
+
53
+
static inline void proto_memory_pcpu_drain(struct proto *proto)
54
+
{
55
+
int val = this_cpu_xchg(*proto->per_cpu_fw_alloc, 0);
56
+
57
+
if (val)
58
+
atomic_long_add(val, proto->memory_allocated);
59
+
}
60
+
61
+
static inline void
62
+
sk_memory_allocated_add(const struct sock *sk, int val)
63
+
{
64
+
struct proto *proto = sk->sk_prot;
65
+
66
+
val = this_cpu_add_return(*proto->per_cpu_fw_alloc, val);
67
+
68
+
if (unlikely(val >= READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv)))
69
+
proto_memory_pcpu_drain(proto);
70
+
}
71
+
72
+
static inline void
73
+
sk_memory_allocated_sub(const struct sock *sk, int val)
74
+
{
75
+
struct proto *proto = sk->sk_prot;
76
+
77
+
val = this_cpu_sub_return(*proto->per_cpu_fw_alloc, val);
78
+
79
+
if (unlikely(val <= -READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv)))
80
+
proto_memory_pcpu_drain(proto);
81
+
}
82
+
83
+
#endif /* _PROTO_MEMORY_H */
-78
include/net/sock.h
-78
include/net/sock.h
···
1371
1371
#endif
1372
1372
}
1373
1373
1374
-
static inline bool sk_has_memory_pressure(const struct sock *sk)
1375
-
{
1376
-
return sk->sk_prot->memory_pressure != NULL;
1377
-
}
1378
-
1379
-
static inline bool sk_under_global_memory_pressure(const struct sock *sk)
1380
-
{
1381
-
return sk->sk_prot->memory_pressure &&
1382
-
!!READ_ONCE(*sk->sk_prot->memory_pressure);
1383
-
}
1384
-
1385
-
static inline bool sk_under_memory_pressure(const struct sock *sk)
1386
-
{
1387
-
if (!sk->sk_prot->memory_pressure)
1388
-
return false;
1389
-
1390
-
if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
1391
-
mem_cgroup_under_socket_pressure(sk->sk_memcg))
1392
-
return true;
1393
-
1394
-
return !!READ_ONCE(*sk->sk_prot->memory_pressure);
1395
-
}
1396
-
1397
-
static inline long
1398
-
proto_memory_allocated(const struct proto *prot)
1399
-
{
1400
-
return max(0L, atomic_long_read(prot->memory_allocated));
1401
-
}
1402
-
1403
-
static inline long
1404
-
sk_memory_allocated(const struct sock *sk)
1405
-
{
1406
-
return proto_memory_allocated(sk->sk_prot);
1407
-
}
1408
-
1409
-
/* 1 MB per cpu, in page units */
1410
-
#define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT))
1411
-
extern int sysctl_mem_pcpu_rsv;
1412
-
1413
-
static inline void proto_memory_pcpu_drain(struct proto *proto)
1414
-
{
1415
-
int val = this_cpu_xchg(*proto->per_cpu_fw_alloc, 0);
1416
-
1417
-
if (val)
1418
-
atomic_long_add(val, proto->memory_allocated);
1419
-
}
1420
-
1421
-
static inline void
1422
-
sk_memory_allocated_add(const struct sock *sk, int val)
1423
-
{
1424
-
struct proto *proto = sk->sk_prot;
1425
-
1426
-
val = this_cpu_add_return(*proto->per_cpu_fw_alloc, val);
1427
-
1428
-
if (unlikely(val >= READ_ONCE(sysctl_mem_pcpu_rsv)))
1429
-
proto_memory_pcpu_drain(proto);
1430
-
}
1431
-
1432
-
static inline void
1433
-
sk_memory_allocated_sub(const struct sock *sk, int val)
1434
-
{
1435
-
struct proto *proto = sk->sk_prot;
1436
-
1437
-
val = this_cpu_sub_return(*proto->per_cpu_fw_alloc, val);
1438
-
1439
-
if (unlikely(val <= -READ_ONCE(sysctl_mem_pcpu_rsv)))
1440
-
proto_memory_pcpu_drain(proto);
1441
-
}
1442
-
1443
1374
#define SK_ALLOC_PERCPU_COUNTER_BATCH 16
1444
1375
1445
1376
static inline void sk_sockets_allocated_dec(struct sock *sk)
···
1396
1465
{
1397
1466
return percpu_counter_sum_positive(prot->sockets_allocated);
1398
1467
}
1399
-
1400
-
static inline bool
1401
-
proto_memory_pressure(struct proto *prot)
1402
-
{
1403
-
if (!prot->memory_pressure)
1404
-
return false;
1405
-
return !!READ_ONCE(*prot->memory_pressure);
1406
-
}
1407
-
1408
1468
1409
1469
#ifdef CONFIG_PROC_FS
1410
1470
#define PROTO_INUSE_NR 64 /* should be enough for the first time */
+1
-9
include/net/tcp.h
+1
-9
include/net/tcp.h
···
296
296
return seq3 - seq2 >= seq1 - seq2;
297
297
}
298
298
299
-
static inline bool tcp_out_of_memory(struct sock *sk)
300
-
{
301
-
if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
302
-
sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))
303
-
return true;
304
-
return false;
305
-
}
306
-
307
299
static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
308
300
{
309
301
sk_wmem_queued_add(sk, -skb->truesize);
···
308
316
309
317
void sk_forced_mem_schedule(struct sock *sk, int size);
310
318
311
-
bool tcp_check_oom(struct sock *sk, int shift);
319
+
bool tcp_check_oom(const struct sock *sk, int shift);
312
320
313
321
314
322
extern struct proto tcp_prot;
-1
net/core/dev.c
-1
net/core/dev.c
···
4450
4450
*************************************************************************/
4451
4451
static DEFINE_PER_CPU(struct task_struct *, backlog_napi);
4452
4452
4453
-
unsigned int sysctl_skb_defer_max __read_mostly = 64;
4454
4453
int weight_p __read_mostly = 64; /* old backlog weight */
4455
4454
int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
4456
4455
int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
-1
net/core/dev.h
-1
net/core/dev.h
+5
-2
net/core/hotdata.c
+5
-2
net/core/hotdata.c
···
1
1
// SPDX-License-Identifier: GPL-2.0-or-later
2
-
#include <net/hotdata.h>
3
2
#include <linux/cache.h>
4
3
#include <linux/jiffies.h>
5
4
#include <linux/list.h>
6
-
5
+
#include <net/hotdata.h>
6
+
#include <net/proto_memory.h>
7
7
8
8
struct net_hotdata net_hotdata __cacheline_aligned = {
9
9
.offload_base = LIST_HEAD_INIT(net_hotdata.offload_base),
···
18
18
.max_backlog = 1000,
19
19
.dev_tx_weight = 64,
20
20
.dev_rx_weight = 64,
21
+
.sysctl_max_skb_frags = MAX_SKB_FRAGS,
22
+
.sysctl_skb_defer_max = 64,
23
+
.sysctl_mem_pcpu_rsv = SK_MEMORY_PCPU_RESERVE
21
24
};
22
25
EXPORT_SYMBOL(net_hotdata);
+2
-5
net/core/skbuff.c
+2
-5
net/core/skbuff.c
···
109
109
#define SKB_SMALL_HEAD_HEADROOM \
110
110
SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE)
111
111
112
-
int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
113
-
EXPORT_SYMBOL(sysctl_max_skb_frags);
114
-
115
112
/* kcm_write_msgs() relies on casting paged frags to bio_vec to use
116
113
* iov_iter_bvec(). These static asserts ensure the cast is valid is long as the
117
114
* netmem is a page.
···
6985
6988
DEBUG_NET_WARN_ON_ONCE(skb->destructor);
6986
6989
6987
6990
sd = &per_cpu(softnet_data, cpu);
6988
-
defer_max = READ_ONCE(sysctl_skb_defer_max);
6991
+
defer_max = READ_ONCE(net_hotdata.sysctl_skb_defer_max);
6989
6992
if (READ_ONCE(sd->defer_count) >= defer_max)
6990
6993
goto nodefer;
6991
6994
···
7037
7040
ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter,
7038
7041
ssize_t maxsize, gfp_t gfp)
7039
7042
{
7040
-
size_t frag_limit = READ_ONCE(sysctl_max_skb_frags);
7043
+
size_t frag_limit = READ_ONCE(net_hotdata.sysctl_max_skb_frags);
7041
7044
struct page *pages[8], **ppages = pages;
7042
7045
ssize_t spliced = 0, ret = 0;
7043
7046
unsigned int i;
+1
-1
net/core/sock.c
+1
-1
net/core/sock.c
···
127
127
#include <net/net_namespace.h>
128
128
#include <net/request_sock.h>
129
129
#include <net/sock.h>
130
+
#include <net/proto_memory.h>
130
131
#include <linux/net_tstamp.h>
131
132
#include <net/xfrm.h>
132
133
#include <linux/ipsec.h>
···
284
283
EXPORT_SYMBOL(sysctl_rmem_max);
285
284
__u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
286
285
__u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
287
-
int sysctl_mem_pcpu_rsv __read_mostly = SK_MEMORY_PCPU_RESERVE;
288
286
289
287
int sysctl_tstamp_allow_data __read_mostly = 1;
290
288
+4
-3
net/core/sysctl_net_core.c
+4
-3
net/core/sysctl_net_core.c
···
24
24
#include <net/busy_poll.h>
25
25
#include <net/pkt_sched.h>
26
26
#include <net/hotdata.h>
27
+
#include <net/proto_memory.h>
27
28
#include <net/rps.h>
28
29
29
30
#include "dev.h"
···
416
415
},
417
416
{
418
417
.procname = "mem_pcpu_rsv",
419
-
.data = &sysctl_mem_pcpu_rsv,
418
+
.data = &net_hotdata.sysctl_mem_pcpu_rsv,
420
419
.maxlen = sizeof(int),
421
420
.mode = 0644,
422
421
.proc_handler = proc_dointvec_minmax,
···
596
595
},
597
596
{
598
597
.procname = "max_skb_frags",
599
-
.data = &sysctl_max_skb_frags,
598
+
.data = &net_hotdata.sysctl_max_skb_frags,
600
599
.maxlen = sizeof(int),
601
600
.mode = 0644,
602
601
.proc_handler = proc_dointvec_minmax,
···
655
654
},
656
655
{
657
656
.procname = "skb_defer_max",
658
-
.data = &sysctl_skb_defer_max,
657
+
.data = &net_hotdata.sysctl_skb_defer_max,
659
658
.maxlen = sizeof(unsigned int),
660
659
.mode = 0644,
661
660
.proc_handler = proc_dointvec_minmax,
+1
net/ipv4/proc.c
+1
net/ipv4/proc.c
+12
-2
net/ipv4/tcp.c
+12
-2
net/ipv4/tcp.c
···
272
272
#include <net/inet_common.h>
273
273
#include <net/tcp.h>
274
274
#include <net/mptcp.h>
275
+
#include <net/proto_memory.h>
275
276
#include <net/xfrm.h>
276
277
#include <net/ip.h>
277
278
#include <net/sock.h>
···
281
280
#include <linux/uaccess.h>
282
281
#include <asm/ioctls.h>
283
282
#include <net/busy_poll.h>
283
+
#include <net/hotdata.h>
284
284
#include <net/rps.h>
285
285
286
286
/* Track pending CMSGs. */
···
1190
1188
1191
1189
if (!skb_can_coalesce(skb, i, pfrag->page,
1192
1190
pfrag->offset)) {
1193
-
if (i >= READ_ONCE(sysctl_max_skb_frags)) {
1191
+
if (i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) {
1194
1192
tcp_mark_push(tp, skb);
1195
1193
goto new_segment;
1196
1194
}
···
2753
2751
READ_ONCE(sysctl_tcp_max_orphans);
2754
2752
}
2755
2753
2756
-
bool tcp_check_oom(struct sock *sk, int shift)
2754
+
static bool tcp_out_of_memory(const struct sock *sk)
2755
+
{
2756
+
if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
2757
+
sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))
2758
+
return true;
2759
+
return false;
2760
+
}
2761
+
2762
+
bool tcp_check_oom(const struct sock *sk, int shift)
2757
2763
{
2758
2764
bool too_many_orphans, out_of_socket_memory;
2759
2765
+1
net/ipv4/tcp_input.c
+1
net/ipv4/tcp_input.c
+1
net/ipv4/tcp_output.c
+1
net/ipv4/tcp_output.c
+2
-1
net/mptcp/protocol.c
+2
-1
net/mptcp/protocol.c
···
20
20
#include <net/transp_v6.h>
21
21
#endif
22
22
#include <net/mptcp.h>
23
+
#include <net/hotdata.h>
23
24
#include <net/xfrm.h>
24
25
#include <asm/ioctls.h>
25
26
#include "protocol.h"
···
1273
1272
1274
1273
i = skb_shinfo(skb)->nr_frags;
1275
1274
can_coalesce = skb_can_coalesce(skb, i, dfrag->page, offset);
1276
-
if (!can_coalesce && i >= READ_ONCE(sysctl_max_skb_frags)) {
1275
+
if (!can_coalesce && i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) {
1277
1276
tcp_mark_push(tcp_sk(ssk), skb);
1278
1277
goto alloc_skb;
1279
1278
}