Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'rtnetlink-per-netns-rtnl'
Kuniyuki Iwashima says:
====================
rtnetlink: Per-netns RTNL.
rtnl_lock() is a "Big Kernel Lock" in the networking slow path and
serialised all rtnetlink requests until 4.13.
Since RTNL_FLAG_DOIT_UNLOCKED and RTNL_FLAG_DUMP_UNLOCKED have been
introduced in 4.14 and 6.9, respectively, rtnetlink message handlers
are ready to be converted to RTNL-less/free.
15 out of 44 dumpit()s have been converted to RCU so far, and the
progress is pretty good. We can now dump various major network
resources without RTNL.
12 out of 87 doit()s have been converted, but most of the converted
doit()s are also on the reader side of RTNL; their message types are
RTM_GET*.
So, most of RTM_(NEW|DEL|SET)* operations are still serialised by RTNL.
For example, one of our services creates 2K netns and a small number
of network interfaces in each netns that require too many writer-side
rtnetlink requests, and setting up a single host takes 10+ minutes.
RTNL is still a huge pain for network configuration paths, and we need
more granular locking, given converting all doit()s would be unfeasible.
Actually, most RTNL users do not need to freeze multiple netns, and such
users can be protected by per-netns RTNL mutex. The exceptions would be
RTM_NEWLINK, RTM_DELLINK, and RTM_SETLINK. (See [0] and [1])
This series is the first step of the per-netns RTNL conversion that
gradually replaces rtnl_lock() with rtnl_net_lock(net) under
CONFIG_DEBUG_NET_SMALL_RTNL.
[0]: https://netdev.bots.linux.dev/netconf/2024/index.html
[1]: https://lpc.events/event/18/contributions/1959/
====================
Link: https://patch.msgid.link/20241004221031.77743-1-kuniyu@amazon.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
+289
-7
+62
-7
include/linux/rtnetlink.h
+62
-7
include/linux/rtnetlink.h
···
7
7
#include <linux/netdevice.h>
8
8
#include <linux/wait.h>
9
9
#include <linux/refcount.h>
10
-
#include <linux/cleanup.h>
11
10
#include <uapi/linux/rtnetlink.h>
12
11
13
12
extern int rtnetlink_send(struct sk_buff *skb, struct net *net, u32 pid, u32 group, int echo);
···
46
47
extern int rtnl_lock_killable(void);
47
48
extern bool refcount_dec_and_rtnl_lock(refcount_t *r);
48
49
49
-
DEFINE_LOCK_GUARD_0(rtnl, rtnl_lock(), rtnl_unlock())
50
-
51
50
extern wait_queue_head_t netdev_unregistering_wq;
52
51
extern atomic_t dev_unreg_count;
53
52
extern struct rw_semaphore pernet_ops_rwsem;
54
53
extern struct rw_semaphore net_rwsem;
54
+
55
+
#define ASSERT_RTNL() \
56
+
WARN_ONCE(!rtnl_is_locked(), \
57
+
"RTNL: assertion failed at %s (%d)\n", __FILE__, __LINE__)
55
58
56
59
#ifdef CONFIG_PROVE_LOCKING
57
60
extern bool lockdep_rtnl_is_held(void);
···
96
95
#define rcu_replace_pointer_rtnl(rp, p) \
97
96
rcu_replace_pointer(rp, p, lockdep_rtnl_is_held())
98
97
98
+
#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
99
+
void __rtnl_net_lock(struct net *net);
100
+
void __rtnl_net_unlock(struct net *net);
101
+
void rtnl_net_lock(struct net *net);
102
+
void rtnl_net_unlock(struct net *net);
103
+
int rtnl_net_lock_cmp_fn(const struct lockdep_map *a, const struct lockdep_map *b);
104
+
105
+
bool rtnl_net_is_locked(struct net *net);
106
+
107
+
#define ASSERT_RTNL_NET(net) \
108
+
WARN_ONCE(!rtnl_net_is_locked(net), \
109
+
"RTNL_NET: assertion failed at %s (%d)\n", \
110
+
__FILE__, __LINE__)
111
+
112
+
bool lockdep_rtnl_net_is_held(struct net *net);
113
+
114
+
#define rcu_dereference_rtnl_net(net, p) \
115
+
rcu_dereference_check(p, lockdep_rtnl_net_is_held(net))
116
+
#define rtnl_net_dereference(net, p) \
117
+
rcu_dereference_protected(p, lockdep_rtnl_net_is_held(net))
118
+
#define rcu_replace_pointer_rtnl_net(net, rp, p) \
119
+
rcu_replace_pointer(rp, p, lockdep_rtnl_net_is_held(net))
120
+
#else
121
+
static inline void __rtnl_net_lock(struct net *net) {}
122
+
static inline void __rtnl_net_unlock(struct net *net) {}
123
+
124
+
static inline void rtnl_net_lock(struct net *net)
125
+
{
126
+
rtnl_lock();
127
+
}
128
+
129
+
static inline void rtnl_net_unlock(struct net *net)
130
+
{
131
+
rtnl_unlock();
132
+
}
133
+
134
+
static inline void ASSERT_RTNL_NET(struct net *net)
135
+
{
136
+
ASSERT_RTNL();
137
+
}
138
+
139
+
static inline void *rcu_dereference_rtnl_net(struct net *net, void *p)
140
+
{
141
+
return rcu_dereference_rtnl(p);
142
+
}
143
+
144
+
static inline void *rtnl_net_dereference(struct net *net, void *p)
145
+
{
146
+
return rtnl_dereference(p);
147
+
}
148
+
149
+
static inline void *rcu_replace_pointer_rtnl_net(struct net *net,
150
+
void *rp, void *p)
151
+
{
152
+
return rcu_replace_pointer_rtnl(rp, p);
153
+
}
154
+
#endif
155
+
99
156
static inline struct netdev_queue *dev_ingress_queue(struct net_device *dev)
100
157
{
101
158
return rtnl_dereference(dev->ingress_queue);
···
180
121
void rtnetlink_init(void);
181
122
void __rtnl_unlock(void);
182
123
void rtnl_kfree_skbs(struct sk_buff *head, struct sk_buff *tail);
183
-
184
-
#define ASSERT_RTNL() \
185
-
WARN_ONCE(!rtnl_is_locked(), \
186
-
"RTNL: assertion failed at %s (%d)\n", __FILE__, __LINE__)
187
124
188
125
extern int ndo_dflt_fdb_dump(struct sk_buff *skb,
189
126
struct netlink_callback *cb,
+4
include/net/net_namespace.h
+4
include/net/net_namespace.h
···
188
188
#if IS_ENABLED(CONFIG_SMC)
189
189
struct netns_smc smc;
190
190
#endif
191
+
#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
192
+
/* Move to a better place when the config guard is removed. */
193
+
struct mutex rtnl_mutex;
194
+
#endif
191
195
} __randomize_layout;
192
196
193
197
#include <linux/seq_file_net.h>
+15
net/Kconfig.debug
+15
net/Kconfig.debug
···
24
24
help
25
25
Enable extra sanity checks in networking.
26
26
This is mostly used by fuzzers, but is safe to select.
27
+
28
+
config DEBUG_NET_SMALL_RTNL
29
+
bool "Add extra per-netns mutex inside RTNL"
30
+
depends on DEBUG_KERNEL && NET && LOCK_DEBUGGING_SUPPORT
31
+
select PROVE_LOCKING
32
+
default n
33
+
help
34
+
rtnl_lock() is being replaced with rtnl_net_lock() that
35
+
acquires the global RTNL and a small per-netns RTNL mutex.
36
+
37
+
During the conversion, rtnl_net_lock() just adds an extra
38
+
mutex in every RTNL scope and slows down the operations.
39
+
40
+
Once the conversion completes, rtnl_lock() will be removed
41
+
and rtnetlink will gain per-netns scalability.
+1
net/core/Makefile
+1
net/core/Makefile
+6
net/core/net_namespace.c
+6
net/core/net_namespace.c
···
334
334
idr_init(&net->netns_ids);
335
335
spin_lock_init(&net->nsid_lock);
336
336
mutex_init(&net->ipv4.ra_mutex);
337
+
338
+
#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
339
+
mutex_init(&net->rtnl_mutex);
340
+
lock_set_cmp_fn(&net->rtnl_mutex, rtnl_net_lock_cmp_fn, NULL);
341
+
#endif
342
+
337
343
preinit_net_sysctl(net);
338
344
}
339
345
+70
net/core/rtnetlink.c
+70
net/core/rtnetlink.c
···
179
179
EXPORT_SYMBOL(lockdep_rtnl_is_held);
180
180
#endif /* #ifdef CONFIG_PROVE_LOCKING */
181
181
182
+
#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
183
+
void __rtnl_net_lock(struct net *net)
184
+
{
185
+
ASSERT_RTNL();
186
+
187
+
mutex_lock(&net->rtnl_mutex);
188
+
}
189
+
EXPORT_SYMBOL(__rtnl_net_lock);
190
+
191
+
void __rtnl_net_unlock(struct net *net)
192
+
{
193
+
ASSERT_RTNL();
194
+
195
+
mutex_unlock(&net->rtnl_mutex);
196
+
}
197
+
EXPORT_SYMBOL(__rtnl_net_unlock);
198
+
199
+
void rtnl_net_lock(struct net *net)
200
+
{
201
+
rtnl_lock();
202
+
__rtnl_net_lock(net);
203
+
}
204
+
EXPORT_SYMBOL(rtnl_net_lock);
205
+
206
+
void rtnl_net_unlock(struct net *net)
207
+
{
208
+
__rtnl_net_unlock(net);
209
+
rtnl_unlock();
210
+
}
211
+
EXPORT_SYMBOL(rtnl_net_unlock);
212
+
213
+
static int rtnl_net_cmp_locks(const struct net *net_a, const struct net *net_b)
214
+
{
215
+
if (net_eq(net_a, net_b))
216
+
return 0;
217
+
218
+
/* always init_net first */
219
+
if (net_eq(net_a, &init_net))
220
+
return -1;
221
+
222
+
if (net_eq(net_b, &init_net))
223
+
return 1;
224
+
225
+
/* otherwise lock in ascending order */
226
+
return net_a < net_b ? -1 : 1;
227
+
}
228
+
229
+
int rtnl_net_lock_cmp_fn(const struct lockdep_map *a, const struct lockdep_map *b)
230
+
{
231
+
const struct net *net_a, *net_b;
232
+
233
+
net_a = container_of(a, struct net, rtnl_mutex.dep_map);
234
+
net_b = container_of(b, struct net, rtnl_mutex.dep_map);
235
+
236
+
return rtnl_net_cmp_locks(net_a, net_b);
237
+
}
238
+
239
+
bool rtnl_net_is_locked(struct net *net)
240
+
{
241
+
return rtnl_is_locked() && mutex_is_locked(&net->rtnl_mutex);
242
+
}
243
+
EXPORT_SYMBOL(rtnl_net_is_locked);
244
+
245
+
bool lockdep_rtnl_net_is_held(struct net *net)
246
+
{
247
+
return lockdep_rtnl_is_held() && lockdep_is_held(&net->rtnl_mutex);
248
+
}
249
+
EXPORT_SYMBOL(lockdep_rtnl_net_is_held);
250
+
#endif
251
+
182
252
static struct rtnl_link __rcu *__rcu *rtnl_msg_handlers[RTNL_FAMILY_MAX + 1];
183
253
184
254
static inline int rtm_msgindex(int msgtype)
+131
net/core/rtnl_net_debug.c
+131
net/core/rtnl_net_debug.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-or-later
2
+
/* Copyright Amazon.com Inc. or its affiliates. */
3
+
4
+
#include <linux/init.h>
5
+
#include <linux/netdevice.h>
6
+
#include <linux/notifier.h>
7
+
#include <linux/rtnetlink.h>
8
+
#include <net/net_namespace.h>
9
+
#include <net/netns/generic.h>
10
+
11
+
static int rtnl_net_debug_event(struct notifier_block *nb,
12
+
unsigned long event, void *ptr)
13
+
{
14
+
struct net_device *dev = netdev_notifier_info_to_dev(ptr);
15
+
struct net *net = dev_net(dev);
16
+
enum netdev_cmd cmd = event;
17
+
18
+
/* Keep enum and don't add default to trigger -Werror=switch */
19
+
switch (cmd) {
20
+
case NETDEV_UP:
21
+
case NETDEV_DOWN:
22
+
case NETDEV_REBOOT:
23
+
case NETDEV_CHANGE:
24
+
case NETDEV_REGISTER:
25
+
case NETDEV_UNREGISTER:
26
+
case NETDEV_CHANGEMTU:
27
+
case NETDEV_CHANGEADDR:
28
+
case NETDEV_PRE_CHANGEADDR:
29
+
case NETDEV_GOING_DOWN:
30
+
case NETDEV_CHANGENAME:
31
+
case NETDEV_FEAT_CHANGE:
32
+
case NETDEV_BONDING_FAILOVER:
33
+
case NETDEV_PRE_UP:
34
+
case NETDEV_PRE_TYPE_CHANGE:
35
+
case NETDEV_POST_TYPE_CHANGE:
36
+
case NETDEV_POST_INIT:
37
+
case NETDEV_PRE_UNINIT:
38
+
case NETDEV_RELEASE:
39
+
case NETDEV_NOTIFY_PEERS:
40
+
case NETDEV_JOIN:
41
+
case NETDEV_CHANGEUPPER:
42
+
case NETDEV_RESEND_IGMP:
43
+
case NETDEV_PRECHANGEMTU:
44
+
case NETDEV_CHANGEINFODATA:
45
+
case NETDEV_BONDING_INFO:
46
+
case NETDEV_PRECHANGEUPPER:
47
+
case NETDEV_CHANGELOWERSTATE:
48
+
case NETDEV_UDP_TUNNEL_PUSH_INFO:
49
+
case NETDEV_UDP_TUNNEL_DROP_INFO:
50
+
case NETDEV_CHANGE_TX_QUEUE_LEN:
51
+
case NETDEV_CVLAN_FILTER_PUSH_INFO:
52
+
case NETDEV_CVLAN_FILTER_DROP_INFO:
53
+
case NETDEV_SVLAN_FILTER_PUSH_INFO:
54
+
case NETDEV_SVLAN_FILTER_DROP_INFO:
55
+
case NETDEV_OFFLOAD_XSTATS_ENABLE:
56
+
case NETDEV_OFFLOAD_XSTATS_DISABLE:
57
+
case NETDEV_OFFLOAD_XSTATS_REPORT_USED:
58
+
case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA:
59
+
case NETDEV_XDP_FEAT_CHANGE:
60
+
ASSERT_RTNL();
61
+
break;
62
+
63
+
/* Once an event fully supports RTNL_NET, move it here
64
+
* and remove "if (0)" below.
65
+
*
66
+
* case NETDEV_XXX:
67
+
* ASSERT_RTNL_NET(net);
68
+
* break;
69
+
*/
70
+
}
71
+
72
+
/* Just to avoid unused-variable error for dev and net. */
73
+
if (0)
74
+
ASSERT_RTNL_NET(net);
75
+
76
+
return NOTIFY_DONE;
77
+
}
78
+
79
+
static int rtnl_net_debug_net_id;
80
+
81
+
static int __net_init rtnl_net_debug_net_init(struct net *net)
82
+
{
83
+
struct notifier_block *nb;
84
+
85
+
nb = net_generic(net, rtnl_net_debug_net_id);
86
+
nb->notifier_call = rtnl_net_debug_event;
87
+
88
+
return register_netdevice_notifier_net(net, nb);
89
+
}
90
+
91
+
static void __net_exit rtnl_net_debug_net_exit(struct net *net)
92
+
{
93
+
struct notifier_block *nb;
94
+
95
+
nb = net_generic(net, rtnl_net_debug_net_id);
96
+
unregister_netdevice_notifier_net(net, nb);
97
+
}
98
+
99
+
static struct pernet_operations rtnl_net_debug_net_ops __net_initdata = {
100
+
.init = rtnl_net_debug_net_init,
101
+
.exit = rtnl_net_debug_net_exit,
102
+
.id = &rtnl_net_debug_net_id,
103
+
.size = sizeof(struct notifier_block),
104
+
};
105
+
106
+
static struct notifier_block rtnl_net_debug_block = {
107
+
.notifier_call = rtnl_net_debug_event,
108
+
};
109
+
110
+
static int __init rtnl_net_debug_init(void)
111
+
{
112
+
int ret;
113
+
114
+
ret = register_pernet_device(&rtnl_net_debug_net_ops);
115
+
if (ret)
116
+
return ret;
117
+
118
+
ret = register_netdevice_notifier(&rtnl_net_debug_block);
119
+
if (ret)
120
+
unregister_pernet_subsys(&rtnl_net_debug_net_ops);
121
+
122
+
return ret;
123
+
}
124
+
125
+
static void __exit rtnl_net_debug_exit(void)
126
+
{
127
+
unregister_netdevice_notifier(&rtnl_net_debug_block);
128
+
unregister_pernet_device(&rtnl_net_debug_net_ops);
129
+
}
130
+
131
+
subsys_initcall(rtnl_net_debug_init);