Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'l2tp-sk_user_data'
James Chapman says:
====================
l2tp: don't use the tunnel socket's sk_user_data in datapath
This series refactors l2tp to not use the tunnel socket's sk_user_data
in the datapath. The main reasons for doing this are
* to allow for simplifying internal socket cleanup code (to be done
in a later series)
* to support multiple L2TPv3 UDP tunnels using the same 5-tuple
address
When handling received UDP frames, l2tp's current approach is to look
up a session in a per-tunnel list. l2tp uses the tunnel socket's
sk_user_data to derive the tunnel context from the receiving socket.
But this results in the socket and tunnel lifetimes being very tightly
coupled and the tunnel/socket cleanup paths being complicated. The
latter has historically been a source of l2tp bugs and makes the code
more difficult to maintain. Also, if sockets are aliased, we can't
trust that the socket's sk_user_data references the right tunnel
anyway. Hence the desire to not use sk_user_data in the datapath.
The new approach is to lookup sessions in a per-net session list
without first deriving the tunnel:
* For L2TPv2, the l2tp header has separate tunnel ID and session ID
fields which can be trivially combined to make a unique 32-bit key
for per-net session lookup.
* For L2TPv3, there is no tunnel ID in the packet header, only a
session ID, which should be unique over all tunnels so can be used
as a key for per-net session lookup. However, this only works when
the L2TPv3 session ID really is unique over all tunnels. At least
one L2TPv3 application is known to use the same session ID in
different L2TPv3 UDP tunnels, relying on UDP address/ports to
distinguish them. This worked previously because sessions in UDP
tunnels were kept in a per-tunnel list. To retain support for this,
L2TPv3 session ID collisions are managed using a separate per-net
session hlist, keyed by ID and sk. When looking up a session by ID,
if there's more than one match, sk is used to find the right one.
L2TPv3 sessions in IP-encap tunnels are already looked up by session
ID in a per-net list. This work has UDP sessions also use the per-net
session list, while allowing for session ID collisions. The existing
per-tunnel hlist becomes a plain list since it is used only in
management and cleanup paths to walk a list of sessions in a given
tunnel.
For better performance, the per-net session lists use IDR. Separate
IDRs are used for L2TPv2 and L2TPv3 sessions to avoid potential key
collisions.
These changes pass l2tp regression tests and improve data forwarding
performance by about 10% in some of my test setups.
====================
Signed-off-by: David S. Miller <davem@davemloft.net>
+323
-280
+287
-244
net/l2tp/l2tp_core.c
+287
-244
net/l2tp/l2tp_core.c
···
39
39
#include <linux/ip.h>
40
40
#include <linux/udp.h>
41
41
#include <linux/l2tp.h>
42
-
#include <linux/hash.h>
43
42
#include <linux/sort.h>
44
43
#include <linux/file.h>
45
44
#include <linux/nsproxy.h>
···
106
107
/* Lock for write access to l2tp_tunnel_idr */
107
108
spinlock_t l2tp_tunnel_idr_lock;
108
109
struct idr l2tp_tunnel_idr;
109
-
struct hlist_head l2tp_session_hlist[L2TP_HASH_SIZE_2];
110
-
/* Lock for write access to l2tp_session_hlist */
111
-
spinlock_t l2tp_session_hlist_lock;
110
+
/* Lock for write access to l2tp_v[23]_session_idr/htable */
111
+
spinlock_t l2tp_session_idr_lock;
112
+
struct idr l2tp_v2_session_idr;
113
+
struct idr l2tp_v3_session_idr;
114
+
struct hlist_head l2tp_v3_session_htable[16];
112
115
};
116
+
117
+
static inline u32 l2tp_v2_session_key(u16 tunnel_id, u16 session_id)
118
+
{
119
+
return ((u32)tunnel_id) << 16 | session_id;
120
+
}
121
+
122
+
static inline unsigned long l2tp_v3_session_hashkey(struct sock *sk, u32 session_id)
123
+
{
124
+
return ((unsigned long)sk) + session_id;
125
+
}
113
126
114
127
#if IS_ENABLED(CONFIG_IPV6)
115
128
static bool l2tp_sk_is_v6(struct sock *sk)
···
134
123
static inline struct l2tp_net *l2tp_pernet(const struct net *net)
135
124
{
136
125
return net_generic(net, l2tp_net_id);
137
-
}
138
-
139
-
/* Session hash global list for L2TPv3.
140
-
* The session_id SHOULD be random according to RFC3931, but several
141
-
* L2TP implementations use incrementing session_ids. So we do a real
142
-
* hash on the session_id, rather than a simple bitmask.
143
-
*/
144
-
static inline struct hlist_head *
145
-
l2tp_session_id_hash_2(struct l2tp_net *pn, u32 session_id)
146
-
{
147
-
return &pn->l2tp_session_hlist[hash_32(session_id, L2TP_HASH_BITS_2)];
148
-
}
149
-
150
-
/* Session hash list.
151
-
* The session_id SHOULD be random according to RFC2661, but several
152
-
* L2TP implementations (Cisco and Microsoft) use incrementing
153
-
* session_ids. So we do a real hash on the session_id, rather than a
154
-
* simple bitmask.
155
-
*/
156
-
static inline struct hlist_head *
157
-
l2tp_session_id_hash(struct l2tp_tunnel *tunnel, u32 session_id)
158
-
{
159
-
return &tunnel->session_hlist[hash_32(session_id, L2TP_HASH_BITS)];
160
126
}
161
127
162
128
static void l2tp_tunnel_free(struct l2tp_tunnel *tunnel)
···
228
240
}
229
241
EXPORT_SYMBOL_GPL(l2tp_tunnel_get_nth);
230
242
231
-
struct l2tp_session *l2tp_tunnel_get_session(struct l2tp_tunnel *tunnel,
232
-
u32 session_id)
243
+
struct l2tp_session *l2tp_v3_session_get(const struct net *net, struct sock *sk, u32 session_id)
233
244
{
234
-
struct hlist_head *session_list;
245
+
const struct l2tp_net *pn = l2tp_pernet(net);
235
246
struct l2tp_session *session;
236
247
237
-
session_list = l2tp_session_id_hash(tunnel, session_id);
238
-
239
248
rcu_read_lock_bh();
240
-
hlist_for_each_entry_rcu(session, session_list, hlist)
241
-
if (session->session_id == session_id) {
242
-
l2tp_session_inc_refcount(session);
243
-
rcu_read_unlock_bh();
249
+
session = idr_find(&pn->l2tp_v3_session_idr, session_id);
250
+
if (session && !hash_hashed(&session->hlist) &&
251
+
refcount_inc_not_zero(&session->ref_count)) {
252
+
rcu_read_unlock_bh();
253
+
return session;
254
+
}
244
255
245
-
return session;
246
-
}
247
-
rcu_read_unlock_bh();
256
+
/* If we get here and session is non-NULL, the session_id
257
+
* collides with one in another tunnel. If sk is non-NULL,
258
+
* find the session matching sk.
259
+
*/
260
+
if (session && sk) {
261
+
unsigned long key = l2tp_v3_session_hashkey(sk, session->session_id);
248
262
249
-
return NULL;
250
-
}
251
-
EXPORT_SYMBOL_GPL(l2tp_tunnel_get_session);
252
-
253
-
struct l2tp_session *l2tp_session_get(const struct net *net, u32 session_id)
254
-
{
255
-
struct hlist_head *session_list;
256
-
struct l2tp_session *session;
257
-
258
-
session_list = l2tp_session_id_hash_2(l2tp_pernet(net), session_id);
259
-
260
-
rcu_read_lock_bh();
261
-
hlist_for_each_entry_rcu(session, session_list, global_hlist)
262
-
if (session->session_id == session_id) {
263
-
l2tp_session_inc_refcount(session);
264
-
rcu_read_unlock_bh();
265
-
266
-
return session;
267
-
}
268
-
rcu_read_unlock_bh();
269
-
270
-
return NULL;
271
-
}
272
-
EXPORT_SYMBOL_GPL(l2tp_session_get);
273
-
274
-
struct l2tp_session *l2tp_session_get_nth(struct l2tp_tunnel *tunnel, int nth)
275
-
{
276
-
int hash;
277
-
struct l2tp_session *session;
278
-
int count = 0;
279
-
280
-
rcu_read_lock_bh();
281
-
for (hash = 0; hash < L2TP_HASH_SIZE; hash++) {
282
-
hlist_for_each_entry_rcu(session, &tunnel->session_hlist[hash], hlist) {
283
-
if (++count > nth) {
284
-
l2tp_session_inc_refcount(session);
263
+
hash_for_each_possible_rcu(pn->l2tp_v3_session_htable, session,
264
+
hlist, key) {
265
+
if (session->tunnel->sock == sk &&
266
+
refcount_inc_not_zero(&session->ref_count)) {
285
267
rcu_read_unlock_bh();
286
268
return session;
287
269
}
288
270
}
289
271
}
272
+
rcu_read_unlock_bh();
290
273
274
+
return NULL;
275
+
}
276
+
EXPORT_SYMBOL_GPL(l2tp_v3_session_get);
277
+
278
+
struct l2tp_session *l2tp_v2_session_get(const struct net *net, u16 tunnel_id, u16 session_id)
279
+
{
280
+
u32 session_key = l2tp_v2_session_key(tunnel_id, session_id);
281
+
const struct l2tp_net *pn = l2tp_pernet(net);
282
+
struct l2tp_session *session;
283
+
284
+
rcu_read_lock_bh();
285
+
session = idr_find(&pn->l2tp_v2_session_idr, session_key);
286
+
if (session && refcount_inc_not_zero(&session->ref_count)) {
287
+
rcu_read_unlock_bh();
288
+
return session;
289
+
}
290
+
rcu_read_unlock_bh();
291
+
292
+
return NULL;
293
+
}
294
+
EXPORT_SYMBOL_GPL(l2tp_v2_session_get);
295
+
296
+
struct l2tp_session *l2tp_session_get(const struct net *net, struct sock *sk, int pver,
297
+
u32 tunnel_id, u32 session_id)
298
+
{
299
+
if (pver == L2TP_HDR_VER_2)
300
+
return l2tp_v2_session_get(net, tunnel_id, session_id);
301
+
else
302
+
return l2tp_v3_session_get(net, sk, session_id);
303
+
}
304
+
EXPORT_SYMBOL_GPL(l2tp_session_get);
305
+
306
+
struct l2tp_session *l2tp_session_get_nth(struct l2tp_tunnel *tunnel, int nth)
307
+
{
308
+
struct l2tp_session *session;
309
+
int count = 0;
310
+
311
+
rcu_read_lock_bh();
312
+
list_for_each_entry_rcu(session, &tunnel->session_list, list) {
313
+
if (++count > nth) {
314
+
l2tp_session_inc_refcount(session);
315
+
rcu_read_unlock_bh();
316
+
return session;
317
+
}
318
+
}
291
319
rcu_read_unlock_bh();
292
320
293
321
return NULL;
···
317
313
const char *ifname)
318
314
{
319
315
struct l2tp_net *pn = l2tp_pernet(net);
320
-
int hash;
316
+
unsigned long tunnel_id, tmp;
321
317
struct l2tp_session *session;
318
+
struct l2tp_tunnel *tunnel;
322
319
323
320
rcu_read_lock_bh();
324
-
for (hash = 0; hash < L2TP_HASH_SIZE_2; hash++) {
325
-
hlist_for_each_entry_rcu(session, &pn->l2tp_session_hlist[hash], global_hlist) {
326
-
if (!strcmp(session->ifname, ifname)) {
327
-
l2tp_session_inc_refcount(session);
328
-
rcu_read_unlock_bh();
321
+
idr_for_each_entry_ul(&pn->l2tp_tunnel_idr, tunnel, tmp, tunnel_id) {
322
+
if (tunnel) {
323
+
list_for_each_entry_rcu(session, &tunnel->session_list, list) {
324
+
if (!strcmp(session->ifname, ifname)) {
325
+
l2tp_session_inc_refcount(session);
326
+
rcu_read_unlock_bh();
329
327
330
-
return session;
328
+
return session;
329
+
}
331
330
}
332
331
}
333
332
}
334
-
335
333
rcu_read_unlock_bh();
336
334
337
335
return NULL;
338
336
}
339
337
EXPORT_SYMBOL_GPL(l2tp_session_get_by_ifname);
340
338
339
+
static void l2tp_session_coll_list_add(struct l2tp_session_coll_list *clist,
340
+
struct l2tp_session *session)
341
+
{
342
+
l2tp_session_inc_refcount(session);
343
+
WARN_ON_ONCE(session->coll_list);
344
+
session->coll_list = clist;
345
+
spin_lock(&clist->lock);
346
+
list_add(&session->clist, &clist->list);
347
+
spin_unlock(&clist->lock);
348
+
}
349
+
350
+
static int l2tp_session_collision_add(struct l2tp_net *pn,
351
+
struct l2tp_session *session1,
352
+
struct l2tp_session *session2)
353
+
{
354
+
struct l2tp_session_coll_list *clist;
355
+
356
+
lockdep_assert_held(&pn->l2tp_session_idr_lock);
357
+
358
+
if (!session2)
359
+
return -EEXIST;
360
+
361
+
/* If existing session is in IP-encap tunnel, refuse new session */
362
+
if (session2->tunnel->encap == L2TP_ENCAPTYPE_IP)
363
+
return -EEXIST;
364
+
365
+
clist = session2->coll_list;
366
+
if (!clist) {
367
+
/* First collision. Allocate list to manage the collided sessions
368
+
* and add the existing session to the list.
369
+
*/
370
+
clist = kmalloc(sizeof(*clist), GFP_ATOMIC);
371
+
if (!clist)
372
+
return -ENOMEM;
373
+
374
+
spin_lock_init(&clist->lock);
375
+
INIT_LIST_HEAD(&clist->list);
376
+
refcount_set(&clist->ref_count, 1);
377
+
l2tp_session_coll_list_add(clist, session2);
378
+
}
379
+
380
+
/* If existing session isn't already in the session hlist, add it. */
381
+
if (!hash_hashed(&session2->hlist))
382
+
hash_add(pn->l2tp_v3_session_htable, &session2->hlist,
383
+
session2->hlist_key);
384
+
385
+
/* Add new session to the hlist and collision list */
386
+
hash_add(pn->l2tp_v3_session_htable, &session1->hlist,
387
+
session1->hlist_key);
388
+
refcount_inc(&clist->ref_count);
389
+
l2tp_session_coll_list_add(clist, session1);
390
+
391
+
return 0;
392
+
}
393
+
394
+
static void l2tp_session_collision_del(struct l2tp_net *pn,
395
+
struct l2tp_session *session)
396
+
{
397
+
struct l2tp_session_coll_list *clist = session->coll_list;
398
+
unsigned long session_key = session->session_id;
399
+
struct l2tp_session *session2;
400
+
401
+
lockdep_assert_held(&pn->l2tp_session_idr_lock);
402
+
403
+
hash_del(&session->hlist);
404
+
405
+
if (clist) {
406
+
/* Remove session from its collision list. If there
407
+
* are other sessions with the same ID, replace this
408
+
* session's IDR entry with that session, otherwise
409
+
* remove the IDR entry. If this is the last session,
410
+
* the collision list data is freed.
411
+
*/
412
+
spin_lock(&clist->lock);
413
+
list_del_init(&session->clist);
414
+
session2 = list_first_entry_or_null(&clist->list, struct l2tp_session, clist);
415
+
if (session2) {
416
+
void *old = idr_replace(&pn->l2tp_v3_session_idr, session2, session_key);
417
+
418
+
WARN_ON_ONCE(IS_ERR_VALUE(old));
419
+
} else {
420
+
void *removed = idr_remove(&pn->l2tp_v3_session_idr, session_key);
421
+
422
+
WARN_ON_ONCE(removed != session);
423
+
}
424
+
session->coll_list = NULL;
425
+
spin_unlock(&clist->lock);
426
+
if (refcount_dec_and_test(&clist->ref_count))
427
+
kfree(clist);
428
+
l2tp_session_dec_refcount(session);
429
+
}
430
+
}
431
+
341
432
int l2tp_session_register(struct l2tp_session *session,
342
433
struct l2tp_tunnel *tunnel)
343
434
{
344
-
struct l2tp_session *session_walk;
345
-
struct hlist_head *g_head;
346
-
struct hlist_head *head;
347
-
struct l2tp_net *pn;
435
+
struct l2tp_net *pn = l2tp_pernet(tunnel->l2tp_net);
436
+
u32 session_key;
348
437
int err;
349
438
350
-
head = l2tp_session_id_hash(tunnel, session->session_id);
351
-
352
-
spin_lock_bh(&tunnel->hlist_lock);
439
+
spin_lock_bh(&tunnel->list_lock);
353
440
if (!tunnel->acpt_newsess) {
354
441
err = -ENODEV;
355
442
goto err_tlock;
356
443
}
357
444
358
-
hlist_for_each_entry(session_walk, head, hlist)
359
-
if (session_walk->session_id == session->session_id) {
360
-
err = -EEXIST;
361
-
goto err_tlock;
362
-
}
363
-
364
445
if (tunnel->version == L2TP_HDR_VER_3) {
365
-
pn = l2tp_pernet(tunnel->l2tp_net);
366
-
g_head = l2tp_session_id_hash_2(pn, session->session_id);
367
-
368
-
spin_lock_bh(&pn->l2tp_session_hlist_lock);
369
-
446
+
session_key = session->session_id;
447
+
spin_lock_bh(&pn->l2tp_session_idr_lock);
448
+
err = idr_alloc_u32(&pn->l2tp_v3_session_idr, NULL,
449
+
&session_key, session_key, GFP_ATOMIC);
370
450
/* IP encap expects session IDs to be globally unique, while
371
-
* UDP encap doesn't.
451
+
* UDP encap doesn't. This isn't per the RFC, which says that
452
+
* sessions are identified only by the session ID, but is to
453
+
* support existing userspace which depends on it.
372
454
*/
373
-
hlist_for_each_entry(session_walk, g_head, global_hlist)
374
-
if (session_walk->session_id == session->session_id &&
375
-
(session_walk->tunnel->encap == L2TP_ENCAPTYPE_IP ||
376
-
tunnel->encap == L2TP_ENCAPTYPE_IP)) {
377
-
err = -EEXIST;
378
-
goto err_tlock_pnlock;
379
-
}
455
+
if (err == -ENOSPC && tunnel->encap == L2TP_ENCAPTYPE_UDP) {
456
+
struct l2tp_session *session2;
380
457
381
-
l2tp_tunnel_inc_refcount(tunnel);
382
-
hlist_add_head_rcu(&session->global_hlist, g_head);
383
-
384
-
spin_unlock_bh(&pn->l2tp_session_hlist_lock);
458
+
session2 = idr_find(&pn->l2tp_v3_session_idr,
459
+
session_key);
460
+
err = l2tp_session_collision_add(pn, session, session2);
461
+
}
462
+
spin_unlock_bh(&pn->l2tp_session_idr_lock);
385
463
} else {
386
-
l2tp_tunnel_inc_refcount(tunnel);
464
+
session_key = l2tp_v2_session_key(tunnel->tunnel_id,
465
+
session->session_id);
466
+
spin_lock_bh(&pn->l2tp_session_idr_lock);
467
+
err = idr_alloc_u32(&pn->l2tp_v2_session_idr, NULL,
468
+
&session_key, session_key, GFP_ATOMIC);
469
+
spin_unlock_bh(&pn->l2tp_session_idr_lock);
387
470
}
388
471
389
-
hlist_add_head_rcu(&session->hlist, head);
390
-
spin_unlock_bh(&tunnel->hlist_lock);
472
+
if (err) {
473
+
if (err == -ENOSPC)
474
+
err = -EEXIST;
475
+
goto err_tlock;
476
+
}
477
+
478
+
l2tp_tunnel_inc_refcount(tunnel);
479
+
480
+
list_add(&session->list, &tunnel->session_list);
481
+
spin_unlock_bh(&tunnel->list_lock);
482
+
483
+
spin_lock_bh(&pn->l2tp_session_idr_lock);
484
+
if (tunnel->version == L2TP_HDR_VER_3)
485
+
idr_replace(&pn->l2tp_v3_session_idr, session, session_key);
486
+
else
487
+
idr_replace(&pn->l2tp_v2_session_idr, session, session_key);
488
+
spin_unlock_bh(&pn->l2tp_session_idr_lock);
391
489
392
490
trace_register_session(session);
393
491
394
492
return 0;
395
493
396
-
err_tlock_pnlock:
397
-
spin_unlock_bh(&pn->l2tp_session_hlist_lock);
398
494
err_tlock:
399
-
spin_unlock_bh(&tunnel->hlist_lock);
495
+
spin_unlock_bh(&tunnel->list_lock);
400
496
401
497
return err;
402
498
}
···
889
785
}
890
786
}
891
787
892
-
/* Internal UDP receive frame. Do the real work of receiving an L2TP data frame
893
-
* here. The skb is not on a list when we get here.
894
-
* Returns 0 if the packet was a data packet and was successfully passed on.
895
-
* Returns 1 if the packet was not a good data packet and could not be
896
-
* forwarded. All such packets are passed up to userspace to deal with.
897
-
*/
898
-
static int l2tp_udp_recv_core(struct l2tp_tunnel *tunnel, struct sk_buff *skb)
788
+
/* UDP encapsulation receive handler. See net/ipv4/udp.c for details. */
789
+
int l2tp_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
899
790
{
900
791
struct l2tp_session *session = NULL;
901
-
struct l2tp_tunnel *orig_tunnel = tunnel;
792
+
struct l2tp_tunnel *tunnel = NULL;
793
+
struct net *net = sock_net(sk);
902
794
unsigned char *ptr, *optr;
903
795
u16 hdrflags;
904
-
u32 tunnel_id, session_id;
905
796
u16 version;
906
797
int length;
907
798
···
906
807
__skb_pull(skb, sizeof(struct udphdr));
907
808
908
809
/* Short packet? */
909
-
if (!pskb_may_pull(skb, L2TP_HDR_SIZE_MAX)) {
910
-
pr_debug_ratelimited("%s: recv short packet (len=%d)\n",
911
-
tunnel->name, skb->len);
912
-
goto invalid;
913
-
}
810
+
if (!pskb_may_pull(skb, L2TP_HDR_SIZE_MAX))
811
+
goto pass;
914
812
915
813
/* Point to L2TP header */
916
814
optr = skb->data;
···
930
834
ptr += 2;
931
835
932
836
if (version == L2TP_HDR_VER_2) {
837
+
u16 tunnel_id, session_id;
838
+
933
839
/* If length is present, skip it */
934
840
if (hdrflags & L2TP_HDRFLAG_L)
935
841
ptr += 2;
···
939
841
/* Extract tunnel and session ID */
940
842
tunnel_id = ntohs(*(__be16 *)ptr);
941
843
ptr += 2;
942
-
943
-
if (tunnel_id != tunnel->tunnel_id) {
944
-
/* We are receiving trafic for another tunnel, probably
945
-
* because we have several tunnels between the same
946
-
* IP/port quadruple, look it up.
947
-
*/
948
-
struct l2tp_tunnel *alt_tunnel;
949
-
950
-
alt_tunnel = l2tp_tunnel_get(tunnel->l2tp_net, tunnel_id);
951
-
if (!alt_tunnel)
952
-
goto pass;
953
-
tunnel = alt_tunnel;
954
-
}
955
-
956
844
session_id = ntohs(*(__be16 *)ptr);
957
845
ptr += 2;
846
+
847
+
session = l2tp_v2_session_get(net, tunnel_id, session_id);
958
848
} else {
849
+
u32 session_id;
850
+
959
851
ptr += 2; /* skip reserved bits */
960
-
tunnel_id = tunnel->tunnel_id;
961
852
session_id = ntohl(*(__be32 *)ptr);
962
853
ptr += 4;
854
+
855
+
session = l2tp_v3_session_get(net, sk, session_id);
963
856
}
964
857
965
-
/* Check protocol version */
966
-
if (version != tunnel->version) {
967
-
pr_debug_ratelimited("%s: recv protocol version mismatch: got %d expected %d\n",
968
-
tunnel->name, version, tunnel->version);
969
-
goto invalid;
970
-
}
971
-
972
-
/* Find the session context */
973
-
session = l2tp_tunnel_get_session(tunnel, session_id);
974
858
if (!session || !session->recv_skb) {
975
859
if (session)
976
860
l2tp_session_dec_refcount(session);
977
861
978
862
/* Not found? Pass to userspace to deal with */
979
-
pr_debug_ratelimited("%s: no session found (%u/%u). Passing up.\n",
980
-
tunnel->name, tunnel_id, session_id);
981
863
goto pass;
982
864
}
983
865
984
-
if (tunnel->version == L2TP_HDR_VER_3 &&
866
+
tunnel = session->tunnel;
867
+
868
+
/* Check protocol version */
869
+
if (version != tunnel->version)
870
+
goto invalid;
871
+
872
+
if (version == L2TP_HDR_VER_3 &&
985
873
l2tp_v3_ensure_opt_in_linear(session, skb, &ptr, &optr)) {
986
874
l2tp_session_dec_refcount(session);
987
875
goto invalid;
···
975
891
976
892
l2tp_recv_common(session, skb, ptr, optr, hdrflags, length);
977
893
l2tp_session_dec_refcount(session);
978
-
979
-
if (tunnel != orig_tunnel)
980
-
l2tp_tunnel_dec_refcount(tunnel);
981
894
982
895
return 0;
983
896
···
985
904
/* Put UDP header back */
986
905
__skb_push(skb, sizeof(struct udphdr));
987
906
988
-
if (tunnel != orig_tunnel)
989
-
l2tp_tunnel_dec_refcount(tunnel);
990
-
991
-
return 1;
992
-
}
993
-
994
-
/* UDP encapsulation receive and error receive handlers.
995
-
* See net/ipv4/udp.c for details.
996
-
*
997
-
* Note that these functions are called from inside an
998
-
* RCU-protected region, but without the socket being locked.
999
-
*
1000
-
* Hence we use rcu_dereference_sk_user_data to access the
1001
-
* tunnel data structure rather the usual l2tp_sk_to_tunnel
1002
-
* accessor function.
1003
-
*/
1004
-
int l2tp_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
1005
-
{
1006
-
struct l2tp_tunnel *tunnel;
1007
-
1008
-
tunnel = rcu_dereference_sk_user_data(sk);
1009
-
if (!tunnel)
1010
-
goto pass_up;
1011
-
if (WARN_ON(tunnel->magic != L2TP_TUNNEL_MAGIC))
1012
-
goto pass_up;
1013
-
1014
-
if (l2tp_udp_recv_core(tunnel, skb))
1015
-
goto pass_up;
1016
-
1017
-
return 0;
1018
-
1019
-
pass_up:
1020
907
return 1;
1021
908
}
1022
909
EXPORT_SYMBOL_GPL(l2tp_udp_encap_recv);
1023
910
911
+
/* UDP encapsulation receive error handler. See net/ipv4/udp.c for details. */
1024
912
static void l2tp_udp_encap_err_recv(struct sock *sk, struct sk_buff *skb, int err,
1025
913
__be16 port, u32 info, u8 *payload)
1026
914
{
1027
-
struct l2tp_tunnel *tunnel;
1028
-
1029
-
tunnel = rcu_dereference_sk_user_data(sk);
1030
-
if (!tunnel || tunnel->fd < 0)
1031
-
return;
1032
-
1033
915
sk->sk_err = err;
1034
916
sk_error_report(sk);
1035
917
···
1250
1206
return;
1251
1207
}
1252
1208
1253
-
/* Remove an l2tp session from l2tp_core's hash lists. */
1209
+
/* Remove an l2tp session from l2tp_core's lists. */
1254
1210
static void l2tp_session_unhash(struct l2tp_session *session)
1255
1211
{
1256
1212
struct l2tp_tunnel *tunnel = session->tunnel;
1257
1213
1258
-
/* Remove the session from core hashes */
1259
1214
if (tunnel) {
1260
-
/* Remove from the per-tunnel hash */
1261
-
spin_lock_bh(&tunnel->hlist_lock);
1262
-
hlist_del_init_rcu(&session->hlist);
1263
-
spin_unlock_bh(&tunnel->hlist_lock);
1215
+
struct l2tp_net *pn = l2tp_pernet(tunnel->l2tp_net);
1216
+
struct l2tp_session *removed = session;
1264
1217
1265
-
/* For L2TPv3 we have a per-net hash: remove from there, too */
1266
-
if (tunnel->version != L2TP_HDR_VER_2) {
1267
-
struct l2tp_net *pn = l2tp_pernet(tunnel->l2tp_net);
1218
+
/* Remove from the per-tunnel list */
1219
+
spin_lock_bh(&tunnel->list_lock);
1220
+
list_del_init(&session->list);
1221
+
spin_unlock_bh(&tunnel->list_lock);
1268
1222
1269
-
spin_lock_bh(&pn->l2tp_session_hlist_lock);
1270
-
hlist_del_init_rcu(&session->global_hlist);
1271
-
spin_unlock_bh(&pn->l2tp_session_hlist_lock);
1223
+
/* Remove from per-net IDR */
1224
+
spin_lock_bh(&pn->l2tp_session_idr_lock);
1225
+
if (tunnel->version == L2TP_HDR_VER_3) {
1226
+
if (hash_hashed(&session->hlist))
1227
+
l2tp_session_collision_del(pn, session);
1228
+
else
1229
+
removed = idr_remove(&pn->l2tp_v3_session_idr,
1230
+
session->session_id);
1231
+
} else {
1232
+
u32 session_key = l2tp_v2_session_key(tunnel->tunnel_id,
1233
+
session->session_id);
1234
+
removed = idr_remove(&pn->l2tp_v2_session_idr,
1235
+
session_key);
1272
1236
}
1237
+
WARN_ON_ONCE(removed && removed != session);
1238
+
spin_unlock_bh(&pn->l2tp_session_idr_lock);
1273
1239
1274
1240
synchronize_rcu();
1275
1241
}
···
1290
1236
static void l2tp_tunnel_closeall(struct l2tp_tunnel *tunnel)
1291
1237
{
1292
1238
struct l2tp_session *session;
1293
-
int hash;
1239
+
struct list_head __rcu *pos;
1240
+
struct list_head *tmp;
1294
1241
1295
-
spin_lock_bh(&tunnel->hlist_lock);
1242
+
spin_lock_bh(&tunnel->list_lock);
1296
1243
tunnel->acpt_newsess = false;
1297
-
for (hash = 0; hash < L2TP_HASH_SIZE; hash++) {
1298
-
again:
1299
-
hlist_for_each_entry_rcu(session, &tunnel->session_hlist[hash], hlist) {
1300
-
hlist_del_init_rcu(&session->hlist);
1301
-
1302
-
spin_unlock_bh(&tunnel->hlist_lock);
1303
-
l2tp_session_delete(session);
1304
-
spin_lock_bh(&tunnel->hlist_lock);
1305
-
1306
-
/* Now restart from the beginning of this hash
1307
-
* chain. We always remove a session from the
1308
-
* list so we are guaranteed to make forward
1309
-
* progress.
1310
-
*/
1311
-
goto again;
1312
-
}
1244
+
list_for_each_safe(pos, tmp, &tunnel->session_list) {
1245
+
session = list_entry(pos, struct l2tp_session, list);
1246
+
list_del_init(&session->list);
1247
+
spin_unlock_bh(&tunnel->list_lock);
1248
+
l2tp_session_delete(session);
1249
+
spin_lock_bh(&tunnel->list_lock);
1313
1250
}
1314
-
spin_unlock_bh(&tunnel->hlist_lock);
1251
+
spin_unlock_bh(&tunnel->list_lock);
1315
1252
}
1316
1253
1317
1254
/* Tunnel socket destroy hook for UDP encapsulation */
···
1496
1451
1497
1452
tunnel->magic = L2TP_TUNNEL_MAGIC;
1498
1453
sprintf(&tunnel->name[0], "tunl %u", tunnel_id);
1499
-
spin_lock_init(&tunnel->hlist_lock);
1454
+
spin_lock_init(&tunnel->list_lock);
1500
1455
tunnel->acpt_newsess = true;
1456
+
INIT_LIST_HEAD(&tunnel->session_list);
1501
1457
1502
1458
tunnel->encap = encap;
1503
1459
···
1507
1461
1508
1462
/* Init delete workqueue struct */
1509
1463
INIT_WORK(&tunnel->del_work, l2tp_tunnel_del_work);
1510
-
1511
-
INIT_LIST_HEAD(&tunnel->list);
1512
1464
1513
1465
err = 0;
1514
1466
err:
···
1695
1651
1696
1652
skb_queue_head_init(&session->reorder_q);
1697
1653
1654
+
session->hlist_key = l2tp_v3_session_hashkey(tunnel->sock, session->session_id);
1698
1655
INIT_HLIST_NODE(&session->hlist);
1699
-
INIT_HLIST_NODE(&session->global_hlist);
1656
+
INIT_LIST_HEAD(&session->clist);
1657
+
INIT_LIST_HEAD(&session->list);
1700
1658
1701
1659
if (cfg) {
1702
1660
session->pwtype = cfg->pw_type;
···
1731
1685
static __net_init int l2tp_init_net(struct net *net)
1732
1686
{
1733
1687
struct l2tp_net *pn = net_generic(net, l2tp_net_id);
1734
-
int hash;
1735
1688
1736
1689
idr_init(&pn->l2tp_tunnel_idr);
1737
1690
spin_lock_init(&pn->l2tp_tunnel_idr_lock);
1738
1691
1739
-
for (hash = 0; hash < L2TP_HASH_SIZE_2; hash++)
1740
-
INIT_HLIST_HEAD(&pn->l2tp_session_hlist[hash]);
1741
-
1742
-
spin_lock_init(&pn->l2tp_session_hlist_lock);
1692
+
idr_init(&pn->l2tp_v2_session_idr);
1693
+
idr_init(&pn->l2tp_v3_session_idr);
1694
+
spin_lock_init(&pn->l2tp_session_idr_lock);
1743
1695
1744
1696
return 0;
1745
1697
}
···
1747
1703
struct l2tp_net *pn = l2tp_pernet(net);
1748
1704
struct l2tp_tunnel *tunnel = NULL;
1749
1705
unsigned long tunnel_id, tmp;
1750
-
int hash;
1751
1706
1752
1707
rcu_read_lock_bh();
1753
1708
idr_for_each_entry_ul(&pn->l2tp_tunnel_idr, tunnel, tmp, tunnel_id) {
···
1759
1716
flush_workqueue(l2tp_wq);
1760
1717
rcu_barrier();
1761
1718
1762
-
for (hash = 0; hash < L2TP_HASH_SIZE_2; hash++)
1763
-
WARN_ON_ONCE(!hlist_empty(&pn->l2tp_session_hlist[hash]));
1719
+
idr_destroy(&pn->l2tp_v2_session_idr);
1720
+
idr_destroy(&pn->l2tp_v3_session_idr);
1764
1721
idr_destroy(&pn->l2tp_tunnel_idr);
1765
1722
}
1766
1723
+21
-22
net/l2tp/l2tp_core.h
+21
-22
net/l2tp/l2tp_core.h
···
19
19
#define L2TP_TUNNEL_MAGIC 0x42114DDA
20
20
#define L2TP_SESSION_MAGIC 0x0C04EB7D
21
21
22
-
/* Per tunnel session hash table size */
23
-
#define L2TP_HASH_BITS 4
24
-
#define L2TP_HASH_SIZE BIT(L2TP_HASH_BITS)
25
-
26
-
/* System-wide session hash table size */
27
-
#define L2TP_HASH_BITS_2 8
28
-
#define L2TP_HASH_SIZE_2 BIT(L2TP_HASH_BITS_2)
29
-
30
22
struct sk_buff;
31
23
32
24
struct l2tp_stats {
···
53
61
char *ifname;
54
62
};
55
63
64
+
struct l2tp_session_coll_list {
65
+
spinlock_t lock; /* for access to list */
66
+
struct list_head list;
67
+
refcount_t ref_count;
68
+
};
69
+
56
70
/* Represents a session (pseudowire) instance.
57
71
* Tracks runtime state including cookies, dataplane packet sequencing, and IO statistics.
58
-
* Is linked into a per-tunnel session hashlist; and in the case of an L2TPv3 session into
59
-
* an additional per-net ("global") hashlist.
72
+
* Is linked into a per-tunnel session list and a per-net ("global") IDR tree.
60
73
*/
61
74
#define L2TP_SESSION_NAME_MAX 32
62
75
struct l2tp_session {
···
85
88
u32 nr_oos; /* NR of last OOS packet */
86
89
int nr_oos_count; /* for OOS recovery */
87
90
int nr_oos_count_max;
88
-
struct hlist_node hlist; /* hash list node */
91
+
struct list_head list; /* per-tunnel list node */
89
92
refcount_t ref_count;
93
+
struct hlist_node hlist; /* per-net session hlist */
94
+
unsigned long hlist_key; /* key for session hlist */
95
+
struct l2tp_session_coll_list *coll_list; /* session collision list */
96
+
struct list_head clist; /* for coll_list */
90
97
91
98
char name[L2TP_SESSION_NAME_MAX]; /* for logging */
92
99
char ifname[IFNAMSIZ];
···
103
102
int reorder_skip; /* set if skip to next nr */
104
103
enum l2tp_pwtype pwtype;
105
104
struct l2tp_stats stats;
106
-
struct hlist_node global_hlist; /* global hash list node */
107
105
108
106
/* Session receive handler for data packets.
109
107
* Each pseudowire implementation should implement this callback in order to
···
114
114
/* Session close handler.
115
115
* Each pseudowire implementation may implement this callback in order to carry
116
116
* out pseudowire-specific shutdown actions.
117
-
* The callback is called by core after unhashing the session and purging its
117
+
* The callback is called by core after unlisting the session and purging its
118
118
* reorder queue.
119
119
*/
120
120
void (*session_close)(struct l2tp_session *session);
···
150
150
/* Represents a tunnel instance.
151
151
* Tracks runtime state including IO statistics.
152
152
* Holds the tunnel socket (either passed from userspace or directly created by the kernel).
153
-
* Maintains a hashlist of sessions belonging to the tunnel instance.
153
+
* Maintains a list of sessions belonging to the tunnel instance.
154
154
* Is linked into a per-net list of tunnels.
155
155
*/
156
156
#define L2TP_TUNNEL_NAME_MAX 20
···
160
160
unsigned long dead;
161
161
162
162
struct rcu_head rcu;
163
-
spinlock_t hlist_lock; /* write-protection for session_hlist */
163
+
spinlock_t list_lock; /* write-protection for session_list */
164
164
bool acpt_newsess; /* indicates whether this tunnel accepts
165
-
* new sessions. Protected by hlist_lock.
165
+
* new sessions. Protected by list_lock.
166
166
*/
167
-
struct hlist_head session_hlist[L2TP_HASH_SIZE];
168
-
/* hashed list of sessions, hashed by id */
167
+
struct list_head session_list; /* list of sessions */
169
168
u32 tunnel_id;
170
169
u32 peer_tunnel_id;
171
170
int version; /* 2=>L2TPv2, 3=>L2TPv3 */
···
173
174
enum l2tp_encap_type encap;
174
175
struct l2tp_stats stats;
175
176
176
-
struct list_head list; /* list node on per-namespace list of tunnels */
177
177
struct net *l2tp_net; /* the net we belong to */
178
178
179
179
refcount_t ref_count;
···
222
224
*/
223
225
struct l2tp_tunnel *l2tp_tunnel_get(const struct net *net, u32 tunnel_id);
224
226
struct l2tp_tunnel *l2tp_tunnel_get_nth(const struct net *net, int nth);
225
-
struct l2tp_session *l2tp_tunnel_get_session(struct l2tp_tunnel *tunnel,
226
-
u32 session_id);
227
227
228
-
struct l2tp_session *l2tp_session_get(const struct net *net, u32 session_id);
228
+
struct l2tp_session *l2tp_v3_session_get(const struct net *net, struct sock *sk, u32 session_id);
229
+
struct l2tp_session *l2tp_v2_session_get(const struct net *net, u16 tunnel_id, u16 session_id);
230
+
struct l2tp_session *l2tp_session_get(const struct net *net, struct sock *sk, int pver,
231
+
u32 tunnel_id, u32 session_id);
229
232
struct l2tp_session *l2tp_session_get_nth(struct l2tp_tunnel *tunnel, int nth);
230
233
struct l2tp_session *l2tp_session_get_by_ifname(const struct net *net,
231
234
const char *ifname);
+5
-8
net/l2tp/l2tp_debugfs.c
+5
-8
net/l2tp/l2tp_debugfs.c
···
123
123
struct l2tp_tunnel *tunnel = v;
124
124
struct l2tp_session *session;
125
125
int session_count = 0;
126
-
int hash;
127
126
128
127
rcu_read_lock_bh();
129
-
for (hash = 0; hash < L2TP_HASH_SIZE; hash++) {
130
-
hlist_for_each_entry_rcu(session, &tunnel->session_hlist[hash], hlist) {
131
-
/* Session ID of zero is a dummy/reserved value used by pppol2tp */
132
-
if (session->session_id == 0)
133
-
continue;
128
+
list_for_each_entry_rcu(session, &tunnel->session_list, list) {
129
+
/* Session ID of zero is a dummy/reserved value used by pppol2tp */
130
+
if (session->session_id == 0)
131
+
continue;
134
132
135
-
session_count++;
136
-
}
133
+
session_count++;
137
134
}
138
135
rcu_read_unlock_bh();
139
136
+1
-1
net/l2tp/l2tp_ip.c
+1
-1
net/l2tp/l2tp_ip.c
+1
-1
net/l2tp/l2tp_ip6.c
+1
-1
net/l2tp/l2tp_ip6.c
+4
-2
net/l2tp/l2tp_netlink.c
+4
-2
net/l2tp/l2tp_netlink.c
···
61
61
session_id = nla_get_u32(info->attrs[L2TP_ATTR_SESSION_ID]);
62
62
tunnel = l2tp_tunnel_get(net, tunnel_id);
63
63
if (tunnel) {
64
-
session = l2tp_tunnel_get_session(tunnel, session_id);
64
+
session = l2tp_session_get(net, tunnel->sock, tunnel->version,
65
+
tunnel_id, session_id);
65
66
l2tp_tunnel_dec_refcount(tunnel);
66
67
}
67
68
}
···
636
635
&cfg);
637
636
638
637
if (ret >= 0) {
639
-
session = l2tp_tunnel_get_session(tunnel, session_id);
638
+
session = l2tp_session_get(net, tunnel->sock, tunnel->version,
639
+
tunnel_id, session_id);
640
640
if (session) {
641
641
ret = l2tp_session_notify(&l2tp_nl_family, info, session,
642
642
L2TP_CMD_SESSION_CREATE);
+4
-2
net/l2tp/l2tp_ppp.c
+4
-2
net/l2tp/l2tp_ppp.c
···
753
753
if (tunnel->peer_tunnel_id == 0)
754
754
tunnel->peer_tunnel_id = info.peer_tunnel_id;
755
755
756
-
session = l2tp_tunnel_get_session(tunnel, info.session_id);
756
+
session = l2tp_session_get(sock_net(sk), tunnel->sock, tunnel->version,
757
+
info.tunnel_id, info.session_id);
757
758
if (session) {
758
759
drop_refcnt = true;
759
760
···
1046
1045
/* If session_id is set, search the corresponding session in the
1047
1046
* context of this tunnel and record the session's statistics.
1048
1047
*/
1049
-
session = l2tp_tunnel_get_session(tunnel, stats->session_id);
1048
+
session = l2tp_session_get(tunnel->l2tp_net, tunnel->sock, tunnel->version,
1049
+
tunnel->tunnel_id, stats->session_id);
1050
1050
if (!session)
1051
1051
return -EBADR;
1052
1052