Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'net-few-critical-helpers-are-inlined-again'
Eric Dumazet says:
====================
net: few critical helpers are inlined again
Recent devmem additions increased stack depth. Some helpers
that were inlined in the past are now out-of-line.
====================
Link: https://patch.msgid.link/20260122045720.1221017-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+47
-40
+5
-5
include/linux/skbuff_ref.h
+5
-5
include/linux/skbuff_ref.h
···
15
15
*
16
16
* Takes an additional reference on the paged fragment @frag.
17
17
*/
18
-
static inline void __skb_frag_ref(skb_frag_t *frag)
18
+
static __always_inline void __skb_frag_ref(skb_frag_t *frag)
19
19
{
20
20
get_netmem(skb_frag_netmem(frag));
21
21
}
···
27
27
*
28
28
* Takes an additional reference on the @f'th paged fragment of @skb.
29
29
*/
30
-
static inline void skb_frag_ref(struct sk_buff *skb, int f)
30
+
static __always_inline void skb_frag_ref(struct sk_buff *skb, int f)
31
31
{
32
32
__skb_frag_ref(&skb_shinfo(skb)->frags[f]);
33
33
}
34
34
35
35
bool napi_pp_put_page(netmem_ref netmem);
36
36
37
-
static inline void skb_page_unref(netmem_ref netmem, bool recycle)
37
+
static __always_inline void skb_page_unref(netmem_ref netmem, bool recycle)
38
38
{
39
39
#ifdef CONFIG_PAGE_POOL
40
40
if (recycle && napi_pp_put_page(netmem))
···
51
51
* Releases a reference on the paged fragment @frag
52
52
* or recycles the page via the page_pool API.
53
53
*/
54
-
static inline void __skb_frag_unref(skb_frag_t *frag, bool recycle)
54
+
static __always_inline void __skb_frag_unref(skb_frag_t *frag, bool recycle)
55
55
{
56
56
skb_page_unref(skb_frag_netmem(frag), recycle);
57
57
}
···
63
63
*
64
64
* Releases a reference on the @f'th paged fragment of @skb.
65
65
*/
66
-
static inline void skb_frag_unref(struct sk_buff *skb, int f)
66
+
static __always_inline void skb_frag_unref(struct sk_buff *skb, int f)
67
67
{
68
68
struct skb_shared_info *shinfo = skb_shinfo(skb);
69
69
+30
-2
include/net/netmem.h
+30
-2
include/net/netmem.h
···
389
389
return netmem_to_nmdesc(netmem)->dma_addr;
390
390
}
391
391
392
-
void get_netmem(netmem_ref netmem);
393
-
void put_netmem(netmem_ref netmem);
392
+
#if defined(CONFIG_NET_DEVMEM)
393
+
static inline bool net_is_devmem_iov(const struct net_iov *niov)
394
+
{
395
+
return niov->type == NET_IOV_DMABUF;
396
+
}
397
+
#else
398
+
static inline bool net_is_devmem_iov(const struct net_iov *niov)
399
+
{
400
+
return false;
401
+
}
402
+
#endif
403
+
404
+
void __get_netmem(netmem_ref netmem);
405
+
void __put_netmem(netmem_ref netmem);
406
+
407
+
static __always_inline void get_netmem(netmem_ref netmem)
408
+
{
409
+
if (netmem_is_net_iov(netmem))
410
+
__get_netmem(netmem);
411
+
else
412
+
get_page(netmem_to_page(netmem));
413
+
}
414
+
415
+
static __always_inline void put_netmem(netmem_ref netmem)
416
+
{
417
+
if (netmem_is_net_iov(netmem))
418
+
__put_netmem(netmem);
419
+
else
420
+
put_page(netmem_to_page(netmem));
421
+
}
394
422
395
423
#define netmem_dma_unmap_addr_set(NETMEM, PTR, ADDR_NAME, VAL) \
396
424
do { \
-5
net/core/devmem.c
-5
net/core/devmem.c
···
30
30
31
31
static const struct memory_provider_ops dmabuf_devmem_ops;
32
32
33
-
bool net_is_devmem_iov(struct net_iov *niov)
34
-
{
35
-
return niov->type == NET_IOV_DMABUF;
36
-
}
37
-
38
33
static void net_devmem_dmabuf_free_chunk_owner(struct gen_pool *genpool,
39
34
struct gen_pool_chunk *chunk,
40
35
void *not_used)
+1
-6
net/core/devmem.h
+1
-6
net/core/devmem.h
···
141
141
net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding);
142
142
void net_devmem_free_dmabuf(struct net_iov *ppiov);
143
143
144
-
bool net_is_devmem_iov(struct net_iov *niov);
144
+
145
145
struct net_devmem_dmabuf_binding *
146
146
net_devmem_get_binding(struct sock *sk, unsigned int dmabuf_id);
147
147
struct net_iov *
···
212
212
static inline u32 net_devmem_iov_binding_id(const struct net_iov *niov)
213
213
{
214
214
return 0;
215
-
}
216
-
217
-
static inline bool net_is_devmem_iov(struct net_iov *niov)
218
-
{
219
-
return false;
220
215
}
221
216
222
217
static inline struct net_devmem_dmabuf_binding *
+1
-1
net/core/gro.c
+1
-1
net/core/gro.c
+10
-21
net/core/skbuff.c
+10
-21
net/core/skbuff.c
···
7423
7423
}
7424
7424
EXPORT_SYMBOL(csum_and_copy_from_iter_full);
7425
7425
7426
-
void get_netmem(netmem_ref netmem)
7426
+
void __get_netmem(netmem_ref netmem)
7427
7427
{
7428
-
struct net_iov *niov;
7428
+
struct net_iov *niov = netmem_to_net_iov(netmem);
7429
7429
7430
-
if (netmem_is_net_iov(netmem)) {
7431
-
niov = netmem_to_net_iov(netmem);
7432
-
if (net_is_devmem_iov(niov))
7433
-
net_devmem_get_net_iov(netmem_to_net_iov(netmem));
7434
-
return;
7435
-
}
7436
-
get_page(netmem_to_page(netmem));
7430
+
if (net_is_devmem_iov(niov))
7431
+
net_devmem_get_net_iov(netmem_to_net_iov(netmem));
7437
7432
}
7438
-
EXPORT_SYMBOL(get_netmem);
7433
+
EXPORT_SYMBOL(__get_netmem);
7439
7434
7440
-
void put_netmem(netmem_ref netmem)
7435
+
void __put_netmem(netmem_ref netmem)
7441
7436
{
7442
-
struct net_iov *niov;
7437
+
struct net_iov *niov = netmem_to_net_iov(netmem);
7443
7438
7444
-
if (netmem_is_net_iov(netmem)) {
7445
-
niov = netmem_to_net_iov(netmem);
7446
-
if (net_is_devmem_iov(niov))
7447
-
net_devmem_put_net_iov(netmem_to_net_iov(netmem));
7448
-
return;
7449
-
}
7450
-
7451
-
put_page(netmem_to_page(netmem));
7439
+
if (net_is_devmem_iov(niov))
7440
+
net_devmem_put_net_iov(netmem_to_net_iov(netmem));
7452
7441
}
7453
-
EXPORT_SYMBOL(put_netmem);
7442
+
EXPORT_SYMBOL(__put_netmem);