tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

tcp: avoid atomic operations on sk->sk_rmem_alloc

TCP uses generic skb_set_owner_r() and sock_rfree()
for received packets, with socket lock being owned.

Switch to private versions, avoiding two atomic operations
per packet.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Neal Cardwell <ncardwell@google.com>
Reviewed-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Link: https://patch.msgid.link/20250320121604.3342831-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

authored by

Eric Dumazet and committed by
Jakub Kicinski 0de2a5c4 c1dacb45

+35 -6
+15
include/net/tcp.h
··· 779 779 780 780 /* tcp.c */ 781 781 void tcp_get_info(struct sock *, struct tcp_info *); 782 + void tcp_sock_rfree(struct sk_buff *skb); 782 783 783 784 /* Read 'sendfile()'-style from a TCP socket */ 784 785 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, ··· 2898 2897 const struct request_sock *req, const struct sk_buff *skb, 2899 2898 const void *saddr, const void *daddr, 2900 2899 int family, int dif, int sdif); 2900 + 2901 + /* version of skb_set_owner_r() avoiding one atomic_add() */ 2902 + static inline void tcp_skb_set_owner_r(struct sk_buff *skb, struct sock *sk) 2903 + { 2904 + skb_orphan(skb); 2905 + skb->sk = sk; 2906 + skb->destructor = tcp_sock_rfree; 2907 + 2908 + sock_owned_by_me(sk); 2909 + atomic_set(&sk->sk_rmem_alloc, 2910 + atomic_read(&sk->sk_rmem_alloc) + skb->truesize); 2911 + 2912 + sk_forward_alloc_add(sk, -skb->truesize); 2913 + } 2901 2914 2902 2915 #endif /* _TCP_H */
+16 -2
net/ipv4/tcp.c
··· 1525 1525 __tcp_cleanup_rbuf(sk, copied); 1526 1526 } 1527 1527 1528 + /* private version of sock_rfree() avoiding one atomic_sub() */ 1529 + void tcp_sock_rfree(struct sk_buff *skb) 1530 + { 1531 + struct sock *sk = skb->sk; 1532 + unsigned int len = skb->truesize; 1533 + 1534 + sock_owned_by_me(sk); 1535 + atomic_set(&sk->sk_rmem_alloc, 1536 + atomic_read(&sk->sk_rmem_alloc) - len); 1537 + 1538 + sk_forward_alloc_add(sk, len); 1539 + sk_mem_reclaim(sk); 1540 + } 1541 + 1528 1542 static void tcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb) 1529 1543 { 1530 1544 __skb_unlink(skb, &sk->sk_receive_queue); 1531 - if (likely(skb->destructor == sock_rfree)) { 1532 - sock_rfree(skb); 1545 + if (likely(skb->destructor == tcp_sock_rfree)) { 1546 + tcp_sock_rfree(skb); 1533 1547 skb->destructor = NULL; 1534 1548 skb->sk = NULL; 1535 1549 return skb_attempt_defer_free(skb);
+1 -1
net/ipv4/tcp_fastopen.c
··· 189 189 tcp_segs_in(tp, skb); 190 190 __skb_pull(skb, tcp_hdrlen(skb)); 191 191 sk_forced_mem_schedule(sk, skb->truesize); 192 - skb_set_owner_r(skb, sk); 192 + tcp_skb_set_owner_r(skb, sk); 193 193 194 194 TCP_SKB_CB(skb)->seq++; 195 195 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_SYN;
+3 -3
net/ipv4/tcp_input.c
··· 5171 5171 if (tcp_is_sack(tp)) 5172 5172 tcp_grow_window(sk, skb, false); 5173 5173 skb_condense(skb); 5174 - skb_set_owner_r(skb, sk); 5174 + tcp_skb_set_owner_r(skb, sk); 5175 5175 } 5176 5176 } 5177 5177 ··· 5187 5187 tcp_rcv_nxt_update(tcp_sk(sk), TCP_SKB_CB(skb)->end_seq); 5188 5188 if (!eaten) { 5189 5189 tcp_add_receive_queue(sk, skb); 5190 - skb_set_owner_r(skb, sk); 5190 + tcp_skb_set_owner_r(skb, sk); 5191 5191 } 5192 5192 return eaten; 5193 5193 } ··· 5504 5504 __skb_queue_before(list, skb, nskb); 5505 5505 else 5506 5506 __skb_queue_tail(&tmp, nskb); /* defer rbtree insertion */ 5507 - skb_set_owner_r(nskb, sk); 5507 + tcp_skb_set_owner_r(nskb, sk); 5508 5508 mptcp_skb_ext_move(nskb, skb); 5509 5509 5510 5510 /* Copy data, releasing collapsed skbs. */