tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge branch 'xdp-a-fistful-of-generic-changes-pt-i'

Alexander Lobakin says:

====================
xdp: a fistful of generic changes pt. I

XDP for idpf is currently 6 chapters:
* convert Rx to libeth;
* convert Tx and stats to libeth;
* generic XDP and XSk code changes (you are here);
* generic XDP and XSk code additions;
* actual XDP for idpf via new libeth_xdp;
* XSk for idpf (via ^).

Part III does the following:
* improve &xdp_buff_xsk cacheline placement;
* does some cleanups with marking read-only bpf_prog and xdp_buff
arguments const for some generic functions;
* allows attaching already registered XDP memory model to RxQ info;
* makes system percpu page_pools valid XDP memory models;
* starts using netmems in the XDP core code (1 function);
* allows mixing pages from several page_pools within one XDP frame;
* optimizes &xdp_frame layout and removes no-more-used field.

Bullets 4-6 are the most important ones. All of them are prereqs to
libeth_xdp.
====================

Link: https://patch.msgid.link/20241203173733.3181246-1-aleksander.lobakin@intel.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

Jakub Kicinski 9d1a8c6f 1daa6591

+266 -82
+6 -6
include/linux/bpf.h
··· 2591 2591 int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, 2592 2592 struct bpf_map *map, bool exclude_ingress); 2593 2593 int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, 2594 - struct bpf_prog *xdp_prog); 2594 + const struct bpf_prog *xdp_prog); 2595 2595 int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, 2596 - struct bpf_prog *xdp_prog, struct bpf_map *map, 2597 - bool exclude_ingress); 2596 + const struct bpf_prog *xdp_prog, 2597 + struct bpf_map *map, bool exclude_ingress); 2598 2598 2599 2599 void __cpu_map_flush(struct list_head *flush_list); 2600 2600 int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf, ··· 2864 2864 2865 2865 static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, 2866 2866 struct sk_buff *skb, 2867 - struct bpf_prog *xdp_prog) 2867 + const struct bpf_prog *xdp_prog) 2868 2868 { 2869 2869 return 0; 2870 2870 } 2871 2871 2872 2872 static inline 2873 2873 int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, 2874 - struct bpf_prog *xdp_prog, struct bpf_map *map, 2875 - bool exclude_ingress) 2874 + const struct bpf_prog *xdp_prog, 2875 + struct bpf_map *map, bool exclude_ingress) 2876 2876 { 2877 2877 return 0; 2878 2878 }
+5 -4
include/linux/filter.h
··· 1179 1179 * This does not appear to be a real limitation for existing software. 1180 1180 */ 1181 1181 int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb, 1182 - struct xdp_buff *xdp, struct bpf_prog *prog); 1182 + struct xdp_buff *xdp, const struct bpf_prog *prog); 1183 1183 int xdp_do_redirect(struct net_device *dev, 1184 1184 struct xdp_buff *xdp, 1185 - struct bpf_prog *prog); 1185 + const struct bpf_prog *prog); 1186 1186 int xdp_do_redirect_frame(struct net_device *dev, 1187 1187 struct xdp_buff *xdp, 1188 1188 struct xdp_frame *xdpf, 1189 - struct bpf_prog *prog); 1189 + const struct bpf_prog *prog); 1190 1190 void xdp_do_flush(void); 1191 1191 1192 - void bpf_warn_invalid_xdp_action(struct net_device *dev, struct bpf_prog *prog, u32 act); 1192 + void bpf_warn_invalid_xdp_action(const struct net_device *dev, 1193 + const struct bpf_prog *prog, u32 act); 1193 1194 1194 1195 #ifdef CONFIG_INET 1195 1196 struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk,
+4 -3
include/linux/netdevice.h
··· 3322 3322 }; 3323 3323 3324 3324 DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data); 3325 + DECLARE_PER_CPU(struct page_pool *, system_page_pool); 3325 3326 3326 3327 #ifndef CONFIG_PREEMPT_RT 3327 3328 static inline int dev_recursion_level(void) ··· 3959 3958 } 3960 3959 3961 3960 u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp, 3962 - struct bpf_prog *xdp_prog); 3963 - void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog); 3964 - int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff **pskb); 3961 + const struct bpf_prog *xdp_prog); 3962 + void generic_xdp_tx(struct sk_buff *skb, const struct bpf_prog *xdp_prog); 3963 + int do_xdp_generic(const struct bpf_prog *xdp_prog, struct sk_buff **pskb); 3965 3964 int netif_rx(struct sk_buff *skb); 3966 3965 int __netif_rx(struct sk_buff *skb); 3967 3966
+1 -1
include/linux/skbuff.h
··· 3627 3627 int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, 3628 3628 unsigned int headroom); 3629 3629 int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb, 3630 - struct bpf_prog *prog); 3630 + const struct bpf_prog *prog); 3631 3631 3632 3632 /** 3633 3633 * skb_frag_address - gets the address of the data contained in a paged fragment
+76 -2
include/net/netmem.h
··· 72 72 return (__force unsigned long)netmem & NET_IOV; 73 73 } 74 74 75 + /** 76 + * __netmem_to_page - unsafely get pointer to the &page backing @netmem 77 + * @netmem: netmem reference to convert 78 + * 79 + * Unsafe version of netmem_to_page(). When @netmem is always page-backed, 80 + * e.g. when it's a header buffer, performs faster and generates smaller 81 + * object code (no check for the LSB, no WARN). When @netmem points to IOV, 82 + * provokes undefined behaviour. 83 + * 84 + * Return: pointer to the &page (garbage if @netmem is not page-backed). 85 + */ 86 + static inline struct page *__netmem_to_page(netmem_ref netmem) 87 + { 88 + return (__force struct page *)netmem; 89 + } 90 + 75 91 /* This conversion fails (returns NULL) if the netmem_ref is not struct page 76 92 * backed. 77 93 */ ··· 96 80 if (WARN_ON_ONCE(netmem_is_net_iov(netmem))) 97 81 return NULL; 98 82 99 - return (__force struct page *)netmem; 83 + return __netmem_to_page(netmem); 100 84 } 101 85 102 86 static inline struct net_iov *netmem_to_net_iov(netmem_ref netmem) ··· 117 101 static inline netmem_ref page_to_netmem(struct page *page) 118 102 { 119 103 return (__force netmem_ref)page; 104 + } 105 + 106 + /** 107 + * virt_to_netmem - convert virtual memory pointer to a netmem reference 108 + * @data: host memory pointer to convert 109 + * 110 + * Return: netmem reference to the &page backing this virtual address. 111 + */ 112 + static inline netmem_ref virt_to_netmem(const void *data) 113 + { 114 + return page_to_netmem(virt_to_page(data)); 120 115 } 121 116 122 117 static inline int netmem_ref_count(netmem_ref netmem) ··· 152 125 static inline struct net_iov *__netmem_clear_lsb(netmem_ref netmem) 153 126 { 154 127 return (struct net_iov *)((__force unsigned long)netmem & ~NET_IOV); 128 + } 129 + 130 + /** 131 + * __netmem_get_pp - unsafely get pointer to the &page_pool backing @netmem 132 + * @netmem: netmem reference to get the pointer from 133 + * 134 + * Unsafe version of netmem_get_pp(). When @netmem is always page-backed, 135 + * e.g. when it's a header buffer, performs faster and generates smaller 136 + * object code (avoids clearing the LSB). When @netmem points to IOV, 137 + * provokes invalid memory access. 138 + * 139 + * Return: pointer to the &page_pool (garbage if @netmem is not page-backed). 140 + */ 141 + static inline struct page_pool *__netmem_get_pp(netmem_ref netmem) 142 + { 143 + return __netmem_to_page(netmem)->pp; 155 144 } 156 145 157 146 static inline struct page_pool *netmem_get_pp(netmem_ref netmem) ··· 201 158 return page_to_netmem(compound_head(netmem_to_page(netmem))); 202 159 } 203 160 161 + /** 162 + * __netmem_address - unsafely get pointer to the memory backing @netmem 163 + * @netmem: netmem reference to get the pointer for 164 + * 165 + * Unsafe version of netmem_address(). When @netmem is always page-backed, 166 + * e.g. when it's a header buffer, performs faster and generates smaller 167 + * object code (no check for the LSB). When @netmem points to IOV, provokes 168 + * undefined behaviour. 169 + * 170 + * Return: pointer to the memory (garbage if @netmem is not page-backed). 171 + */ 172 + static inline void *__netmem_address(netmem_ref netmem) 173 + { 174 + return page_address(__netmem_to_page(netmem)); 175 + } 176 + 204 177 static inline void *netmem_address(netmem_ref netmem) 205 178 { 206 179 if (netmem_is_net_iov(netmem)) 207 180 return NULL; 208 181 209 - return page_address(netmem_to_page(netmem)); 182 + return __netmem_address(netmem); 183 + } 184 + 185 + /** 186 + * netmem_is_pfmemalloc - check if @netmem was allocated under memory pressure 187 + * @netmem: netmem reference to check 188 + * 189 + * Return: true if @netmem is page-backed and the page was allocated under 190 + * memory pressure, false otherwise. 191 + */ 192 + static inline bool netmem_is_pfmemalloc(netmem_ref netmem) 193 + { 194 + if (netmem_is_net_iov(netmem)) 195 + return false; 196 + 197 + return page_is_pfmemalloc(netmem_to_page(netmem)); 210 198 } 211 199 212 200 static inline unsigned long netmem_get_dma_addr(netmem_ref netmem)
+4 -4
include/net/page_pool/types.h
··· 259 259 void page_pool_destroy(struct page_pool *pool); 260 260 void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *), 261 261 const struct xdp_mem_info *mem); 262 - void page_pool_put_page_bulk(struct page_pool *pool, void **data, 263 - int count); 262 + void page_pool_put_netmem_bulk(struct page_pool *pool, netmem_ref *data, 263 + u32 count); 264 264 #else 265 265 static inline void page_pool_destroy(struct page_pool *pool) 266 266 { ··· 272 272 { 273 273 } 274 274 275 - static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data, 276 - int count) 275 + static inline void page_pool_put_netmem_bulk(struct page_pool *pool, 276 + netmem_ref *data, u32 count) 277 277 { 278 278 } 279 279 #endif
+50 -13
include/net/xdp.h
··· 88 88 u32 flags; /* supported values defined in xdp_buff_flags */ 89 89 }; 90 90 91 - static __always_inline bool xdp_buff_has_frags(struct xdp_buff *xdp) 91 + static __always_inline bool xdp_buff_has_frags(const struct xdp_buff *xdp) 92 92 { 93 93 return !!(xdp->flags & XDP_FLAGS_HAS_FRAGS); 94 94 } ··· 103 103 xdp->flags &= ~XDP_FLAGS_HAS_FRAGS; 104 104 } 105 105 106 - static __always_inline bool xdp_buff_is_frag_pfmemalloc(struct xdp_buff *xdp) 106 + static __always_inline bool 107 + xdp_buff_is_frag_pfmemalloc(const struct xdp_buff *xdp) 107 108 { 108 109 return !!(xdp->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC); 109 110 } ··· 145 144 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 146 145 147 146 static inline struct skb_shared_info * 148 - xdp_get_shared_info_from_buff(struct xdp_buff *xdp) 147 + xdp_get_shared_info_from_buff(const struct xdp_buff *xdp) 149 148 { 150 149 return (struct skb_shared_info *)xdp_data_hard_end(xdp); 151 150 } 152 151 153 - static __always_inline unsigned int xdp_get_buff_len(struct xdp_buff *xdp) 152 + static __always_inline unsigned int 153 + xdp_get_buff_len(const struct xdp_buff *xdp) 154 154 { 155 155 unsigned int len = xdp->data_end - xdp->data; 156 - struct skb_shared_info *sinfo; 156 + const struct skb_shared_info *sinfo; 157 157 158 158 if (likely(!xdp_buff_has_frags(xdp))) 159 159 goto out; ··· 179 177 u32 flags; /* supported values defined in xdp_buff_flags */ 180 178 }; 181 179 182 - static __always_inline bool xdp_frame_has_frags(struct xdp_frame *frame) 180 + static __always_inline bool xdp_frame_has_frags(const struct xdp_frame *frame) 183 181 { 184 182 return !!(frame->flags & XDP_FLAGS_HAS_FRAGS); 185 183 } 186 184 187 - static __always_inline bool xdp_frame_is_frag_pfmemalloc(struct xdp_frame *frame) 185 + static __always_inline bool 186 + xdp_frame_is_frag_pfmemalloc(const struct xdp_frame *frame) 188 187 { 189 188 return !!(frame->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC); 190 189 } ··· 194 191 struct xdp_frame_bulk { 195 192 int count; 196 193 void *xa; 197 - void *q[XDP_BULK_QUEUE_SIZE]; 194 + netmem_ref q[XDP_BULK_QUEUE_SIZE]; 198 195 }; 199 196 200 197 static __always_inline void xdp_frame_bulk_init(struct xdp_frame_bulk *bq) ··· 204 201 } 205 202 206 203 static inline struct skb_shared_info * 207 - xdp_get_shared_info_from_frame(struct xdp_frame *frame) 204 + xdp_get_shared_info_from_frame(const struct xdp_frame *frame) 208 205 { 209 206 void *data_hard_start = frame->data - frame->headroom - sizeof(*frame); 210 207 ··· 252 249 struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf); 253 250 254 251 static inline 255 - void xdp_convert_frame_to_buff(struct xdp_frame *frame, struct xdp_buff *xdp) 252 + void xdp_convert_frame_to_buff(const struct xdp_frame *frame, 253 + struct xdp_buff *xdp) 256 254 { 257 255 xdp->data_hard_start = frame->data - frame->headroom - sizeof(*frame); 258 256 xdp->data = frame->data; ··· 264 260 } 265 261 266 262 static inline 267 - int xdp_update_frame_from_buff(struct xdp_buff *xdp, 263 + int xdp_update_frame_from_buff(const struct xdp_buff *xdp, 268 264 struct xdp_frame *xdp_frame) 269 265 { 270 266 int metasize, headroom; ··· 321 317 void xdp_return_frame_bulk(struct xdp_frame *xdpf, 322 318 struct xdp_frame_bulk *bq); 323 319 324 - static __always_inline unsigned int xdp_get_frame_len(struct xdp_frame *xdpf) 320 + static __always_inline unsigned int 321 + xdp_get_frame_len(const struct xdp_frame *xdpf) 325 322 { 326 - struct skb_shared_info *sinfo; 323 + const struct skb_shared_info *sinfo; 327 324 unsigned int len = xdpf->len; 328 325 329 326 if (likely(!xdp_frame_has_frags(xdpf))) ··· 356 351 int xdp_reg_mem_model(struct xdp_mem_info *mem, 357 352 enum xdp_mem_type type, void *allocator); 358 353 void xdp_unreg_mem_model(struct xdp_mem_info *mem); 354 + int xdp_reg_page_pool(struct page_pool *pool); 355 + void xdp_unreg_page_pool(const struct page_pool *pool); 356 + void xdp_rxq_info_attach_page_pool(struct xdp_rxq_info *xdp_rxq, 357 + const struct page_pool *pool); 358 + 359 + /** 360 + * xdp_rxq_info_attach_mem_model - attach registered mem info to RxQ info 361 + * @xdp_rxq: XDP RxQ info to attach the memory info to 362 + * @mem: already registered memory info 363 + * 364 + * If the driver registers its memory providers manually, it must use this 365 + * function instead of xdp_rxq_info_reg_mem_model(). 366 + */ 367 + static inline void 368 + xdp_rxq_info_attach_mem_model(struct xdp_rxq_info *xdp_rxq, 369 + const struct xdp_mem_info *mem) 370 + { 371 + xdp_rxq->mem = *mem; 372 + } 373 + 374 + /** 375 + * xdp_rxq_info_detach_mem_model - detach registered mem info from RxQ info 376 + * @xdp_rxq: XDP RxQ info to detach the memory info from 377 + * 378 + * If the driver registers its memory providers manually and then attaches it 379 + * via xdp_rxq_info_attach_mem_model(), it must call this function before 380 + * xdp_rxq_info_unreg(). 381 + */ 382 + static inline void xdp_rxq_info_detach_mem_model(struct xdp_rxq_info *xdp_rxq) 383 + { 384 + xdp_rxq->mem = (struct xdp_mem_info){ }; 385 + } 359 386 360 387 /* Drivers not supporting XDP metadata can use this helper, which 361 388 * rejects any room expansion for metadata as a result.
+6 -5
include/net/xdp_sock_drv.h
··· 101 101 return xp_alloc(pool); 102 102 } 103 103 104 - static inline bool xsk_is_eop_desc(struct xdp_desc *desc) 104 + static inline bool xsk_is_eop_desc(const struct xdp_desc *desc) 105 105 { 106 106 return !xp_mb_desc(desc); 107 107 } ··· 143 143 list_add_tail(&frag->list_node, &frag->pool->xskb_list); 144 144 } 145 145 146 - static inline struct xdp_buff *xsk_buff_get_frag(struct xdp_buff *first) 146 + static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first) 147 147 { 148 148 struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp); 149 149 struct xdp_buff *ret = NULL; ··· 200 200 XDP_TXMD_FLAGS_CHECKSUM | \ 201 201 0) 202 202 203 - static inline bool xsk_buff_valid_tx_metadata(struct xsk_tx_metadata *meta) 203 + static inline bool 204 + xsk_buff_valid_tx_metadata(const struct xsk_tx_metadata *meta) 204 205 { 205 206 return !(meta->flags & ~XDP_TXMD_FLAGS_VALID); 206 207 } ··· 338 337 return NULL; 339 338 } 340 339 341 - static inline bool xsk_is_eop_desc(struct xdp_desc *desc) 340 + static inline bool xsk_is_eop_desc(const struct xdp_desc *desc) 342 341 { 343 342 return false; 344 343 } ··· 361 360 { 362 361 } 363 362 364 - static inline struct xdp_buff *xsk_buff_get_frag(struct xdp_buff *first) 363 + static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first) 365 364 { 366 365 return NULL; 367 366 }
+2 -2
include/net/xsk_buff_pool.h
··· 29 29 dma_addr_t frame_dma; 30 30 struct xsk_buff_pool *pool; 31 31 struct list_head list_node; 32 - }; 32 + } __aligned_largest; 33 33 34 34 #define XSK_CHECK_PRIV_TYPE(t) BUILD_BUG_ON(sizeof(t) > offsetofend(struct xdp_buff_xsk, cb)) 35 35 #define XSK_TX_COMPL_FITS(t) BUILD_BUG_ON(sizeof(struct xsk_tx_metadata_compl) > sizeof(t)) ··· 183 183 !(pool->dma_pages[addr >> PAGE_SHIFT] & XSK_NEXT_PG_CONTIG_MASK); 184 184 } 185 185 186 - static inline bool xp_mb_desc(struct xdp_desc *desc) 186 + static inline bool xp_mb_desc(const struct xdp_desc *desc) 187 187 { 188 188 return desc->options & XDP_PKT_CONTD; 189 189 }
+4 -4
kernel/bpf/devmap.c
··· 678 678 } 679 679 680 680 int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, 681 - struct bpf_prog *xdp_prog) 681 + const struct bpf_prog *xdp_prog) 682 682 { 683 683 int err; 684 684 ··· 701 701 702 702 static int dev_map_redirect_clone(struct bpf_dtab_netdev *dst, 703 703 struct sk_buff *skb, 704 - struct bpf_prog *xdp_prog) 704 + const struct bpf_prog *xdp_prog) 705 705 { 706 706 struct sk_buff *nskb; 707 707 int err; ··· 720 720 } 721 721 722 722 int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, 723 - struct bpf_prog *xdp_prog, struct bpf_map *map, 724 - bool exclude_ingress) 723 + const struct bpf_prog *xdp_prog, 724 + struct bpf_map *map, bool exclude_ingress) 725 725 { 726 726 struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); 727 727 struct bpf_dtab_netdev *dst, *last_dst = NULL;
+14 -6
net/core/dev.c
··· 460 460 * PP consumers must pay attention to run APIs in the appropriate context 461 461 * (e.g. NAPI context). 462 462 */ 463 - static DEFINE_PER_CPU(struct page_pool *, system_page_pool); 463 + DEFINE_PER_CPU(struct page_pool *, system_page_pool); 464 464 465 465 #ifdef CONFIG_LOCKDEP 466 466 /* ··· 4931 4931 } 4932 4932 4933 4933 u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp, 4934 - struct bpf_prog *xdp_prog) 4934 + const struct bpf_prog *xdp_prog) 4935 4935 { 4936 4936 void *orig_data, *orig_data_end, *hard_start; 4937 4937 struct netdev_rx_queue *rxqueue; ··· 5033 5033 } 5034 5034 5035 5035 static int 5036 - netif_skb_check_for_xdp(struct sk_buff **pskb, struct bpf_prog *prog) 5036 + netif_skb_check_for_xdp(struct sk_buff **pskb, const struct bpf_prog *prog) 5037 5037 { 5038 5038 struct sk_buff *skb = *pskb; 5039 5039 int err, hroom, troom; ··· 5057 5057 5058 5058 static u32 netif_receive_generic_xdp(struct sk_buff **pskb, 5059 5059 struct xdp_buff *xdp, 5060 - struct bpf_prog *xdp_prog) 5060 + const struct bpf_prog *xdp_prog) 5061 5061 { 5062 5062 struct sk_buff *skb = *pskb; 5063 5063 u32 mac_len, act = XDP_DROP; ··· 5110 5110 * and DDOS attacks will be more effective. In-driver-XDP use dedicated TX 5111 5111 * queues, so they do not have this starvation issue. 5112 5112 */ 5113 - void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog) 5113 + void generic_xdp_tx(struct sk_buff *skb, const struct bpf_prog *xdp_prog) 5114 5114 { 5115 5115 struct net_device *dev = skb->dev; 5116 5116 struct netdev_queue *txq; ··· 5135 5135 5136 5136 static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key); 5137 5137 5138 - int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff **pskb) 5138 + int do_xdp_generic(const struct bpf_prog *xdp_prog, struct sk_buff **pskb) 5139 5139 { 5140 5140 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 5141 5141 ··· 12152 12152 .nid = cpu_to_mem(cpuid), 12153 12153 }; 12154 12154 struct page_pool *pp_ptr; 12155 + int err; 12155 12156 12156 12157 pp_ptr = page_pool_create_percpu(&page_pool_params, cpuid); 12157 12158 if (IS_ERR(pp_ptr)) 12158 12159 return -ENOMEM; 12160 + 12161 + err = xdp_reg_page_pool(pp_ptr); 12162 + if (err) { 12163 + page_pool_destroy(pp_ptr); 12164 + return err; 12165 + } 12159 12166 12160 12167 per_cpu(system_page_pool, cpuid) = pp_ptr; 12161 12168 #endif ··· 12297 12290 if (!pp_ptr) 12298 12291 continue; 12299 12292 12293 + xdp_unreg_page_pool(pp_ptr); 12300 12294 page_pool_destroy(pp_ptr); 12301 12295 per_cpu(system_page_pool, i) = NULL; 12302 12296 }
+16 -13
net/core/filter.c
··· 4348 4348 EXPORT_SYMBOL_GPL(xdp_master_redirect); 4349 4349 4350 4350 static inline int __xdp_do_redirect_xsk(struct bpf_redirect_info *ri, 4351 - struct net_device *dev, 4351 + const struct net_device *dev, 4352 4352 struct xdp_buff *xdp, 4353 - struct bpf_prog *xdp_prog) 4353 + const struct bpf_prog *xdp_prog) 4354 4354 { 4355 4355 enum bpf_map_type map_type = ri->map_type; 4356 4356 void *fwd = ri->tgt_value; ··· 4371 4371 return err; 4372 4372 } 4373 4373 4374 - static __always_inline int __xdp_do_redirect_frame(struct bpf_redirect_info *ri, 4375 - struct net_device *dev, 4376 - struct xdp_frame *xdpf, 4377 - struct bpf_prog *xdp_prog) 4374 + static __always_inline int 4375 + __xdp_do_redirect_frame(struct bpf_redirect_info *ri, struct net_device *dev, 4376 + struct xdp_frame *xdpf, 4377 + const struct bpf_prog *xdp_prog) 4378 4378 { 4379 4379 enum bpf_map_type map_type = ri->map_type; 4380 4380 void *fwd = ri->tgt_value; ··· 4443 4443 } 4444 4444 4445 4445 int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, 4446 - struct bpf_prog *xdp_prog) 4446 + const struct bpf_prog *xdp_prog) 4447 4447 { 4448 4448 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); 4449 4449 enum bpf_map_type map_type = ri->map_type; ··· 4457 4457 EXPORT_SYMBOL_GPL(xdp_do_redirect); 4458 4458 4459 4459 int xdp_do_redirect_frame(struct net_device *dev, struct xdp_buff *xdp, 4460 - struct xdp_frame *xdpf, struct bpf_prog *xdp_prog) 4460 + struct xdp_frame *xdpf, 4461 + const struct bpf_prog *xdp_prog) 4461 4462 { 4462 4463 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); 4463 4464 enum bpf_map_type map_type = ri->map_type; ··· 4473 4472 static int xdp_do_generic_redirect_map(struct net_device *dev, 4474 4473 struct sk_buff *skb, 4475 4474 struct xdp_buff *xdp, 4476 - struct bpf_prog *xdp_prog, void *fwd, 4477 - enum bpf_map_type map_type, u32 map_id, 4478 - u32 flags) 4475 + const struct bpf_prog *xdp_prog, 4476 + void *fwd, enum bpf_map_type map_type, 4477 + u32 map_id, u32 flags) 4479 4478 { 4480 4479 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); 4481 4480 struct bpf_map *map; ··· 4529 4528 } 4530 4529 4531 4530 int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb, 4532 - struct xdp_buff *xdp, struct bpf_prog *xdp_prog) 4531 + struct xdp_buff *xdp, 4532 + const struct bpf_prog *xdp_prog) 4533 4533 { 4534 4534 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); 4535 4535 enum bpf_map_type map_type = ri->map_type; ··· 9077 9075 return __is_valid_xdp_access(off, size); 9078 9076 } 9079 9077 9080 - void bpf_warn_invalid_xdp_action(struct net_device *dev, struct bpf_prog *prog, u32 act) 9078 + void bpf_warn_invalid_xdp_action(const struct net_device *dev, 9079 + const struct bpf_prog *prog, u32 act) 9081 9080 { 9082 9081 const u32 act_max = XDP_REDIRECT; 9083 9082
+15 -15
net/core/page_pool.c
··· 840 840 EXPORT_SYMBOL(page_pool_put_unrefed_page); 841 841 842 842 /** 843 - * page_pool_put_page_bulk() - release references on multiple pages 843 + * page_pool_put_netmem_bulk() - release references on multiple netmems 844 844 * @pool: pool from which pages were allocated 845 - * @data: array holding page pointers 846 - * @count: number of pages in @data 845 + * @data: array holding netmem references 846 + * @count: number of entries in @data 847 847 * 848 - * Tries to refill a number of pages into the ptr_ring cache holding ptr_ring 849 - * producer lock. If the ptr_ring is full, page_pool_put_page_bulk() 850 - * will release leftover pages to the page allocator. 851 - * page_pool_put_page_bulk() is suitable to be run inside the driver NAPI tx 848 + * Tries to refill a number of netmems into the ptr_ring cache holding ptr_ring 849 + * producer lock. If the ptr_ring is full, page_pool_put_netmem_bulk() 850 + * will release leftover netmems to the memory provider. 851 + * page_pool_put_netmem_bulk() is suitable to be run inside the driver NAPI tx 852 852 * completion loop for the XDP_REDIRECT use case. 853 853 * 854 854 * Please note the caller must not use data area after running 855 - * page_pool_put_page_bulk(), as this function overwrites it. 855 + * page_pool_put_netmem_bulk(), as this function overwrites it. 856 856 */ 857 - void page_pool_put_page_bulk(struct page_pool *pool, void **data, 858 - int count) 857 + void page_pool_put_netmem_bulk(struct page_pool *pool, netmem_ref *data, 858 + u32 count) 859 859 { 860 860 int i, bulk_len = 0; 861 861 bool allow_direct; ··· 864 864 allow_direct = page_pool_napi_local(pool); 865 865 866 866 for (i = 0; i < count; i++) { 867 - netmem_ref netmem = page_to_netmem(virt_to_head_page(data[i])); 867 + netmem_ref netmem = netmem_compound_head(data[i]); 868 868 869 869 /* It is not the last user for the page frag case */ 870 870 if (!page_pool_is_last_ref(netmem)) ··· 873 873 netmem = __page_pool_put_page(pool, netmem, -1, allow_direct); 874 874 /* Approved for bulk recycling in ptr_ring cache */ 875 875 if (netmem) 876 - data[bulk_len++] = (__force void *)netmem; 876 + data[bulk_len++] = netmem; 877 877 } 878 878 879 879 if (!bulk_len) ··· 882 882 /* Bulk producer into ptr_ring page_pool cache */ 883 883 in_softirq = page_pool_producer_lock(pool); 884 884 for (i = 0; i < bulk_len; i++) { 885 - if (__ptr_ring_produce(&pool->ring, data[i])) { 885 + if (__ptr_ring_produce(&pool->ring, (__force void *)data[i])) { 886 886 /* ring full */ 887 887 recycle_stat_inc(pool, ring_full); 888 888 break; ··· 899 899 * since put_page() with refcnt == 1 can be an expensive operation 900 900 */ 901 901 for (; i < bulk_len; i++) 902 - page_pool_return_page(pool, (__force netmem_ref)data[i]); 902 + page_pool_return_page(pool, data[i]); 903 903 } 904 - EXPORT_SYMBOL(page_pool_put_page_bulk); 904 + EXPORT_SYMBOL(page_pool_put_netmem_bulk); 905 905 906 906 static netmem_ref page_pool_drain_frag(struct page_pool *pool, 907 907 netmem_ref netmem)
+1 -1
net/core/skbuff.c
··· 1009 1009 EXPORT_SYMBOL(skb_pp_cow_data); 1010 1010 1011 1011 int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb, 1012 - struct bpf_prog *prog) 1012 + const struct bpf_prog *prog) 1013 1013 { 1014 1014 if (!prog->aux->xdp_has_frags) 1015 1015 return -EINVAL;
+62 -3
net/core/xdp.c
··· 358 358 if (IS_ERR(xdp_alloc)) 359 359 return PTR_ERR(xdp_alloc); 360 360 361 + if (type == MEM_TYPE_XSK_BUFF_POOL && allocator) 362 + xsk_pool_set_rxq_info(allocator, xdp_rxq); 363 + 361 364 if (trace_mem_connect_enabled() && xdp_alloc) 362 365 trace_mem_connect(xdp_alloc, xdp_rxq); 363 366 return 0; 364 367 } 365 368 366 369 EXPORT_SYMBOL_GPL(xdp_rxq_info_reg_mem_model); 370 + 371 + /** 372 + * xdp_reg_page_pool - register &page_pool as a memory provider for XDP 373 + * @pool: &page_pool to register 374 + * 375 + * Can be used to register pools manually without connecting to any XDP RxQ 376 + * info, so that the XDP layer will be aware of them. Then, they can be 377 + * attached to an RxQ info manually via xdp_rxq_info_attach_page_pool(). 378 + * 379 + * Return: %0 on success, -errno on error. 380 + */ 381 + int xdp_reg_page_pool(struct page_pool *pool) 382 + { 383 + struct xdp_mem_info mem; 384 + 385 + return xdp_reg_mem_model(&mem, MEM_TYPE_PAGE_POOL, pool); 386 + } 387 + EXPORT_SYMBOL_GPL(xdp_reg_page_pool); 388 + 389 + /** 390 + * xdp_unreg_page_pool - unregister &page_pool from the memory providers list 391 + * @pool: &page_pool to unregister 392 + * 393 + * A shorthand for manual unregistering page pools. If the pool was previously 394 + * attached to an RxQ info, it must be detached first. 395 + */ 396 + void xdp_unreg_page_pool(const struct page_pool *pool) 397 + { 398 + struct xdp_mem_info mem = { 399 + .type = MEM_TYPE_PAGE_POOL, 400 + .id = pool->xdp_mem_id, 401 + }; 402 + 403 + xdp_unreg_mem_model(&mem); 404 + } 405 + EXPORT_SYMBOL_GPL(xdp_unreg_page_pool); 406 + 407 + /** 408 + * xdp_rxq_info_attach_page_pool - attach registered pool to RxQ info 409 + * @xdp_rxq: XDP RxQ info to attach the pool to 410 + * @pool: pool to attach 411 + * 412 + * If the pool was registered manually, this function must be called instead 413 + * of xdp_rxq_info_reg_mem_model() to connect it to the RxQ info. 414 + */ 415 + void xdp_rxq_info_attach_page_pool(struct xdp_rxq_info *xdp_rxq, 416 + const struct page_pool *pool) 417 + { 418 + struct xdp_mem_info mem = { 419 + .type = MEM_TYPE_PAGE_POOL, 420 + .id = pool->xdp_mem_id, 421 + }; 422 + 423 + xdp_rxq_info_attach_mem_model(xdp_rxq, &mem); 424 + } 425 + EXPORT_SYMBOL_GPL(xdp_rxq_info_attach_page_pool); 367 426 368 427 /* XDP RX runs under NAPI protection, and in different delivery error 369 428 * scenarios (e.g. queue full), it is possible to return the xdp_frame ··· 518 459 if (unlikely(!xa || !bq->count)) 519 460 return; 520 461 521 - page_pool_put_page_bulk(xa->page_pool, bq->q, bq->count); 462 + page_pool_put_netmem_bulk(xa->page_pool, bq->q, bq->count); 522 463 /* bq->xa is not cleared to save lookup, if mem.id same in next bulk */ 523 464 bq->count = 0; 524 465 } ··· 559 500 for (i = 0; i < sinfo->nr_frags; i++) { 560 501 skb_frag_t *frag = &sinfo->frags[i]; 561 502 562 - bq->q[bq->count++] = skb_frag_address(frag); 503 + bq->q[bq->count++] = skb_frag_netmem(frag); 563 504 if (bq->count == XDP_BULK_QUEUE_SIZE) 564 505 xdp_flush_frame_bulk(bq); 565 506 } 566 507 } 567 - bq->q[bq->count++] = xdpf->data; 508 + bq->q[bq->count++] = virt_to_netmem(xdpf->data); 568 509 } 569 510 EXPORT_SYMBOL_GPL(xdp_return_frame_bulk); 570 511