Merge branch 'bpf-next/xdp_pull_data' into 'bpf-next/master'

+5

include/net/xdp.h

··· 115 115 xdp->flags |= XDP_FLAGS_FRAGS_PF_MEMALLOC; 116 116 } 117 117 118 + static __always_inline void xdp_buff_clear_frag_pfmemalloc(struct xdp_buff *xdp) 119 + { 120 + xdp->flags &= ~XDP_FLAGS_FRAGS_PF_MEMALLOC; 121 + } 122 + 118 123 static __always_inline void 119 124 xdp_init_buff(struct xdp_buff *xdp, u32 frame_sz, struct xdp_rxq_info *rxq) 120 125 {

+18 -3

include/net/xdp_sock_drv.h

··· 160 160 return ret; 161 161 } 162 162 163 - static inline void xsk_buff_del_tail(struct xdp_buff *tail) 163 + static inline void xsk_buff_del_frag(struct xdp_buff *xdp) 164 164 { 165 - struct xdp_buff_xsk *xskb = container_of(tail, struct xdp_buff_xsk, xdp); 165 + struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); 166 166 167 167 list_del(&xskb->list_node); 168 + } 169 + 170 + static inline struct xdp_buff *xsk_buff_get_head(struct xdp_buff *first) 171 + { 172 + struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp); 173 + struct xdp_buff_xsk *frag; 174 + 175 + frag = list_first_entry(&xskb->pool->xskb_list, struct xdp_buff_xsk, 176 + list_node); 177 + return &frag->xdp; 168 178 } 169 179 170 180 static inline struct xdp_buff *xsk_buff_get_tail(struct xdp_buff *first) ··· 399 389 return NULL; 400 390 } 401 391 402 - static inline void xsk_buff_del_tail(struct xdp_buff *tail) 392 + static inline void xsk_buff_del_frag(struct xdp_buff *xdp) 403 393 { 394 + } 395 + 396 + static inline struct xdp_buff *xsk_buff_get_head(struct xdp_buff *first) 397 + { 398 + return NULL; 404 399 } 405 400 406 401 static inline struct xdp_buff *xsk_buff_get_tail(struct xdp_buff *first)

+13

kernel/bpf/verifier.c

··· 12234 12234 KF_bpf_dynptr_from_skb, 12235 12235 KF_bpf_dynptr_from_xdp, 12236 12236 KF_bpf_dynptr_from_skb_meta, 12237 + KF_bpf_xdp_pull_data, 12237 12238 KF_bpf_dynptr_slice, 12238 12239 KF_bpf_dynptr_slice_rdwr, 12239 12240 KF_bpf_dynptr_clone, ··· 12287 12286 BTF_ID(func, bpf_dynptr_from_skb) 12288 12287 BTF_ID(func, bpf_dynptr_from_xdp) 12289 12288 BTF_ID(func, bpf_dynptr_from_skb_meta) 12289 + BTF_ID(func, bpf_xdp_pull_data) 12290 12290 #else 12291 + BTF_ID_UNUSED 12291 12292 BTF_ID_UNUSED 12292 12293 BTF_ID_UNUSED 12293 12294 BTF_ID_UNUSED ··· 12368 12365 static bool is_kfunc_bpf_preempt_enable(struct bpf_kfunc_call_arg_meta *meta) 12369 12366 { 12370 12367 return meta->func_id == special_kfunc_list[KF_bpf_preempt_enable]; 12368 + } 12369 + 12370 + static bool is_kfunc_pkt_changing(struct bpf_kfunc_call_arg_meta *meta) 12371 + { 12372 + return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data]; 12371 12373 } 12372 12374 12373 12375 static enum kfunc_ptr_arg_type ··· 14134 14126 } 14135 14127 } 14136 14128 } 14129 + 14130 + if (is_kfunc_pkt_changing(&meta)) 14131 + clear_all_pkt_pointers(env); 14137 14132 14138 14133 nargs = btf_type_vlen(meta.func_proto); 14139 14134 args = (const struct btf_param *)(meta.func_proto + 1); ··· 17854 17843 */ 17855 17844 if (ret == 0 && is_kfunc_sleepable(&meta)) 17856 17845 mark_subprog_might_sleep(env, t); 17846 + if (ret == 0 && is_kfunc_pkt_changing(&meta)) 17847 + mark_subprog_changes_pkt_data(env, t); 17857 17848 } 17858 17849 return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL); 17859 17850

+23 -14

net/bpf/test_run.c

··· 665 665 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 666 666 void *data; 667 667 668 - if (user_size < ETH_HLEN || user_size > PAGE_SIZE - headroom - tailroom) 668 + if (user_size > PAGE_SIZE - headroom - tailroom) 669 669 return ERR_PTR(-EINVAL); 670 670 671 671 size = SKB_DATA_ALIGN(size); ··· 1001 1001 kattr->test.cpu || kattr->test.batch_size) 1002 1002 return -EINVAL; 1003 1003 1004 + if (size < ETH_HLEN) 1005 + return -EINVAL; 1006 + 1004 1007 data = bpf_test_init(kattr, kattr->test.data_size_in, 1005 1008 size, NET_SKB_PAD + NET_IP_ALIGN, 1006 1009 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); ··· 1210 1207 { 1211 1208 bool do_live = (kattr->test.flags & BPF_F_TEST_XDP_LIVE_FRAMES); 1212 1209 u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1210 + u32 retval = 0, meta_sz = 0, duration, max_linear_sz, size; 1211 + u32 linear_sz = kattr->test.data_size_in; 1213 1212 u32 batch_size = kattr->test.batch_size; 1214 - u32 retval = 0, duration, max_data_sz; 1215 - u32 size = kattr->test.data_size_in; 1216 1213 u32 headroom = XDP_PACKET_HEADROOM; 1217 1214 u32 repeat = kattr->test.repeat; 1218 1215 struct netdev_rx_queue *rxqueue; ··· 1249 1246 1250 1247 if (ctx) { 1251 1248 /* There can't be user provided data before the meta data */ 1252 - if (ctx->data_meta || ctx->data_end != size || 1249 + if (ctx->data_meta || ctx->data_end > kattr->test.data_size_in || 1253 1250 ctx->data > ctx->data_end || 1254 1251 unlikely(xdp_metalen_invalid(ctx->data)) || 1255 1252 (do_live && (kattr->test.data_out || kattr->test.ctx_out))) 1256 1253 goto free_ctx; 1257 1254 /* Meta data is allocated from the headroom */ 1258 1255 headroom -= ctx->data; 1256 + 1257 + meta_sz = ctx->data; 1258 + linear_sz = ctx->data_end; 1259 1259 } 1260 1260 1261 - max_data_sz = PAGE_SIZE - headroom - tailroom; 1262 - if (size > max_data_sz) { 1263 - /* disallow live data mode for jumbo frames */ 1264 - if (do_live) 1265 - goto free_ctx; 1266 - size = max_data_sz; 1267 - } 1261 + max_linear_sz = PAGE_SIZE - headroom - tailroom; 1262 + linear_sz = min_t(u32, linear_sz, max_linear_sz); 1268 1263 1269 - data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom); 1264 + /* disallow live data mode for jumbo frames */ 1265 + if (do_live && kattr->test.data_size_in > linear_sz) 1266 + goto free_ctx; 1267 + 1268 + if (kattr->test.data_size_in - meta_sz < ETH_HLEN) 1269 + return -EINVAL; 1270 + 1271 + data = bpf_test_init(kattr, linear_sz, max_linear_sz, headroom, tailroom); 1270 1272 if (IS_ERR(data)) { 1271 1273 ret = PTR_ERR(data); 1272 1274 goto free_ctx; 1273 1275 } 1274 1276 1275 1277 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); 1276 - rxqueue->xdp_rxq.frag_size = headroom + max_data_sz + tailroom; 1278 + rxqueue->xdp_rxq.frag_size = PAGE_SIZE; 1277 1279 xdp_init_buff(&xdp, rxqueue->xdp_rxq.frag_size, &rxqueue->xdp_rxq); 1278 - xdp_prepare_buff(&xdp, data, headroom, size, true); 1280 + xdp_prepare_buff(&xdp, data, headroom, linear_sz, true); 1279 1281 sinfo = xdp_get_shared_info_from_buff(&xdp); 1280 1282 1281 1283 ret = xdp_convert_md_to_buff(ctx, &xdp); 1282 1284 if (ret) 1283 1285 goto free_data; 1284 1286 1287 + size = linear_sz; 1285 1288 if (unlikely(kattr->test.data_size_in > size)) { 1286 1289 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 1287 1290

+118 -17

net/core/filter.c

··· 4153 4153 return 0; 4154 4154 } 4155 4155 4156 - static void bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink, 4157 - enum xdp_mem_type mem_type, bool release) 4156 + static struct xdp_buff *bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink, 4157 + bool tail, bool release) 4158 4158 { 4159 - struct xdp_buff *zc_frag = xsk_buff_get_tail(xdp); 4159 + struct xdp_buff *zc_frag = tail ? xsk_buff_get_tail(xdp) : 4160 + xsk_buff_get_head(xdp); 4160 4161 4161 4162 if (release) { 4162 - xsk_buff_del_tail(zc_frag); 4163 - __xdp_return(0, mem_type, false, zc_frag); 4163 + xsk_buff_del_frag(zc_frag); 4164 4164 } else { 4165 - zc_frag->data_end -= shrink; 4165 + if (tail) 4166 + zc_frag->data_end -= shrink; 4167 + else 4168 + zc_frag->data += shrink; 4166 4169 } 4170 + 4171 + return zc_frag; 4167 4172 } 4168 4173 4169 4174 static bool bpf_xdp_shrink_data(struct xdp_buff *xdp, skb_frag_t *frag, 4170 - int shrink) 4175 + int shrink, bool tail) 4171 4176 { 4172 4177 enum xdp_mem_type mem_type = xdp->rxq->mem.type; 4173 4178 bool release = skb_frag_size(frag) == shrink; 4179 + netmem_ref netmem = skb_frag_netmem(frag); 4180 + struct xdp_buff *zc_frag = NULL; 4174 4181 4175 4182 if (mem_type == MEM_TYPE_XSK_BUFF_POOL) { 4176 - bpf_xdp_shrink_data_zc(xdp, shrink, mem_type, release); 4177 - goto out; 4183 + netmem = 0; 4184 + zc_frag = bpf_xdp_shrink_data_zc(xdp, shrink, tail, release); 4178 4185 } 4179 4186 4180 - if (release) 4181 - __xdp_return(skb_frag_netmem(frag), mem_type, false, NULL); 4187 + if (release) { 4188 + __xdp_return(netmem, mem_type, false, zc_frag); 4189 + } else { 4190 + if (!tail) 4191 + skb_frag_off_add(frag, shrink); 4192 + skb_frag_size_sub(frag, shrink); 4193 + } 4182 4194 4183 - out: 4184 4195 return release; 4185 4196 } 4186 4197 ··· 4209 4198 4210 4199 len_free += shrink; 4211 4200 offset -= shrink; 4212 - if (bpf_xdp_shrink_data(xdp, frag, shrink)) { 4201 + if (bpf_xdp_shrink_data(xdp, frag, shrink, true)) 4213 4202 n_frags_free++; 4214 - } else { 4215 - skb_frag_size_sub(frag, shrink); 4216 - break; 4217 - } 4218 4203 } 4219 4204 sinfo->nr_frags -= n_frags_free; 4220 4205 sinfo->xdp_frags_size -= len_free; 4221 4206 4222 4207 if (unlikely(!sinfo->nr_frags)) { 4223 4208 xdp_buff_clear_frags_flag(xdp); 4209 + xdp_buff_clear_frag_pfmemalloc(xdp); 4224 4210 xdp->data_end -= offset; 4225 4211 } 4226 4212 ··· 12218 12210 return 0; 12219 12211 } 12220 12212 12213 + /** 12214 + * bpf_xdp_pull_data() - Pull in non-linear xdp data. 12215 + * @x: &xdp_md associated with the XDP buffer 12216 + * @len: length of data to be made directly accessible in the linear part 12217 + * 12218 + * Pull in data in case the XDP buffer associated with @x is non-linear and 12219 + * not all @len are in the linear data area. 12220 + * 12221 + * Direct packet access allows reading and writing linear XDP data through 12222 + * packet pointers (i.e., &xdp_md->data + offsets). The amount of data which 12223 + * ends up in the linear part of the xdp_buff depends on the NIC and its 12224 + * configuration. When a frag-capable XDP program wants to directly access 12225 + * headers that may be in the non-linear area, call this kfunc to make sure 12226 + * the data is available in the linear area. Alternatively, use dynptr or 12227 + * bpf_xdp_{load,store}_bytes() to access data without pulling. 12228 + * 12229 + * This kfunc can also be used with bpf_xdp_adjust_head() to decapsulate 12230 + * headers in the non-linear data area. 12231 + * 12232 + * A call to this kfunc may reduce headroom. If there is not enough tailroom 12233 + * in the linear data area, metadata and data will be shifted down. 12234 + * 12235 + * A call to this kfunc is susceptible to change the buffer geometry. 12236 + * Therefore, at load time, all checks on pointers previously done by the 12237 + * verifier are invalidated and must be performed again, if the kfunc is used 12238 + * in combination with direct packet access. 12239 + * 12240 + * Return: 12241 + * * %0 - success 12242 + * * %-EINVAL - invalid len 12243 + */ 12244 + __bpf_kfunc int bpf_xdp_pull_data(struct xdp_md *x, u32 len) 12245 + { 12246 + struct xdp_buff *xdp = (struct xdp_buff *)x; 12247 + struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); 12248 + int i, delta, shift, headroom, tailroom, n_frags_free = 0; 12249 + void *data_hard_end = xdp_data_hard_end(xdp); 12250 + int data_len = xdp->data_end - xdp->data; 12251 + void *start; 12252 + 12253 + if (len <= data_len) 12254 + return 0; 12255 + 12256 + if (unlikely(len > xdp_get_buff_len(xdp))) 12257 + return -EINVAL; 12258 + 12259 + start = xdp_data_meta_unsupported(xdp) ? xdp->data : xdp->data_meta; 12260 + 12261 + headroom = start - xdp->data_hard_start - sizeof(struct xdp_frame); 12262 + tailroom = data_hard_end - xdp->data_end; 12263 + 12264 + delta = len - data_len; 12265 + if (unlikely(delta > tailroom + headroom)) 12266 + return -EINVAL; 12267 + 12268 + shift = delta - tailroom; 12269 + if (shift > 0) { 12270 + memmove(start - shift, start, xdp->data_end - start); 12271 + 12272 + xdp->data_meta -= shift; 12273 + xdp->data -= shift; 12274 + xdp->data_end -= shift; 12275 + } 12276 + 12277 + for (i = 0; i < sinfo->nr_frags && delta; i++) { 12278 + skb_frag_t *frag = &sinfo->frags[i]; 12279 + u32 shrink = min_t(u32, delta, skb_frag_size(frag)); 12280 + 12281 + memcpy(xdp->data_end, skb_frag_address(frag), shrink); 12282 + 12283 + xdp->data_end += shrink; 12284 + sinfo->xdp_frags_size -= shrink; 12285 + delta -= shrink; 12286 + if (bpf_xdp_shrink_data(xdp, frag, shrink, false)) 12287 + n_frags_free++; 12288 + } 12289 + 12290 + if (unlikely(n_frags_free)) { 12291 + memmove(sinfo->frags, sinfo->frags + n_frags_free, 12292 + (sinfo->nr_frags - n_frags_free) * sizeof(skb_frag_t)); 12293 + 12294 + sinfo->nr_frags -= n_frags_free; 12295 + 12296 + if (!sinfo->nr_frags) { 12297 + xdp_buff_clear_frags_flag(xdp); 12298 + xdp_buff_clear_frag_pfmemalloc(xdp); 12299 + } 12300 + } 12301 + 12302 + return 0; 12303 + } 12304 + 12221 12305 __bpf_kfunc_end_defs(); 12222 12306 12223 12307 int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags, ··· 12337 12237 12338 12238 BTF_KFUNCS_START(bpf_kfunc_check_set_xdp) 12339 12239 BTF_ID_FLAGS(func, bpf_dynptr_from_xdp) 12240 + BTF_ID_FLAGS(func, bpf_xdp_pull_data) 12340 12241 BTF_KFUNCS_END(bpf_kfunc_check_set_xdp) 12341 12242 12342 12243 BTF_KFUNCS_START(bpf_kfunc_check_set_sock_addr)

+1 -3

tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c

··· 97 97 /* Meta data must be 255 bytes or smaller */ 98 98 test_xdp_context_error(prog_fd, opts, 0, 256, sizeof(data), 0, 0, 0); 99 99 100 - /* Total size of data must match data_end - data_meta */ 101 - test_xdp_context_error(prog_fd, opts, 0, sizeof(__u32), 102 - sizeof(data) - 1, 0, 0, 0); 100 + /* Total size of data must be data_end - data_meta or larger */ 103 101 test_xdp_context_error(prog_fd, opts, 0, sizeof(__u32), 104 102 sizeof(data) + 1, 0, 0, 0); 105 103

+179

tools/testing/selftests/bpf/prog_tests/xdp_pull_data.c

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + #include <test_progs.h> 4 + #include <network_helpers.h> 5 + #include "test_xdp_pull_data.skel.h" 6 + 7 + #define PULL_MAX (1 << 31) 8 + #define PULL_PLUS_ONE (1 << 30) 9 + 10 + #define XDP_PACKET_HEADROOM 256 11 + 12 + /* Find headroom and tailroom occupied by struct xdp_frame and struct 13 + * skb_shared_info so that we can calculate the maximum pull lengths for 14 + * test cases. They might not be the real size of the structures due to 15 + * cache alignment. 16 + */ 17 + static int find_xdp_sizes(struct test_xdp_pull_data *skel, int frame_sz) 18 + { 19 + LIBBPF_OPTS(bpf_test_run_opts, topts); 20 + struct xdp_md ctx = {}; 21 + int prog_fd, err; 22 + __u8 *buf; 23 + 24 + buf = calloc(frame_sz, sizeof(__u8)); 25 + if (!ASSERT_OK_PTR(buf, "calloc buf")) 26 + return -ENOMEM; 27 + 28 + topts.data_in = buf; 29 + topts.data_out = buf; 30 + topts.data_size_in = frame_sz; 31 + topts.data_size_out = frame_sz; 32 + /* Pass a data_end larger than the linear space available to make sure 33 + * bpf_prog_test_run_xdp() will fill the linear data area so that 34 + * xdp_find_sizes can infer the size of struct skb_shared_info 35 + */ 36 + ctx.data_end = frame_sz; 37 + topts.ctx_in = &ctx; 38 + topts.ctx_out = &ctx; 39 + topts.ctx_size_in = sizeof(ctx); 40 + topts.ctx_size_out = sizeof(ctx); 41 + 42 + prog_fd = bpf_program__fd(skel->progs.xdp_find_sizes); 43 + err = bpf_prog_test_run_opts(prog_fd, &topts); 44 + ASSERT_OK(err, "bpf_prog_test_run_opts"); 45 + 46 + free(buf); 47 + 48 + return err; 49 + } 50 + 51 + /* xdp_pull_data_prog will directly read a marker 0xbb stored at buf[1024] 52 + * so caller expecting XDP_PASS should always pass pull_len no less than 1024 53 + */ 54 + static void run_test(struct test_xdp_pull_data *skel, int retval, 55 + int frame_sz, int buff_len, int meta_len, int data_len, 56 + int pull_len) 57 + { 58 + LIBBPF_OPTS(bpf_test_run_opts, topts); 59 + struct xdp_md ctx = {}; 60 + int prog_fd, err; 61 + __u8 *buf; 62 + 63 + buf = calloc(buff_len, sizeof(__u8)); 64 + if (!ASSERT_OK_PTR(buf, "calloc buf")) 65 + return; 66 + 67 + buf[meta_len + 1023] = 0xaa; 68 + buf[meta_len + 1024] = 0xbb; 69 + buf[meta_len + 1025] = 0xcc; 70 + 71 + topts.data_in = buf; 72 + topts.data_out = buf; 73 + topts.data_size_in = buff_len; 74 + topts.data_size_out = buff_len; 75 + ctx.data = meta_len; 76 + ctx.data_end = meta_len + data_len; 77 + topts.ctx_in = &ctx; 78 + topts.ctx_out = &ctx; 79 + topts.ctx_size_in = sizeof(ctx); 80 + topts.ctx_size_out = sizeof(ctx); 81 + 82 + skel->bss->data_len = data_len; 83 + if (pull_len & PULL_MAX) { 84 + int headroom = XDP_PACKET_HEADROOM - meta_len - skel->bss->xdpf_sz; 85 + int tailroom = frame_sz - XDP_PACKET_HEADROOM - 86 + data_len - skel->bss->sinfo_sz; 87 + 88 + pull_len = pull_len & PULL_PLUS_ONE ? 1 : 0; 89 + pull_len += headroom + tailroom + data_len; 90 + } 91 + skel->bss->pull_len = pull_len; 92 + 93 + prog_fd = bpf_program__fd(skel->progs.xdp_pull_data_prog); 94 + err = bpf_prog_test_run_opts(prog_fd, &topts); 95 + ASSERT_OK(err, "bpf_prog_test_run_opts"); 96 + ASSERT_EQ(topts.retval, retval, "xdp_pull_data_prog retval"); 97 + 98 + if (retval == XDP_DROP) 99 + goto out; 100 + 101 + ASSERT_EQ(ctx.data_end, meta_len + pull_len, "linear data size"); 102 + ASSERT_EQ(topts.data_size_out, buff_len, "linear + non-linear data size"); 103 + /* Make sure data around xdp->data_end was not messed up by 104 + * bpf_xdp_pull_data() 105 + */ 106 + ASSERT_EQ(buf[meta_len + 1023], 0xaa, "data[1023]"); 107 + ASSERT_EQ(buf[meta_len + 1024], 0xbb, "data[1024]"); 108 + ASSERT_EQ(buf[meta_len + 1025], 0xcc, "data[1025]"); 109 + out: 110 + free(buf); 111 + } 112 + 113 + static void test_xdp_pull_data_basic(void) 114 + { 115 + u32 pg_sz, max_meta_len, max_data_len; 116 + struct test_xdp_pull_data *skel; 117 + 118 + skel = test_xdp_pull_data__open_and_load(); 119 + if (!ASSERT_OK_PTR(skel, "test_xdp_pull_data__open_and_load")) 120 + return; 121 + 122 + pg_sz = sysconf(_SC_PAGE_SIZE); 123 + 124 + if (find_xdp_sizes(skel, pg_sz)) 125 + goto out; 126 + 127 + max_meta_len = XDP_PACKET_HEADROOM - skel->bss->xdpf_sz; 128 + max_data_len = pg_sz - XDP_PACKET_HEADROOM - skel->bss->sinfo_sz; 129 + 130 + /* linear xdp pkt, pull 0 byte */ 131 + run_test(skel, XDP_PASS, pg_sz, 2048, 0, 2048, 2048); 132 + 133 + /* multi-buf pkt, pull results in linear xdp pkt */ 134 + run_test(skel, XDP_PASS, pg_sz, 2048, 0, 1024, 2048); 135 + 136 + /* multi-buf pkt, pull 1 byte to linear data area */ 137 + run_test(skel, XDP_PASS, pg_sz, 9000, 0, 1024, 1025); 138 + 139 + /* multi-buf pkt, pull 0 byte to linear data area */ 140 + run_test(skel, XDP_PASS, pg_sz, 9000, 0, 1025, 1025); 141 + 142 + /* multi-buf pkt, empty linear data area, pull requires memmove */ 143 + run_test(skel, XDP_PASS, pg_sz, 9000, 0, 0, PULL_MAX); 144 + 145 + /* multi-buf pkt, no headroom */ 146 + run_test(skel, XDP_PASS, pg_sz, 9000, max_meta_len, 1024, PULL_MAX); 147 + 148 + /* multi-buf pkt, no tailroom, pull requires memmove */ 149 + run_test(skel, XDP_PASS, pg_sz, 9000, 0, max_data_len, PULL_MAX); 150 + 151 + /* Test cases with invalid pull length */ 152 + 153 + /* linear xdp pkt, pull more than total data len */ 154 + run_test(skel, XDP_DROP, pg_sz, 2048, 0, 2048, 2049); 155 + 156 + /* multi-buf pkt with no space left in linear data area */ 157 + run_test(skel, XDP_DROP, pg_sz, 9000, max_meta_len, max_data_len, 158 + PULL_MAX | PULL_PLUS_ONE); 159 + 160 + /* multi-buf pkt, empty linear data area */ 161 + run_test(skel, XDP_DROP, pg_sz, 9000, 0, 0, PULL_MAX | PULL_PLUS_ONE); 162 + 163 + /* multi-buf pkt, no headroom */ 164 + run_test(skel, XDP_DROP, pg_sz, 9000, max_meta_len, 1024, 165 + PULL_MAX | PULL_PLUS_ONE); 166 + 167 + /* multi-buf pkt, no tailroom */ 168 + run_test(skel, XDP_DROP, pg_sz, 9000, 0, max_data_len, 169 + PULL_MAX | PULL_PLUS_ONE); 170 + 171 + out: 172 + test_xdp_pull_data__destroy(skel); 173 + } 174 + 175 + void test_xdp_pull_data(void) 176 + { 177 + if (test__start_subtest("xdp_pull_data")) 178 + test_xdp_pull_data_basic(); 179 + }

+48

tools/testing/selftests/bpf/progs/test_xdp_pull_data.c

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + #include "vmlinux.h" 4 + #include <bpf/bpf_helpers.h> 5 + 6 + int xdpf_sz; 7 + int sinfo_sz; 8 + int data_len; 9 + int pull_len; 10 + 11 + #define XDP_PACKET_HEADROOM 256 12 + 13 + SEC("xdp.frags") 14 + int xdp_find_sizes(struct xdp_md *ctx) 15 + { 16 + xdpf_sz = sizeof(struct xdp_frame); 17 + sinfo_sz = __PAGE_SIZE - XDP_PACKET_HEADROOM - 18 + (ctx->data_end - ctx->data); 19 + 20 + return XDP_PASS; 21 + } 22 + 23 + SEC("xdp.frags") 24 + int xdp_pull_data_prog(struct xdp_md *ctx) 25 + { 26 + __u8 *data_end = (void *)(long)ctx->data_end; 27 + __u8 *data = (void *)(long)ctx->data; 28 + __u8 *val_p; 29 + int err; 30 + 31 + if (data_len != data_end - data) 32 + return XDP_DROP; 33 + 34 + err = bpf_xdp_pull_data(ctx, pull_len); 35 + if (err) 36 + return XDP_DROP; 37 + 38 + val_p = (void *)(long)ctx->data + 1024; 39 + if (val_p + 1 > (void *)(long)ctx->data_end) 40 + return XDP_DROP; 41 + 42 + if (*val_p != 0xbb) 43 + return XDP_DROP; 44 + 45 + return XDP_PASS; 46 + } 47 + 48 + char _license[] SEC("license") = "GPL";

+74 -15

tools/testing/selftests/net/lib/xdp_native.bpf.c

··· 14 14 #define MAX_PAYLOAD_LEN 5000 15 15 #define MAX_HDR_LEN 64 16 16 17 + extern int bpf_xdp_pull_data(struct xdp_md *xdp, __u32 len) __ksym __weak; 18 + 17 19 enum { 18 20 XDP_MODE = 0, 19 21 XDP_PORT = 1, ··· 70 68 71 69 static struct udphdr *filter_udphdr(struct xdp_md *ctx, __u16 port) 72 70 { 73 - void *data_end = (void *)(long)ctx->data_end; 74 - void *data = (void *)(long)ctx->data; 75 71 struct udphdr *udph = NULL; 76 - struct ethhdr *eth = data; 72 + void *data, *data_end; 73 + struct ethhdr *eth; 74 + int err; 75 + 76 + err = bpf_xdp_pull_data(ctx, sizeof(*eth)); 77 + if (err) 78 + return NULL; 79 + 80 + data_end = (void *)(long)ctx->data_end; 81 + data = eth = (void *)(long)ctx->data; 77 82 78 83 if (data + sizeof(*eth) > data_end) 79 84 return NULL; 80 85 81 86 if (eth->h_proto == bpf_htons(ETH_P_IP)) { 82 - struct iphdr *iph = data + sizeof(*eth); 87 + struct iphdr *iph; 88 + 89 + err = bpf_xdp_pull_data(ctx, sizeof(*eth) + sizeof(*iph) + 90 + sizeof(*udph)); 91 + if (err) 92 + return NULL; 93 + 94 + data_end = (void *)(long)ctx->data_end; 95 + data = (void *)(long)ctx->data; 96 + 97 + iph = data + sizeof(*eth); 83 98 84 99 if (iph + 1 > (struct iphdr *)data_end || 85 100 iph->protocol != IPPROTO_UDP) 86 101 return NULL; 87 102 88 - udph = (void *)eth + sizeof(*iph) + sizeof(*eth); 89 - } else if (eth->h_proto == bpf_htons(ETH_P_IPV6)) { 90 - struct ipv6hdr *ipv6h = data + sizeof(*eth); 103 + udph = data + sizeof(*iph) + sizeof(*eth); 104 + } else if (eth->h_proto == bpf_htons(ETH_P_IPV6)) { 105 + struct ipv6hdr *ipv6h; 106 + 107 + err = bpf_xdp_pull_data(ctx, sizeof(*eth) + sizeof(*ipv6h) + 108 + sizeof(*udph)); 109 + if (err) 110 + return NULL; 111 + 112 + data_end = (void *)(long)ctx->data_end; 113 + data = (void *)(long)ctx->data; 114 + 115 + ipv6h = data + sizeof(*eth); 91 116 92 117 if (ipv6h + 1 > (struct ipv6hdr *)data_end || 93 118 ipv6h->nexthdr != IPPROTO_UDP) 94 119 return NULL; 95 120 96 - udph = (void *)eth + sizeof(*ipv6h) + sizeof(*eth); 121 + udph = data + sizeof(*ipv6h) + sizeof(*eth); 97 122 } else { 98 123 return NULL; 99 124 } ··· 174 145 175 146 static int xdp_mode_tx_handler(struct xdp_md *ctx, __u16 port) 176 147 { 177 - void *data_end = (void *)(long)ctx->data_end; 178 - void *data = (void *)(long)ctx->data; 179 148 struct udphdr *udph = NULL; 180 - struct ethhdr *eth = data; 149 + void *data, *data_end; 150 + struct ethhdr *eth; 151 + int err; 152 + 153 + err = bpf_xdp_pull_data(ctx, sizeof(*eth)); 154 + if (err) 155 + return XDP_PASS; 156 + 157 + data_end = (void *)(long)ctx->data_end; 158 + data = eth = (void *)(long)ctx->data; 181 159 182 160 if (data + sizeof(*eth) > data_end) 183 161 return XDP_PASS; 184 162 185 163 if (eth->h_proto == bpf_htons(ETH_P_IP)) { 186 - struct iphdr *iph = data + sizeof(*eth); 187 - __be32 tmp_ip = iph->saddr; 164 + struct iphdr *iph; 165 + __be32 tmp_ip; 166 + 167 + err = bpf_xdp_pull_data(ctx, sizeof(*eth) + sizeof(*iph) + 168 + sizeof(*udph)); 169 + if (err) 170 + return XDP_PASS; 171 + 172 + data_end = (void *)(long)ctx->data_end; 173 + data = (void *)(long)ctx->data; 174 + 175 + iph = data + sizeof(*eth); 188 176 189 177 if (iph + 1 > (struct iphdr *)data_end || 190 178 iph->protocol != IPPROTO_UDP) ··· 215 169 return XDP_PASS; 216 170 217 171 record_stats(ctx, STATS_RX); 172 + eth = data; 218 173 swap_machdr((void *)eth); 219 174 175 + tmp_ip = iph->saddr; 220 176 iph->saddr = iph->daddr; 221 177 iph->daddr = tmp_ip; 222 178 ··· 226 178 227 179 return XDP_TX; 228 180 229 - } else if (eth->h_proto == bpf_htons(ETH_P_IPV6)) { 230 - struct ipv6hdr *ipv6h = data + sizeof(*eth); 181 + } else if (eth->h_proto == bpf_htons(ETH_P_IPV6)) { 231 182 struct in6_addr tmp_ipv6; 183 + struct ipv6hdr *ipv6h; 184 + 185 + err = bpf_xdp_pull_data(ctx, sizeof(*eth) + sizeof(*ipv6h) + 186 + sizeof(*udph)); 187 + if (err) 188 + return XDP_PASS; 189 + 190 + data_end = (void *)(long)ctx->data_end; 191 + data = (void *)(long)ctx->data; 192 + 193 + ipv6h = data + sizeof(*eth); 232 194 233 195 if (ipv6h + 1 > (struct ipv6hdr *)data_end || 234 196 ipv6h->nexthdr != IPPROTO_UDP) ··· 252 194 return XDP_PASS; 253 195 254 196 record_stats(ctx, STATS_RX); 197 + eth = data; 255 198 swap_machdr((void *)eth); 256 199 257 200 __builtin_memcpy(&tmp_ipv6, &ipv6h->saddr, sizeof(tmp_ipv6));

Configure Feed

Configure Feed