Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'selftests-drv-net-test-xdp-native-support'
Mohsin Bashir says:
====================
selftests: drv-net: Test XDP native support
This patch series add tests to validate XDP native support for PASS,
DROP, ABORT, and TX actions, as well as headroom and tailroom adjustment.
For adjustment tests, validate support for both the extension and
shrinking cases across various packet sizes and offset values.
The pass criteria for head/tail adjustment tests require that at-least
one adjustment value works for at-least one packet size. This ensure
that the variability in maximum supported head/tail adjustment offset
across different drivers is being incorporated.
The results reported in this series are based on netdevsim. However,
the series is tested against multiple other drivers including fbnic.
Note: The XDP support for fbnic will be added later.
====================
Link: https://patch.msgid.link/20250719083059.3209169-1-mohsin.bashr@gmail.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+1298
-1
+20
-1
drivers/net/netdevsim/netdev.c
+20
-1
drivers/net/netdevsim/netdev.c
···
387
387
static int nsim_rcv(struct nsim_rq *rq, int budget)
388
388
{
389
389
struct net_device *dev = rq->napi.dev;
390
+
struct bpf_prog *xdp_prog;
391
+
struct netdevsim *ns;
390
392
struct sk_buff *skb;
391
393
unsigned int skblen;
392
394
int i, ret;
395
+
396
+
ns = netdev_priv(dev);
397
+
xdp_prog = READ_ONCE(ns->xdp.prog);
393
398
394
399
for (i = 0; i < budget; i++) {
395
400
if (skb_queue_empty(&rq->skb_queue))
396
401
break;
397
402
398
403
skb = skb_dequeue(&rq->skb_queue);
404
+
405
+
if (xdp_prog) {
406
+
/* skb might be freed directly by XDP, save the len */
407
+
skblen = skb->len;
408
+
409
+
if (skb->ip_summed == CHECKSUM_PARTIAL)
410
+
skb_checksum_help(skb);
411
+
ret = do_xdp_generic(xdp_prog, &skb);
412
+
if (ret != XDP_PASS) {
413
+
dev_dstats_rx_add(dev, skblen);
414
+
continue;
415
+
}
416
+
}
417
+
399
418
/* skb might be discard at netif_receive_skb, save the len */
400
419
skblen = skb->len;
401
420
skb_mark_napi_id(skb, &rq->napi);
···
955
936
NETIF_F_TSO;
956
937
dev->pcpu_stat_type = NETDEV_PCPU_STAT_DSTATS;
957
938
dev->max_mtu = ETH_MAX_MTU;
958
-
dev->xdp_features = NETDEV_XDP_ACT_HW_OFFLOAD;
939
+
dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_HW_OFFLOAD;
959
940
}
960
941
961
942
static int nsim_queue_init(struct netdevsim *ns)
+1
tools/testing/selftests/drivers/net/Makefile
+1
tools/testing/selftests/drivers/net/Makefile
+656
tools/testing/selftests/drivers/net/xdp.py
+656
tools/testing/selftests/drivers/net/xdp.py
···
1
+
#!/usr/bin/env python3
2
+
# SPDX-License-Identifier: GPL-2.0
3
+
4
+
"""
5
+
This file contains tests to verify native XDP support in network drivers.
6
+
The tests utilize the BPF program `xdp_native.bpf.o` from the `selftests.net.lib`
7
+
directory, with each test focusing on a specific aspect of XDP functionality.
8
+
"""
9
+
import random
10
+
import string
11
+
from dataclasses import dataclass
12
+
from enum import Enum
13
+
14
+
from lib.py import ksft_run, ksft_exit, ksft_eq, ksft_ne, ksft_pr
15
+
from lib.py import KsftFailEx, NetDrvEpEnv, EthtoolFamily, NlError
16
+
from lib.py import bkg, cmd, rand_port
17
+
from lib.py import ip, bpftool, defer
18
+
19
+
20
+
class TestConfig(Enum):
21
+
"""Enum for XDP configuration options."""
22
+
MODE = 0 # Configures the BPF program for a specific test
23
+
PORT = 1 # Port configuration to communicate with the remote host
24
+
ADJST_OFFSET = 2 # Tail/Head adjustment offset for extension/shrinking
25
+
ADJST_TAG = 3 # Adjustment tag to annotate the start and end of extension
26
+
27
+
28
+
class XDPAction(Enum):
29
+
"""Enum for XDP actions."""
30
+
PASS = 0 # Pass the packet up to the stack
31
+
DROP = 1 # Drop the packet
32
+
TX = 2 # Route the packet to the remote host
33
+
TAIL_ADJST = 3 # Adjust the tail of the packet
34
+
HEAD_ADJST = 4 # Adjust the head of the packet
35
+
36
+
37
+
class XDPStats(Enum):
38
+
"""Enum for XDP statistics."""
39
+
RX = 0 # Count of valid packets received for testing
40
+
PASS = 1 # Count of packets passed up to the stack
41
+
DROP = 2 # Count of packets dropped
42
+
TX = 3 # Count of incoming packets routed to the remote host
43
+
ABORT = 4 # Count of packets that were aborted
44
+
45
+
46
+
@dataclass
47
+
class BPFProgInfo:
48
+
"""Data class to store information about a BPF program."""
49
+
name: str # Name of the BPF program
50
+
file: str # BPF program object file
51
+
xdp_sec: str = "xdp" # XDP section name (e.g., "xdp" or "xdp.frags")
52
+
mtu: int = 1500 # Maximum Transmission Unit, default is 1500
53
+
54
+
55
+
def _exchg_udp(cfg, port, test_string):
56
+
"""
57
+
Exchanges UDP packets between a local and remote host using the socat tool.
58
+
59
+
Args:
60
+
cfg: Configuration object containing network settings.
61
+
port: Port number to use for the UDP communication.
62
+
test_string: String that the remote host will send.
63
+
64
+
Returns:
65
+
The string received by the test host.
66
+
"""
67
+
cfg.require_cmd("socat", remote=True)
68
+
69
+
rx_udp_cmd = f"socat -{cfg.addr_ipver} -T 2 -u UDP-RECV:{port},reuseport STDOUT"
70
+
tx_udp_cmd = f"echo -n {test_string} | socat -t 2 -u STDIN UDP:{cfg.baddr}:{port}"
71
+
72
+
with bkg(rx_udp_cmd, exit_wait=True) as nc:
73
+
cmd(tx_udp_cmd, host=cfg.remote, shell=True)
74
+
75
+
return nc.stdout.strip()
76
+
77
+
78
+
def _test_udp(cfg, port, size=256):
79
+
"""
80
+
Tests UDP packet exchange between a local and remote host.
81
+
82
+
Args:
83
+
cfg: Configuration object containing network settings.
84
+
port: Port number to use for the UDP communication.
85
+
size: The length of the test string to be exchanged, default is 256 characters.
86
+
87
+
Returns:
88
+
bool: True if the received string matches the sent string, False otherwise.
89
+
"""
90
+
test_str = "".join(random.choice(string.ascii_lowercase) for _ in range(size))
91
+
recvd_str = _exchg_udp(cfg, port, test_str)
92
+
93
+
return recvd_str == test_str
94
+
95
+
96
+
def _load_xdp_prog(cfg, bpf_info):
97
+
"""
98
+
Loads an XDP program onto a network interface.
99
+
100
+
Args:
101
+
cfg: Configuration object containing network settings.
102
+
bpf_info: BPFProgInfo object containing information about the BPF program.
103
+
104
+
Returns:
105
+
dict: A dictionary containing the XDP program ID, name, and associated map IDs.
106
+
"""
107
+
abs_path = cfg.net_lib_dir / bpf_info.file
108
+
prog_info = {}
109
+
110
+
cmd(f"ip link set dev {cfg.remote_ifname} mtu {bpf_info.mtu}", shell=True, host=cfg.remote)
111
+
defer(ip, f"link set dev {cfg.remote_ifname} mtu 1500", host=cfg.remote)
112
+
113
+
cmd(
114
+
f"ip link set dev {cfg.ifname} mtu {bpf_info.mtu} xdp obj {abs_path} sec {bpf_info.xdp_sec}",
115
+
shell=True
116
+
)
117
+
defer(ip, f"link set dev {cfg.ifname} mtu 1500 xdp off")
118
+
119
+
xdp_info = ip(f"-d link show dev {cfg.ifname}", json=True)[0]
120
+
prog_info["id"] = xdp_info["xdp"]["prog"]["id"]
121
+
prog_info["name"] = xdp_info["xdp"]["prog"]["name"]
122
+
prog_id = prog_info["id"]
123
+
124
+
map_ids = bpftool(f"prog show id {prog_id}", json=True)["map_ids"]
125
+
prog_info["maps"] = {}
126
+
for map_id in map_ids:
127
+
name = bpftool(f"map show id {map_id}", json=True)["name"]
128
+
prog_info["maps"][name] = map_id
129
+
130
+
return prog_info
131
+
132
+
133
+
def format_hex_bytes(value):
134
+
"""
135
+
Helper function that converts an integer into a formatted hexadecimal byte string.
136
+
137
+
Args:
138
+
value: An integer representing the number to be converted.
139
+
140
+
Returns:
141
+
A string representing hexadecimal equivalent of value, with bytes separated by spaces.
142
+
"""
143
+
hex_str = value.to_bytes(4, byteorder='little', signed=True)
144
+
return ' '.join(f'{byte:02x}' for byte in hex_str)
145
+
146
+
147
+
def _set_xdp_map(map_name, key, value):
148
+
"""
149
+
Updates an XDP map with a given key-value pair using bpftool.
150
+
151
+
Args:
152
+
map_name: The name of the XDP map to update.
153
+
key: The key to update in the map, formatted as a hexadecimal string.
154
+
value: The value to associate with the key, formatted as a hexadecimal string.
155
+
"""
156
+
key_formatted = format_hex_bytes(key)
157
+
value_formatted = format_hex_bytes(value)
158
+
bpftool(
159
+
f"map update name {map_name} key hex {key_formatted} value hex {value_formatted}"
160
+
)
161
+
162
+
163
+
def _get_stats(xdp_map_id):
164
+
"""
165
+
Retrieves and formats statistics from an XDP map.
166
+
167
+
Args:
168
+
xdp_map_id: The ID of the XDP map from which to retrieve statistics.
169
+
170
+
Returns:
171
+
A dictionary containing formatted packet statistics for various XDP actions.
172
+
The keys are based on the XDPStats Enum values.
173
+
174
+
Raises:
175
+
KsftFailEx: If the stats retrieval fails.
176
+
"""
177
+
stats_dump = bpftool(f"map dump id {xdp_map_id}", json=True)
178
+
if not stats_dump:
179
+
raise KsftFailEx(f"Failed to get stats for map {xdp_map_id}")
180
+
181
+
stats_formatted = {}
182
+
for key in range(0, 5):
183
+
val = stats_dump[key]["formatted"]["value"]
184
+
if stats_dump[key]["formatted"]["key"] == XDPStats.RX.value:
185
+
stats_formatted[XDPStats.RX.value] = val
186
+
elif stats_dump[key]["formatted"]["key"] == XDPStats.PASS.value:
187
+
stats_formatted[XDPStats.PASS.value] = val
188
+
elif stats_dump[key]["formatted"]["key"] == XDPStats.DROP.value:
189
+
stats_formatted[XDPStats.DROP.value] = val
190
+
elif stats_dump[key]["formatted"]["key"] == XDPStats.TX.value:
191
+
stats_formatted[XDPStats.TX.value] = val
192
+
elif stats_dump[key]["formatted"]["key"] == XDPStats.ABORT.value:
193
+
stats_formatted[XDPStats.ABORT.value] = val
194
+
195
+
return stats_formatted
196
+
197
+
198
+
def _test_pass(cfg, bpf_info, msg_sz):
199
+
"""
200
+
Tests the XDP_PASS action by exchanging UDP packets.
201
+
202
+
Args:
203
+
cfg: Configuration object containing network settings.
204
+
bpf_info: BPFProgInfo object containing information about the BPF program.
205
+
msg_sz: Size of the test message to send.
206
+
"""
207
+
208
+
prog_info = _load_xdp_prog(cfg, bpf_info)
209
+
port = rand_port()
210
+
211
+
_set_xdp_map("map_xdp_setup", TestConfig.MODE.value, XDPAction.PASS.value)
212
+
_set_xdp_map("map_xdp_setup", TestConfig.PORT.value, port)
213
+
214
+
ksft_eq(_test_udp(cfg, port, msg_sz), True, "UDP packet exchange failed")
215
+
stats = _get_stats(prog_info["maps"]["map_xdp_stats"])
216
+
217
+
ksft_ne(stats[XDPStats.RX.value], 0, "RX stats should not be zero")
218
+
ksft_eq(stats[XDPStats.RX.value], stats[XDPStats.PASS.value], "RX and PASS stats mismatch")
219
+
220
+
221
+
def test_xdp_native_pass_sb(cfg):
222
+
"""
223
+
Tests the XDP_PASS action for single buffer case.
224
+
225
+
Args:
226
+
cfg: Configuration object containing network settings.
227
+
"""
228
+
bpf_info = BPFProgInfo("xdp_prog", "xdp_native.bpf.o", "xdp", 1500)
229
+
230
+
_test_pass(cfg, bpf_info, 256)
231
+
232
+
233
+
def test_xdp_native_pass_mb(cfg):
234
+
"""
235
+
Tests the XDP_PASS action for a multi-buff size.
236
+
237
+
Args:
238
+
cfg: Configuration object containing network settings.
239
+
"""
240
+
bpf_info = BPFProgInfo("xdp_prog_frags", "xdp_native.bpf.o", "xdp.frags", 9000)
241
+
242
+
_test_pass(cfg, bpf_info, 8000)
243
+
244
+
245
+
def _test_drop(cfg, bpf_info, msg_sz):
246
+
"""
247
+
Tests the XDP_DROP action by exchanging UDP packets.
248
+
249
+
Args:
250
+
cfg: Configuration object containing network settings.
251
+
bpf_info: BPFProgInfo object containing information about the BPF program.
252
+
msg_sz: Size of the test message to send.
253
+
"""
254
+
255
+
prog_info = _load_xdp_prog(cfg, bpf_info)
256
+
port = rand_port()
257
+
258
+
_set_xdp_map("map_xdp_setup", TestConfig.MODE.value, XDPAction.DROP.value)
259
+
_set_xdp_map("map_xdp_setup", TestConfig.PORT.value, port)
260
+
261
+
ksft_eq(_test_udp(cfg, port, msg_sz), False, "UDP packet exchange should fail")
262
+
stats = _get_stats(prog_info["maps"]["map_xdp_stats"])
263
+
264
+
ksft_ne(stats[XDPStats.RX.value], 0, "RX stats should be zero")
265
+
ksft_eq(stats[XDPStats.RX.value], stats[XDPStats.DROP.value], "RX and DROP stats mismatch")
266
+
267
+
268
+
def test_xdp_native_drop_sb(cfg):
269
+
"""
270
+
Tests the XDP_DROP action for a signle-buff case.
271
+
272
+
Args:
273
+
cfg: Configuration object containing network settings.
274
+
"""
275
+
bpf_info = BPFProgInfo("xdp_prog", "xdp_native.bpf.o", "xdp", 1500)
276
+
277
+
_test_drop(cfg, bpf_info, 256)
278
+
279
+
280
+
def test_xdp_native_drop_mb(cfg):
281
+
"""
282
+
Tests the XDP_DROP action for a multi-buff case.
283
+
284
+
Args:
285
+
cfg: Configuration object containing network settings.
286
+
"""
287
+
bpf_info = BPFProgInfo("xdp_prog_frags", "xdp_native.bpf.o", "xdp.frags", 9000)
288
+
289
+
_test_drop(cfg, bpf_info, 8000)
290
+
291
+
292
+
def test_xdp_native_tx_mb(cfg):
293
+
"""
294
+
Tests the XDP_TX action for a multi-buff case.
295
+
296
+
Args:
297
+
cfg: Configuration object containing network settings.
298
+
"""
299
+
cfg.require_cmd("socat", remote=True)
300
+
301
+
bpf_info = BPFProgInfo("xdp_prog_frags", "xdp_native.bpf.o", "xdp.frags", 9000)
302
+
prog_info = _load_xdp_prog(cfg, bpf_info)
303
+
port = rand_port()
304
+
305
+
_set_xdp_map("map_xdp_setup", TestConfig.MODE.value, XDPAction.TX.value)
306
+
_set_xdp_map("map_xdp_setup", TestConfig.PORT.value, port)
307
+
308
+
test_string = ''.join(random.choice(string.ascii_lowercase) for _ in range(8000))
309
+
rx_udp = f"socat -{cfg.addr_ipver} -T 2 -u UDP-RECV:{port},reuseport STDOUT"
310
+
tx_udp = f"echo {test_string} | socat -t 2 -u STDIN UDP:{cfg.baddr}:{port}"
311
+
312
+
with bkg(rx_udp, host=cfg.remote, exit_wait=True) as rnc:
313
+
cmd(tx_udp, host=cfg.remote, shell=True)
314
+
315
+
stats = _get_stats(prog_info['maps']['map_xdp_stats'])
316
+
317
+
ksft_eq(rnc.stdout.strip(), test_string, "UDP packet exchange failed")
318
+
ksft_eq(stats[XDPStats.TX.value], 1, "TX stats mismatch")
319
+
320
+
321
+
def _validate_res(res, offset_lst, pkt_sz_lst):
322
+
"""
323
+
Validates the result of a test.
324
+
325
+
Args:
326
+
res: The result of the test, which should be a dictionary with a "status" key.
327
+
328
+
Raises:
329
+
KsftFailEx: If the test fails to pass any combination of offset and packet size.
330
+
"""
331
+
if "status" not in res:
332
+
raise KsftFailEx("Missing 'status' key in result dictionary")
333
+
334
+
# Validate that not a single case was successful
335
+
if res["status"] == "fail":
336
+
if res["offset"] == offset_lst[0] and res["pkt_sz"] == pkt_sz_lst[0]:
337
+
raise KsftFailEx(f"{res['reason']}")
338
+
339
+
# Get the previous offset and packet size to report the successful run
340
+
tmp_idx = offset_lst.index(res["offset"])
341
+
prev_offset = offset_lst[tmp_idx - 1]
342
+
if tmp_idx == 0:
343
+
tmp_idx = pkt_sz_lst.index(res["pkt_sz"])
344
+
prev_pkt_sz = pkt_sz_lst[tmp_idx - 1]
345
+
else:
346
+
prev_pkt_sz = res["pkt_sz"]
347
+
348
+
# Use these values for error reporting
349
+
ksft_pr(
350
+
f"Failed run: pkt_sz {res['pkt_sz']}, offset {res['offset']}. "
351
+
f"Last successful run: pkt_sz {prev_pkt_sz}, offset {prev_offset}. "
352
+
f"Reason: {res['reason']}"
353
+
)
354
+
355
+
356
+
def _check_for_failures(recvd_str, stats):
357
+
"""
358
+
Checks for common failures while adjusting headroom or tailroom.
359
+
360
+
Args:
361
+
recvd_str: The string received from the remote host after sending a test string.
362
+
stats: A dictionary containing formatted packet statistics for various XDP actions.
363
+
364
+
Returns:
365
+
str: A string describing the failure reason if a failure is detected, otherwise None.
366
+
"""
367
+
368
+
# Any adjustment failure result in an abort hence, we track this counter
369
+
if stats[XDPStats.ABORT.value] != 0:
370
+
return "Adjustment failed"
371
+
372
+
# Since we are using aggregate stats for a single test across all offsets and packet sizes
373
+
# we can't use RX stats only to track data exchange failure without taking a previous
374
+
# snapshot. An easier way is to simply check for non-zero length of received string.
375
+
if len(recvd_str) == 0:
376
+
return "Data exchange failed"
377
+
378
+
# Check for RX and PASS stats mismatch. Ideally, they should be equal for a successful run
379
+
if stats[XDPStats.RX.value] != stats[XDPStats.PASS.value]:
380
+
return "RX stats mismatch"
381
+
382
+
return None
383
+
384
+
385
+
def _test_xdp_native_tail_adjst(cfg, pkt_sz_lst, offset_lst):
386
+
"""
387
+
Tests the XDP tail adjustment functionality.
388
+
389
+
This function loads the appropriate XDP program based on the provided
390
+
program name and configures the XDP map for tail adjustment. It then
391
+
validates the tail adjustment by sending and receiving UDP packets
392
+
with specified packet sizes and offsets.
393
+
394
+
Args:
395
+
cfg: Configuration object containing network settings.
396
+
prog: Name of the XDP program to load.
397
+
pkt_sz_lst: List of packet sizes to test.
398
+
offset_lst: List of offsets to validate support for tail adjustment.
399
+
400
+
Returns:
401
+
dict: A dictionary with test status and failure details if applicable.
402
+
"""
403
+
port = rand_port()
404
+
bpf_info = BPFProgInfo("xdp_prog_frags", "xdp_native.bpf.o", "xdp.frags", 9000)
405
+
406
+
prog_info = _load_xdp_prog(cfg, bpf_info)
407
+
408
+
# Configure the XDP map for tail adjustment
409
+
_set_xdp_map("map_xdp_setup", TestConfig.MODE.value, XDPAction.TAIL_ADJST.value)
410
+
_set_xdp_map("map_xdp_setup", TestConfig.PORT.value, port)
411
+
412
+
for offset in offset_lst:
413
+
tag = format(random.randint(65, 90), "02x")
414
+
415
+
_set_xdp_map("map_xdp_setup", TestConfig.ADJST_OFFSET.value, offset)
416
+
if offset > 0:
417
+
_set_xdp_map("map_xdp_setup", TestConfig.ADJST_TAG.value, int(tag, 16))
418
+
419
+
for pkt_sz in pkt_sz_lst:
420
+
test_str = "".join(random.choice(string.ascii_lowercase) for _ in range(pkt_sz))
421
+
recvd_str = _exchg_udp(cfg, port, test_str)
422
+
stats = _get_stats(prog_info["maps"]["map_xdp_stats"])
423
+
424
+
failure = _check_for_failures(recvd_str, stats)
425
+
if failure is not None:
426
+
return {
427
+
"status": "fail",
428
+
"reason": failure,
429
+
"offset": offset,
430
+
"pkt_sz": pkt_sz,
431
+
}
432
+
433
+
# Validate data content based on offset direction
434
+
expected_data = None
435
+
if offset > 0:
436
+
expected_data = test_str + (offset * chr(int(tag, 16)))
437
+
else:
438
+
expected_data = test_str[0:pkt_sz + offset]
439
+
440
+
if recvd_str != expected_data:
441
+
return {
442
+
"status": "fail",
443
+
"reason": "Data mismatch",
444
+
"offset": offset,
445
+
"pkt_sz": pkt_sz,
446
+
}
447
+
448
+
return {"status": "pass"}
449
+
450
+
451
+
def test_xdp_native_adjst_tail_grow_data(cfg):
452
+
"""
453
+
Tests the XDP tail adjustment by growing packet data.
454
+
455
+
Args:
456
+
cfg: Configuration object containing network settings.
457
+
"""
458
+
pkt_sz_lst = [512, 1024, 2048]
459
+
offset_lst = [1, 16, 32, 64, 128, 256]
460
+
res = _test_xdp_native_tail_adjst(
461
+
cfg,
462
+
pkt_sz_lst,
463
+
offset_lst,
464
+
)
465
+
466
+
_validate_res(res, offset_lst, pkt_sz_lst)
467
+
468
+
469
+
def test_xdp_native_adjst_tail_shrnk_data(cfg):
470
+
"""
471
+
Tests the XDP tail adjustment by shrinking packet data.
472
+
473
+
Args:
474
+
cfg: Configuration object containing network settings.
475
+
"""
476
+
pkt_sz_lst = [512, 1024, 2048]
477
+
offset_lst = [-16, -32, -64, -128, -256]
478
+
res = _test_xdp_native_tail_adjst(
479
+
cfg,
480
+
pkt_sz_lst,
481
+
offset_lst,
482
+
)
483
+
484
+
_validate_res(res, offset_lst, pkt_sz_lst)
485
+
486
+
487
+
def get_hds_thresh(cfg):
488
+
"""
489
+
Retrieves the header data split (HDS) threshold for a network interface.
490
+
491
+
Args:
492
+
cfg: Configuration object containing network settings.
493
+
494
+
Returns:
495
+
The HDS threshold value. If the threshold is not supported or an error occurs,
496
+
a default value of 1500 is returned.
497
+
"""
498
+
netnl = cfg.netnl
499
+
hds_thresh = 1500
500
+
501
+
try:
502
+
rings = netnl.rings_get({'header': {'dev-index': cfg.ifindex}})
503
+
if 'hds-thresh' not in rings:
504
+
ksft_pr(f'hds-thresh not supported. Using default: {hds_thresh}')
505
+
return hds_thresh
506
+
hds_thresh = rings['hds-thresh']
507
+
except NlError as e:
508
+
ksft_pr(f"Failed to get rings: {e}. Using default: {hds_thresh}")
509
+
510
+
return hds_thresh
511
+
512
+
513
+
def _test_xdp_native_head_adjst(cfg, prog, pkt_sz_lst, offset_lst):
514
+
"""
515
+
Tests the XDP head adjustment action for a multi-buffer case.
516
+
517
+
Args:
518
+
cfg: Configuration object containing network settings.
519
+
netnl: Network namespace or link object (not used in this function).
520
+
521
+
This function sets up the packet size and offset lists, then performs
522
+
the head adjustment test by sending and receiving UDP packets.
523
+
"""
524
+
cfg.require_cmd("socat", remote=True)
525
+
526
+
prog_info = _load_xdp_prog(cfg, BPFProgInfo(prog, "xdp_native.bpf.o", "xdp.frags", 9000))
527
+
port = rand_port()
528
+
529
+
_set_xdp_map("map_xdp_setup", TestConfig.MODE.value, XDPAction.HEAD_ADJST.value)
530
+
_set_xdp_map("map_xdp_setup", TestConfig.PORT.value, port)
531
+
532
+
hds_thresh = get_hds_thresh(cfg)
533
+
for offset in offset_lst:
534
+
for pkt_sz in pkt_sz_lst:
535
+
# The "head" buffer must contain at least the Ethernet header
536
+
# after we eat into it. We send large-enough packets, but if HDS
537
+
# is enabled head will only contain headers. Don't try to eat
538
+
# more than 28 bytes (UDPv4 + eth hdr left: (14 + 20 + 8) - 14)
539
+
l2_cut_off = 28 if cfg.addr_ipver == 4 else 48
540
+
if pkt_sz > hds_thresh and offset > l2_cut_off:
541
+
ksft_pr(
542
+
f"Failed run: pkt_sz ({pkt_sz}) > HDS threshold ({hds_thresh}) and "
543
+
f"offset {offset} > {l2_cut_off}"
544
+
)
545
+
return {"status": "pass"}
546
+
547
+
test_str = ''.join(random.choice(string.ascii_lowercase) for _ in range(pkt_sz))
548
+
tag = format(random.randint(65, 90), '02x')
549
+
550
+
_set_xdp_map("map_xdp_setup",
551
+
TestConfig.ADJST_OFFSET.value,
552
+
offset)
553
+
_set_xdp_map("map_xdp_setup", TestConfig.ADJST_TAG.value, int(tag, 16))
554
+
_set_xdp_map("map_xdp_setup", TestConfig.ADJST_OFFSET.value, offset)
555
+
556
+
recvd_str = _exchg_udp(cfg, port, test_str)
557
+
558
+
# Check for failures around adjustment and data exchange
559
+
failure = _check_for_failures(recvd_str, _get_stats(prog_info['maps']['map_xdp_stats']))
560
+
if failure is not None:
561
+
return {
562
+
"status": "fail",
563
+
"reason": failure,
564
+
"offset": offset,
565
+
"pkt_sz": pkt_sz
566
+
}
567
+
568
+
# Validate data content based on offset direction
569
+
expected_data = None
570
+
if offset < 0:
571
+
expected_data = chr(int(tag, 16)) * (0 - offset) + test_str
572
+
else:
573
+
expected_data = test_str[offset:]
574
+
575
+
if recvd_str != expected_data:
576
+
return {
577
+
"status": "fail",
578
+
"reason": "Data mismatch",
579
+
"offset": offset,
580
+
"pkt_sz": pkt_sz
581
+
}
582
+
583
+
return {"status": "pass"}
584
+
585
+
586
+
def test_xdp_native_adjst_head_grow_data(cfg):
587
+
"""
588
+
Tests the XDP headroom growth support.
589
+
590
+
Args:
591
+
cfg: Configuration object containing network settings.
592
+
593
+
This function sets up the packet size and offset lists, then calls the
594
+
_test_xdp_native_head_adjst_mb function to perform the actual test. The
595
+
test is passed if the headroom is successfully extended for given packet
596
+
sizes and offsets.
597
+
"""
598
+
pkt_sz_lst = [512, 1024, 2048]
599
+
600
+
# Negative values result in headroom shrinking, resulting in growing of payload
601
+
offset_lst = [-16, -32, -64, -128, -256]
602
+
res = _test_xdp_native_head_adjst(cfg, "xdp_prog_frags", pkt_sz_lst, offset_lst)
603
+
604
+
_validate_res(res, offset_lst, pkt_sz_lst)
605
+
606
+
607
+
def test_xdp_native_adjst_head_shrnk_data(cfg):
608
+
"""
609
+
Tests the XDP headroom shrinking support.
610
+
611
+
Args:
612
+
cfg: Configuration object containing network settings.
613
+
614
+
This function sets up the packet size and offset lists, then calls the
615
+
_test_xdp_native_head_adjst_mb function to perform the actual test. The
616
+
test is passed if the headroom is successfully shrunk for given packet
617
+
sizes and offsets.
618
+
"""
619
+
pkt_sz_lst = [512, 1024, 2048]
620
+
621
+
# Positive values result in headroom growing, resulting in shrinking of payload
622
+
offset_lst = [16, 32, 64, 128, 256]
623
+
res = _test_xdp_native_head_adjst(cfg, "xdp_prog_frags", pkt_sz_lst, offset_lst)
624
+
625
+
_validate_res(res, offset_lst, pkt_sz_lst)
626
+
627
+
628
+
def main():
629
+
"""
630
+
Main function to execute the XDP tests.
631
+
632
+
This function runs a series of tests to validate the XDP support for
633
+
both the single and multi-buffer. It uses the NetDrvEpEnv context
634
+
manager to manage the network driver environment and the ksft_run
635
+
function to execute the tests.
636
+
"""
637
+
with NetDrvEpEnv(__file__) as cfg:
638
+
cfg.netnl = EthtoolFamily()
639
+
ksft_run(
640
+
[
641
+
test_xdp_native_pass_sb,
642
+
test_xdp_native_pass_mb,
643
+
test_xdp_native_drop_sb,
644
+
test_xdp_native_drop_mb,
645
+
test_xdp_native_tx_mb,
646
+
test_xdp_native_adjst_tail_grow_data,
647
+
test_xdp_native_adjst_tail_shrnk_data,
648
+
test_xdp_native_adjst_head_grow_data,
649
+
test_xdp_native_adjst_head_shrnk_data,
650
+
],
651
+
args=(cfg,))
652
+
ksft_exit()
653
+
654
+
655
+
if __name__ == "__main__":
656
+
main()
+621
tools/testing/selftests/net/lib/xdp_native.bpf.c
+621
tools/testing/selftests/net/lib/xdp_native.bpf.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
3
+
#include <stddef.h>
4
+
#include <linux/bpf.h>
5
+
#include <linux/in.h>
6
+
#include <linux/if_ether.h>
7
+
#include <linux/ip.h>
8
+
#include <linux/ipv6.h>
9
+
#include <linux/udp.h>
10
+
#include <bpf/bpf_endian.h>
11
+
#include <bpf/bpf_helpers.h>
12
+
13
+
#define MAX_ADJST_OFFSET 256
14
+
#define MAX_PAYLOAD_LEN 5000
15
+
#define MAX_HDR_LEN 64
16
+
17
+
enum {
18
+
XDP_MODE = 0,
19
+
XDP_PORT = 1,
20
+
XDP_ADJST_OFFSET = 2,
21
+
XDP_ADJST_TAG = 3,
22
+
} xdp_map_setup_keys;
23
+
24
+
enum {
25
+
XDP_MODE_PASS = 0,
26
+
XDP_MODE_DROP = 1,
27
+
XDP_MODE_TX = 2,
28
+
XDP_MODE_TAIL_ADJST = 3,
29
+
XDP_MODE_HEAD_ADJST = 4,
30
+
} xdp_map_modes;
31
+
32
+
enum {
33
+
STATS_RX = 0,
34
+
STATS_PASS = 1,
35
+
STATS_DROP = 2,
36
+
STATS_TX = 3,
37
+
STATS_ABORT = 4,
38
+
} xdp_stats;
39
+
40
+
struct {
41
+
__uint(type, BPF_MAP_TYPE_ARRAY);
42
+
__uint(max_entries, 5);
43
+
__type(key, __u32);
44
+
__type(value, __s32);
45
+
} map_xdp_setup SEC(".maps");
46
+
47
+
struct {
48
+
__uint(type, BPF_MAP_TYPE_ARRAY);
49
+
__uint(max_entries, 5);
50
+
__type(key, __u32);
51
+
__type(value, __u64);
52
+
} map_xdp_stats SEC(".maps");
53
+
54
+
static __u32 min(__u32 a, __u32 b)
55
+
{
56
+
return a < b ? a : b;
57
+
}
58
+
59
+
static void record_stats(struct xdp_md *ctx, __u32 stat_type)
60
+
{
61
+
__u64 *count;
62
+
63
+
count = bpf_map_lookup_elem(&map_xdp_stats, &stat_type);
64
+
65
+
if (count)
66
+
__sync_fetch_and_add(count, 1);
67
+
}
68
+
69
+
static struct udphdr *filter_udphdr(struct xdp_md *ctx, __u16 port)
70
+
{
71
+
void *data_end = (void *)(long)ctx->data_end;
72
+
void *data = (void *)(long)ctx->data;
73
+
struct udphdr *udph = NULL;
74
+
struct ethhdr *eth = data;
75
+
76
+
if (data + sizeof(*eth) > data_end)
77
+
return NULL;
78
+
79
+
if (eth->h_proto == bpf_htons(ETH_P_IP)) {
80
+
struct iphdr *iph = data + sizeof(*eth);
81
+
82
+
if (iph + 1 > (struct iphdr *)data_end ||
83
+
iph->protocol != IPPROTO_UDP)
84
+
return NULL;
85
+
86
+
udph = (void *)eth + sizeof(*iph) + sizeof(*eth);
87
+
} else if (eth->h_proto == bpf_htons(ETH_P_IPV6)) {
88
+
struct ipv6hdr *ipv6h = data + sizeof(*eth);
89
+
90
+
if (ipv6h + 1 > (struct ipv6hdr *)data_end ||
91
+
ipv6h->nexthdr != IPPROTO_UDP)
92
+
return NULL;
93
+
94
+
udph = (void *)eth + sizeof(*ipv6h) + sizeof(*eth);
95
+
} else {
96
+
return NULL;
97
+
}
98
+
99
+
if (udph + 1 > (struct udphdr *)data_end)
100
+
return NULL;
101
+
102
+
if (udph->dest != bpf_htons(port))
103
+
return NULL;
104
+
105
+
record_stats(ctx, STATS_RX);
106
+
107
+
return udph;
108
+
}
109
+
110
+
static int xdp_mode_pass(struct xdp_md *ctx, __u16 port)
111
+
{
112
+
struct udphdr *udph = NULL;
113
+
114
+
udph = filter_udphdr(ctx, port);
115
+
if (!udph)
116
+
return XDP_PASS;
117
+
118
+
record_stats(ctx, STATS_PASS);
119
+
120
+
return XDP_PASS;
121
+
}
122
+
123
+
static int xdp_mode_drop_handler(struct xdp_md *ctx, __u16 port)
124
+
{
125
+
struct udphdr *udph = NULL;
126
+
127
+
udph = filter_udphdr(ctx, port);
128
+
if (!udph)
129
+
return XDP_PASS;
130
+
131
+
record_stats(ctx, STATS_DROP);
132
+
133
+
return XDP_DROP;
134
+
}
135
+
136
+
static void swap_machdr(void *data)
137
+
{
138
+
struct ethhdr *eth = data;
139
+
__u8 tmp_mac[ETH_ALEN];
140
+
141
+
__builtin_memcpy(tmp_mac, eth->h_source, ETH_ALEN);
142
+
__builtin_memcpy(eth->h_source, eth->h_dest, ETH_ALEN);
143
+
__builtin_memcpy(eth->h_dest, tmp_mac, ETH_ALEN);
144
+
}
145
+
146
+
static int xdp_mode_tx_handler(struct xdp_md *ctx, __u16 port)
147
+
{
148
+
void *data_end = (void *)(long)ctx->data_end;
149
+
void *data = (void *)(long)ctx->data;
150
+
struct udphdr *udph = NULL;
151
+
struct ethhdr *eth = data;
152
+
153
+
if (data + sizeof(*eth) > data_end)
154
+
return XDP_PASS;
155
+
156
+
if (eth->h_proto == bpf_htons(ETH_P_IP)) {
157
+
struct iphdr *iph = data + sizeof(*eth);
158
+
__be32 tmp_ip = iph->saddr;
159
+
160
+
if (iph + 1 > (struct iphdr *)data_end ||
161
+
iph->protocol != IPPROTO_UDP)
162
+
return XDP_PASS;
163
+
164
+
udph = data + sizeof(*iph) + sizeof(*eth);
165
+
166
+
if (udph + 1 > (struct udphdr *)data_end)
167
+
return XDP_PASS;
168
+
if (udph->dest != bpf_htons(port))
169
+
return XDP_PASS;
170
+
171
+
record_stats(ctx, STATS_RX);
172
+
swap_machdr((void *)eth);
173
+
174
+
iph->saddr = iph->daddr;
175
+
iph->daddr = tmp_ip;
176
+
177
+
record_stats(ctx, STATS_TX);
178
+
179
+
return XDP_TX;
180
+
181
+
} else if (eth->h_proto == bpf_htons(ETH_P_IPV6)) {
182
+
struct ipv6hdr *ipv6h = data + sizeof(*eth);
183
+
struct in6_addr tmp_ipv6;
184
+
185
+
if (ipv6h + 1 > (struct ipv6hdr *)data_end ||
186
+
ipv6h->nexthdr != IPPROTO_UDP)
187
+
return XDP_PASS;
188
+
189
+
udph = data + sizeof(*ipv6h) + sizeof(*eth);
190
+
191
+
if (udph + 1 > (struct udphdr *)data_end)
192
+
return XDP_PASS;
193
+
if (udph->dest != bpf_htons(port))
194
+
return XDP_PASS;
195
+
196
+
record_stats(ctx, STATS_RX);
197
+
swap_machdr((void *)eth);
198
+
199
+
__builtin_memcpy(&tmp_ipv6, &ipv6h->saddr, sizeof(tmp_ipv6));
200
+
__builtin_memcpy(&ipv6h->saddr, &ipv6h->daddr,
201
+
sizeof(tmp_ipv6));
202
+
__builtin_memcpy(&ipv6h->daddr, &tmp_ipv6, sizeof(tmp_ipv6));
203
+
204
+
record_stats(ctx, STATS_TX);
205
+
206
+
return XDP_TX;
207
+
}
208
+
209
+
return XDP_PASS;
210
+
}
211
+
212
+
static void *update_pkt(struct xdp_md *ctx, __s16 offset, __u32 *udp_csum)
213
+
{
214
+
void *data_end = (void *)(long)ctx->data_end;
215
+
void *data = (void *)(long)ctx->data;
216
+
struct udphdr *udph = NULL;
217
+
struct ethhdr *eth = data;
218
+
__u32 len, len_new;
219
+
220
+
if (data + sizeof(*eth) > data_end)
221
+
return NULL;
222
+
223
+
if (eth->h_proto == bpf_htons(ETH_P_IP)) {
224
+
struct iphdr *iph = data + sizeof(*eth);
225
+
__u16 total_len;
226
+
227
+
if (iph + 1 > (struct iphdr *)data_end)
228
+
return NULL;
229
+
230
+
iph->tot_len = bpf_htons(bpf_ntohs(iph->tot_len) + offset);
231
+
232
+
udph = (void *)eth + sizeof(*iph) + sizeof(*eth);
233
+
if (!udph || udph + 1 > (struct udphdr *)data_end)
234
+
return NULL;
235
+
236
+
len_new = bpf_htons(bpf_ntohs(udph->len) + offset);
237
+
} else if (eth->h_proto == bpf_htons(ETH_P_IPV6)) {
238
+
struct ipv6hdr *ipv6h = data + sizeof(*eth);
239
+
__u16 payload_len;
240
+
241
+
if (ipv6h + 1 > (struct ipv6hdr *)data_end)
242
+
return NULL;
243
+
244
+
udph = (void *)eth + sizeof(*ipv6h) + sizeof(*eth);
245
+
if (!udph || udph + 1 > (struct udphdr *)data_end)
246
+
return NULL;
247
+
248
+
*udp_csum = ~((__u32)udph->check);
249
+
250
+
len = ipv6h->payload_len;
251
+
len_new = bpf_htons(bpf_ntohs(len) + offset);
252
+
ipv6h->payload_len = len_new;
253
+
254
+
*udp_csum = bpf_csum_diff(&len, sizeof(len), &len_new,
255
+
sizeof(len_new), *udp_csum);
256
+
257
+
len = udph->len;
258
+
len_new = bpf_htons(bpf_ntohs(udph->len) + offset);
259
+
*udp_csum = bpf_csum_diff(&len, sizeof(len), &len_new,
260
+
sizeof(len_new), *udp_csum);
261
+
} else {
262
+
return NULL;
263
+
}
264
+
265
+
udph->len = len_new;
266
+
267
+
return udph;
268
+
}
269
+
270
+
static __u16 csum_fold_helper(__u32 csum)
271
+
{
272
+
return ~((csum & 0xffff) + (csum >> 16)) ? : 0xffff;
273
+
}
274
+
275
+
static int xdp_adjst_tail_shrnk_data(struct xdp_md *ctx, __u16 offset,
276
+
__u32 hdr_len)
277
+
{
278
+
char tmp_buff[MAX_ADJST_OFFSET];
279
+
__u32 buff_pos, udp_csum = 0;
280
+
struct udphdr *udph = NULL;
281
+
__u32 buff_len;
282
+
283
+
udph = update_pkt(ctx, 0 - offset, &udp_csum);
284
+
if (!udph)
285
+
return -1;
286
+
287
+
buff_len = bpf_xdp_get_buff_len(ctx);
288
+
289
+
offset = (offset & 0x1ff) >= MAX_ADJST_OFFSET ? MAX_ADJST_OFFSET :
290
+
offset & 0xff;
291
+
if (offset == 0)
292
+
return -1;
293
+
294
+
/* Make sure we have enough data to avoid eating the header */
295
+
if (buff_len - offset < hdr_len)
296
+
return -1;
297
+
298
+
buff_pos = buff_len - offset;
299
+
if (bpf_xdp_load_bytes(ctx, buff_pos, tmp_buff, offset) < 0)
300
+
return -1;
301
+
302
+
udp_csum = bpf_csum_diff((__be32 *)tmp_buff, offset, 0, 0, udp_csum);
303
+
udph->check = (__u16)csum_fold_helper(udp_csum);
304
+
305
+
if (bpf_xdp_adjust_tail(ctx, 0 - offset) < 0)
306
+
return -1;
307
+
308
+
return 0;
309
+
}
310
+
311
+
static int xdp_adjst_tail_grow_data(struct xdp_md *ctx, __u16 offset)
312
+
{
313
+
char tmp_buff[MAX_ADJST_OFFSET];
314
+
__u32 buff_pos, udp_csum = 0;
315
+
__u32 buff_len, hdr_len, key;
316
+
struct udphdr *udph;
317
+
__s32 *val;
318
+
__u8 tag;
319
+
320
+
/* Proceed to update the packet headers before attempting to adjuste
321
+
* the tail. Once the tail is adjusted we lose access to the offset
322
+
* amount of data at the end of the packet which is crucial to update
323
+
* the checksum.
324
+
* Since any failure beyond this would abort the packet, we should
325
+
* not worry about passing a packet up the stack with wrong headers
326
+
*/
327
+
udph = update_pkt(ctx, offset, &udp_csum);
328
+
if (!udph)
329
+
return -1;
330
+
331
+
key = XDP_ADJST_TAG;
332
+
val = bpf_map_lookup_elem(&map_xdp_setup, &key);
333
+
if (!val)
334
+
return -1;
335
+
336
+
tag = (__u8)(*val);
337
+
338
+
for (int i = 0; i < MAX_ADJST_OFFSET; i++)
339
+
__builtin_memcpy(&tmp_buff[i], &tag, 1);
340
+
341
+
offset = (offset & 0x1ff) >= MAX_ADJST_OFFSET ? MAX_ADJST_OFFSET :
342
+
offset & 0xff;
343
+
if (offset == 0)
344
+
return -1;
345
+
346
+
udp_csum = bpf_csum_diff(0, 0, (__be32 *)tmp_buff, offset, udp_csum);
347
+
udph->check = (__u16)csum_fold_helper(udp_csum);
348
+
349
+
buff_len = bpf_xdp_get_buff_len(ctx);
350
+
351
+
if (bpf_xdp_adjust_tail(ctx, offset) < 0) {
352
+
bpf_printk("Failed to adjust tail\n");
353
+
return -1;
354
+
}
355
+
356
+
if (bpf_xdp_store_bytes(ctx, buff_len, tmp_buff, offset) < 0)
357
+
return -1;
358
+
359
+
return 0;
360
+
}
361
+
362
+
static int xdp_adjst_tail(struct xdp_md *ctx, __u16 port)
363
+
{
364
+
void *data = (void *)(long)ctx->data;
365
+
struct udphdr *udph = NULL;
366
+
__s32 *adjust_offset, *val;
367
+
__u32 key, hdr_len;
368
+
void *offset_ptr;
369
+
__u8 tag;
370
+
int ret;
371
+
372
+
udph = filter_udphdr(ctx, port);
373
+
if (!udph)
374
+
return XDP_PASS;
375
+
376
+
hdr_len = (void *)udph - data + sizeof(struct udphdr);
377
+
key = XDP_ADJST_OFFSET;
378
+
adjust_offset = bpf_map_lookup_elem(&map_xdp_setup, &key);
379
+
if (!adjust_offset)
380
+
return XDP_PASS;
381
+
382
+
if (*adjust_offset < 0)
383
+
ret = xdp_adjst_tail_shrnk_data(ctx,
384
+
(__u16)(0 - *adjust_offset),
385
+
hdr_len);
386
+
else
387
+
ret = xdp_adjst_tail_grow_data(ctx, (__u16)(*adjust_offset));
388
+
if (ret)
389
+
goto abort_pkt;
390
+
391
+
record_stats(ctx, STATS_PASS);
392
+
return XDP_PASS;
393
+
394
+
abort_pkt:
395
+
record_stats(ctx, STATS_ABORT);
396
+
return XDP_ABORTED;
397
+
}
398
+
399
+
static int xdp_adjst_head_shrnk_data(struct xdp_md *ctx, __u64 hdr_len,
400
+
__u32 offset)
401
+
{
402
+
char tmp_buff[MAX_ADJST_OFFSET];
403
+
struct udphdr *udph;
404
+
void *offset_ptr;
405
+
__u32 udp_csum = 0;
406
+
407
+
/* Update the length information in the IP and UDP headers before
408
+
* adjusting the headroom. This simplifies accessing the relevant
409
+
* fields in the IP and UDP headers for fragmented packets. Any
410
+
* failure beyond this point will result in the packet being aborted,
411
+
* so we don't need to worry about incorrect length information for
412
+
* passed packets.
413
+
*/
414
+
udph = update_pkt(ctx, (__s16)(0 - offset), &udp_csum);
415
+
if (!udph)
416
+
return -1;
417
+
418
+
offset = (offset & 0x1ff) >= MAX_ADJST_OFFSET ? MAX_ADJST_OFFSET :
419
+
offset & 0xff;
420
+
if (offset == 0)
421
+
return -1;
422
+
423
+
if (bpf_xdp_load_bytes(ctx, hdr_len, tmp_buff, offset) < 0)
424
+
return -1;
425
+
426
+
udp_csum = bpf_csum_diff((__be32 *)tmp_buff, offset, 0, 0, udp_csum);
427
+
428
+
udph->check = (__u16)csum_fold_helper(udp_csum);
429
+
430
+
if (bpf_xdp_load_bytes(ctx, 0, tmp_buff, MAX_ADJST_OFFSET) < 0)
431
+
return -1;
432
+
433
+
if (bpf_xdp_adjust_head(ctx, offset) < 0)
434
+
return -1;
435
+
436
+
if (offset > MAX_ADJST_OFFSET)
437
+
return -1;
438
+
439
+
if (hdr_len > MAX_ADJST_OFFSET || hdr_len == 0)
440
+
return -1;
441
+
442
+
/* Added here to handle clang complain about negative value */
443
+
hdr_len = hdr_len & 0xff;
444
+
445
+
if (hdr_len == 0)
446
+
return -1;
447
+
448
+
if (bpf_xdp_store_bytes(ctx, 0, tmp_buff, hdr_len) < 0)
449
+
return -1;
450
+
451
+
return 0;
452
+
}
453
+
454
+
static int xdp_adjst_head_grow_data(struct xdp_md *ctx, __u64 hdr_len,
455
+
__u32 offset)
456
+
{
457
+
char hdr_buff[MAX_HDR_LEN];
458
+
char data_buff[MAX_ADJST_OFFSET];
459
+
void *offset_ptr;
460
+
__s32 *val;
461
+
__u32 key;
462
+
__u8 tag;
463
+
__u32 udp_csum = 0;
464
+
struct udphdr *udph;
465
+
466
+
udph = update_pkt(ctx, (__s16)(offset), &udp_csum);
467
+
if (!udph)
468
+
return -1;
469
+
470
+
key = XDP_ADJST_TAG;
471
+
val = bpf_map_lookup_elem(&map_xdp_setup, &key);
472
+
if (!val)
473
+
return -1;
474
+
475
+
tag = (__u8)(*val);
476
+
for (int i = 0; i < MAX_ADJST_OFFSET; i++)
477
+
__builtin_memcpy(&data_buff[i], &tag, 1);
478
+
479
+
offset = (offset & 0x1ff) >= MAX_ADJST_OFFSET ? MAX_ADJST_OFFSET :
480
+
offset & 0xff;
481
+
if (offset == 0)
482
+
return -1;
483
+
484
+
udp_csum = bpf_csum_diff(0, 0, (__be32 *)data_buff, offset, udp_csum);
485
+
udph->check = (__u16)csum_fold_helper(udp_csum);
486
+
487
+
if (hdr_len > MAX_ADJST_OFFSET || hdr_len == 0)
488
+
return -1;
489
+
490
+
/* Added here to handle clang complain about negative value */
491
+
hdr_len = hdr_len & 0xff;
492
+
493
+
if (hdr_len == 0)
494
+
return -1;
495
+
496
+
if (bpf_xdp_load_bytes(ctx, 0, hdr_buff, hdr_len) < 0)
497
+
return -1;
498
+
499
+
if (offset > MAX_ADJST_OFFSET)
500
+
return -1;
501
+
502
+
if (bpf_xdp_adjust_head(ctx, 0 - offset) < 0)
503
+
return -1;
504
+
505
+
if (bpf_xdp_store_bytes(ctx, 0, hdr_buff, hdr_len) < 0)
506
+
return -1;
507
+
508
+
if (bpf_xdp_store_bytes(ctx, hdr_len, data_buff, offset) < 0)
509
+
return -1;
510
+
511
+
return 0;
512
+
}
513
+
514
+
static int xdp_head_adjst(struct xdp_md *ctx, __u16 port)
515
+
{
516
+
void *data_end = (void *)(long)ctx->data_end;
517
+
void *data = (void *)(long)ctx->data;
518
+
struct udphdr *udph_ptr = NULL;
519
+
__u32 key, size, hdr_len;
520
+
__s32 *val;
521
+
int res;
522
+
523
+
/* Filter packets based on UDP port */
524
+
udph_ptr = filter_udphdr(ctx, port);
525
+
if (!udph_ptr)
526
+
return XDP_PASS;
527
+
528
+
hdr_len = (void *)udph_ptr - data + sizeof(struct udphdr);
529
+
530
+
key = XDP_ADJST_OFFSET;
531
+
val = bpf_map_lookup_elem(&map_xdp_setup, &key);
532
+
if (!val)
533
+
return XDP_PASS;
534
+
535
+
switch (*val) {
536
+
case -16:
537
+
case 16:
538
+
size = 16;
539
+
break;
540
+
case -32:
541
+
case 32:
542
+
size = 32;
543
+
break;
544
+
case -64:
545
+
case 64:
546
+
size = 64;
547
+
break;
548
+
case -128:
549
+
case 128:
550
+
size = 128;
551
+
break;
552
+
case -256:
553
+
case 256:
554
+
size = 256;
555
+
break;
556
+
default:
557
+
bpf_printk("Invalid adjustment offset: %d\n", *val);
558
+
goto abort;
559
+
}
560
+
561
+
if (*val < 0)
562
+
res = xdp_adjst_head_grow_data(ctx, hdr_len, size);
563
+
else
564
+
res = xdp_adjst_head_shrnk_data(ctx, hdr_len, size);
565
+
566
+
if (res)
567
+
goto abort;
568
+
569
+
record_stats(ctx, STATS_PASS);
570
+
return XDP_PASS;
571
+
572
+
abort:
573
+
record_stats(ctx, STATS_ABORT);
574
+
return XDP_ABORTED;
575
+
}
576
+
577
+
static int xdp_prog_common(struct xdp_md *ctx)
578
+
{
579
+
__u32 key, *port;
580
+
__s32 *mode;
581
+
582
+
key = XDP_MODE;
583
+
mode = bpf_map_lookup_elem(&map_xdp_setup, &key);
584
+
if (!mode)
585
+
return XDP_PASS;
586
+
587
+
key = XDP_PORT;
588
+
port = bpf_map_lookup_elem(&map_xdp_setup, &key);
589
+
if (!port)
590
+
return XDP_PASS;
591
+
592
+
switch (*mode) {
593
+
case XDP_MODE_PASS:
594
+
return xdp_mode_pass(ctx, (__u16)(*port));
595
+
case XDP_MODE_DROP:
596
+
return xdp_mode_drop_handler(ctx, (__u16)(*port));
597
+
case XDP_MODE_TX:
598
+
return xdp_mode_tx_handler(ctx, (__u16)(*port));
599
+
case XDP_MODE_TAIL_ADJST:
600
+
return xdp_adjst_tail(ctx, (__u16)(*port));
601
+
case XDP_MODE_HEAD_ADJST:
602
+
return xdp_head_adjst(ctx, (__u16)(*port));
603
+
}
604
+
605
+
/* Default action is to simple pass */
606
+
return XDP_PASS;
607
+
}
608
+
609
+
SEC("xdp")
610
+
int xdp_prog(struct xdp_md *ctx)
611
+
{
612
+
return xdp_prog_common(ctx);
613
+
}
614
+
615
+
SEC("xdp.frags")
616
+
int xdp_prog_frags(struct xdp_md *ctx)
617
+
{
618
+
return xdp_prog_common(ctx);
619
+
}
620
+
621
+
char _license[] SEC("license") = "GPL";