Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2
3#include <stddef.h>
4#include <linux/bpf.h>
5#include <linux/in.h>
6#include <linux/if_ether.h>
7#include <linux/ip.h>
8#include <linux/ipv6.h>
9#include <linux/udp.h>
10#include <linux/tcp.h>
11#include <bpf/bpf_endian.h>
12#include <bpf/bpf_helpers.h>
13
14enum {
15 XDP_PORT = 1,
16 XDP_PROTO = 4,
17} xdp_map_setup_keys;
18
19struct {
20 __uint(type, BPF_MAP_TYPE_ARRAY);
21 __uint(max_entries, 5);
22 __type(key, __u32);
23 __type(value, __s32);
24} map_xdp_setup SEC(".maps");
25
26/* RSS hash results: key 0 = hash, key 1 = hash type,
27 * key 2 = packet count, key 3 = error count.
28 */
29enum {
30 RSS_KEY_HASH = 0,
31 RSS_KEY_TYPE = 1,
32 RSS_KEY_PKT_CNT = 2,
33 RSS_KEY_ERR_CNT = 3,
34};
35
36struct {
37 __uint(type, BPF_MAP_TYPE_ARRAY);
38 __type(key, __u32);
39 __type(value, __u32);
40 __uint(max_entries, 4);
41} map_rss SEC(".maps");
42
43/* Mirror of enum xdp_rss_hash_type from include/net/xdp.h.
44 * Needed because the enum is not part of UAPI headers.
45 */
46enum xdp_rss_hash_type {
47 XDP_RSS_L3_IPV4 = 1U << 0,
48 XDP_RSS_L3_IPV6 = 1U << 1,
49 XDP_RSS_L3_DYNHDR = 1U << 2,
50 XDP_RSS_L4 = 1U << 3,
51 XDP_RSS_L4_TCP = 1U << 4,
52 XDP_RSS_L4_UDP = 1U << 5,
53 XDP_RSS_L4_SCTP = 1U << 6,
54 XDP_RSS_L4_IPSEC = 1U << 7,
55 XDP_RSS_L4_ICMP = 1U << 8,
56};
57
58extern int bpf_xdp_metadata_rx_hash(const struct xdp_md *ctx, __u32 *hash,
59 enum xdp_rss_hash_type *rss_type) __ksym;
60
61static __always_inline __u16 get_dest_port(void *l4, void *data_end,
62 __u8 protocol)
63{
64 if (protocol == IPPROTO_UDP) {
65 struct udphdr *udp = l4;
66
67 if ((void *)(udp + 1) > data_end)
68 return 0;
69 return udp->dest;
70 } else if (protocol == IPPROTO_TCP) {
71 struct tcphdr *tcp = l4;
72
73 if ((void *)(tcp + 1) > data_end)
74 return 0;
75 return tcp->dest;
76 }
77
78 return 0;
79}
80
81SEC("xdp")
82int xdp_rss_hash(struct xdp_md *ctx)
83{
84 void *data_end = (void *)(long)ctx->data_end;
85 void *data = (void *)(long)ctx->data;
86 enum xdp_rss_hash_type rss_type = 0;
87 struct ethhdr *eth = data;
88 __u8 l4_proto = 0;
89 __u32 hash = 0;
90 __u32 key, val;
91 void *l4 = NULL;
92 __u32 *cnt;
93 int ret;
94
95 if ((void *)(eth + 1) > data_end)
96 return XDP_PASS;
97
98 if (eth->h_proto == bpf_htons(ETH_P_IP)) {
99 struct iphdr *iph = (void *)(eth + 1);
100
101 if ((void *)(iph + 1) > data_end)
102 return XDP_PASS;
103 l4_proto = iph->protocol;
104 l4 = (void *)(iph + 1);
105 } else if (eth->h_proto == bpf_htons(ETH_P_IPV6)) {
106 struct ipv6hdr *ip6h = (void *)(eth + 1);
107
108 if ((void *)(ip6h + 1) > data_end)
109 return XDP_PASS;
110 l4_proto = ip6h->nexthdr;
111 l4 = (void *)(ip6h + 1);
112 }
113
114 if (!l4)
115 return XDP_PASS;
116
117 /* Filter on the configured protocol (map_xdp_setup key XDP_PROTO).
118 * When set, only process packets matching the requested L4 protocol.
119 */
120 key = XDP_PROTO;
121 __s32 *proto_cfg = bpf_map_lookup_elem(&map_xdp_setup, &key);
122
123 if (proto_cfg && *proto_cfg != 0 && l4_proto != (__u8)*proto_cfg)
124 return XDP_PASS;
125
126 /* Filter on the configured port (map_xdp_setup key XDP_PORT).
127 * Only applies to protocols with ports (UDP, TCP).
128 */
129 key = XDP_PORT;
130 __s32 *port_cfg = bpf_map_lookup_elem(&map_xdp_setup, &key);
131
132 if (port_cfg && *port_cfg != 0) {
133 __u16 dest = get_dest_port(l4, data_end, l4_proto);
134
135 if (!dest || bpf_ntohs(dest) != (__u16)*port_cfg)
136 return XDP_PASS;
137 }
138
139 ret = bpf_xdp_metadata_rx_hash(ctx, &hash, &rss_type);
140 if (ret < 0) {
141 key = RSS_KEY_ERR_CNT;
142 cnt = bpf_map_lookup_elem(&map_rss, &key);
143 if (cnt)
144 __sync_fetch_and_add(cnt, 1);
145 return XDP_PASS;
146 }
147
148 key = RSS_KEY_HASH;
149 bpf_map_update_elem(&map_rss, &key, &hash, BPF_ANY);
150
151 key = RSS_KEY_TYPE;
152 val = (__u32)rss_type;
153 bpf_map_update_elem(&map_rss, &key, &val, BPF_ANY);
154
155 key = RSS_KEY_PKT_CNT;
156 cnt = bpf_map_lookup_elem(&map_rss, &key);
157 if (cnt)
158 __sync_fetch_and_add(cnt, 1);
159
160 return XDP_PASS;
161}
162
163char _license[] SEC("license") = "GPL";