Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * This is a module which is used for queueing packets and communicating with
4 * userspace via nfnetlink.
5 *
6 * (C) 2005 by Harald Welte <laforge@netfilter.org>
7 * (C) 2007 by Patrick McHardy <kaber@trash.net>
8 *
9 * Based on the old ipv4-only ip_queue.c:
10 * (C) 2000-2002 James Morris <jmorris@intercode.com.au>
11 * (C) 2003-2005 Netfilter Core Team <coreteam@netfilter.org>
12 */
13
14#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15
16#include <linux/module.h>
17#include <linux/skbuff.h>
18#include <linux/init.h>
19#include <linux/spinlock.h>
20#include <linux/slab.h>
21#include <linux/notifier.h>
22#include <linux/netdevice.h>
23#include <linux/netfilter.h>
24#include <linux/proc_fs.h>
25#include <linux/netfilter_ipv4.h>
26#include <linux/netfilter_ipv6.h>
27#include <linux/netfilter_bridge.h>
28#include <linux/netfilter/nfnetlink.h>
29#include <linux/netfilter/nfnetlink_queue.h>
30#include <linux/netfilter/nf_conntrack_common.h>
31#include <linux/list.h>
32#include <linux/cgroup-defs.h>
33#include <linux/rhashtable.h>
34#include <linux/jhash.h>
35#include <net/gso.h>
36#include <net/sock.h>
37#include <net/tcp_states.h>
38#include <net/netfilter/nf_queue.h>
39#include <net/netns/generic.h>
40
41#include <linux/atomic.h>
42
43#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
44#include "../bridge/br_private.h"
45#endif
46
47#if IS_ENABLED(CONFIG_NF_CONNTRACK)
48#include <net/netfilter/nf_conntrack.h>
49#endif
50
51#define NFQNL_QMAX_DEFAULT 1024
52#define NFQNL_HASH_MIN 8
53#define NFQNL_HASH_MAX 32768
54
55/* We're using struct nlattr which has 16bit nla_len. Note that nla_len
56 * includes the header length. Thus, the maximum packet length that we
57 * support is 65531 bytes. We send truncated packets if the specified length
58 * is larger than that. Userspace can check for presence of NFQA_CAP_LEN
59 * attribute to detect truncation.
60 */
61#define NFQNL_MAX_COPY_RANGE (0xffff - NLA_HDRLEN)
62
63struct nfqnl_instance {
64 struct hlist_node hlist; /* global list of queues */
65 struct rhashtable nfqnl_packet_map;
66 struct rcu_work rwork;
67
68 u32 peer_portid;
69 unsigned int queue_maxlen;
70 unsigned int copy_range;
71 unsigned int queue_dropped;
72 unsigned int queue_user_dropped;
73
74
75 u_int16_t queue_num; /* number of this queue */
76 u_int8_t copy_mode;
77 u_int32_t flags; /* Set using NFQA_CFG_FLAGS */
78/*
79 * Following fields are dirtied for each queued packet,
80 * keep them in same cache line if possible.
81 */
82 spinlock_t lock ____cacheline_aligned_in_smp;
83 unsigned int queue_total;
84 unsigned int id_sequence; /* 'sequence' of pkt ids */
85 struct list_head queue_list; /* packets in queue */
86};
87
88typedef int (*nfqnl_cmpfn)(struct nf_queue_entry *, unsigned long);
89
90static struct workqueue_struct *nfq_cleanup_wq __read_mostly;
91static unsigned int nfnl_queue_net_id __read_mostly;
92
93#define INSTANCE_BUCKETS 16
94struct nfnl_queue_net {
95 spinlock_t instances_lock;
96 struct hlist_head instance_table[INSTANCE_BUCKETS];
97};
98
99static struct nfnl_queue_net *nfnl_queue_pernet(struct net *net)
100{
101 return net_generic(net, nfnl_queue_net_id);
102}
103
104static inline u_int8_t instance_hashfn(u_int16_t queue_num)
105{
106 return ((queue_num >> 8) ^ queue_num) % INSTANCE_BUCKETS;
107}
108
109static const struct rhashtable_params nfqnl_rhashtable_params = {
110 .head_offset = offsetof(struct nf_queue_entry, hash_node),
111 .key_offset = offsetof(struct nf_queue_entry, id),
112 .key_len = sizeof(u32),
113 .automatic_shrinking = true,
114 .min_size = NFQNL_HASH_MIN,
115 .max_size = NFQNL_HASH_MAX,
116};
117
118static struct nfqnl_instance *
119instance_lookup(struct nfnl_queue_net *q, u_int16_t queue_num)
120{
121 struct hlist_head *head;
122 struct nfqnl_instance *inst;
123
124 head = &q->instance_table[instance_hashfn(queue_num)];
125 hlist_for_each_entry_rcu(inst, head, hlist) {
126 if (inst->queue_num == queue_num)
127 return inst;
128 }
129 return NULL;
130}
131
132static struct nfqnl_instance *
133instance_create(struct nfnl_queue_net *q, u_int16_t queue_num, u32 portid)
134{
135 struct nfqnl_instance *inst;
136 unsigned int h;
137 int err;
138
139 inst = kzalloc_obj(*inst, GFP_KERNEL_ACCOUNT);
140 if (!inst)
141 return ERR_PTR(-ENOMEM);
142
143 inst->queue_num = queue_num;
144 inst->peer_portid = portid;
145 inst->queue_maxlen = NFQNL_QMAX_DEFAULT;
146 inst->copy_range = NFQNL_MAX_COPY_RANGE;
147 inst->copy_mode = NFQNL_COPY_NONE;
148 spin_lock_init(&inst->lock);
149 INIT_LIST_HEAD(&inst->queue_list);
150
151 err = rhashtable_init(&inst->nfqnl_packet_map, &nfqnl_rhashtable_params);
152 if (err < 0)
153 goto out_free;
154
155 spin_lock(&q->instances_lock);
156 if (instance_lookup(q, queue_num)) {
157 err = -EEXIST;
158 goto out_unlock;
159 }
160
161 if (!try_module_get(THIS_MODULE)) {
162 err = -EAGAIN;
163 goto out_unlock;
164 }
165
166 h = instance_hashfn(queue_num);
167 hlist_add_head_rcu(&inst->hlist, &q->instance_table[h]);
168
169 spin_unlock(&q->instances_lock);
170
171 return inst;
172
173out_unlock:
174 spin_unlock(&q->instances_lock);
175 rhashtable_destroy(&inst->nfqnl_packet_map);
176out_free:
177 kfree(inst);
178 return ERR_PTR(err);
179}
180
181static void nfqnl_flush(struct nfqnl_instance *queue, nfqnl_cmpfn cmpfn,
182 unsigned long data);
183
184static void instance_destroy_work(struct work_struct *work)
185{
186 struct nfqnl_instance *inst;
187
188 inst = container_of(to_rcu_work(work), struct nfqnl_instance,
189 rwork);
190 rcu_read_lock();
191 nfqnl_flush(inst, NULL, 0);
192 rcu_read_unlock();
193
194 rhashtable_destroy(&inst->nfqnl_packet_map);
195
196 kfree(inst);
197 module_put(THIS_MODULE);
198}
199
200static void
201__instance_destroy(struct nfqnl_instance *inst)
202{
203 hlist_del_rcu(&inst->hlist);
204
205 INIT_RCU_WORK(&inst->rwork, instance_destroy_work);
206 queue_rcu_work(nfq_cleanup_wq, &inst->rwork);
207}
208
209static void
210instance_destroy(struct nfnl_queue_net *q, struct nfqnl_instance *inst)
211{
212 spin_lock(&q->instances_lock);
213 __instance_destroy(inst);
214 spin_unlock(&q->instances_lock);
215}
216
217static int
218__enqueue_entry(struct nfqnl_instance *queue, struct nf_queue_entry *entry)
219{
220 int err;
221
222 err = rhashtable_insert_fast(&queue->nfqnl_packet_map, &entry->hash_node,
223 nfqnl_rhashtable_params);
224 if (unlikely(err))
225 return err;
226
227 list_add_tail(&entry->list, &queue->queue_list);
228 queue->queue_total++;
229
230 return 0;
231}
232
233static void
234__dequeue_entry(struct nfqnl_instance *queue, struct nf_queue_entry *entry)
235{
236 rhashtable_remove_fast(&queue->nfqnl_packet_map, &entry->hash_node,
237 nfqnl_rhashtable_params);
238 list_del(&entry->list);
239 queue->queue_total--;
240}
241
242static struct nf_queue_entry *
243find_dequeue_entry(struct nfqnl_instance *queue, unsigned int id)
244{
245 struct nf_queue_entry *entry;
246
247 spin_lock_bh(&queue->lock);
248 entry = rhashtable_lookup_fast(&queue->nfqnl_packet_map, &id,
249 nfqnl_rhashtable_params);
250
251 if (entry)
252 __dequeue_entry(queue, entry);
253
254 spin_unlock_bh(&queue->lock);
255
256 return entry;
257}
258
259static unsigned int nf_iterate(struct sk_buff *skb,
260 struct nf_hook_state *state,
261 const struct nf_hook_entries *hooks,
262 unsigned int *index)
263{
264 const struct nf_hook_entry *hook;
265 unsigned int verdict, i = *index;
266
267 while (i < hooks->num_hook_entries) {
268 hook = &hooks->hooks[i];
269repeat:
270 verdict = nf_hook_entry_hookfn(hook, skb, state);
271 if (verdict != NF_ACCEPT) {
272 *index = i;
273 if (verdict != NF_REPEAT)
274 return verdict;
275 goto repeat;
276 }
277 i++;
278 }
279
280 *index = i;
281 return NF_ACCEPT;
282}
283
284static struct nf_hook_entries *nf_hook_entries_head(const struct net *net, u8 pf, u8 hooknum)
285{
286 switch (pf) {
287#ifdef CONFIG_NETFILTER_FAMILY_BRIDGE
288 case NFPROTO_BRIDGE:
289 return rcu_dereference(net->nf.hooks_bridge[hooknum]);
290#endif
291 case NFPROTO_IPV4:
292 return rcu_dereference(net->nf.hooks_ipv4[hooknum]);
293 case NFPROTO_IPV6:
294 return rcu_dereference(net->nf.hooks_ipv6[hooknum]);
295 default:
296 WARN_ON_ONCE(1);
297 return NULL;
298 }
299
300 return NULL;
301}
302
303static int nf_ip_reroute(struct sk_buff *skb, const struct nf_queue_entry *entry)
304{
305#ifdef CONFIG_INET
306 const struct ip_rt_info *rt_info = nf_queue_entry_reroute(entry);
307
308 if (entry->state.hook == NF_INET_LOCAL_OUT) {
309 const struct iphdr *iph = ip_hdr(skb);
310
311 if (!(iph->tos == rt_info->tos &&
312 skb->mark == rt_info->mark &&
313 iph->daddr == rt_info->daddr &&
314 iph->saddr == rt_info->saddr))
315 return ip_route_me_harder(entry->state.net, entry->state.sk,
316 skb, RTN_UNSPEC);
317 }
318#endif
319 return 0;
320}
321
322static int nf_ip6_reroute(struct sk_buff *skb,
323 const struct nf_queue_entry *entry)
324{
325 struct ip6_rt_info *rt_info = nf_queue_entry_reroute(entry);
326
327 if (entry->state.hook == NF_INET_LOCAL_OUT) {
328 const struct ipv6hdr *iph = ipv6_hdr(skb);
329
330 if (!ipv6_addr_equal(&iph->daddr, &rt_info->daddr) ||
331 !ipv6_addr_equal(&iph->saddr, &rt_info->saddr) ||
332 skb->mark != rt_info->mark)
333 return nf_ip6_route_me_harder(entry->state.net,
334 entry->state.sk, skb);
335 }
336 return 0;
337}
338
339static int nf_reroute(struct sk_buff *skb, struct nf_queue_entry *entry)
340{
341 int ret = 0;
342
343 switch (entry->state.pf) {
344 case AF_INET:
345 ret = nf_ip_reroute(skb, entry);
346 break;
347 case AF_INET6:
348 ret = nf_ip6_reroute(skb, entry);
349 break;
350 }
351 return ret;
352}
353
354/* caller must hold rcu read-side lock */
355static void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict)
356{
357 const struct nf_hook_entry *hook_entry;
358 const struct nf_hook_entries *hooks;
359 struct sk_buff *skb = entry->skb;
360 const struct net *net;
361 unsigned int i;
362 int err;
363 u8 pf;
364
365 net = entry->state.net;
366 pf = entry->state.pf;
367
368 hooks = nf_hook_entries_head(net, pf, entry->state.hook);
369
370 i = entry->hook_index;
371 if (!hooks || i >= hooks->num_hook_entries) {
372 kfree_skb_reason(skb, SKB_DROP_REASON_NETFILTER_DROP);
373 nf_queue_entry_free(entry);
374 return;
375 }
376
377 hook_entry = &hooks->hooks[i];
378
379 /* Continue traversal iff userspace said ok... */
380 if (verdict == NF_REPEAT)
381 verdict = nf_hook_entry_hookfn(hook_entry, skb, &entry->state);
382
383 if (verdict == NF_ACCEPT) {
384 if (nf_reroute(skb, entry) < 0)
385 verdict = NF_DROP;
386 }
387
388 if (verdict == NF_ACCEPT) {
389next_hook:
390 ++i;
391 verdict = nf_iterate(skb, &entry->state, hooks, &i);
392 }
393
394 switch (verdict & NF_VERDICT_MASK) {
395 case NF_ACCEPT:
396 case NF_STOP:
397 local_bh_disable();
398 entry->state.okfn(entry->state.net, entry->state.sk, skb);
399 local_bh_enable();
400 break;
401 case NF_QUEUE:
402 err = nf_queue(skb, &entry->state, i, verdict);
403 if (err == 1)
404 goto next_hook;
405 break;
406 case NF_STOLEN:
407 break;
408 default:
409 kfree_skb(skb);
410 }
411
412 nf_queue_entry_free(entry);
413}
414
415/* return true if the entry has an unconfirmed conntrack attached that isn't owned by us
416 * exclusively.
417 */
418static bool nf_ct_drop_unconfirmed(const struct nf_queue_entry *entry, bool *is_unconfirmed)
419{
420#if IS_ENABLED(CONFIG_NF_CONNTRACK)
421 struct nf_conn *ct = (void *)skb_nfct(entry->skb);
422
423 if (!ct || nf_ct_is_confirmed(ct))
424 return false;
425
426 if (is_unconfirmed)
427 *is_unconfirmed = true;
428
429 /* in some cases skb_clone() can occur after initial conntrack
430 * pickup, but conntrack assumes exclusive skb->_nfct ownership for
431 * unconfirmed entries.
432 *
433 * This happens for br_netfilter and with ip multicast routing.
434 * This can't be solved with serialization here because one clone
435 * could have been queued for local delivery or could be transmitted
436 * in parallel on another CPU.
437 */
438 return refcount_read(&ct->ct_general.use) > 1;
439#endif
440 return false;
441}
442
443static void nfqnl_reinject(struct nf_queue_entry *entry, unsigned int verdict)
444{
445 const struct nf_ct_hook *ct_hook;
446
447 if (verdict == NF_ACCEPT ||
448 verdict == NF_REPEAT ||
449 verdict == NF_STOP) {
450 unsigned int ct_verdict = verdict;
451
452 rcu_read_lock();
453 ct_hook = rcu_dereference(nf_ct_hook);
454 if (ct_hook)
455 ct_verdict = ct_hook->update(entry->state.net, entry->skb);
456 rcu_read_unlock();
457
458 switch (ct_verdict & NF_VERDICT_MASK) {
459 case NF_ACCEPT:
460 /* follow userspace verdict, could be REPEAT */
461 break;
462 case NF_STOLEN:
463 nf_queue_entry_free(entry);
464 return;
465 default:
466 verdict = ct_verdict & NF_VERDICT_MASK;
467 break;
468 }
469 }
470
471 if (verdict != NF_DROP && entry->nf_ct_is_unconfirmed) {
472 /* If first queued segment was already reinjected then
473 * there is a good chance the ct entry is now confirmed.
474 *
475 * Handle the rare cases:
476 * - out-of-order verdict
477 * - threaded userspace reinjecting in parallel
478 * - first segment was dropped
479 *
480 * In all of those cases we can't handle this packet
481 * because we can't be sure that another CPU won't modify
482 * nf_conn->ext in parallel which isn't allowed.
483 */
484 if (nf_ct_drop_unconfirmed(entry, NULL))
485 verdict = NF_DROP;
486 }
487
488 nf_reinject(entry, verdict);
489}
490
491static void
492nfqnl_flush(struct nfqnl_instance *queue, nfqnl_cmpfn cmpfn, unsigned long data)
493{
494 struct nf_queue_entry *entry, *next;
495
496 spin_lock_bh(&queue->lock);
497 list_for_each_entry_safe(entry, next, &queue->queue_list, list) {
498 if (!cmpfn || cmpfn(entry, data)) {
499 __dequeue_entry(queue, entry);
500 nfqnl_reinject(entry, NF_DROP);
501 }
502 }
503 spin_unlock_bh(&queue->lock);
504}
505
506static int
507nfqnl_put_packet_info(struct sk_buff *nlskb, struct sk_buff *packet,
508 bool csum_verify)
509{
510 __u32 flags = 0;
511
512 if (packet->ip_summed == CHECKSUM_PARTIAL)
513 flags = NFQA_SKB_CSUMNOTREADY;
514 else if (csum_verify)
515 flags = NFQA_SKB_CSUM_NOTVERIFIED;
516
517 if (skb_is_gso(packet))
518 flags |= NFQA_SKB_GSO;
519
520 return flags ? nla_put_be32(nlskb, NFQA_SKB_INFO, htonl(flags)) : 0;
521}
522
523static int nfqnl_put_sk_uidgid(struct sk_buff *skb, struct sock *sk)
524{
525 const struct socket *sock;
526 const struct file *file;
527 const struct cred *cred;
528
529 if (!sk_fullsock(sk))
530 return 0;
531
532 /* The sk pointer remains valid as long as the skb is.
533 * The sk_socket and file pointer may become NULL
534 * if the socket is closed.
535 * Both structures (including file->cred) are RCU freed
536 * which means they can be accessed within a RCU read section.
537 */
538 sock = READ_ONCE(sk->sk_socket);
539 file = sock ? READ_ONCE(sock->file) : NULL;
540 if (file) {
541 cred = file->f_cred;
542 if (nla_put_be32(skb, NFQA_UID,
543 htonl(from_kuid_munged(&init_user_ns, cred->fsuid))))
544 goto nla_put_failure;
545 if (nla_put_be32(skb, NFQA_GID,
546 htonl(from_kgid_munged(&init_user_ns, cred->fsgid))))
547 goto nla_put_failure;
548 }
549 return 0;
550
551nla_put_failure:
552 return -1;
553}
554
555static int nfqnl_put_sk_classid(struct sk_buff *skb, struct sock *sk)
556{
557#if IS_ENABLED(CONFIG_CGROUP_NET_CLASSID)
558 if (sk && sk_fullsock(sk)) {
559 u32 classid = sock_cgroup_classid(&sk->sk_cgrp_data);
560
561 if (classid && nla_put_be32(skb, NFQA_CGROUP_CLASSID, htonl(classid)))
562 return -1;
563 }
564#endif
565 return 0;
566}
567
568static int nfqnl_get_sk_secctx(struct sk_buff *skb, struct lsm_context *ctx)
569{
570 int seclen = 0;
571#if IS_ENABLED(CONFIG_NETWORK_SECMARK)
572 if (skb->secmark)
573 seclen = security_secid_to_secctx(skb->secmark, ctx);
574#endif
575 return seclen;
576}
577
578static u32 nfqnl_get_bridge_size(struct nf_queue_entry *entry)
579{
580 struct sk_buff *entskb = entry->skb;
581 u32 nlalen = 0;
582 u32 mac_len;
583
584 if (entry->state.pf != PF_BRIDGE || !skb_mac_header_was_set(entskb))
585 return 0;
586
587 if (skb_vlan_tag_present(entskb))
588 nlalen += nla_total_size(nla_total_size(sizeof(__be16)) +
589 nla_total_size(sizeof(__be16)));
590
591 mac_len = skb_mac_header_len(entskb);
592 if (mac_len > 0)
593 nlalen += nla_total_size(mac_len);
594
595 return nlalen;
596}
597
598static int nfqnl_put_bridge(struct nf_queue_entry *entry, struct sk_buff *skb)
599{
600 struct sk_buff *entskb = entry->skb;
601 u32 mac_len;
602
603 if (entry->state.pf != PF_BRIDGE || !skb_mac_header_was_set(entskb))
604 return 0;
605
606 if (skb_vlan_tag_present(entskb)) {
607 struct nlattr *nest;
608
609 nest = nla_nest_start(skb, NFQA_VLAN);
610 if (!nest)
611 goto nla_put_failure;
612
613 if (nla_put_be16(skb, NFQA_VLAN_TCI, htons(entskb->vlan_tci)) ||
614 nla_put_be16(skb, NFQA_VLAN_PROTO, entskb->vlan_proto))
615 goto nla_put_failure;
616
617 nla_nest_end(skb, nest);
618 }
619
620 mac_len = skb_mac_header_len(entskb);
621 if (mac_len > 0 &&
622 nla_put(skb, NFQA_L2HDR, mac_len, skb_mac_header(entskb)))
623 goto nla_put_failure;
624
625 return 0;
626
627nla_put_failure:
628 return -1;
629}
630
631static int nf_queue_checksum_help(struct sk_buff *entskb)
632{
633 if (skb_csum_is_sctp(entskb))
634 return skb_crc32c_csum_help(entskb);
635
636 return skb_checksum_help(entskb);
637}
638
639static struct sk_buff *
640nfqnl_build_packet_message(struct net *net, struct nfqnl_instance *queue,
641 struct nf_queue_entry *entry,
642 __be32 **packet_id_ptr)
643{
644 size_t size;
645 size_t data_len = 0, cap_len = 0;
646 unsigned int hlen = 0;
647 struct sk_buff *skb;
648 struct nlattr *nla;
649 struct nfqnl_msg_packet_hdr *pmsg;
650 struct nlmsghdr *nlh;
651 struct sk_buff *entskb = entry->skb;
652 struct net_device *indev;
653 struct net_device *outdev;
654 struct nf_conn *ct = NULL;
655 enum ip_conntrack_info ctinfo = 0;
656 const struct nfnl_ct_hook *nfnl_ct;
657 bool csum_verify;
658 struct lsm_context ctx = { NULL, 0, 0 };
659 int seclen = 0;
660 ktime_t tstamp;
661
662 size = nlmsg_total_size(sizeof(struct nfgenmsg))
663 + nla_total_size(sizeof(struct nfqnl_msg_packet_hdr))
664 + nla_total_size(sizeof(u_int32_t)) /* ifindex */
665 + nla_total_size(sizeof(u_int32_t)) /* ifindex */
666#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
667 + nla_total_size(sizeof(u_int32_t)) /* ifindex */
668 + nla_total_size(sizeof(u_int32_t)) /* ifindex */
669#endif
670 + nla_total_size(sizeof(u_int32_t)) /* mark */
671 + nla_total_size(sizeof(u_int32_t)) /* priority */
672 + nla_total_size(sizeof(struct nfqnl_msg_packet_hw))
673 + nla_total_size(sizeof(u_int32_t)) /* skbinfo */
674#if IS_ENABLED(CONFIG_CGROUP_NET_CLASSID)
675 + nla_total_size(sizeof(u_int32_t)) /* classid */
676#endif
677 + nla_total_size(sizeof(u_int32_t)); /* cap_len */
678
679 tstamp = skb_tstamp_cond(entskb, false);
680 if (tstamp)
681 size += nla_total_size(sizeof(struct nfqnl_msg_packet_timestamp));
682
683 size += nfqnl_get_bridge_size(entry);
684
685 if (entry->state.hook <= NF_INET_FORWARD ||
686 (entry->state.hook == NF_INET_POST_ROUTING && entskb->sk == NULL))
687 csum_verify = !skb_csum_unnecessary(entskb);
688 else
689 csum_verify = false;
690
691 outdev = entry->state.out;
692
693 switch ((enum nfqnl_config_mode)READ_ONCE(queue->copy_mode)) {
694 case NFQNL_COPY_META:
695 case NFQNL_COPY_NONE:
696 break;
697
698 case NFQNL_COPY_PACKET:
699 if (!(queue->flags & NFQA_CFG_F_GSO) &&
700 entskb->ip_summed == CHECKSUM_PARTIAL &&
701 nf_queue_checksum_help(entskb))
702 return NULL;
703
704 data_len = READ_ONCE(queue->copy_range);
705 if (data_len > entskb->len)
706 data_len = entskb->len;
707
708 hlen = skb_zerocopy_headlen(entskb);
709 hlen = min_t(unsigned int, hlen, data_len);
710 size += sizeof(struct nlattr) + hlen;
711 cap_len = entskb->len;
712 break;
713 }
714
715 nfnl_ct = rcu_dereference(nfnl_ct_hook);
716
717#if IS_ENABLED(CONFIG_NF_CONNTRACK)
718 if (queue->flags & NFQA_CFG_F_CONNTRACK) {
719 if (nfnl_ct != NULL) {
720 ct = nf_ct_get(entskb, &ctinfo);
721 if (ct != NULL)
722 size += nfnl_ct->build_size(ct);
723 }
724 }
725#endif
726
727 if (queue->flags & NFQA_CFG_F_UID_GID) {
728 size += (nla_total_size(sizeof(u_int32_t)) /* uid */
729 + nla_total_size(sizeof(u_int32_t))); /* gid */
730 }
731
732 if ((queue->flags & NFQA_CFG_F_SECCTX) && entskb->sk) {
733 seclen = nfqnl_get_sk_secctx(entskb, &ctx);
734 if (seclen < 0)
735 return NULL;
736 if (seclen)
737 size += nla_total_size(seclen);
738 }
739
740 skb = alloc_skb(size, GFP_ATOMIC);
741 if (!skb) {
742 skb_tx_error(entskb);
743 goto nlmsg_failure;
744 }
745
746 nlh = nfnl_msg_put(skb, 0, 0,
747 nfnl_msg_type(NFNL_SUBSYS_QUEUE, NFQNL_MSG_PACKET),
748 0, entry->state.pf, NFNETLINK_V0,
749 htons(queue->queue_num));
750 if (!nlh) {
751 skb_tx_error(entskb);
752 kfree_skb(skb);
753 goto nlmsg_failure;
754 }
755
756 nla = __nla_reserve(skb, NFQA_PACKET_HDR, sizeof(*pmsg));
757 pmsg = nla_data(nla);
758 pmsg->hw_protocol = entskb->protocol;
759 pmsg->hook = entry->state.hook;
760 *packet_id_ptr = &pmsg->packet_id;
761
762 indev = entry->state.in;
763 if (indev) {
764#if !IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
765 if (nla_put_be32(skb, NFQA_IFINDEX_INDEV, htonl(indev->ifindex)))
766 goto nla_put_failure;
767#else
768 if (entry->state.pf == PF_BRIDGE) {
769 /* Case 1: indev is physical input device, we need to
770 * look for bridge group (when called from
771 * netfilter_bridge) */
772 if (nla_put_be32(skb, NFQA_IFINDEX_PHYSINDEV,
773 htonl(indev->ifindex)) ||
774 /* this is the bridge group "brX" */
775 /* rcu_read_lock()ed by __nf_queue */
776 nla_put_be32(skb, NFQA_IFINDEX_INDEV,
777 htonl(br_port_get_rcu(indev)->br->dev->ifindex)))
778 goto nla_put_failure;
779 } else {
780 int physinif;
781
782 /* Case 2: indev is bridge group, we need to look for
783 * physical device (when called from ipv4) */
784 if (nla_put_be32(skb, NFQA_IFINDEX_INDEV,
785 htonl(indev->ifindex)))
786 goto nla_put_failure;
787
788 physinif = nf_bridge_get_physinif(entskb);
789 if (physinif &&
790 nla_put_be32(skb, NFQA_IFINDEX_PHYSINDEV,
791 htonl(physinif)))
792 goto nla_put_failure;
793 }
794#endif
795 }
796
797 if (outdev) {
798#if !IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
799 if (nla_put_be32(skb, NFQA_IFINDEX_OUTDEV, htonl(outdev->ifindex)))
800 goto nla_put_failure;
801#else
802 if (entry->state.pf == PF_BRIDGE) {
803 /* Case 1: outdev is physical output device, we need to
804 * look for bridge group (when called from
805 * netfilter_bridge) */
806 if (nla_put_be32(skb, NFQA_IFINDEX_PHYSOUTDEV,
807 htonl(outdev->ifindex)) ||
808 /* this is the bridge group "brX" */
809 /* rcu_read_lock()ed by __nf_queue */
810 nla_put_be32(skb, NFQA_IFINDEX_OUTDEV,
811 htonl(br_port_get_rcu(outdev)->br->dev->ifindex)))
812 goto nla_put_failure;
813 } else {
814 int physoutif;
815
816 /* Case 2: outdev is bridge group, we need to look for
817 * physical output device (when called from ipv4) */
818 if (nla_put_be32(skb, NFQA_IFINDEX_OUTDEV,
819 htonl(outdev->ifindex)))
820 goto nla_put_failure;
821
822 physoutif = nf_bridge_get_physoutif(entskb);
823 if (physoutif &&
824 nla_put_be32(skb, NFQA_IFINDEX_PHYSOUTDEV,
825 htonl(physoutif)))
826 goto nla_put_failure;
827 }
828#endif
829 }
830
831 if (entskb->mark &&
832 nla_put_be32(skb, NFQA_MARK, htonl(entskb->mark)))
833 goto nla_put_failure;
834
835 if (entskb->priority &&
836 nla_put_be32(skb, NFQA_PRIORITY, htonl(entskb->priority)))
837 goto nla_put_failure;
838
839 if (indev && entskb->dev &&
840 skb_mac_header_was_set(entskb) &&
841 skb_mac_header_len(entskb) != 0) {
842 struct nfqnl_msg_packet_hw phw;
843 int len;
844
845 memset(&phw, 0, sizeof(phw));
846 len = dev_parse_header(entskb, phw.hw_addr);
847 if (len) {
848 phw.hw_addrlen = htons(len);
849 if (nla_put(skb, NFQA_HWADDR, sizeof(phw), &phw))
850 goto nla_put_failure;
851 }
852 }
853
854 if (nfqnl_put_bridge(entry, skb) < 0)
855 goto nla_put_failure;
856
857 if (entry->state.hook <= NF_INET_FORWARD && tstamp) {
858 struct nfqnl_msg_packet_timestamp ts;
859 struct timespec64 kts = ktime_to_timespec64(tstamp);
860
861 ts.sec = cpu_to_be64(kts.tv_sec);
862 ts.usec = cpu_to_be64(kts.tv_nsec / NSEC_PER_USEC);
863
864 if (nla_put(skb, NFQA_TIMESTAMP, sizeof(ts), &ts))
865 goto nla_put_failure;
866 }
867
868 if ((queue->flags & NFQA_CFG_F_UID_GID) && entskb->sk &&
869 nfqnl_put_sk_uidgid(skb, entskb->sk) < 0)
870 goto nla_put_failure;
871
872 if (nfqnl_put_sk_classid(skb, entskb->sk) < 0)
873 goto nla_put_failure;
874
875 if (seclen > 0 && nla_put(skb, NFQA_SECCTX, ctx.len, ctx.context))
876 goto nla_put_failure;
877
878 if (ct && nfnl_ct->build(skb, ct, ctinfo, NFQA_CT, NFQA_CT_INFO) < 0)
879 goto nla_put_failure;
880
881 if (cap_len > data_len &&
882 nla_put_be32(skb, NFQA_CAP_LEN, htonl(cap_len)))
883 goto nla_put_failure;
884
885 if (nfqnl_put_packet_info(skb, entskb, csum_verify))
886 goto nla_put_failure;
887
888 if (data_len) {
889 struct nlattr *nla;
890
891 if (skb_tailroom(skb) < sizeof(*nla) + hlen)
892 goto nla_put_failure;
893
894 nla = skb_put(skb, sizeof(*nla));
895 nla->nla_type = NFQA_PAYLOAD;
896 nla->nla_len = nla_attr_size(data_len);
897
898 if (skb_zerocopy(skb, entskb, data_len, hlen))
899 goto nla_put_failure;
900 }
901
902 nlh->nlmsg_len = skb->len;
903 if (seclen >= 0)
904 security_release_secctx(&ctx);
905 return skb;
906
907nla_put_failure:
908 skb_tx_error(entskb);
909 kfree_skb(skb);
910 net_err_ratelimited("nf_queue: error creating packet message\n");
911nlmsg_failure:
912 if (seclen >= 0)
913 security_release_secctx(&ctx);
914 return NULL;
915}
916
917static int
918__nfqnl_enqueue_packet(struct net *net, struct nfqnl_instance *queue,
919 struct nf_queue_entry *entry)
920{
921 struct sk_buff *nskb;
922 int err = -ENOBUFS;
923 __be32 *packet_id_ptr;
924 int failopen = 0;
925
926 nskb = nfqnl_build_packet_message(net, queue, entry, &packet_id_ptr);
927 if (nskb == NULL) {
928 err = -ENOMEM;
929 goto err_out;
930 }
931 spin_lock_bh(&queue->lock);
932
933 if (queue->queue_total >= queue->queue_maxlen)
934 goto err_out_queue_drop;
935
936 entry->id = ++queue->id_sequence;
937 *packet_id_ptr = htonl(entry->id);
938
939 /* Insert into hash BEFORE unicast. If failure don't send to userspace. */
940 err = __enqueue_entry(queue, entry);
941 if (unlikely(err))
942 goto err_out_queue_drop;
943
944 /* nfnetlink_unicast will either free the nskb or add it to a socket */
945 err = nfnetlink_unicast(nskb, net, queue->peer_portid);
946 if (err < 0) {
947 /* Unicast failed - remove entry we just inserted */
948 __dequeue_entry(queue, entry);
949
950 if (queue->flags & NFQA_CFG_F_FAIL_OPEN) {
951 failopen = 1;
952 err = 0;
953 } else {
954 queue->queue_user_dropped++;
955 }
956 goto err_out_unlock;
957 }
958
959 spin_unlock_bh(&queue->lock);
960 return 0;
961
962err_out_queue_drop:
963 if (queue->flags & NFQA_CFG_F_FAIL_OPEN) {
964 failopen = 1;
965 err = 0;
966 } else {
967 queue->queue_dropped++;
968
969 if (queue->queue_total >= queue->queue_maxlen)
970 net_warn_ratelimited("nf_queue: full at %d entries, dropping packets(s)\n",
971 queue->queue_total);
972 else
973 net_warn_ratelimited("nf_queue: hash insert failed: %d\n", err);
974 }
975 kfree_skb(nskb);
976err_out_unlock:
977 spin_unlock_bh(&queue->lock);
978 if (failopen)
979 nfqnl_reinject(entry, NF_ACCEPT);
980err_out:
981 return err;
982}
983
984static struct nf_queue_entry *
985nf_queue_entry_dup(struct nf_queue_entry *e)
986{
987 struct nf_queue_entry *entry = kmemdup(e, e->size, GFP_ATOMIC);
988
989 if (!entry)
990 return NULL;
991
992 if (nf_queue_entry_get_refs(entry))
993 return entry;
994
995 kfree(entry);
996 return NULL;
997}
998
999#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
1000/* When called from bridge netfilter, skb->data must point to MAC header
1001 * before calling skb_gso_segment(). Else, original MAC header is lost
1002 * and segmented skbs will be sent to wrong destination.
1003 */
1004static void nf_bridge_adjust_skb_data(struct sk_buff *skb)
1005{
1006 if (nf_bridge_info_get(skb))
1007 __skb_push(skb, skb_mac_header_len(skb));
1008}
1009
1010static void nf_bridge_adjust_segmented_data(struct sk_buff *skb)
1011{
1012 if (nf_bridge_info_get(skb))
1013 __skb_pull(skb, skb_mac_header_len(skb));
1014}
1015#else
1016#define nf_bridge_adjust_skb_data(s) do {} while (0)
1017#define nf_bridge_adjust_segmented_data(s) do {} while (0)
1018#endif
1019
1020static int
1021__nfqnl_enqueue_packet_gso(struct net *net, struct nfqnl_instance *queue,
1022 struct sk_buff *skb, struct nf_queue_entry *entry)
1023{
1024 int ret = -ENOMEM;
1025 struct nf_queue_entry *entry_seg;
1026
1027 nf_bridge_adjust_segmented_data(skb);
1028
1029 if (skb->next == NULL) { /* last packet, no need to copy entry */
1030 struct sk_buff *gso_skb = entry->skb;
1031 entry->skb = skb;
1032 ret = __nfqnl_enqueue_packet(net, queue, entry);
1033 if (ret)
1034 entry->skb = gso_skb;
1035 return ret;
1036 }
1037
1038 skb_mark_not_on_list(skb);
1039
1040 entry_seg = nf_queue_entry_dup(entry);
1041 if (entry_seg) {
1042 entry_seg->skb = skb;
1043 ret = __nfqnl_enqueue_packet(net, queue, entry_seg);
1044 if (ret)
1045 nf_queue_entry_free(entry_seg);
1046 }
1047 return ret;
1048}
1049
1050static int
1051nfqnl_enqueue_packet(struct nf_queue_entry *entry, unsigned int queuenum)
1052{
1053 struct sk_buff *skb, *segs, *nskb;
1054 bool ct_is_unconfirmed = false;
1055 struct nfqnl_instance *queue;
1056 unsigned int queued;
1057 int err = -ENOBUFS;
1058 struct net *net = entry->state.net;
1059 struct nfnl_queue_net *q = nfnl_queue_pernet(net);
1060
1061 /* rcu_read_lock()ed by nf_hook_thresh */
1062 queue = instance_lookup(q, queuenum);
1063 if (!queue)
1064 return -ESRCH;
1065
1066 if (queue->copy_mode == NFQNL_COPY_NONE)
1067 return -EINVAL;
1068
1069 skb = entry->skb;
1070
1071 switch (entry->state.pf) {
1072 case NFPROTO_IPV4:
1073 skb->protocol = htons(ETH_P_IP);
1074 break;
1075 case NFPROTO_IPV6:
1076 skb->protocol = htons(ETH_P_IPV6);
1077 break;
1078 }
1079
1080 /* Check if someone already holds another reference to
1081 * unconfirmed ct. If so, we cannot queue the skb:
1082 * concurrent modifications of nf_conn->ext are not
1083 * allowed and we can't know if another CPU isn't
1084 * processing the same nf_conn entry in parallel.
1085 */
1086 if (nf_ct_drop_unconfirmed(entry, &ct_is_unconfirmed))
1087 return -EINVAL;
1088
1089 if (!skb_is_gso(skb) || ((queue->flags & NFQA_CFG_F_GSO) && !skb_is_gso_sctp(skb)))
1090 return __nfqnl_enqueue_packet(net, queue, entry);
1091
1092 nf_bridge_adjust_skb_data(skb);
1093 segs = skb_gso_segment(skb, 0);
1094 /* Does not use PTR_ERR to limit the number of error codes that can be
1095 * returned by nf_queue. For instance, callers rely on -ESRCH to
1096 * mean 'ignore this hook'.
1097 */
1098 if (IS_ERR_OR_NULL(segs))
1099 goto out_err;
1100 queued = 0;
1101 err = 0;
1102
1103 skb_list_walk_safe(segs, segs, nskb) {
1104 if (ct_is_unconfirmed && queued > 0) {
1105 /* skb_gso_segment() increments the ct refcount.
1106 * This is a problem for unconfirmed (not in hash)
1107 * entries, those can race when reinjections happen
1108 * in parallel.
1109 *
1110 * Annotate this for all queued entries except the
1111 * first one.
1112 *
1113 * As long as the first one is reinjected first it
1114 * will do the confirmation for us.
1115 */
1116 entry->nf_ct_is_unconfirmed = ct_is_unconfirmed;
1117 }
1118
1119 if (err == 0)
1120 err = __nfqnl_enqueue_packet_gso(net, queue,
1121 segs, entry);
1122 if (err == 0)
1123 queued++;
1124 else
1125 kfree_skb(segs);
1126 }
1127
1128 if (queued) {
1129 if (err) /* some segments are already queued */
1130 nf_queue_entry_free(entry);
1131 kfree_skb(skb);
1132 return 0;
1133 }
1134 out_err:
1135 nf_bridge_adjust_segmented_data(skb);
1136 return err;
1137}
1138
1139static int
1140nfqnl_mangle(void *data, unsigned int data_len, struct nf_queue_entry *e, int diff)
1141{
1142 struct sk_buff *nskb;
1143
1144 if (diff < 0) {
1145 unsigned int min_len = skb_transport_offset(e->skb);
1146
1147 if (data_len < min_len)
1148 return -EINVAL;
1149
1150 if (pskb_trim(e->skb, data_len))
1151 return -ENOMEM;
1152 } else if (diff > 0) {
1153 if (data_len > 0xFFFF)
1154 return -EINVAL;
1155 if (diff > skb_tailroom(e->skb)) {
1156 nskb = skb_copy_expand(e->skb, skb_headroom(e->skb),
1157 diff, GFP_ATOMIC);
1158 if (!nskb)
1159 return -ENOMEM;
1160 kfree_skb(e->skb);
1161 e->skb = nskb;
1162 }
1163 skb_put(e->skb, diff);
1164 }
1165 if (skb_ensure_writable(e->skb, data_len))
1166 return -ENOMEM;
1167 skb_copy_to_linear_data(e->skb, data, data_len);
1168 e->skb->ip_summed = CHECKSUM_NONE;
1169 return 0;
1170}
1171
1172static int
1173nfqnl_set_mode(struct nfqnl_instance *queue,
1174 unsigned char mode, unsigned int range)
1175{
1176 int status = 0;
1177
1178 spin_lock_bh(&queue->lock);
1179 switch (mode) {
1180 case NFQNL_COPY_NONE:
1181 case NFQNL_COPY_META:
1182 queue->copy_mode = mode;
1183 queue->copy_range = 0;
1184 break;
1185
1186 case NFQNL_COPY_PACKET:
1187 queue->copy_mode = mode;
1188 if (range == 0 || range > NFQNL_MAX_COPY_RANGE)
1189 queue->copy_range = NFQNL_MAX_COPY_RANGE;
1190 else
1191 queue->copy_range = range;
1192 break;
1193
1194 default:
1195 status = -EINVAL;
1196
1197 }
1198 spin_unlock_bh(&queue->lock);
1199
1200 return status;
1201}
1202
1203static int
1204dev_cmp(struct nf_queue_entry *entry, unsigned long ifindex)
1205{
1206#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
1207 int physinif, physoutif;
1208
1209 physinif = nf_bridge_get_physinif(entry->skb);
1210 physoutif = nf_bridge_get_physoutif(entry->skb);
1211
1212 if (physinif == ifindex || physoutif == ifindex)
1213 return 1;
1214#endif
1215 if (entry->state.in)
1216 if (entry->state.in->ifindex == ifindex)
1217 return 1;
1218 if (entry->state.out)
1219 if (entry->state.out->ifindex == ifindex)
1220 return 1;
1221
1222 return 0;
1223}
1224
1225/* drop all packets with either indev or outdev == ifindex from all queue
1226 * instances */
1227static void
1228nfqnl_dev_drop(struct net *net, int ifindex)
1229{
1230 int i;
1231 struct nfnl_queue_net *q = nfnl_queue_pernet(net);
1232
1233 rcu_read_lock();
1234
1235 for (i = 0; i < INSTANCE_BUCKETS; i++) {
1236 struct nfqnl_instance *inst;
1237 struct hlist_head *head = &q->instance_table[i];
1238
1239 hlist_for_each_entry_rcu(inst, head, hlist)
1240 nfqnl_flush(inst, dev_cmp, ifindex);
1241 }
1242
1243 rcu_read_unlock();
1244}
1245
1246static int
1247nfqnl_rcv_dev_event(struct notifier_block *this,
1248 unsigned long event, void *ptr)
1249{
1250 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
1251
1252 /* Drop any packets associated with the downed device */
1253 if (event == NETDEV_DOWN)
1254 nfqnl_dev_drop(dev_net(dev), dev->ifindex);
1255 return NOTIFY_DONE;
1256}
1257
1258static struct notifier_block nfqnl_dev_notifier = {
1259 .notifier_call = nfqnl_rcv_dev_event,
1260};
1261
1262static void nfqnl_nf_hook_drop(struct net *net)
1263{
1264 struct nfnl_queue_net *q = nfnl_queue_pernet(net);
1265 int i;
1266
1267 /* This function is also called on net namespace error unwind,
1268 * when pernet_ops->init() failed and ->exit() functions of the
1269 * previous pernet_ops gets called.
1270 *
1271 * This may result in a call to nfqnl_nf_hook_drop() before
1272 * struct nfnl_queue_net was allocated.
1273 */
1274 if (!q)
1275 return;
1276
1277 for (i = 0; i < INSTANCE_BUCKETS; i++) {
1278 struct nfqnl_instance *inst;
1279 struct hlist_head *head = &q->instance_table[i];
1280
1281 hlist_for_each_entry_rcu(inst, head, hlist)
1282 nfqnl_flush(inst, NULL, 0);
1283 }
1284}
1285
1286static int
1287nfqnl_rcv_nl_event(struct notifier_block *this,
1288 unsigned long event, void *ptr)
1289{
1290 struct netlink_notify *n = ptr;
1291 struct nfnl_queue_net *q = nfnl_queue_pernet(n->net);
1292
1293 if (event == NETLINK_URELEASE && n->protocol == NETLINK_NETFILTER) {
1294 int i;
1295
1296 /* destroy all instances for this portid */
1297 spin_lock(&q->instances_lock);
1298 for (i = 0; i < INSTANCE_BUCKETS; i++) {
1299 struct hlist_node *t2;
1300 struct nfqnl_instance *inst;
1301 struct hlist_head *head = &q->instance_table[i];
1302
1303 hlist_for_each_entry_safe(inst, t2, head, hlist) {
1304 if (n->portid == inst->peer_portid)
1305 __instance_destroy(inst);
1306 }
1307 }
1308 spin_unlock(&q->instances_lock);
1309 }
1310 return NOTIFY_DONE;
1311}
1312
1313static struct notifier_block nfqnl_rtnl_notifier = {
1314 .notifier_call = nfqnl_rcv_nl_event,
1315};
1316
1317static const struct nla_policy nfqa_vlan_policy[NFQA_VLAN_MAX + 1] = {
1318 [NFQA_VLAN_TCI] = { .type = NLA_U16},
1319 [NFQA_VLAN_PROTO] = { .type = NLA_U16},
1320};
1321
1322static const struct nla_policy nfqa_verdict_policy[NFQA_MAX+1] = {
1323 [NFQA_VERDICT_HDR] = { .len = sizeof(struct nfqnl_msg_verdict_hdr) },
1324 [NFQA_MARK] = { .type = NLA_U32 },
1325 [NFQA_PAYLOAD] = { .type = NLA_UNSPEC },
1326 [NFQA_CT] = { .type = NLA_UNSPEC },
1327 [NFQA_EXP] = { .type = NLA_UNSPEC },
1328 [NFQA_VLAN] = { .type = NLA_NESTED },
1329 [NFQA_PRIORITY] = { .type = NLA_U32 },
1330};
1331
1332static const struct nla_policy nfqa_verdict_batch_policy[NFQA_MAX+1] = {
1333 [NFQA_VERDICT_HDR] = { .len = sizeof(struct nfqnl_msg_verdict_hdr) },
1334 [NFQA_MARK] = { .type = NLA_U32 },
1335 [NFQA_PRIORITY] = { .type = NLA_U32 },
1336};
1337
1338static struct nfqnl_instance *
1339verdict_instance_lookup(struct nfnl_queue_net *q, u16 queue_num, u32 nlportid)
1340{
1341 struct nfqnl_instance *queue;
1342
1343 queue = instance_lookup(q, queue_num);
1344 if (!queue)
1345 return ERR_PTR(-ENODEV);
1346
1347 if (queue->peer_portid != nlportid)
1348 return ERR_PTR(-EPERM);
1349
1350 return queue;
1351}
1352
1353static struct nfqnl_msg_verdict_hdr*
1354verdicthdr_get(const struct nlattr * const nfqa[])
1355{
1356 struct nfqnl_msg_verdict_hdr *vhdr;
1357 unsigned int verdict;
1358
1359 if (!nfqa[NFQA_VERDICT_HDR])
1360 return NULL;
1361
1362 vhdr = nla_data(nfqa[NFQA_VERDICT_HDR]);
1363 verdict = ntohl(vhdr->verdict) & NF_VERDICT_MASK;
1364 if (verdict > NF_MAX_VERDICT || verdict == NF_STOLEN)
1365 return NULL;
1366 return vhdr;
1367}
1368
1369static int nfq_id_after(unsigned int id, unsigned int max)
1370{
1371 return (int)(id - max) > 0;
1372}
1373
1374static int nfqnl_recv_verdict_batch(struct sk_buff *skb,
1375 const struct nfnl_info *info,
1376 const struct nlattr * const nfqa[])
1377{
1378 struct nfnl_queue_net *q = nfnl_queue_pernet(info->net);
1379 u16 queue_num = ntohs(info->nfmsg->res_id);
1380 struct nf_queue_entry *entry, *tmp;
1381 struct nfqnl_msg_verdict_hdr *vhdr;
1382 struct nfqnl_instance *queue;
1383 unsigned int verdict, maxid;
1384 LIST_HEAD(batch_list);
1385
1386 queue = verdict_instance_lookup(q, queue_num,
1387 NETLINK_CB(skb).portid);
1388 if (IS_ERR(queue))
1389 return PTR_ERR(queue);
1390
1391 vhdr = verdicthdr_get(nfqa);
1392 if (!vhdr)
1393 return -EINVAL;
1394
1395 verdict = ntohl(vhdr->verdict);
1396 maxid = ntohl(vhdr->id);
1397
1398 spin_lock_bh(&queue->lock);
1399
1400 list_for_each_entry_safe(entry, tmp, &queue->queue_list, list) {
1401 if (nfq_id_after(entry->id, maxid))
1402 break;
1403 __dequeue_entry(queue, entry);
1404 list_add_tail(&entry->list, &batch_list);
1405 }
1406
1407 spin_unlock_bh(&queue->lock);
1408
1409 if (list_empty(&batch_list))
1410 return -ENOENT;
1411
1412 list_for_each_entry_safe(entry, tmp, &batch_list, list) {
1413 if (nfqa[NFQA_MARK])
1414 entry->skb->mark = ntohl(nla_get_be32(nfqa[NFQA_MARK]));
1415
1416 if (nfqa[NFQA_PRIORITY])
1417 entry->skb->priority = ntohl(nla_get_be32(nfqa[NFQA_PRIORITY]));
1418
1419 nfqnl_reinject(entry, verdict);
1420 }
1421 return 0;
1422}
1423
1424static struct nf_conn *nfqnl_ct_parse(const struct nfnl_ct_hook *nfnl_ct,
1425 const struct nlmsghdr *nlh,
1426 const struct nlattr * const nfqa[],
1427 struct nf_queue_entry *entry,
1428 enum ip_conntrack_info *ctinfo)
1429{
1430#if IS_ENABLED(CONFIG_NF_CONNTRACK)
1431 struct nf_conn *ct;
1432
1433 ct = nf_ct_get(entry->skb, ctinfo);
1434 if (ct == NULL)
1435 return NULL;
1436
1437 if (nfnl_ct->parse(nfqa[NFQA_CT], ct) < 0)
1438 return NULL;
1439
1440 if (nfqa[NFQA_EXP])
1441 nfnl_ct->attach_expect(nfqa[NFQA_EXP], ct,
1442 NETLINK_CB(entry->skb).portid,
1443 nlmsg_report(nlh));
1444 return ct;
1445#else
1446 return NULL;
1447#endif
1448}
1449
1450static int nfqa_parse_bridge(struct nf_queue_entry *entry,
1451 const struct nlattr * const nfqa[])
1452{
1453 if (nfqa[NFQA_VLAN]) {
1454 struct nlattr *tb[NFQA_VLAN_MAX + 1];
1455 int err;
1456
1457 err = nla_parse_nested_deprecated(tb, NFQA_VLAN_MAX,
1458 nfqa[NFQA_VLAN],
1459 nfqa_vlan_policy, NULL);
1460 if (err < 0)
1461 return err;
1462
1463 if (!tb[NFQA_VLAN_TCI] || !tb[NFQA_VLAN_PROTO])
1464 return -EINVAL;
1465
1466 __vlan_hwaccel_put_tag(entry->skb,
1467 nla_get_be16(tb[NFQA_VLAN_PROTO]),
1468 ntohs(nla_get_be16(tb[NFQA_VLAN_TCI])));
1469 }
1470
1471 if (nfqa[NFQA_L2HDR]) {
1472 u32 mac_header_len = skb_mac_header_len(entry->skb);
1473
1474 if (mac_header_len != nla_len(nfqa[NFQA_L2HDR]))
1475 return -EINVAL;
1476 else if (mac_header_len > 0)
1477 memcpy(skb_mac_header(entry->skb),
1478 nla_data(nfqa[NFQA_L2HDR]),
1479 mac_header_len);
1480 }
1481
1482 return 0;
1483}
1484
1485static int nfqnl_recv_verdict(struct sk_buff *skb, const struct nfnl_info *info,
1486 const struct nlattr * const nfqa[])
1487{
1488 struct nfnl_queue_net *q = nfnl_queue_pernet(info->net);
1489 u_int16_t queue_num = ntohs(info->nfmsg->res_id);
1490 const struct nfnl_ct_hook *nfnl_ct;
1491 struct nfqnl_msg_verdict_hdr *vhdr;
1492 enum ip_conntrack_info ctinfo;
1493 struct nfqnl_instance *queue;
1494 struct nf_queue_entry *entry;
1495 struct nf_conn *ct = NULL;
1496 unsigned int verdict;
1497 int err;
1498
1499 queue = verdict_instance_lookup(q, queue_num,
1500 NETLINK_CB(skb).portid);
1501 if (IS_ERR(queue))
1502 return PTR_ERR(queue);
1503
1504 vhdr = verdicthdr_get(nfqa);
1505 if (!vhdr)
1506 return -EINVAL;
1507
1508 verdict = ntohl(vhdr->verdict);
1509
1510 entry = find_dequeue_entry(queue, ntohl(vhdr->id));
1511 if (entry == NULL)
1512 return -ENOENT;
1513
1514 /* rcu lock already held from nfnl->call_rcu. */
1515 nfnl_ct = rcu_dereference(nfnl_ct_hook);
1516
1517 if (nfqa[NFQA_CT]) {
1518 if (nfnl_ct != NULL)
1519 ct = nfqnl_ct_parse(nfnl_ct, info->nlh, nfqa, entry,
1520 &ctinfo);
1521 }
1522
1523 if (entry->state.pf == PF_BRIDGE) {
1524 err = nfqa_parse_bridge(entry, nfqa);
1525 if (err < 0) {
1526 nfqnl_reinject(entry, NF_DROP);
1527 return err;
1528 }
1529 }
1530
1531 if (nfqa[NFQA_PAYLOAD]) {
1532 u16 payload_len = nla_len(nfqa[NFQA_PAYLOAD]);
1533 int diff = payload_len - entry->skb->len;
1534
1535 if (nfqnl_mangle(nla_data(nfqa[NFQA_PAYLOAD]),
1536 payload_len, entry, diff) < 0)
1537 verdict = NF_DROP;
1538
1539 if (ct && diff)
1540 nfnl_ct->seq_adjust(entry->skb, ct, ctinfo, diff);
1541 }
1542
1543 if (nfqa[NFQA_MARK])
1544 entry->skb->mark = ntohl(nla_get_be32(nfqa[NFQA_MARK]));
1545
1546 if (nfqa[NFQA_PRIORITY])
1547 entry->skb->priority = ntohl(nla_get_be32(nfqa[NFQA_PRIORITY]));
1548
1549 nfqnl_reinject(entry, verdict);
1550 return 0;
1551}
1552
1553static int nfqnl_recv_unsupp(struct sk_buff *skb, const struct nfnl_info *info,
1554 const struct nlattr * const cda[])
1555{
1556 return -ENOTSUPP;
1557}
1558
1559static const struct nla_policy nfqa_cfg_policy[NFQA_CFG_MAX+1] = {
1560 [NFQA_CFG_CMD] = { .len = sizeof(struct nfqnl_msg_config_cmd) },
1561 [NFQA_CFG_PARAMS] = { .len = sizeof(struct nfqnl_msg_config_params) },
1562 [NFQA_CFG_QUEUE_MAXLEN] = { .type = NLA_U32 },
1563 [NFQA_CFG_MASK] = { .type = NLA_U32 },
1564 [NFQA_CFG_FLAGS] = NLA_POLICY_MASK(NLA_BE32, NFQA_CFG_F_MAX - 1),
1565};
1566
1567static const struct nf_queue_handler nfqh = {
1568 .outfn = nfqnl_enqueue_packet,
1569 .nf_hook_drop = nfqnl_nf_hook_drop,
1570};
1571
1572static int nfqnl_recv_config(struct sk_buff *skb, const struct nfnl_info *info,
1573 const struct nlattr * const nfqa[])
1574{
1575 struct nfnl_queue_net *q = nfnl_queue_pernet(info->net);
1576 u_int16_t queue_num = ntohs(info->nfmsg->res_id);
1577 struct nfqnl_msg_config_cmd *cmd = NULL;
1578 struct nfqnl_instance *queue;
1579 __u32 flags = 0, mask = 0;
1580
1581 WARN_ON_ONCE(!lockdep_nfnl_is_held(NFNL_SUBSYS_QUEUE));
1582
1583 if (nfqa[NFQA_CFG_CMD]) {
1584 cmd = nla_data(nfqa[NFQA_CFG_CMD]);
1585
1586 /* Obsolete commands without queue context */
1587 switch (cmd->command) {
1588 case NFQNL_CFG_CMD_PF_BIND: return 0;
1589 case NFQNL_CFG_CMD_PF_UNBIND: return 0;
1590 }
1591 }
1592
1593 /* Check if we support these flags in first place, dependencies should
1594 * be there too not to break atomicity.
1595 */
1596 if (nfqa[NFQA_CFG_FLAGS]) {
1597 if (!nfqa[NFQA_CFG_MASK]) {
1598 /* A mask is needed to specify which flags are being
1599 * changed.
1600 */
1601 return -EINVAL;
1602 }
1603
1604 flags = ntohl(nla_get_be32(nfqa[NFQA_CFG_FLAGS]));
1605 mask = ntohl(nla_get_be32(nfqa[NFQA_CFG_MASK]));
1606
1607 if (flags >= NFQA_CFG_F_MAX)
1608 return -EOPNOTSUPP;
1609
1610#if !IS_ENABLED(CONFIG_NETWORK_SECMARK)
1611 if (flags & mask & NFQA_CFG_F_SECCTX)
1612 return -EOPNOTSUPP;
1613#endif
1614 if ((flags & mask & NFQA_CFG_F_CONNTRACK) &&
1615 !rcu_access_pointer(nfnl_ct_hook)) {
1616#ifdef CONFIG_MODULES
1617 nfnl_unlock(NFNL_SUBSYS_QUEUE);
1618 request_module("ip_conntrack_netlink");
1619 nfnl_lock(NFNL_SUBSYS_QUEUE);
1620 if (rcu_access_pointer(nfnl_ct_hook))
1621 return -EAGAIN;
1622#endif
1623 return -EOPNOTSUPP;
1624 }
1625 }
1626
1627 /* Lookup queue under RCU. After peer_portid check (or for new queue
1628 * in BIND case), the queue is owned by the socket sending this message.
1629 * A socket cannot simultaneously send a message and close, so while
1630 * processing this CONFIG message, nfqnl_rcv_nl_event() (triggered by
1631 * socket close) cannot destroy this queue. Safe to use without RCU.
1632 */
1633 rcu_read_lock();
1634 queue = instance_lookup(q, queue_num);
1635 if (queue && queue->peer_portid != NETLINK_CB(skb).portid) {
1636 rcu_read_unlock();
1637 return -EPERM;
1638 }
1639 rcu_read_unlock();
1640
1641 if (cmd != NULL) {
1642 switch (cmd->command) {
1643 case NFQNL_CFG_CMD_BIND:
1644 if (queue)
1645 return -EBUSY;
1646 queue = instance_create(q, queue_num, NETLINK_CB(skb).portid);
1647 if (IS_ERR(queue))
1648 return PTR_ERR(queue);
1649 break;
1650 case NFQNL_CFG_CMD_UNBIND:
1651 if (!queue)
1652 return -ENODEV;
1653 instance_destroy(q, queue);
1654 return 0;
1655 case NFQNL_CFG_CMD_PF_BIND:
1656 case NFQNL_CFG_CMD_PF_UNBIND:
1657 break;
1658 default:
1659 return -EOPNOTSUPP;
1660 }
1661 }
1662
1663 if (!queue)
1664 return -ENODEV;
1665
1666 if (nfqa[NFQA_CFG_PARAMS]) {
1667 struct nfqnl_msg_config_params *params =
1668 nla_data(nfqa[NFQA_CFG_PARAMS]);
1669
1670 nfqnl_set_mode(queue, params->copy_mode,
1671 ntohl(params->copy_range));
1672 }
1673
1674 if (nfqa[NFQA_CFG_QUEUE_MAXLEN]) {
1675 __be32 *queue_maxlen = nla_data(nfqa[NFQA_CFG_QUEUE_MAXLEN]);
1676
1677 spin_lock_bh(&queue->lock);
1678 queue->queue_maxlen = ntohl(*queue_maxlen);
1679 spin_unlock_bh(&queue->lock);
1680 }
1681
1682 if (nfqa[NFQA_CFG_FLAGS]) {
1683 spin_lock_bh(&queue->lock);
1684 queue->flags &= ~mask;
1685 queue->flags |= flags & mask;
1686 spin_unlock_bh(&queue->lock);
1687 }
1688
1689 return 0;
1690}
1691
1692static const struct nfnl_callback nfqnl_cb[NFQNL_MSG_MAX] = {
1693 [NFQNL_MSG_PACKET] = {
1694 .call = nfqnl_recv_unsupp,
1695 .type = NFNL_CB_RCU,
1696 .attr_count = NFQA_MAX,
1697 },
1698 [NFQNL_MSG_VERDICT] = {
1699 .call = nfqnl_recv_verdict,
1700 .type = NFNL_CB_RCU,
1701 .attr_count = NFQA_MAX,
1702 .policy = nfqa_verdict_policy
1703 },
1704 [NFQNL_MSG_CONFIG] = {
1705 .call = nfqnl_recv_config,
1706 .type = NFNL_CB_MUTEX,
1707 .attr_count = NFQA_CFG_MAX,
1708 .policy = nfqa_cfg_policy
1709 },
1710 [NFQNL_MSG_VERDICT_BATCH] = {
1711 .call = nfqnl_recv_verdict_batch,
1712 .type = NFNL_CB_RCU,
1713 .attr_count = NFQA_MAX,
1714 .policy = nfqa_verdict_batch_policy
1715 },
1716};
1717
1718static const struct nfnetlink_subsystem nfqnl_subsys = {
1719 .name = "nf_queue",
1720 .subsys_id = NFNL_SUBSYS_QUEUE,
1721 .cb_count = NFQNL_MSG_MAX,
1722 .cb = nfqnl_cb,
1723};
1724
1725#ifdef CONFIG_PROC_FS
1726struct iter_state {
1727 struct seq_net_private p;
1728 unsigned int bucket;
1729};
1730
1731static struct hlist_node *get_first(struct seq_file *seq)
1732{
1733 struct iter_state *st = seq->private;
1734 struct net *net;
1735 struct nfnl_queue_net *q;
1736
1737 if (!st)
1738 return NULL;
1739
1740 net = seq_file_net(seq);
1741 q = nfnl_queue_pernet(net);
1742 for (st->bucket = 0; st->bucket < INSTANCE_BUCKETS; st->bucket++) {
1743 if (!hlist_empty(&q->instance_table[st->bucket]))
1744 return q->instance_table[st->bucket].first;
1745 }
1746 return NULL;
1747}
1748
1749static struct hlist_node *get_next(struct seq_file *seq, struct hlist_node *h)
1750{
1751 struct iter_state *st = seq->private;
1752 struct net *net = seq_file_net(seq);
1753
1754 h = h->next;
1755 while (!h) {
1756 struct nfnl_queue_net *q;
1757
1758 if (++st->bucket >= INSTANCE_BUCKETS)
1759 return NULL;
1760
1761 q = nfnl_queue_pernet(net);
1762 h = q->instance_table[st->bucket].first;
1763 }
1764 return h;
1765}
1766
1767static struct hlist_node *get_idx(struct seq_file *seq, loff_t pos)
1768{
1769 struct hlist_node *head;
1770 head = get_first(seq);
1771
1772 if (head)
1773 while (pos && (head = get_next(seq, head)))
1774 pos--;
1775 return pos ? NULL : head;
1776}
1777
1778static void *seq_start(struct seq_file *s, loff_t *pos)
1779 __acquires(nfnl_queue_pernet(seq_file_net(s))->instances_lock)
1780{
1781 spin_lock(&nfnl_queue_pernet(seq_file_net(s))->instances_lock);
1782 return get_idx(s, *pos);
1783}
1784
1785static void *seq_next(struct seq_file *s, void *v, loff_t *pos)
1786{
1787 (*pos)++;
1788 return get_next(s, v);
1789}
1790
1791static void seq_stop(struct seq_file *s, void *v)
1792 __releases(nfnl_queue_pernet(seq_file_net(s))->instances_lock)
1793{
1794 spin_unlock(&nfnl_queue_pernet(seq_file_net(s))->instances_lock);
1795}
1796
1797static int seq_show(struct seq_file *s, void *v)
1798{
1799 const struct nfqnl_instance *inst = v;
1800
1801 seq_printf(s, "%5u %6u %5u %1u %5u %5u %5u %8u %2d\n",
1802 inst->queue_num,
1803 inst->peer_portid, inst->queue_total,
1804 inst->copy_mode, inst->copy_range,
1805 inst->queue_dropped, inst->queue_user_dropped,
1806 inst->id_sequence, 1);
1807 return 0;
1808}
1809
1810static const struct seq_operations nfqnl_seq_ops = {
1811 .start = seq_start,
1812 .next = seq_next,
1813 .stop = seq_stop,
1814 .show = seq_show,
1815};
1816#endif /* PROC_FS */
1817
1818static int __net_init nfnl_queue_net_init(struct net *net)
1819{
1820 unsigned int i;
1821 struct nfnl_queue_net *q = nfnl_queue_pernet(net);
1822
1823 for (i = 0; i < INSTANCE_BUCKETS; i++)
1824 INIT_HLIST_HEAD(&q->instance_table[i]);
1825
1826 spin_lock_init(&q->instances_lock);
1827
1828#ifdef CONFIG_PROC_FS
1829 if (!proc_create_net("nfnetlink_queue", 0440, net->nf.proc_netfilter,
1830 &nfqnl_seq_ops, sizeof(struct iter_state)))
1831 return -ENOMEM;
1832#endif
1833 return 0;
1834}
1835
1836static void __net_exit nfnl_queue_net_exit(struct net *net)
1837{
1838 struct nfnl_queue_net *q = nfnl_queue_pernet(net);
1839 unsigned int i;
1840
1841#ifdef CONFIG_PROC_FS
1842 remove_proc_entry("nfnetlink_queue", net->nf.proc_netfilter);
1843#endif
1844 for (i = 0; i < INSTANCE_BUCKETS; i++)
1845 WARN_ON_ONCE(!hlist_empty(&q->instance_table[i]));
1846}
1847
1848static struct pernet_operations nfnl_queue_net_ops = {
1849 .init = nfnl_queue_net_init,
1850 .exit = nfnl_queue_net_exit,
1851 .id = &nfnl_queue_net_id,
1852 .size = sizeof(struct nfnl_queue_net),
1853};
1854
1855static int __init nfnetlink_queue_init(void)
1856{
1857 int status;
1858
1859 nfq_cleanup_wq = alloc_ordered_workqueue("nfq_workqueue", 0);
1860 if (!nfq_cleanup_wq)
1861 return -ENOMEM;
1862
1863 status = register_pernet_subsys(&nfnl_queue_net_ops);
1864 if (status < 0)
1865 goto cleanup_pernet_subsys;
1866
1867 status = netlink_register_notifier(&nfqnl_rtnl_notifier);
1868 if (status < 0)
1869 goto cleanup_rtnl_notifier;
1870
1871 status = register_netdevice_notifier(&nfqnl_dev_notifier);
1872 if (status < 0)
1873 goto cleanup_dev_notifier;
1874
1875 status = nfnetlink_subsys_register(&nfqnl_subsys);
1876 if (status < 0)
1877 goto cleanup_nfqnl_subsys;
1878
1879 nf_register_queue_handler(&nfqh);
1880
1881 return status;
1882
1883cleanup_nfqnl_subsys:
1884 unregister_netdevice_notifier(&nfqnl_dev_notifier);
1885cleanup_dev_notifier:
1886 netlink_unregister_notifier(&nfqnl_rtnl_notifier);
1887cleanup_rtnl_notifier:
1888 unregister_pernet_subsys(&nfnl_queue_net_ops);
1889cleanup_pernet_subsys:
1890 destroy_workqueue(nfq_cleanup_wq);
1891 return status;
1892}
1893
1894static void __exit nfnetlink_queue_fini(void)
1895{
1896 nf_unregister_queue_handler();
1897 unregister_netdevice_notifier(&nfqnl_dev_notifier);
1898 nfnetlink_subsys_unregister(&nfqnl_subsys);
1899 netlink_unregister_notifier(&nfqnl_rtnl_notifier);
1900 unregister_pernet_subsys(&nfnl_queue_net_ops);
1901 destroy_workqueue(nfq_cleanup_wq);
1902 rcu_barrier(); /* Wait for completion of call_rcu()'s */
1903}
1904
1905MODULE_DESCRIPTION("netfilter packet queue handler");
1906MODULE_AUTHOR("Harald Welte <laforge@netfilter.org>");
1907MODULE_LICENSE("GPL");
1908MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_QUEUE);
1909
1910module_init(nfnetlink_queue_init);
1911module_exit(nfnetlink_queue_fini);