Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Definitions for the Interfaces handler.
8 *
9 * Version: @(#)dev.h 1.0.10 08/12/93
10 *
11 * Authors: Ross Biro
12 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
13 * Corey Minyard <wf-rch!minyard@relay.EU.net>
14 * Donald J. Becker, <becker@cesdis.gsfc.nasa.gov>
15 * Alan Cox, <alan@lxorguk.ukuu.org.uk>
16 * Bjorn Ekwall. <bj0rn@blox.se>
17 * Pekka Riikonen <priikone@poseidon.pspt.fi>
18 *
19 * Moved to /usr/include/linux for NET3
20 */
21#ifndef _LINUX_NETDEVICE_H
22#define _LINUX_NETDEVICE_H
23
24#include <linux/timer.h>
25#include <linux/bug.h>
26#include <linux/delay.h>
27#include <linux/atomic.h>
28#include <linux/prefetch.h>
29#include <asm/cache.h>
30#include <asm/byteorder.h>
31#include <asm/local.h>
32
33#include <linux/percpu.h>
34#include <linux/rculist.h>
35#include <linux/workqueue.h>
36#include <linux/dynamic_queue_limits.h>
37
38#include <net/net_namespace.h>
39#ifdef CONFIG_DCB
40#include <net/dcbnl.h>
41#endif
42#include <net/netprio_cgroup.h>
43#include <linux/netdev_features.h>
44#include <linux/neighbour.h>
45#include <linux/netdevice_xmit.h>
46#include <uapi/linux/netdevice.h>
47#include <uapi/linux/if_bonding.h>
48#include <uapi/linux/pkt_cls.h>
49#include <uapi/linux/netdev.h>
50#include <linux/hashtable.h>
51#include <linux/rbtree.h>
52#include <net/net_trackers.h>
53#include <net/net_debug.h>
54#include <net/dropreason-core.h>
55#include <net/neighbour_tables.h>
56
57struct netpoll_info;
58struct device;
59struct ethtool_ops;
60struct kernel_hwtstamp_config;
61struct phy_device;
62struct dsa_port;
63struct ip_tunnel_parm_kern;
64struct macsec_context;
65struct macsec_ops;
66struct netdev_config;
67struct netdev_name_node;
68struct sd_flow_limit;
69struct sfp_bus;
70/* 802.11 specific */
71struct wireless_dev;
72/* 802.15.4 specific */
73struct wpan_dev;
74struct mpls_dev;
75/* UDP Tunnel offloads */
76struct udp_tunnel_info;
77struct udp_tunnel_nic_info;
78struct udp_tunnel_nic;
79struct bpf_prog;
80struct xdp_buff;
81struct xdp_frame;
82struct xdp_metadata_ops;
83struct xdp_md;
84struct ethtool_netdev_state;
85struct phy_link_topology;
86struct hwtstamp_provider;
87
88typedef u32 xdp_features_t;
89
90void synchronize_net(void);
91void netdev_set_default_ethtool_ops(struct net_device *dev,
92 const struct ethtool_ops *ops);
93void netdev_sw_irq_coalesce_default_on(struct net_device *dev);
94
95/* Backlog congestion levels */
96#define NET_RX_SUCCESS 0 /* keep 'em coming, baby */
97#define NET_RX_DROP 1 /* packet dropped */
98
99#define MAX_NEST_DEV 8
100
101/*
102 * Transmit return codes: transmit return codes originate from three different
103 * namespaces:
104 *
105 * - qdisc return codes
106 * - driver transmit return codes
107 * - errno values
108 *
109 * Drivers are allowed to return any one of those in their hard_start_xmit()
110 * function. Real network devices commonly used with qdiscs should only return
111 * the driver transmit return codes though - when qdiscs are used, the actual
112 * transmission happens asynchronously, so the value is not propagated to
113 * higher layers. Virtual network devices transmit synchronously; in this case
114 * the driver transmit return codes are consumed by dev_queue_xmit(), and all
115 * others are propagated to higher layers.
116 */
117
118/* qdisc ->enqueue() return codes. */
119#define NET_XMIT_SUCCESS 0x00
120#define NET_XMIT_DROP 0x01 /* skb dropped */
121#define NET_XMIT_CN 0x02 /* congestion notification */
122#define NET_XMIT_MASK 0x0f /* qdisc flags in net/sch_generic.h */
123
124/* NET_XMIT_CN is special. It does not guarantee that this packet is lost. It
125 * indicates that the device will soon be dropping packets, or already drops
126 * some packets of the same priority; prompting us to send less aggressively. */
127#define net_xmit_eval(e) ((e) == NET_XMIT_CN ? 0 : (e))
128#define net_xmit_errno(e) ((e) != NET_XMIT_CN ? -ENOBUFS : 0)
129
130/* Driver transmit return codes */
131#define NETDEV_TX_MASK 0xf0
132
133enum netdev_tx {
134 __NETDEV_TX_MIN = INT_MIN, /* make sure enum is signed */
135 NETDEV_TX_OK = 0x00, /* driver took care of packet */
136 NETDEV_TX_BUSY = 0x10, /* driver tx path was busy*/
137};
138typedef enum netdev_tx netdev_tx_t;
139
140/*
141 * Current order: NETDEV_TX_MASK > NET_XMIT_MASK >= 0 is significant;
142 * hard_start_xmit() return < NET_XMIT_MASK means skb was consumed.
143 */
144static inline bool dev_xmit_complete(int rc)
145{
146 /*
147 * Positive cases with an skb consumed by a driver:
148 * - successful transmission (rc == NETDEV_TX_OK)
149 * - error while transmitting (rc < 0)
150 * - error while queueing to a different device (rc & NET_XMIT_MASK)
151 */
152 if (likely(rc < NET_XMIT_MASK))
153 return true;
154
155 return false;
156}
157
158/*
159 * Compute the worst-case header length according to the protocols
160 * used.
161 */
162
163#if defined(CONFIG_HYPERV_NET)
164# define LL_MAX_HEADER 128
165#elif defined(CONFIG_WLAN) || IS_ENABLED(CONFIG_AX25)
166# if defined(CONFIG_MAC80211_MESH)
167# define LL_MAX_HEADER 128
168# else
169# define LL_MAX_HEADER 96
170# endif
171#else
172# define LL_MAX_HEADER 32
173#endif
174
175#if !IS_ENABLED(CONFIG_NET_IPIP) && !IS_ENABLED(CONFIG_NET_IPGRE) && \
176 !IS_ENABLED(CONFIG_IPV6_SIT) && !IS_ENABLED(CONFIG_IPV6_TUNNEL)
177#define MAX_HEADER LL_MAX_HEADER
178#else
179#define MAX_HEADER (LL_MAX_HEADER + 48)
180#endif
181
182/*
183 * Old network device statistics. Fields are native words
184 * (unsigned long) so they can be read and written atomically.
185 */
186
187#define NET_DEV_STAT(FIELD) \
188 union { \
189 unsigned long FIELD; \
190 atomic_long_t __##FIELD; \
191 }
192
193struct net_device_stats {
194 NET_DEV_STAT(rx_packets);
195 NET_DEV_STAT(tx_packets);
196 NET_DEV_STAT(rx_bytes);
197 NET_DEV_STAT(tx_bytes);
198 NET_DEV_STAT(rx_errors);
199 NET_DEV_STAT(tx_errors);
200 NET_DEV_STAT(rx_dropped);
201 NET_DEV_STAT(tx_dropped);
202 NET_DEV_STAT(multicast);
203 NET_DEV_STAT(collisions);
204 NET_DEV_STAT(rx_length_errors);
205 NET_DEV_STAT(rx_over_errors);
206 NET_DEV_STAT(rx_crc_errors);
207 NET_DEV_STAT(rx_frame_errors);
208 NET_DEV_STAT(rx_fifo_errors);
209 NET_DEV_STAT(rx_missed_errors);
210 NET_DEV_STAT(tx_aborted_errors);
211 NET_DEV_STAT(tx_carrier_errors);
212 NET_DEV_STAT(tx_fifo_errors);
213 NET_DEV_STAT(tx_heartbeat_errors);
214 NET_DEV_STAT(tx_window_errors);
215 NET_DEV_STAT(rx_compressed);
216 NET_DEV_STAT(tx_compressed);
217};
218#undef NET_DEV_STAT
219
220/* per-cpu stats, allocated on demand.
221 * Try to fit them in a single cache line, for dev_get_stats() sake.
222 */
223struct net_device_core_stats {
224 unsigned long rx_dropped;
225 unsigned long tx_dropped;
226 unsigned long rx_nohandler;
227 unsigned long rx_otherhost_dropped;
228} __aligned(4 * sizeof(unsigned long));
229
230#include <linux/cache.h>
231#include <linux/skbuff.h>
232
233struct neighbour;
234struct neigh_parms;
235struct sk_buff;
236
237struct netdev_hw_addr {
238 struct list_head list;
239 struct rb_node node;
240 unsigned char addr[MAX_ADDR_LEN];
241 unsigned char type;
242#define NETDEV_HW_ADDR_T_LAN 1
243#define NETDEV_HW_ADDR_T_SAN 2
244#define NETDEV_HW_ADDR_T_UNICAST 3
245#define NETDEV_HW_ADDR_T_MULTICAST 4
246 bool global_use;
247 int sync_cnt;
248 int refcount;
249 int synced;
250 struct rcu_head rcu_head;
251};
252
253struct netdev_hw_addr_list {
254 struct list_head list;
255 int count;
256
257 /* Auxiliary tree for faster lookup on addition and deletion */
258 struct rb_root tree;
259};
260
261#define netdev_hw_addr_list_count(l) ((l)->count)
262#define netdev_hw_addr_list_empty(l) (netdev_hw_addr_list_count(l) == 0)
263#define netdev_hw_addr_list_for_each(ha, l) \
264 list_for_each_entry(ha, &(l)->list, list)
265
266#define netdev_uc_count(dev) netdev_hw_addr_list_count(&(dev)->uc)
267#define netdev_uc_empty(dev) netdev_hw_addr_list_empty(&(dev)->uc)
268#define netdev_for_each_uc_addr(ha, dev) \
269 netdev_hw_addr_list_for_each(ha, &(dev)->uc)
270#define netdev_for_each_synced_uc_addr(_ha, _dev) \
271 netdev_for_each_uc_addr((_ha), (_dev)) \
272 if ((_ha)->sync_cnt)
273
274#define netdev_mc_count(dev) netdev_hw_addr_list_count(&(dev)->mc)
275#define netdev_mc_empty(dev) netdev_hw_addr_list_empty(&(dev)->mc)
276#define netdev_for_each_mc_addr(ha, dev) \
277 netdev_hw_addr_list_for_each(ha, &(dev)->mc)
278#define netdev_for_each_synced_mc_addr(_ha, _dev) \
279 netdev_for_each_mc_addr((_ha), (_dev)) \
280 if ((_ha)->sync_cnt)
281
282struct hh_cache {
283 unsigned int hh_len;
284 seqlock_t hh_lock;
285
286 /* cached hardware header; allow for machine alignment needs. */
287#define HH_DATA_MOD 16
288#define HH_DATA_OFF(__len) \
289 (HH_DATA_MOD - (((__len - 1) & (HH_DATA_MOD - 1)) + 1))
290#define HH_DATA_ALIGN(__len) \
291 (((__len)+(HH_DATA_MOD-1))&~(HH_DATA_MOD - 1))
292 unsigned long hh_data[HH_DATA_ALIGN(LL_MAX_HEADER) / sizeof(long)];
293};
294
295/* Reserve HH_DATA_MOD byte-aligned hard_header_len, but at least that much.
296 * Alternative is:
297 * dev->hard_header_len ? (dev->hard_header_len +
298 * (HH_DATA_MOD - 1)) & ~(HH_DATA_MOD - 1) : 0
299 *
300 * We could use other alignment values, but we must maintain the
301 * relationship HH alignment <= LL alignment.
302 */
303#define LL_RESERVED_SPACE(dev) \
304 ((((dev)->hard_header_len + READ_ONCE((dev)->needed_headroom)) \
305 & ~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
306#define LL_RESERVED_SPACE_EXTRA(dev,extra) \
307 ((((dev)->hard_header_len + READ_ONCE((dev)->needed_headroom) + (extra)) \
308 & ~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
309
310struct header_ops {
311 int (*create) (struct sk_buff *skb, struct net_device *dev,
312 unsigned short type, const void *daddr,
313 const void *saddr, unsigned int len);
314 int (*parse)(const struct sk_buff *skb,
315 const struct net_device *dev,
316 unsigned char *haddr);
317 int (*cache)(const struct neighbour *neigh, struct hh_cache *hh, __be16 type);
318 void (*cache_update)(struct hh_cache *hh,
319 const struct net_device *dev,
320 const unsigned char *haddr);
321 bool (*validate)(const char *ll_header, unsigned int len);
322 __be16 (*parse_protocol)(const struct sk_buff *skb);
323};
324
325/* These flag bits are private to the generic network queueing
326 * layer; they may not be explicitly referenced by any other
327 * code.
328 */
329
330enum netdev_state_t {
331 __LINK_STATE_START,
332 __LINK_STATE_PRESENT,
333 __LINK_STATE_NOCARRIER,
334 __LINK_STATE_LINKWATCH_PENDING,
335 __LINK_STATE_DORMANT,
336 __LINK_STATE_TESTING,
337};
338
339struct gro_list {
340 struct list_head list;
341 int count;
342};
343
344/*
345 * size of gro hash buckets, must be <= the number of bits in
346 * gro_node::bitmask
347 */
348#define GRO_HASH_BUCKETS 8
349
350/**
351 * struct gro_node - structure to support Generic Receive Offload
352 * @bitmask: bitmask to indicate used buckets in @hash
353 * @hash: hashtable of pending aggregated skbs, separated by flows
354 * @rx_list: list of pending ``GRO_NORMAL`` skbs
355 * @rx_count: cached current length of @rx_list
356 * @cached_napi_id: napi_struct::napi_id cached for hotpath, 0 for standalone
357 */
358struct gro_node {
359 unsigned long bitmask;
360 struct gro_list hash[GRO_HASH_BUCKETS];
361 struct list_head rx_list;
362 u32 rx_count;
363 u32 cached_napi_id;
364};
365
366/*
367 * Structure for per-NAPI config
368 */
369struct napi_config {
370 u64 gro_flush_timeout;
371 u64 irq_suspend_timeout;
372 u32 defer_hard_irqs;
373 cpumask_t affinity_mask;
374 u8 threaded;
375 unsigned int napi_id;
376};
377
378/*
379 * Structure for NAPI scheduling similar to tasklet but with weighting
380 */
381struct napi_struct {
382 /* This field should be first or softnet_data.backlog needs tweaks. */
383 unsigned long state;
384 /* The poll_list must only be managed by the entity which
385 * changes the state of the NAPI_STATE_SCHED bit. This means
386 * whoever atomically sets that bit can add this napi_struct
387 * to the per-CPU poll_list, and whoever clears that bit
388 * can remove from the list right before clearing the bit.
389 */
390 struct list_head poll_list;
391
392 int weight;
393 u32 defer_hard_irqs_count;
394 int (*poll)(struct napi_struct *, int);
395#ifdef CONFIG_NETPOLL
396 /* CPU actively polling if netpoll is configured */
397 int poll_owner;
398#endif
399 /* CPU on which NAPI has been scheduled for processing */
400 int list_owner;
401 struct net_device *dev;
402 struct sk_buff *skb;
403 struct gro_node gro;
404 struct hrtimer timer;
405 /* all fields past this point are write-protected by netdev_lock */
406 struct task_struct *thread;
407 unsigned long gro_flush_timeout;
408 unsigned long irq_suspend_timeout;
409 u32 defer_hard_irqs;
410 /* control-path-only fields follow */
411 u32 napi_id;
412 struct list_head dev_list;
413 struct hlist_node napi_hash_node;
414 int irq;
415 struct irq_affinity_notify notify;
416 int napi_rmap_idx;
417 int index;
418 struct napi_config *config;
419};
420
421enum {
422 NAPI_STATE_SCHED, /* Poll is scheduled */
423 NAPI_STATE_MISSED, /* reschedule a napi */
424 NAPI_STATE_DISABLE, /* Disable pending */
425 NAPI_STATE_NPSVC, /* Netpoll - don't dequeue from poll_list */
426 NAPI_STATE_LISTED, /* NAPI added to system lists */
427 NAPI_STATE_NO_BUSY_POLL, /* Do not add in napi_hash, no busy polling */
428 NAPI_STATE_IN_BUSY_POLL, /* Do not rearm NAPI interrupt */
429 NAPI_STATE_PREFER_BUSY_POLL, /* prefer busy-polling over softirq processing*/
430 NAPI_STATE_THREADED, /* The poll is performed inside its own thread*/
431 NAPI_STATE_SCHED_THREADED, /* Napi is currently scheduled in threaded mode */
432 NAPI_STATE_HAS_NOTIFIER, /* Napi has an IRQ notifier */
433 NAPI_STATE_THREADED_BUSY_POLL, /* The threaded NAPI poller will busy poll */
434};
435
436enum {
437 NAPIF_STATE_SCHED = BIT(NAPI_STATE_SCHED),
438 NAPIF_STATE_MISSED = BIT(NAPI_STATE_MISSED),
439 NAPIF_STATE_DISABLE = BIT(NAPI_STATE_DISABLE),
440 NAPIF_STATE_NPSVC = BIT(NAPI_STATE_NPSVC),
441 NAPIF_STATE_LISTED = BIT(NAPI_STATE_LISTED),
442 NAPIF_STATE_NO_BUSY_POLL = BIT(NAPI_STATE_NO_BUSY_POLL),
443 NAPIF_STATE_IN_BUSY_POLL = BIT(NAPI_STATE_IN_BUSY_POLL),
444 NAPIF_STATE_PREFER_BUSY_POLL = BIT(NAPI_STATE_PREFER_BUSY_POLL),
445 NAPIF_STATE_THREADED = BIT(NAPI_STATE_THREADED),
446 NAPIF_STATE_SCHED_THREADED = BIT(NAPI_STATE_SCHED_THREADED),
447 NAPIF_STATE_HAS_NOTIFIER = BIT(NAPI_STATE_HAS_NOTIFIER),
448 NAPIF_STATE_THREADED_BUSY_POLL = BIT(NAPI_STATE_THREADED_BUSY_POLL),
449};
450
451enum gro_result {
452 GRO_MERGED,
453 GRO_MERGED_FREE,
454 GRO_HELD,
455 GRO_NORMAL,
456 GRO_CONSUMED,
457};
458typedef enum gro_result gro_result_t;
459
460/*
461 * enum rx_handler_result - Possible return values for rx_handlers.
462 * @RX_HANDLER_CONSUMED: skb was consumed by rx_handler, do not process it
463 * further.
464 * @RX_HANDLER_ANOTHER: Do another round in receive path. This is indicated in
465 * case skb->dev was changed by rx_handler.
466 * @RX_HANDLER_EXACT: Force exact delivery, no wildcard.
467 * @RX_HANDLER_PASS: Do nothing, pass the skb as if no rx_handler was called.
468 *
469 * rx_handlers are functions called from inside __netif_receive_skb(), to do
470 * special processing of the skb, prior to delivery to protocol handlers.
471 *
472 * Currently, a net_device can only have a single rx_handler registered. Trying
473 * to register a second rx_handler will return -EBUSY.
474 *
475 * To register a rx_handler on a net_device, use netdev_rx_handler_register().
476 * To unregister a rx_handler on a net_device, use
477 * netdev_rx_handler_unregister().
478 *
479 * Upon return, rx_handler is expected to tell __netif_receive_skb() what to
480 * do with the skb.
481 *
482 * If the rx_handler consumed the skb in some way, it should return
483 * RX_HANDLER_CONSUMED. This is appropriate when the rx_handler arranged for
484 * the skb to be delivered in some other way.
485 *
486 * If the rx_handler changed skb->dev, to divert the skb to another
487 * net_device, it should return RX_HANDLER_ANOTHER. The rx_handler for the
488 * new device will be called if it exists.
489 *
490 * If the rx_handler decides the skb should be ignored, it should return
491 * RX_HANDLER_EXACT. The skb will only be delivered to protocol handlers that
492 * are registered on exact device (ptype->dev == skb->dev).
493 *
494 * If the rx_handler didn't change skb->dev, but wants the skb to be normally
495 * delivered, it should return RX_HANDLER_PASS.
496 *
497 * A device without a registered rx_handler will behave as if rx_handler
498 * returned RX_HANDLER_PASS.
499 */
500
501enum rx_handler_result {
502 RX_HANDLER_CONSUMED,
503 RX_HANDLER_ANOTHER,
504 RX_HANDLER_EXACT,
505 RX_HANDLER_PASS,
506};
507typedef enum rx_handler_result rx_handler_result_t;
508typedef rx_handler_result_t rx_handler_func_t(struct sk_buff **pskb);
509
510void __napi_schedule(struct napi_struct *n);
511void __napi_schedule_irqoff(struct napi_struct *n);
512
513static inline bool napi_disable_pending(struct napi_struct *n)
514{
515 return test_bit(NAPI_STATE_DISABLE, &n->state);
516}
517
518static inline bool napi_prefer_busy_poll(struct napi_struct *n)
519{
520 return test_bit(NAPI_STATE_PREFER_BUSY_POLL, &n->state);
521}
522
523/**
524 * napi_is_scheduled - test if NAPI is scheduled
525 * @n: NAPI context
526 *
527 * This check is "best-effort". With no locking implemented,
528 * a NAPI can be scheduled or terminate right after this check
529 * and produce not precise results.
530 *
531 * NAPI_STATE_SCHED is an internal state, napi_is_scheduled
532 * should not be used normally and napi_schedule should be
533 * used instead.
534 *
535 * Use only if the driver really needs to check if a NAPI
536 * is scheduled for example in the context of delayed timer
537 * that can be skipped if a NAPI is already scheduled.
538 *
539 * Return: True if NAPI is scheduled, False otherwise.
540 */
541static inline bool napi_is_scheduled(struct napi_struct *n)
542{
543 return test_bit(NAPI_STATE_SCHED, &n->state);
544}
545
546bool napi_schedule_prep(struct napi_struct *n);
547
548/**
549 * napi_schedule - schedule NAPI poll
550 * @n: NAPI context
551 *
552 * Schedule NAPI poll routine to be called if it is not already
553 * running.
554 * Return: true if we schedule a NAPI or false if not.
555 * Refer to napi_schedule_prep() for additional reason on why
556 * a NAPI might not be scheduled.
557 */
558static inline bool napi_schedule(struct napi_struct *n)
559{
560 if (napi_schedule_prep(n)) {
561 __napi_schedule(n);
562 return true;
563 }
564
565 return false;
566}
567
568/**
569 * napi_schedule_irqoff - schedule NAPI poll
570 * @n: NAPI context
571 *
572 * Variant of napi_schedule(), assuming hard irqs are masked.
573 */
574static inline void napi_schedule_irqoff(struct napi_struct *n)
575{
576 if (napi_schedule_prep(n))
577 __napi_schedule_irqoff(n);
578}
579
580/**
581 * napi_complete_done - NAPI processing complete
582 * @n: NAPI context
583 * @work_done: number of packets processed
584 *
585 * Mark NAPI processing as complete. Should only be called if poll budget
586 * has not been completely consumed.
587 * Prefer over napi_complete().
588 * Return: false if device should avoid rearming interrupts.
589 */
590bool napi_complete_done(struct napi_struct *n, int work_done);
591
592static inline bool napi_complete(struct napi_struct *n)
593{
594 return napi_complete_done(n, 0);
595}
596
597void netif_threaded_enable(struct net_device *dev);
598int dev_set_threaded(struct net_device *dev,
599 enum netdev_napi_threaded threaded);
600
601void napi_disable(struct napi_struct *n);
602void napi_disable_locked(struct napi_struct *n);
603
604void napi_enable(struct napi_struct *n);
605void napi_enable_locked(struct napi_struct *n);
606
607/**
608 * napi_synchronize - wait until NAPI is not running
609 * @n: NAPI context
610 *
611 * Wait until NAPI is done being scheduled on this context.
612 * Waits till any outstanding processing completes but
613 * does not disable future activations.
614 */
615static inline void napi_synchronize(const struct napi_struct *n)
616{
617 if (IS_ENABLED(CONFIG_SMP))
618 while (test_bit(NAPI_STATE_SCHED, &n->state))
619 msleep(1);
620 else
621 barrier();
622}
623
624/**
625 * napi_if_scheduled_mark_missed - if napi is running, set the
626 * NAPIF_STATE_MISSED
627 * @n: NAPI context
628 *
629 * If napi is running, set the NAPIF_STATE_MISSED, and return true if
630 * NAPI is scheduled.
631 **/
632static inline bool napi_if_scheduled_mark_missed(struct napi_struct *n)
633{
634 unsigned long val, new;
635
636 val = READ_ONCE(n->state);
637 do {
638 if (val & NAPIF_STATE_DISABLE)
639 return true;
640
641 if (!(val & NAPIF_STATE_SCHED))
642 return false;
643
644 new = val | NAPIF_STATE_MISSED;
645 } while (!try_cmpxchg(&n->state, &val, new));
646
647 return true;
648}
649
650enum netdev_queue_state_t {
651 __QUEUE_STATE_DRV_XOFF,
652 __QUEUE_STATE_STACK_XOFF,
653 __QUEUE_STATE_FROZEN,
654};
655
656#define QUEUE_STATE_DRV_XOFF (1 << __QUEUE_STATE_DRV_XOFF)
657#define QUEUE_STATE_STACK_XOFF (1 << __QUEUE_STATE_STACK_XOFF)
658#define QUEUE_STATE_FROZEN (1 << __QUEUE_STATE_FROZEN)
659
660#define QUEUE_STATE_ANY_XOFF (QUEUE_STATE_DRV_XOFF | QUEUE_STATE_STACK_XOFF)
661#define QUEUE_STATE_ANY_XOFF_OR_FROZEN (QUEUE_STATE_ANY_XOFF | \
662 QUEUE_STATE_FROZEN)
663#define QUEUE_STATE_DRV_XOFF_OR_FROZEN (QUEUE_STATE_DRV_XOFF | \
664 QUEUE_STATE_FROZEN)
665
666/*
667 * __QUEUE_STATE_DRV_XOFF is used by drivers to stop the transmit queue. The
668 * netif_tx_* functions below are used to manipulate this flag. The
669 * __QUEUE_STATE_STACK_XOFF flag is used by the stack to stop the transmit
670 * queue independently. The netif_xmit_*stopped functions below are called
671 * to check if the queue has been stopped by the driver or stack (either
672 * of the XOFF bits are set in the state). Drivers should not need to call
673 * netif_xmit*stopped functions, they should only be using netif_tx_*.
674 */
675
676struct netdev_queue {
677/*
678 * read-mostly part
679 */
680 struct net_device *dev;
681 netdevice_tracker dev_tracker;
682
683 struct Qdisc __rcu *qdisc;
684 struct Qdisc __rcu *qdisc_sleeping;
685#ifdef CONFIG_SYSFS
686 struct kobject kobj;
687 const struct attribute_group **groups;
688#endif
689 unsigned long tx_maxrate;
690 /*
691 * Number of TX timeouts for this queue
692 * (/sys/class/net/DEV/Q/trans_timeout)
693 */
694 atomic_long_t trans_timeout;
695
696 /* Subordinate device that the queue has been assigned to */
697 struct net_device *sb_dev;
698#ifdef CONFIG_XDP_SOCKETS
699 /* "ops protected", see comment about net_device::lock */
700 struct xsk_buff_pool *pool;
701#endif
702
703/*
704 * write-mostly part
705 */
706#ifdef CONFIG_BQL
707 struct dql dql;
708#endif
709 spinlock_t _xmit_lock ____cacheline_aligned_in_smp;
710 int xmit_lock_owner;
711 /*
712 * Time (in jiffies) of last Tx
713 */
714 unsigned long trans_start;
715
716 unsigned long state;
717
718/*
719 * slow- / control-path part
720 */
721 /* NAPI instance for the queue
722 * "ops protected", see comment about net_device::lock
723 */
724 struct napi_struct *napi;
725
726#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
727 int numa_node;
728#endif
729} ____cacheline_aligned_in_smp;
730
731extern int sysctl_fb_tunnels_only_for_init_net;
732extern int sysctl_devconf_inherit_init_net;
733
734/*
735 * sysctl_fb_tunnels_only_for_init_net == 0 : For all netns
736 * == 1 : For initns only
737 * == 2 : For none.
738 */
739static inline bool net_has_fallback_tunnels(const struct net *net)
740{
741#if IS_ENABLED(CONFIG_SYSCTL)
742 int fb_tunnels_only_for_init_net = READ_ONCE(sysctl_fb_tunnels_only_for_init_net);
743
744 return !fb_tunnels_only_for_init_net ||
745 (net_eq(net, &init_net) && fb_tunnels_only_for_init_net == 1);
746#else
747 return true;
748#endif
749}
750
751static inline int net_inherit_devconf(void)
752{
753#if IS_ENABLED(CONFIG_SYSCTL)
754 return READ_ONCE(sysctl_devconf_inherit_init_net);
755#else
756 return 0;
757#endif
758}
759
760static inline int netdev_queue_numa_node_read(const struct netdev_queue *q)
761{
762#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
763 return q->numa_node;
764#else
765 return NUMA_NO_NODE;
766#endif
767}
768
769static inline void netdev_queue_numa_node_write(struct netdev_queue *q, int node)
770{
771#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
772 q->numa_node = node;
773#endif
774}
775
776#ifdef CONFIG_RFS_ACCEL
777bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, u32 flow_id,
778 u16 filter_id);
779#endif
780
781/* XPS map type and offset of the xps map within net_device->xps_maps[]. */
782enum xps_map_type {
783 XPS_CPUS = 0,
784 XPS_RXQS,
785 XPS_MAPS_MAX,
786};
787
788#ifdef CONFIG_XPS
789/*
790 * This structure holds an XPS map which can be of variable length. The
791 * map is an array of queues.
792 */
793struct xps_map {
794 unsigned int len;
795 unsigned int alloc_len;
796 struct rcu_head rcu;
797 u16 queues[];
798};
799#define XPS_MAP_SIZE(_num) (sizeof(struct xps_map) + ((_num) * sizeof(u16)))
800#define XPS_MIN_MAP_ALLOC ((L1_CACHE_ALIGN(offsetof(struct xps_map, queues[1])) \
801 - sizeof(struct xps_map)) / sizeof(u16))
802
803/*
804 * This structure holds all XPS maps for device. Maps are indexed by CPU.
805 *
806 * We keep track of the number of cpus/rxqs used when the struct is allocated,
807 * in nr_ids. This will help not accessing out-of-bound memory.
808 *
809 * We keep track of the number of traffic classes used when the struct is
810 * allocated, in num_tc. This will be used to navigate the maps, to ensure we're
811 * not crossing its upper bound, as the original dev->num_tc can be updated in
812 * the meantime.
813 */
814struct xps_dev_maps {
815 struct rcu_head rcu;
816 unsigned int nr_ids;
817 s16 num_tc;
818 struct xps_map __rcu *attr_map[]; /* Either CPUs map or RXQs map */
819};
820
821#define XPS_CPU_DEV_MAPS_SIZE(_tcs) (sizeof(struct xps_dev_maps) + \
822 (nr_cpu_ids * (_tcs) * sizeof(struct xps_map *)))
823
824#define XPS_RXQ_DEV_MAPS_SIZE(_tcs, _rxqs) (sizeof(struct xps_dev_maps) +\
825 (_rxqs * (_tcs) * sizeof(struct xps_map *)))
826
827#endif /* CONFIG_XPS */
828
829#define TC_MAX_QUEUE 16
830#define TC_BITMASK 15
831/* HW offloaded queuing disciplines txq count and offset maps */
832struct netdev_tc_txq {
833 u16 count;
834 u16 offset;
835};
836
837#if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
838/*
839 * This structure is to hold information about the device
840 * configured to run FCoE protocol stack.
841 */
842struct netdev_fcoe_hbainfo {
843 char manufacturer[64];
844 char serial_number[64];
845 char hardware_version[64];
846 char driver_version[64];
847 char optionrom_version[64];
848 char firmware_version[64];
849 char model[256];
850 char model_description[256];
851};
852#endif
853
854#define MAX_PHYS_ITEM_ID_LEN 32
855
856/* This structure holds a unique identifier to identify some
857 * physical item (port for example) used by a netdevice.
858 */
859struct netdev_phys_item_id {
860 unsigned char id[MAX_PHYS_ITEM_ID_LEN];
861 unsigned char id_len;
862};
863
864static inline bool netdev_phys_item_id_same(struct netdev_phys_item_id *a,
865 struct netdev_phys_item_id *b)
866{
867 return a->id_len == b->id_len &&
868 memcmp(a->id, b->id, a->id_len) == 0;
869}
870
871typedef u16 (*select_queue_fallback_t)(struct net_device *dev,
872 struct sk_buff *skb,
873 struct net_device *sb_dev);
874
875enum net_device_path_type {
876 DEV_PATH_ETHERNET = 0,
877 DEV_PATH_VLAN,
878 DEV_PATH_BRIDGE,
879 DEV_PATH_PPPOE,
880 DEV_PATH_DSA,
881 DEV_PATH_MTK_WDMA,
882 DEV_PATH_TUN,
883};
884
885struct net_device_path {
886 enum net_device_path_type type;
887 const struct net_device *dev;
888 union {
889 struct {
890 u16 id;
891 __be16 proto;
892 u8 h_dest[ETH_ALEN];
893 } encap;
894 struct {
895 union {
896 struct in_addr src_v4;
897 struct in6_addr src_v6;
898 };
899 union {
900 struct in_addr dst_v4;
901 struct in6_addr dst_v6;
902 };
903
904 u8 l3_proto;
905 } tun;
906 struct {
907 enum {
908 DEV_PATH_BR_VLAN_KEEP,
909 DEV_PATH_BR_VLAN_TAG,
910 DEV_PATH_BR_VLAN_UNTAG,
911 DEV_PATH_BR_VLAN_UNTAG_HW,
912 } vlan_mode;
913 u16 vlan_id;
914 __be16 vlan_proto;
915 } bridge;
916 struct {
917 int port;
918 u16 proto;
919 } dsa;
920 struct {
921 u8 wdma_idx;
922 u8 queue;
923 u16 wcid;
924 u8 bss;
925 u8 amsdu;
926 } mtk_wdma;
927 };
928};
929
930#define NET_DEVICE_PATH_STACK_MAX 5
931#define NET_DEVICE_PATH_VLAN_MAX 2
932
933struct net_device_path_stack {
934 int num_paths;
935 struct net_device_path path[NET_DEVICE_PATH_STACK_MAX];
936};
937
938struct net_device_path_ctx {
939 const struct net_device *dev;
940 u8 daddr[ETH_ALEN];
941
942 int num_vlans;
943 struct {
944 u16 id;
945 __be16 proto;
946 } vlan[NET_DEVICE_PATH_VLAN_MAX];
947};
948
949enum tc_setup_type {
950 TC_QUERY_CAPS,
951 TC_SETUP_QDISC_MQPRIO,
952 TC_SETUP_CLSU32,
953 TC_SETUP_CLSFLOWER,
954 TC_SETUP_CLSMATCHALL,
955 TC_SETUP_CLSBPF,
956 TC_SETUP_BLOCK,
957 TC_SETUP_QDISC_CBS,
958 TC_SETUP_QDISC_RED,
959 TC_SETUP_QDISC_PRIO,
960 TC_SETUP_QDISC_MQ,
961 TC_SETUP_QDISC_ETF,
962 TC_SETUP_ROOT_QDISC,
963 TC_SETUP_QDISC_GRED,
964 TC_SETUP_QDISC_TAPRIO,
965 TC_SETUP_FT,
966 TC_SETUP_QDISC_ETS,
967 TC_SETUP_QDISC_TBF,
968 TC_SETUP_QDISC_FIFO,
969 TC_SETUP_QDISC_HTB,
970 TC_SETUP_ACT,
971};
972
973/* These structures hold the attributes of bpf state that are being passed
974 * to the netdevice through the bpf op.
975 */
976enum bpf_netdev_command {
977 /* Set or clear a bpf program used in the earliest stages of packet
978 * rx. The prog will have been loaded as BPF_PROG_TYPE_XDP. The callee
979 * is responsible for calling bpf_prog_put on any old progs that are
980 * stored. In case of error, the callee need not release the new prog
981 * reference, but on success it takes ownership and must bpf_prog_put
982 * when it is no longer used.
983 */
984 XDP_SETUP_PROG,
985 XDP_SETUP_PROG_HW,
986 /* BPF program for offload callbacks, invoked at program load time. */
987 BPF_OFFLOAD_MAP_ALLOC,
988 BPF_OFFLOAD_MAP_FREE,
989 XDP_SETUP_XSK_POOL,
990};
991
992struct bpf_prog_offload_ops;
993struct netlink_ext_ack;
994struct xdp_umem;
995struct xdp_dev_bulk_queue;
996struct bpf_xdp_link;
997
998enum bpf_xdp_mode {
999 XDP_MODE_SKB = 0,
1000 XDP_MODE_DRV = 1,
1001 XDP_MODE_HW = 2,
1002 __MAX_XDP_MODE
1003};
1004
1005struct bpf_xdp_entity {
1006 struct bpf_prog *prog;
1007 struct bpf_xdp_link *link;
1008};
1009
1010struct netdev_bpf {
1011 enum bpf_netdev_command command;
1012 union {
1013 /* XDP_SETUP_PROG */
1014 struct {
1015 u32 flags;
1016 struct bpf_prog *prog;
1017 struct netlink_ext_ack *extack;
1018 };
1019 /* BPF_OFFLOAD_MAP_ALLOC, BPF_OFFLOAD_MAP_FREE */
1020 struct {
1021 struct bpf_offloaded_map *offmap;
1022 };
1023 /* XDP_SETUP_XSK_POOL */
1024 struct {
1025 struct xsk_buff_pool *pool;
1026 u16 queue_id;
1027 } xsk;
1028 };
1029};
1030
1031/* Flags for ndo_xsk_wakeup. */
1032#define XDP_WAKEUP_RX (1 << 0)
1033#define XDP_WAKEUP_TX (1 << 1)
1034
1035#ifdef CONFIG_XFRM_OFFLOAD
1036struct xfrmdev_ops {
1037 int (*xdo_dev_state_add)(struct net_device *dev,
1038 struct xfrm_state *x,
1039 struct netlink_ext_ack *extack);
1040 void (*xdo_dev_state_delete)(struct net_device *dev,
1041 struct xfrm_state *x);
1042 void (*xdo_dev_state_free)(struct net_device *dev,
1043 struct xfrm_state *x);
1044 bool (*xdo_dev_offload_ok) (struct sk_buff *skb,
1045 struct xfrm_state *x);
1046 void (*xdo_dev_state_advance_esn) (struct xfrm_state *x);
1047 void (*xdo_dev_state_update_stats) (struct xfrm_state *x);
1048 int (*xdo_dev_policy_add) (struct xfrm_policy *x, struct netlink_ext_ack *extack);
1049 void (*xdo_dev_policy_delete) (struct xfrm_policy *x);
1050 void (*xdo_dev_policy_free) (struct xfrm_policy *x);
1051};
1052#endif
1053
1054struct dev_ifalias {
1055 struct rcu_head rcuhead;
1056 char ifalias[];
1057};
1058
1059struct devlink;
1060struct tlsdev_ops;
1061
1062struct netdev_net_notifier {
1063 struct list_head list;
1064 struct notifier_block *nb;
1065};
1066
1067/*
1068 * This structure defines the management hooks for network devices.
1069 * The following hooks can be defined; unless noted otherwise, they are
1070 * optional and can be filled with a null pointer.
1071 *
1072 * int (*ndo_init)(struct net_device *dev);
1073 * This function is called once when a network device is registered.
1074 * The network device can use this for any late stage initialization
1075 * or semantic validation. It can fail with an error code which will
1076 * be propagated back to register_netdev.
1077 *
1078 * void (*ndo_uninit)(struct net_device *dev);
1079 * This function is called when device is unregistered or when registration
1080 * fails. It is not called if init fails.
1081 *
1082 * int (*ndo_open)(struct net_device *dev);
1083 * This function is called when a network device transitions to the up
1084 * state.
1085 *
1086 * int (*ndo_stop)(struct net_device *dev);
1087 * This function is called when a network device transitions to the down
1088 * state.
1089 *
1090 * netdev_tx_t (*ndo_start_xmit)(struct sk_buff *skb,
1091 * struct net_device *dev);
1092 * Called when a packet needs to be transmitted.
1093 * Returns NETDEV_TX_OK. Can return NETDEV_TX_BUSY, but you should stop
1094 * the queue before that can happen; it's for obsolete devices and weird
1095 * corner cases, but the stack really does a non-trivial amount
1096 * of useless work if you return NETDEV_TX_BUSY.
1097 * Required; cannot be NULL.
1098 *
1099 * netdev_features_t (*ndo_features_check)(struct sk_buff *skb,
1100 * struct net_device *dev
1101 * netdev_features_t features);
1102 * Called by core transmit path to determine if device is capable of
1103 * performing offload operations on a given packet. This is to give
1104 * the device an opportunity to implement any restrictions that cannot
1105 * be otherwise expressed by feature flags. The check is called with
1106 * the set of features that the stack has calculated and it returns
1107 * those the driver believes to be appropriate.
1108 *
1109 * u16 (*ndo_select_queue)(struct net_device *dev, struct sk_buff *skb,
1110 * struct net_device *sb_dev);
1111 * Called to decide which queue to use when device supports multiple
1112 * transmit queues.
1113 *
1114 * void (*ndo_change_rx_flags)(struct net_device *dev, int flags);
1115 * This function is called to allow device receiver to make
1116 * changes to configuration when multicast or promiscuous is enabled.
1117 *
1118 * void (*ndo_set_rx_mode)(struct net_device *dev);
1119 * This function is called device changes address list filtering.
1120 * If driver handles unicast address filtering, it should set
1121 * IFF_UNICAST_FLT in its priv_flags.
1122 *
1123 * int (*ndo_set_mac_address)(struct net_device *dev, void *addr);
1124 * This function is called when the Media Access Control address
1125 * needs to be changed. If this interface is not defined, the
1126 * MAC address can not be changed.
1127 *
1128 * int (*ndo_validate_addr)(struct net_device *dev);
1129 * Test if Media Access Control address is valid for the device.
1130 *
1131 * int (*ndo_do_ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd);
1132 * Old-style ioctl entry point. This is used internally by the
1133 * ieee802154 subsystem but is no longer called by the device
1134 * ioctl handler.
1135 *
1136 * int (*ndo_siocbond)(struct net_device *dev, struct ifreq *ifr, int cmd);
1137 * Used by the bonding driver for its device specific ioctls:
1138 * SIOCBONDENSLAVE, SIOCBONDRELEASE, SIOCBONDSETHWADDR, SIOCBONDCHANGEACTIVE,
1139 * SIOCBONDSLAVEINFOQUERY, and SIOCBONDINFOQUERY
1140 *
1141 * * int (*ndo_eth_ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd);
1142 * Called for ethernet specific ioctls: SIOCGMIIPHY, SIOCGMIIREG,
1143 * SIOCSMIIREG, SIOCSHWTSTAMP and SIOCGHWTSTAMP.
1144 *
1145 * int (*ndo_set_config)(struct net_device *dev, struct ifmap *map);
1146 * Used to set network devices bus interface parameters. This interface
1147 * is retained for legacy reasons; new devices should use the bus
1148 * interface (PCI) for low level management.
1149 *
1150 * int (*ndo_change_mtu)(struct net_device *dev, int new_mtu);
1151 * Called when a user wants to change the Maximum Transfer Unit
1152 * of a device.
1153 *
1154 * void (*ndo_tx_timeout)(struct net_device *dev, unsigned int txqueue);
1155 * Callback used when the transmitter has not made any progress
1156 * for dev->watchdog ticks.
1157 *
1158 * void (*ndo_get_stats64)(struct net_device *dev,
1159 * struct rtnl_link_stats64 *storage);
1160 * struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
1161 * Called when a user wants to get the network device usage
1162 * statistics. Drivers must do one of the following:
1163 * 1. Define @ndo_get_stats64 to fill in a zero-initialised
1164 * rtnl_link_stats64 structure passed by the caller.
1165 * 2. Define @ndo_get_stats to update a net_device_stats structure
1166 * (which should normally be dev->stats) and return a pointer to
1167 * it. The structure may be changed asynchronously only if each
1168 * field is written atomically.
1169 * 3. Update dev->stats asynchronously and atomically, and define
1170 * neither operation.
1171 *
1172 * bool (*ndo_has_offload_stats)(const struct net_device *dev, int attr_id)
1173 * Return true if this device supports offload stats of this attr_id.
1174 *
1175 * int (*ndo_get_offload_stats)(int attr_id, const struct net_device *dev,
1176 * void *attr_data)
1177 * Get statistics for offload operations by attr_id. Write it into the
1178 * attr_data pointer.
1179 *
1180 * int (*ndo_vlan_rx_add_vid)(struct net_device *dev, __be16 proto, u16 vid);
1181 * If device supports VLAN filtering this function is called when a
1182 * VLAN id is registered.
1183 *
1184 * int (*ndo_vlan_rx_kill_vid)(struct net_device *dev, __be16 proto, u16 vid);
1185 * If device supports VLAN filtering this function is called when a
1186 * VLAN id is unregistered.
1187 *
1188 * void (*ndo_poll_controller)(struct net_device *dev);
1189 *
1190 * SR-IOV management functions.
1191 * int (*ndo_set_vf_mac)(struct net_device *dev, int vf, u8* mac);
1192 * int (*ndo_set_vf_vlan)(struct net_device *dev, int vf, u16 vlan,
1193 * u8 qos, __be16 proto);
1194 * int (*ndo_set_vf_rate)(struct net_device *dev, int vf, int min_tx_rate,
1195 * int max_tx_rate);
1196 * int (*ndo_set_vf_spoofchk)(struct net_device *dev, int vf, bool setting);
1197 * int (*ndo_set_vf_trust)(struct net_device *dev, int vf, bool setting);
1198 * int (*ndo_get_vf_config)(struct net_device *dev,
1199 * int vf, struct ifla_vf_info *ivf);
1200 * int (*ndo_set_vf_link_state)(struct net_device *dev, int vf, int link_state);
1201 * int (*ndo_set_vf_port)(struct net_device *dev, int vf,
1202 * struct nlattr *port[]);
1203 *
1204 * Enable or disable the VF ability to query its RSS Redirection Table and
1205 * Hash Key. This is needed since on some devices VF share this information
1206 * with PF and querying it may introduce a theoretical security risk.
1207 * int (*ndo_set_vf_rss_query_en)(struct net_device *dev, int vf, bool setting);
1208 * int (*ndo_get_vf_port)(struct net_device *dev, int vf, struct sk_buff *skb);
1209 * int (*ndo_setup_tc)(struct net_device *dev, enum tc_setup_type type,
1210 * void *type_data);
1211 * Called to setup any 'tc' scheduler, classifier or action on @dev.
1212 * This is always called from the stack with the rtnl lock held and netif
1213 * tx queues stopped. This allows the netdevice to perform queue
1214 * management safely.
1215 *
1216 * Fiber Channel over Ethernet (FCoE) offload functions.
1217 * int (*ndo_fcoe_enable)(struct net_device *dev);
1218 * Called when the FCoE protocol stack wants to start using LLD for FCoE
1219 * so the underlying device can perform whatever needed configuration or
1220 * initialization to support acceleration of FCoE traffic.
1221 *
1222 * int (*ndo_fcoe_disable)(struct net_device *dev);
1223 * Called when the FCoE protocol stack wants to stop using LLD for FCoE
1224 * so the underlying device can perform whatever needed clean-ups to
1225 * stop supporting acceleration of FCoE traffic.
1226 *
1227 * int (*ndo_fcoe_ddp_setup)(struct net_device *dev, u16 xid,
1228 * struct scatterlist *sgl, unsigned int sgc);
1229 * Called when the FCoE Initiator wants to initialize an I/O that
1230 * is a possible candidate for Direct Data Placement (DDP). The LLD can
1231 * perform necessary setup and returns 1 to indicate the device is set up
1232 * successfully to perform DDP on this I/O, otherwise this returns 0.
1233 *
1234 * int (*ndo_fcoe_ddp_done)(struct net_device *dev, u16 xid);
1235 * Called when the FCoE Initiator/Target is done with the DDPed I/O as
1236 * indicated by the FC exchange id 'xid', so the underlying device can
1237 * clean up and reuse resources for later DDP requests.
1238 *
1239 * int (*ndo_fcoe_ddp_target)(struct net_device *dev, u16 xid,
1240 * struct scatterlist *sgl, unsigned int sgc);
1241 * Called when the FCoE Target wants to initialize an I/O that
1242 * is a possible candidate for Direct Data Placement (DDP). The LLD can
1243 * perform necessary setup and returns 1 to indicate the device is set up
1244 * successfully to perform DDP on this I/O, otherwise this returns 0.
1245 *
1246 * int (*ndo_fcoe_get_hbainfo)(struct net_device *dev,
1247 * struct netdev_fcoe_hbainfo *hbainfo);
1248 * Called when the FCoE Protocol stack wants information on the underlying
1249 * device. This information is utilized by the FCoE protocol stack to
1250 * register attributes with Fiber Channel management service as per the
1251 * FC-GS Fabric Device Management Information(FDMI) specification.
1252 *
1253 * int (*ndo_fcoe_get_wwn)(struct net_device *dev, u64 *wwn, int type);
1254 * Called when the underlying device wants to override default World Wide
1255 * Name (WWN) generation mechanism in FCoE protocol stack to pass its own
1256 * World Wide Port Name (WWPN) or World Wide Node Name (WWNN) to the FCoE
1257 * protocol stack to use.
1258 *
1259 * RFS acceleration.
1260 * int (*ndo_rx_flow_steer)(struct net_device *dev, const struct sk_buff *skb,
1261 * u16 rxq_index, u32 flow_id);
1262 * Set hardware filter for RFS. rxq_index is the target queue index;
1263 * flow_id is a flow ID to be passed to rps_may_expire_flow() later.
1264 * Return the filter ID on success, or a negative error code.
1265 *
1266 * Slave management functions (for bridge, bonding, etc).
1267 * int (*ndo_add_slave)(struct net_device *dev, struct net_device *slave_dev);
1268 * Called to make another netdev an underling.
1269 *
1270 * int (*ndo_del_slave)(struct net_device *dev, struct net_device *slave_dev);
1271 * Called to release previously enslaved netdev.
1272 *
1273 * struct net_device *(*ndo_get_xmit_slave)(struct net_device *dev,
1274 * struct sk_buff *skb,
1275 * bool all_slaves);
1276 * Get the xmit slave of master device. If all_slaves is true, function
1277 * assume all the slaves can transmit.
1278 *
1279 * Feature/offload setting functions.
1280 * netdev_features_t (*ndo_fix_features)(struct net_device *dev,
1281 * netdev_features_t features);
1282 * Adjusts the requested feature flags according to device-specific
1283 * constraints, and returns the resulting flags. Must not modify
1284 * the device state.
1285 *
1286 * int (*ndo_set_features)(struct net_device *dev, netdev_features_t features);
1287 * Called to update device configuration to new features. Passed
1288 * feature set might be less than what was returned by ndo_fix_features()).
1289 * Must return >0 or -errno if it changed dev->features itself.
1290 *
1291 * int (*ndo_fdb_add)(struct ndmsg *ndm, struct nlattr *tb[],
1292 * struct net_device *dev,
1293 * const unsigned char *addr, u16 vid, u16 flags,
1294 * bool *notified, struct netlink_ext_ack *extack);
1295 * Adds an FDB entry to dev for addr.
1296 * Callee shall set *notified to true if it sent any appropriate
1297 * notification(s). Otherwise core will send a generic one.
1298 * int (*ndo_fdb_del)(struct ndmsg *ndm, struct nlattr *tb[],
1299 * struct net_device *dev,
1300 * const unsigned char *addr, u16 vid
1301 * bool *notified, struct netlink_ext_ack *extack);
1302 * Deletes the FDB entry from dev corresponding to addr.
1303 * Callee shall set *notified to true if it sent any appropriate
1304 * notification(s). Otherwise core will send a generic one.
1305 * int (*ndo_fdb_del_bulk)(struct nlmsghdr *nlh, struct net_device *dev,
1306 * struct netlink_ext_ack *extack);
1307 * int (*ndo_fdb_dump)(struct sk_buff *skb, struct netlink_callback *cb,
1308 * struct net_device *dev, struct net_device *filter_dev,
1309 * int *idx)
1310 * Used to add FDB entries to dump requests. Implementers should add
1311 * entries to skb and update idx with the number of entries.
1312 *
1313 * int (*ndo_mdb_add)(struct net_device *dev, struct nlattr *tb[],
1314 * u16 nlmsg_flags, struct netlink_ext_ack *extack);
1315 * Adds an MDB entry to dev.
1316 * int (*ndo_mdb_del)(struct net_device *dev, struct nlattr *tb[],
1317 * struct netlink_ext_ack *extack);
1318 * Deletes the MDB entry from dev.
1319 * int (*ndo_mdb_del_bulk)(struct net_device *dev, struct nlattr *tb[],
1320 * struct netlink_ext_ack *extack);
1321 * Bulk deletes MDB entries from dev.
1322 * int (*ndo_mdb_dump)(struct net_device *dev, struct sk_buff *skb,
1323 * struct netlink_callback *cb);
1324 * Dumps MDB entries from dev. The first argument (marker) in the netlink
1325 * callback is used by core rtnetlink code.
1326 *
1327 * int (*ndo_bridge_setlink)(struct net_device *dev, struct nlmsghdr *nlh,
1328 * u16 flags, struct netlink_ext_ack *extack)
1329 * int (*ndo_bridge_getlink)(struct sk_buff *skb, u32 pid, u32 seq,
1330 * struct net_device *dev, u32 filter_mask,
1331 * int nlflags)
1332 * int (*ndo_bridge_dellink)(struct net_device *dev, struct nlmsghdr *nlh,
1333 * u16 flags);
1334 *
1335 * int (*ndo_change_carrier)(struct net_device *dev, bool new_carrier);
1336 * Called to change device carrier. Soft-devices (like dummy, team, etc)
1337 * which do not represent real hardware may define this to allow their
1338 * userspace components to manage their virtual carrier state. Devices
1339 * that determine carrier state from physical hardware properties (eg
1340 * network cables) or protocol-dependent mechanisms (eg
1341 * USB_CDC_NOTIFY_NETWORK_CONNECTION) should NOT implement this function.
1342 *
1343 * int (*ndo_get_phys_port_id)(struct net_device *dev,
1344 * struct netdev_phys_item_id *ppid);
1345 * Called to get ID of physical port of this device. If driver does
1346 * not implement this, it is assumed that the hw is not able to have
1347 * multiple net devices on single physical port.
1348 *
1349 * int (*ndo_get_port_parent_id)(struct net_device *dev,
1350 * struct netdev_phys_item_id *ppid)
1351 * Called to get the parent ID of the physical port of this device.
1352 *
1353 * void* (*ndo_dfwd_add_station)(struct net_device *pdev,
1354 * struct net_device *dev)
1355 * Called by upper layer devices to accelerate switching or other
1356 * station functionality into hardware. 'pdev is the lowerdev
1357 * to use for the offload and 'dev' is the net device that will
1358 * back the offload. Returns a pointer to the private structure
1359 * the upper layer will maintain.
1360 * void (*ndo_dfwd_del_station)(struct net_device *pdev, void *priv)
1361 * Called by upper layer device to delete the station created
1362 * by 'ndo_dfwd_add_station'. 'pdev' is the net device backing
1363 * the station and priv is the structure returned by the add
1364 * operation.
1365 * int (*ndo_set_tx_maxrate)(struct net_device *dev,
1366 * int queue_index, u32 maxrate);
1367 * Called when a user wants to set a max-rate limitation of specific
1368 * TX queue.
1369 * int (*ndo_get_iflink)(const struct net_device *dev);
1370 * Called to get the iflink value of this device.
1371 * int (*ndo_fill_metadata_dst)(struct net_device *dev, struct sk_buff *skb);
1372 * This function is used to get egress tunnel information for given skb.
1373 * This is useful for retrieving outer tunnel header parameters while
1374 * sampling packet.
1375 * void (*ndo_set_rx_headroom)(struct net_device *dev, int needed_headroom);
1376 * This function is used to specify the headroom that the skb must
1377 * consider when allocation skb during packet reception. Setting
1378 * appropriate rx headroom value allows avoiding skb head copy on
1379 * forward. Setting a negative value resets the rx headroom to the
1380 * default value.
1381 * int (*ndo_bpf)(struct net_device *dev, struct netdev_bpf *bpf);
1382 * This function is used to set or query state related to XDP on the
1383 * netdevice and manage BPF offload. See definition of
1384 * enum bpf_netdev_command for details.
1385 * int (*ndo_xdp_xmit)(struct net_device *dev, int n, struct xdp_frame **xdp,
1386 * u32 flags);
1387 * This function is used to submit @n XDP packets for transmit on a
1388 * netdevice. Returns number of frames successfully transmitted, frames
1389 * that got dropped are freed/returned via xdp_return_frame().
1390 * Returns negative number, means general error invoking ndo, meaning
1391 * no frames were xmit'ed and core-caller will free all frames.
1392 * struct net_device *(*ndo_xdp_get_xmit_slave)(struct net_device *dev,
1393 * struct xdp_buff *xdp);
1394 * Get the xmit slave of master device based on the xdp_buff.
1395 * int (*ndo_xsk_wakeup)(struct net_device *dev, u32 queue_id, u32 flags);
1396 * This function is used to wake up the softirq, ksoftirqd or kthread
1397 * responsible for sending and/or receiving packets on a specific
1398 * queue id bound to an AF_XDP socket. The flags field specifies if
1399 * only RX, only Tx, or both should be woken up using the flags
1400 * XDP_WAKEUP_RX and XDP_WAKEUP_TX.
1401 * int (*ndo_tunnel_ctl)(struct net_device *dev, struct ip_tunnel_parm_kern *p,
1402 * int cmd);
1403 * Add, change, delete or get information on an IPv4 tunnel.
1404 * struct net_device *(*ndo_get_peer_dev)(struct net_device *dev);
1405 * If a device is paired with a peer device, return the peer instance.
1406 * The caller must be under RCU read context.
1407 * int (*ndo_fill_forward_path)(struct net_device_path_ctx *ctx, struct net_device_path *path);
1408 * Get the forwarding path to reach the real device from the HW destination address
1409 * ktime_t (*ndo_get_tstamp)(struct net_device *dev,
1410 * const struct skb_shared_hwtstamps *hwtstamps,
1411 * bool cycles);
1412 * Get hardware timestamp based on normal/adjustable time or free running
1413 * cycle counter. This function is required if physical clock supports a
1414 * free running cycle counter.
1415 *
1416 * int (*ndo_hwtstamp_get)(struct net_device *dev,
1417 * struct kernel_hwtstamp_config *kernel_config);
1418 * Get the currently configured hardware timestamping parameters for the
1419 * NIC device.
1420 *
1421 * int (*ndo_hwtstamp_set)(struct net_device *dev,
1422 * struct kernel_hwtstamp_config *kernel_config,
1423 * struct netlink_ext_ack *extack);
1424 * Change the hardware timestamping parameters for NIC device.
1425 */
1426struct net_device_ops {
1427 int (*ndo_init)(struct net_device *dev);
1428 void (*ndo_uninit)(struct net_device *dev);
1429 int (*ndo_open)(struct net_device *dev);
1430 int (*ndo_stop)(struct net_device *dev);
1431 netdev_tx_t (*ndo_start_xmit)(struct sk_buff *skb,
1432 struct net_device *dev);
1433 netdev_features_t (*ndo_features_check)(struct sk_buff *skb,
1434 struct net_device *dev,
1435 netdev_features_t features);
1436 u16 (*ndo_select_queue)(struct net_device *dev,
1437 struct sk_buff *skb,
1438 struct net_device *sb_dev);
1439 void (*ndo_change_rx_flags)(struct net_device *dev,
1440 int flags);
1441 void (*ndo_set_rx_mode)(struct net_device *dev);
1442 int (*ndo_set_mac_address)(struct net_device *dev,
1443 void *addr);
1444 int (*ndo_validate_addr)(struct net_device *dev);
1445 int (*ndo_do_ioctl)(struct net_device *dev,
1446 struct ifreq *ifr, int cmd);
1447 int (*ndo_eth_ioctl)(struct net_device *dev,
1448 struct ifreq *ifr, int cmd);
1449 int (*ndo_siocbond)(struct net_device *dev,
1450 struct ifreq *ifr, int cmd);
1451 int (*ndo_siocwandev)(struct net_device *dev,
1452 struct if_settings *ifs);
1453 int (*ndo_siocdevprivate)(struct net_device *dev,
1454 struct ifreq *ifr,
1455 void __user *data, int cmd);
1456 int (*ndo_set_config)(struct net_device *dev,
1457 struct ifmap *map);
1458 int (*ndo_change_mtu)(struct net_device *dev,
1459 int new_mtu);
1460 int (*ndo_neigh_setup)(struct net_device *dev,
1461 struct neigh_parms *);
1462 void (*ndo_tx_timeout) (struct net_device *dev,
1463 unsigned int txqueue);
1464
1465 void (*ndo_get_stats64)(struct net_device *dev,
1466 struct rtnl_link_stats64 *storage);
1467 bool (*ndo_has_offload_stats)(const struct net_device *dev, int attr_id);
1468 int (*ndo_get_offload_stats)(int attr_id,
1469 const struct net_device *dev,
1470 void *attr_data);
1471 struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
1472
1473 int (*ndo_vlan_rx_add_vid)(struct net_device *dev,
1474 __be16 proto, u16 vid);
1475 int (*ndo_vlan_rx_kill_vid)(struct net_device *dev,
1476 __be16 proto, u16 vid);
1477#ifdef CONFIG_NET_POLL_CONTROLLER
1478 void (*ndo_poll_controller)(struct net_device *dev);
1479 int (*ndo_netpoll_setup)(struct net_device *dev);
1480 void (*ndo_netpoll_cleanup)(struct net_device *dev);
1481#endif
1482 int (*ndo_set_vf_mac)(struct net_device *dev,
1483 int queue, u8 *mac);
1484 int (*ndo_set_vf_vlan)(struct net_device *dev,
1485 int queue, u16 vlan,
1486 u8 qos, __be16 proto);
1487 int (*ndo_set_vf_rate)(struct net_device *dev,
1488 int vf, int min_tx_rate,
1489 int max_tx_rate);
1490 int (*ndo_set_vf_spoofchk)(struct net_device *dev,
1491 int vf, bool setting);
1492 int (*ndo_set_vf_trust)(struct net_device *dev,
1493 int vf, bool setting);
1494 int (*ndo_get_vf_config)(struct net_device *dev,
1495 int vf,
1496 struct ifla_vf_info *ivf);
1497 int (*ndo_set_vf_link_state)(struct net_device *dev,
1498 int vf, int link_state);
1499 int (*ndo_get_vf_stats)(struct net_device *dev,
1500 int vf,
1501 struct ifla_vf_stats
1502 *vf_stats);
1503 int (*ndo_set_vf_port)(struct net_device *dev,
1504 int vf,
1505 struct nlattr *port[]);
1506 int (*ndo_get_vf_port)(struct net_device *dev,
1507 int vf, struct sk_buff *skb);
1508 int (*ndo_get_vf_guid)(struct net_device *dev,
1509 int vf,
1510 struct ifla_vf_guid *node_guid,
1511 struct ifla_vf_guid *port_guid);
1512 int (*ndo_set_vf_guid)(struct net_device *dev,
1513 int vf, u64 guid,
1514 int guid_type);
1515 int (*ndo_set_vf_rss_query_en)(
1516 struct net_device *dev,
1517 int vf, bool setting);
1518 int (*ndo_setup_tc)(struct net_device *dev,
1519 enum tc_setup_type type,
1520 void *type_data);
1521#if IS_ENABLED(CONFIG_FCOE)
1522 int (*ndo_fcoe_enable)(struct net_device *dev);
1523 int (*ndo_fcoe_disable)(struct net_device *dev);
1524 int (*ndo_fcoe_ddp_setup)(struct net_device *dev,
1525 u16 xid,
1526 struct scatterlist *sgl,
1527 unsigned int sgc);
1528 int (*ndo_fcoe_ddp_done)(struct net_device *dev,
1529 u16 xid);
1530 int (*ndo_fcoe_ddp_target)(struct net_device *dev,
1531 u16 xid,
1532 struct scatterlist *sgl,
1533 unsigned int sgc);
1534 int (*ndo_fcoe_get_hbainfo)(struct net_device *dev,
1535 struct netdev_fcoe_hbainfo *hbainfo);
1536#endif
1537
1538#if IS_ENABLED(CONFIG_LIBFCOE)
1539#define NETDEV_FCOE_WWNN 0
1540#define NETDEV_FCOE_WWPN 1
1541 int (*ndo_fcoe_get_wwn)(struct net_device *dev,
1542 u64 *wwn, int type);
1543#endif
1544
1545#ifdef CONFIG_RFS_ACCEL
1546 int (*ndo_rx_flow_steer)(struct net_device *dev,
1547 const struct sk_buff *skb,
1548 u16 rxq_index,
1549 u32 flow_id);
1550#endif
1551 int (*ndo_add_slave)(struct net_device *dev,
1552 struct net_device *slave_dev,
1553 struct netlink_ext_ack *extack);
1554 int (*ndo_del_slave)(struct net_device *dev,
1555 struct net_device *slave_dev);
1556 struct net_device* (*ndo_get_xmit_slave)(struct net_device *dev,
1557 struct sk_buff *skb,
1558 bool all_slaves);
1559 struct net_device* (*ndo_sk_get_lower_dev)(struct net_device *dev,
1560 struct sock *sk);
1561 netdev_features_t (*ndo_fix_features)(struct net_device *dev,
1562 netdev_features_t features);
1563 int (*ndo_set_features)(struct net_device *dev,
1564 netdev_features_t features);
1565 int (*ndo_neigh_construct)(struct net_device *dev,
1566 struct neighbour *n);
1567 void (*ndo_neigh_destroy)(struct net_device *dev,
1568 struct neighbour *n);
1569
1570 int (*ndo_fdb_add)(struct ndmsg *ndm,
1571 struct nlattr *tb[],
1572 struct net_device *dev,
1573 const unsigned char *addr,
1574 u16 vid,
1575 u16 flags,
1576 bool *notified,
1577 struct netlink_ext_ack *extack);
1578 int (*ndo_fdb_del)(struct ndmsg *ndm,
1579 struct nlattr *tb[],
1580 struct net_device *dev,
1581 const unsigned char *addr,
1582 u16 vid,
1583 bool *notified,
1584 struct netlink_ext_ack *extack);
1585 int (*ndo_fdb_del_bulk)(struct nlmsghdr *nlh,
1586 struct net_device *dev,
1587 struct netlink_ext_ack *extack);
1588 int (*ndo_fdb_dump)(struct sk_buff *skb,
1589 struct netlink_callback *cb,
1590 struct net_device *dev,
1591 struct net_device *filter_dev,
1592 int *idx);
1593 int (*ndo_fdb_get)(struct sk_buff *skb,
1594 struct nlattr *tb[],
1595 struct net_device *dev,
1596 const unsigned char *addr,
1597 u16 vid, u32 portid, u32 seq,
1598 struct netlink_ext_ack *extack);
1599 int (*ndo_mdb_add)(struct net_device *dev,
1600 struct nlattr *tb[],
1601 u16 nlmsg_flags,
1602 struct netlink_ext_ack *extack);
1603 int (*ndo_mdb_del)(struct net_device *dev,
1604 struct nlattr *tb[],
1605 struct netlink_ext_ack *extack);
1606 int (*ndo_mdb_del_bulk)(struct net_device *dev,
1607 struct nlattr *tb[],
1608 struct netlink_ext_ack *extack);
1609 int (*ndo_mdb_dump)(struct net_device *dev,
1610 struct sk_buff *skb,
1611 struct netlink_callback *cb);
1612 int (*ndo_mdb_get)(struct net_device *dev,
1613 struct nlattr *tb[], u32 portid,
1614 u32 seq,
1615 struct netlink_ext_ack *extack);
1616 int (*ndo_bridge_setlink)(struct net_device *dev,
1617 struct nlmsghdr *nlh,
1618 u16 flags,
1619 struct netlink_ext_ack *extack);
1620 int (*ndo_bridge_getlink)(struct sk_buff *skb,
1621 u32 pid, u32 seq,
1622 struct net_device *dev,
1623 u32 filter_mask,
1624 int nlflags);
1625 int (*ndo_bridge_dellink)(struct net_device *dev,
1626 struct nlmsghdr *nlh,
1627 u16 flags);
1628 int (*ndo_change_carrier)(struct net_device *dev,
1629 bool new_carrier);
1630 int (*ndo_get_phys_port_id)(struct net_device *dev,
1631 struct netdev_phys_item_id *ppid);
1632 int (*ndo_get_port_parent_id)(struct net_device *dev,
1633 struct netdev_phys_item_id *ppid);
1634 int (*ndo_get_phys_port_name)(struct net_device *dev,
1635 char *name, size_t len);
1636 void* (*ndo_dfwd_add_station)(struct net_device *pdev,
1637 struct net_device *dev);
1638 void (*ndo_dfwd_del_station)(struct net_device *pdev,
1639 void *priv);
1640
1641 int (*ndo_set_tx_maxrate)(struct net_device *dev,
1642 int queue_index,
1643 u32 maxrate);
1644 int (*ndo_get_iflink)(const struct net_device *dev);
1645 int (*ndo_fill_metadata_dst)(struct net_device *dev,
1646 struct sk_buff *skb);
1647 void (*ndo_set_rx_headroom)(struct net_device *dev,
1648 int needed_headroom);
1649 int (*ndo_bpf)(struct net_device *dev,
1650 struct netdev_bpf *bpf);
1651 int (*ndo_xdp_xmit)(struct net_device *dev, int n,
1652 struct xdp_frame **xdp,
1653 u32 flags);
1654 struct net_device * (*ndo_xdp_get_xmit_slave)(struct net_device *dev,
1655 struct xdp_buff *xdp);
1656 int (*ndo_xsk_wakeup)(struct net_device *dev,
1657 u32 queue_id, u32 flags);
1658 int (*ndo_tunnel_ctl)(struct net_device *dev,
1659 struct ip_tunnel_parm_kern *p,
1660 int cmd);
1661 struct net_device * (*ndo_get_peer_dev)(struct net_device *dev);
1662 int (*ndo_fill_forward_path)(struct net_device_path_ctx *ctx,
1663 struct net_device_path *path);
1664 ktime_t (*ndo_get_tstamp)(struct net_device *dev,
1665 const struct skb_shared_hwtstamps *hwtstamps,
1666 bool cycles);
1667 int (*ndo_hwtstamp_get)(struct net_device *dev,
1668 struct kernel_hwtstamp_config *kernel_config);
1669 int (*ndo_hwtstamp_set)(struct net_device *dev,
1670 struct kernel_hwtstamp_config *kernel_config,
1671 struct netlink_ext_ack *extack);
1672
1673#if IS_ENABLED(CONFIG_NET_SHAPER)
1674 /**
1675 * @net_shaper_ops: Device shaping offload operations
1676 * see include/net/net_shapers.h
1677 */
1678 const struct net_shaper_ops *net_shaper_ops;
1679#endif
1680};
1681
1682/**
1683 * enum netdev_priv_flags - &struct net_device priv_flags
1684 *
1685 * These are the &struct net_device, they are only set internally
1686 * by drivers and used in the kernel. These flags are invisible to
1687 * userspace; this means that the order of these flags can change
1688 * during any kernel release.
1689 *
1690 * You should add bitfield booleans after either net_device::priv_flags
1691 * (hotpath) or ::threaded (slowpath) instead of extending these flags.
1692 *
1693 * @IFF_802_1Q_VLAN: 802.1Q VLAN device
1694 * @IFF_EBRIDGE: Ethernet bridging device
1695 * @IFF_BONDING: bonding master or slave
1696 * @IFF_ISATAP: ISATAP interface (RFC4214)
1697 * @IFF_WAN_HDLC: WAN HDLC device
1698 * @IFF_XMIT_DST_RELEASE: dev_hard_start_xmit() is allowed to
1699 * release skb->dst
1700 * @IFF_DONT_BRIDGE: disallow bridging this ether dev
1701 * @IFF_DISABLE_NETPOLL: disable netpoll at run-time
1702 * @IFF_MACVLAN_PORT: device used as macvlan port
1703 * @IFF_BRIDGE_PORT: device used as bridge port
1704 * @IFF_OVS_DATAPATH: device used as Open vSwitch datapath port
1705 * @IFF_TX_SKB_SHARING: The interface supports sharing skbs on transmit
1706 * @IFF_UNICAST_FLT: Supports unicast filtering
1707 * @IFF_TEAM_PORT: device used as team port
1708 * @IFF_SUPP_NOFCS: device supports sending custom FCS
1709 * @IFF_LIVE_ADDR_CHANGE: device supports hardware address
1710 * change when it's running
1711 * @IFF_MACVLAN: Macvlan device
1712 * @IFF_XMIT_DST_RELEASE_PERM: IFF_XMIT_DST_RELEASE not taking into account
1713 * underlying stacked devices
1714 * @IFF_L3MDEV_MASTER: device is an L3 master device
1715 * @IFF_NO_QUEUE: device can run without qdisc attached
1716 * @IFF_OPENVSWITCH: device is a Open vSwitch master
1717 * @IFF_L3MDEV_SLAVE: device is enslaved to an L3 master device
1718 * @IFF_TEAM: device is a team device
1719 * @IFF_RXFH_CONFIGURED: device has had Rx Flow indirection table configured
1720 * @IFF_PHONY_HEADROOM: the headroom value is controlled by an external
1721 * entity (i.e. the master device for bridged veth)
1722 * @IFF_MACSEC: device is a MACsec device
1723 * @IFF_NO_RX_HANDLER: device doesn't support the rx_handler hook
1724 * @IFF_FAILOVER: device is a failover master device
1725 * @IFF_FAILOVER_SLAVE: device is lower dev of a failover master device
1726 * @IFF_L3MDEV_RX_HANDLER: only invoke the rx handler of L3 master device
1727 * @IFF_NO_ADDRCONF: prevent ipv6 addrconf
1728 * @IFF_TX_SKB_NO_LINEAR: device/driver is capable of xmitting frames with
1729 * skb_headlen(skb) == 0 (data starts from frag0)
1730 */
1731enum netdev_priv_flags {
1732 IFF_802_1Q_VLAN = 1<<0,
1733 IFF_EBRIDGE = 1<<1,
1734 IFF_BONDING = 1<<2,
1735 IFF_ISATAP = 1<<3,
1736 IFF_WAN_HDLC = 1<<4,
1737 IFF_XMIT_DST_RELEASE = 1<<5,
1738 IFF_DONT_BRIDGE = 1<<6,
1739 IFF_DISABLE_NETPOLL = 1<<7,
1740 IFF_MACVLAN_PORT = 1<<8,
1741 IFF_BRIDGE_PORT = 1<<9,
1742 IFF_OVS_DATAPATH = 1<<10,
1743 IFF_TX_SKB_SHARING = 1<<11,
1744 IFF_UNICAST_FLT = 1<<12,
1745 IFF_TEAM_PORT = 1<<13,
1746 IFF_SUPP_NOFCS = 1<<14,
1747 IFF_LIVE_ADDR_CHANGE = 1<<15,
1748 IFF_MACVLAN = 1<<16,
1749 IFF_XMIT_DST_RELEASE_PERM = 1<<17,
1750 IFF_L3MDEV_MASTER = 1<<18,
1751 IFF_NO_QUEUE = 1<<19,
1752 IFF_OPENVSWITCH = 1<<20,
1753 IFF_L3MDEV_SLAVE = 1<<21,
1754 IFF_TEAM = 1<<22,
1755 IFF_RXFH_CONFIGURED = 1<<23,
1756 IFF_PHONY_HEADROOM = 1<<24,
1757 IFF_MACSEC = 1<<25,
1758 IFF_NO_RX_HANDLER = 1<<26,
1759 IFF_FAILOVER = 1<<27,
1760 IFF_FAILOVER_SLAVE = 1<<28,
1761 IFF_L3MDEV_RX_HANDLER = 1<<29,
1762 IFF_NO_ADDRCONF = BIT_ULL(30),
1763 IFF_TX_SKB_NO_LINEAR = BIT_ULL(31),
1764};
1765
1766/* Specifies the type of the struct net_device::ml_priv pointer */
1767enum netdev_ml_priv_type {
1768 ML_PRIV_NONE,
1769 ML_PRIV_CAN,
1770};
1771
1772enum netdev_stat_type {
1773 NETDEV_PCPU_STAT_NONE,
1774 NETDEV_PCPU_STAT_LSTATS, /* struct pcpu_lstats */
1775 NETDEV_PCPU_STAT_TSTATS, /* struct pcpu_sw_netstats */
1776 NETDEV_PCPU_STAT_DSTATS, /* struct pcpu_dstats */
1777};
1778
1779enum netdev_reg_state {
1780 NETREG_UNINITIALIZED = 0,
1781 NETREG_REGISTERED, /* completed register_netdevice */
1782 NETREG_UNREGISTERING, /* called unregister_netdevice */
1783 NETREG_UNREGISTERED, /* completed unregister todo */
1784 NETREG_RELEASED, /* called free_netdev */
1785 NETREG_DUMMY, /* dummy device for NAPI poll */
1786};
1787
1788/**
1789 * struct net_device - The DEVICE structure.
1790 *
1791 * Actually, this whole structure is a big mistake. It mixes I/O
1792 * data with strictly "high-level" data, and it has to know about
1793 * almost every data structure used in the INET module.
1794 *
1795 * @priv_flags: flags invisible to userspace defined as bits, see
1796 * enum netdev_priv_flags for the definitions
1797 * @lltx: device supports lockless Tx. Deprecated for real HW
1798 * drivers. Mainly used by logical interfaces, such as
1799 * bonding and tunnels
1800 * @netmem_tx: device support netmem_tx.
1801 *
1802 * @name: This is the first field of the "visible" part of this structure
1803 * (i.e. as seen by users in the "Space.c" file). It is the name
1804 * of the interface.
1805 *
1806 * @name_node: Name hashlist node
1807 * @ifalias: SNMP alias
1808 * @mem_end: Shared memory end
1809 * @mem_start: Shared memory start
1810 * @base_addr: Device I/O address
1811 * @irq: Device IRQ number
1812 *
1813 * @state: Generic network queuing layer state, see netdev_state_t
1814 * @dev_list: The global list of network devices
1815 * @napi_list: List entry used for polling NAPI devices
1816 * @unreg_list: List entry when we are unregistering the
1817 * device; see the function unregister_netdev
1818 * @close_list: List entry used when we are closing the device
1819 * @ptype_all: Device-specific packet handlers for all protocols
1820 * @ptype_specific: Device-specific, protocol-specific packet handlers
1821 *
1822 * @adj_list: Directly linked devices, like slaves for bonding
1823 * @features: Currently active device features
1824 * @hw_features: User-changeable features
1825 *
1826 * @wanted_features: User-requested features
1827 * @vlan_features: Mask of features inheritable by VLAN devices
1828 *
1829 * @hw_enc_features: Mask of features inherited by encapsulating devices
1830 * This field indicates what encapsulation
1831 * offloads the hardware is capable of doing,
1832 * and drivers will need to set them appropriately.
1833 *
1834 * @mpls_features: Mask of features inheritable by MPLS
1835 * @gso_partial_features: value(s) from NETIF_F_GSO\*
1836 * @mangleid_features: Mask of features requiring MANGLEID, will be
1837 * disabled together with the latter.
1838 *
1839 * @ifindex: interface index
1840 * @group: The group the device belongs to
1841 *
1842 * @stats: Statistics struct, which was left as a legacy, use
1843 * rtnl_link_stats64 instead
1844 *
1845 * @core_stats: core networking counters,
1846 * do not use this in drivers
1847 * @carrier_up_count: Number of times the carrier has been up
1848 * @carrier_down_count: Number of times the carrier has been down
1849 *
1850 * @wireless_handlers: List of functions to handle Wireless Extensions,
1851 * instead of ioctl,
1852 * see <net/iw_handler.h> for details.
1853 *
1854 * @netdev_ops: Includes several pointers to callbacks,
1855 * if one wants to override the ndo_*() functions
1856 * @xdp_metadata_ops: Includes pointers to XDP metadata callbacks.
1857 * @xsk_tx_metadata_ops: Includes pointers to AF_XDP TX metadata callbacks.
1858 * @ethtool_ops: Management operations
1859 * @l3mdev_ops: Layer 3 master device operations
1860 * @ndisc_ops: Includes callbacks for different IPv6 neighbour
1861 * discovery handling. Necessary for e.g. 6LoWPAN.
1862 * @xfrmdev_ops: Transformation offload operations
1863 * @tlsdev_ops: Transport Layer Security offload operations
1864 * @header_ops: Includes callbacks for creating,parsing,caching,etc
1865 * of Layer 2 headers.
1866 *
1867 * @flags: Interface flags (a la BSD)
1868 * @xdp_features: XDP capability supported by the device
1869 * @gflags: Global flags ( kept as legacy )
1870 * @priv_len: Size of the ->priv flexible array
1871 * @priv: Flexible array containing private data
1872 * @operstate: RFC2863 operstate
1873 * @link_mode: Mapping policy to operstate
1874 * @if_port: Selectable AUI, TP, ...
1875 * @dma: DMA channel
1876 * @mtu: Interface MTU value
1877 * @min_mtu: Interface Minimum MTU value
1878 * @max_mtu: Interface Maximum MTU value
1879 * @type: Interface hardware type
1880 * @hard_header_len: Maximum hardware header length.
1881 * @min_header_len: Minimum hardware header length
1882 *
1883 * @needed_headroom: Extra headroom the hardware may need, but not in all
1884 * cases can this be guaranteed
1885 * @needed_tailroom: Extra tailroom the hardware may need, but not in all
1886 * cases can this be guaranteed. Some cases also use
1887 * LL_MAX_HEADER instead to allocate the skb
1888 *
1889 * interface address info:
1890 *
1891 * @perm_addr: Permanent hw address
1892 * @addr_assign_type: Hw address assignment type
1893 * @addr_len: Hardware address length
1894 * @upper_level: Maximum depth level of upper devices.
1895 * @lower_level: Maximum depth level of lower devices.
1896 * @threaded: napi threaded state.
1897 * @neigh_priv_len: Used in neigh_alloc()
1898 * @dev_id: Used to differentiate devices that share
1899 * the same link layer address
1900 * @dev_port: Used to differentiate devices that share
1901 * the same function
1902 * @addr_list_lock: XXX: need comments on this one
1903 * @name_assign_type: network interface name assignment type
1904 * @uc_promisc: Counter that indicates promiscuous mode
1905 * has been enabled due to the need to listen to
1906 * additional unicast addresses in a device that
1907 * does not implement ndo_set_rx_mode()
1908 * @uc: unicast mac addresses
1909 * @mc: multicast mac addresses
1910 * @dev_addrs: list of device hw addresses
1911 * @queues_kset: Group of all Kobjects in the Tx and RX queues
1912 * @promiscuity: Number of times the NIC is told to work in
1913 * promiscuous mode; if it becomes 0 the NIC will
1914 * exit promiscuous mode
1915 * @allmulti: Counter, enables or disables allmulticast mode
1916 *
1917 * @vlan_info: VLAN info
1918 * @dsa_ptr: dsa specific data
1919 * @tipc_ptr: TIPC specific data
1920 * @atalk_ptr: AppleTalk link
1921 * @ip_ptr: IPv4 specific data
1922 * @ip6_ptr: IPv6 specific data
1923 * @ax25_ptr: AX.25 specific data
1924 * @ieee80211_ptr: IEEE 802.11 specific data, assign before registering
1925 * @ieee802154_ptr: IEEE 802.15.4 low-rate Wireless Personal Area Network
1926 * device struct
1927 * @mpls_ptr: mpls_dev struct pointer
1928 * @mctp_ptr: MCTP specific data
1929 * @psp_dev: PSP crypto device registered for this netdev
1930 *
1931 * @dev_addr: Hw address (before bcast,
1932 * because most packets are unicast)
1933 *
1934 * @_rx: Array of RX queues
1935 * @num_rx_queues: Number of RX queues
1936 * allocated at register_netdev() time
1937 * @real_num_rx_queues: Number of RX queues currently active in device
1938 * @xdp_prog: XDP sockets filter program pointer
1939 *
1940 * @rx_handler: handler for received packets
1941 * @rx_handler_data: XXX: need comments on this one
1942 * @tcx_ingress: BPF & clsact qdisc specific data for ingress processing
1943 * @ingress_queue: XXX: need comments on this one
1944 * @nf_hooks_ingress: netfilter hooks executed for ingress packets
1945 * @broadcast: hw bcast address
1946 *
1947 * @rx_cpu_rmap: CPU reverse-mapping for RX completion interrupts,
1948 * indexed by RX queue number. Assigned by driver.
1949 * This must only be set if the ndo_rx_flow_steer
1950 * operation is defined
1951 * @index_hlist: Device index hash chain
1952 *
1953 * @_tx: Array of TX queues
1954 * @num_tx_queues: Number of TX queues allocated at alloc_netdev_mq() time
1955 * @real_num_tx_queues: Number of TX queues currently active in device
1956 * @qdisc: Root qdisc from userspace point of view
1957 * @tx_queue_len: Max frames per queue allowed
1958 * @tx_global_lock: XXX: need comments on this one
1959 * @xdp_bulkq: XDP device bulk queue
1960 * @xps_maps: all CPUs/RXQs maps for XPS device
1961 *
1962 * @xps_maps: XXX: need comments on this one
1963 * @tcx_egress: BPF & clsact qdisc specific data for egress processing
1964 * @nf_hooks_egress: netfilter hooks executed for egress packets
1965 * @qdisc_hash: qdisc hash table
1966 * @watchdog_timeo: Represents the timeout that is used by
1967 * the watchdog (see dev_watchdog())
1968 * @watchdog_timer: List of timers
1969 *
1970 * @proto_down_reason: reason a netdev interface is held down
1971 * @pcpu_refcnt: Number of references to this device
1972 * @dev_refcnt: Number of references to this device
1973 * @refcnt_tracker: Tracker directory for tracked references to this device
1974 * @todo_list: Delayed register/unregister
1975 * @link_watch_list: XXX: need comments on this one
1976 *
1977 * @reg_state: Register/unregister state machine
1978 * @dismantle: Device is going to be freed
1979 * @needs_free_netdev: Should unregister perform free_netdev?
1980 * @priv_destructor: Called from unregister
1981 * @npinfo: XXX: need comments on this one
1982 * @nd_net: Network namespace this network device is inside
1983 * protected by @lock
1984 *
1985 * @ml_priv: Mid-layer private
1986 * @ml_priv_type: Mid-layer private type
1987 *
1988 * @pcpu_stat_type: Type of device statistics which the core should
1989 * allocate/free: none, lstats, tstats, dstats. none
1990 * means the driver is handling statistics allocation/
1991 * freeing internally.
1992 * @lstats: Loopback statistics: packets, bytes
1993 * @tstats: Tunnel statistics: RX/TX packets, RX/TX bytes
1994 * @dstats: Dummy statistics: RX/TX/drop packets, RX/TX bytes
1995 *
1996 * @garp_port: GARP
1997 * @mrp_port: MRP
1998 *
1999 * @dm_private: Drop monitor private
2000 *
2001 * @dev: Class/net/name entry
2002 * @sysfs_groups: Space for optional device, statistics and wireless
2003 * sysfs groups
2004 *
2005 * @sysfs_rx_queue_group: Space for optional per-rx queue attributes
2006 * @rtnl_link_ops: Rtnl_link_ops
2007 * @stat_ops: Optional ops for queue-aware statistics
2008 * @queue_mgmt_ops: Optional ops for queue management
2009 *
2010 * @gso_max_size: Maximum size of generic segmentation offload
2011 * @tso_max_size: Device (as in HW) limit on the max TSO request size
2012 * @gso_max_segs: Maximum number of segments that can be passed to the
2013 * NIC for GSO
2014 * @tso_max_segs: Device (as in HW) limit on the max TSO segment count
2015 * @gso_ipv4_max_size: Maximum size of generic segmentation offload,
2016 * for IPv4.
2017 *
2018 * @dcbnl_ops: Data Center Bridging netlink ops
2019 * @num_tc: Number of traffic classes in the net device
2020 * @tc_to_txq: XXX: need comments on this one
2021 * @prio_tc_map: XXX: need comments on this one
2022 *
2023 * @fcoe_ddp_xid: Max exchange id for FCoE LRO by ddp
2024 *
2025 * @priomap: XXX: need comments on this one
2026 * @link_topo: Physical link topology tracking attached PHYs
2027 * @phydev: Physical device may attach itself
2028 * for hardware timestamping
2029 * @sfp_bus: attached &struct sfp_bus structure.
2030 *
2031 * @qdisc_tx_busylock: lockdep class annotating Qdisc->busylock spinlock
2032 *
2033 * @proto_down: protocol port state information can be sent to the
2034 * switch driver and used to set the phys state of the
2035 * switch port.
2036 *
2037 * @irq_affinity_auto: driver wants the core to store and re-assign the IRQ
2038 * affinity. Set by netif_enable_irq_affinity(), then
2039 * the driver must create a persistent napi by
2040 * netif_napi_add_config() and finally bind the napi to
2041 * IRQ (via netif_napi_set_irq()).
2042 *
2043 * @rx_cpu_rmap_auto: driver wants the core to manage the ARFS rmap.
2044 * Set by calling netif_enable_cpu_rmap().
2045 *
2046 * @see_all_hwtstamp_requests: device wants to see calls to
2047 * ndo_hwtstamp_set() for all timestamp requests
2048 * regardless of source, even if those aren't
2049 * HWTSTAMP_SOURCE_NETDEV
2050 * @change_proto_down: device supports setting carrier via IFLA_PROTO_DOWN
2051 * @netns_immutable: interface can't change network namespaces
2052 * @fcoe_mtu: device supports maximum FCoE MTU, 2158 bytes
2053 *
2054 * @net_notifier_list: List of per-net netdev notifier block
2055 * that follow this device when it is moved
2056 * to another network namespace.
2057 *
2058 * @macsec_ops: MACsec offloading ops
2059 *
2060 * @udp_tunnel_nic_info: static structure describing the UDP tunnel
2061 * offload capabilities of the device
2062 * @udp_tunnel_nic: UDP tunnel offload state
2063 * @ethtool: ethtool related state
2064 * @xdp_state: stores info on attached XDP BPF programs
2065 *
2066 * @nested_level: Used as a parameter of spin_lock_nested() of
2067 * dev->addr_list_lock.
2068 * @unlink_list: As netif_addr_lock() can be called recursively,
2069 * keep a list of interfaces to be deleted.
2070 * @gro_max_size: Maximum size of aggregated packet in generic
2071 * receive offload (GRO)
2072 * @gro_ipv4_max_size: Maximum size of aggregated packet in generic
2073 * receive offload (GRO), for IPv4.
2074 * @xdp_zc_max_segs: Maximum number of segments supported by AF_XDP
2075 * zero copy driver
2076 *
2077 * @dev_addr_shadow: Copy of @dev_addr to catch direct writes.
2078 * @linkwatch_dev_tracker: refcount tracker used by linkwatch.
2079 * @watchdog_dev_tracker: refcount tracker used by watchdog.
2080 * @dev_registered_tracker: tracker for reference held while
2081 * registered
2082 * @offload_xstats_l3: L3 HW stats for this netdevice.
2083 *
2084 * @devlink_port: Pointer to related devlink port structure.
2085 * Assigned by a driver before netdev registration using
2086 * SET_NETDEV_DEVLINK_PORT macro. This pointer is static
2087 * during the time netdevice is registered.
2088 *
2089 * @dpll_pin: Pointer to the SyncE source pin of a DPLL subsystem,
2090 * where the clock is recovered.
2091 *
2092 * @max_pacing_offload_horizon: max EDT offload horizon in nsec.
2093 * @napi_config: An array of napi_config structures containing per-NAPI
2094 * settings.
2095 * @num_napi_configs: number of allocated NAPI config structs,
2096 * always >= max(num_rx_queues, num_tx_queues).
2097 * @gro_flush_timeout: timeout for GRO layer in NAPI
2098 * @napi_defer_hard_irqs: If not zero, provides a counter that would
2099 * allow to avoid NIC hard IRQ, on busy queues.
2100 *
2101 * @neighbours: List heads pointing to this device's neighbours'
2102 * dev_list, one per address-family.
2103 * @hwprov: Tracks which PTP performs hardware packet time stamping.
2104 *
2105 * FIXME: cleanup struct net_device such that network protocol info
2106 * moves out.
2107 */
2108
2109struct net_device {
2110 /* Cacheline organization can be found documented in
2111 * Documentation/networking/net_cachelines/net_device.rst.
2112 * Please update the document when adding new fields.
2113 */
2114
2115 /* TX read-mostly hotpath */
2116 __cacheline_group_begin(net_device_read_tx);
2117 struct_group(priv_flags_fast,
2118 unsigned long priv_flags:32;
2119 unsigned long lltx:1;
2120 unsigned long netmem_tx:1;
2121 );
2122 const struct net_device_ops *netdev_ops;
2123 const struct header_ops *header_ops;
2124 struct netdev_queue *_tx;
2125 netdev_features_t gso_partial_features;
2126 unsigned int real_num_tx_queues;
2127 unsigned int gso_max_size;
2128 unsigned int gso_ipv4_max_size;
2129 u16 gso_max_segs;
2130 s16 num_tc;
2131 /* Note : dev->mtu is often read without holding a lock.
2132 * Writers usually hold RTNL.
2133 * It is recommended to use READ_ONCE() to annotate the reads,
2134 * and to use WRITE_ONCE() to annotate the writes.
2135 */
2136 unsigned int mtu;
2137 unsigned short needed_headroom;
2138 struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE];
2139#ifdef CONFIG_XPS
2140 struct xps_dev_maps __rcu *xps_maps[XPS_MAPS_MAX];
2141#endif
2142#ifdef CONFIG_NETFILTER_EGRESS
2143 struct nf_hook_entries __rcu *nf_hooks_egress;
2144#endif
2145#ifdef CONFIG_NET_XGRESS
2146 struct bpf_mprog_entry __rcu *tcx_egress;
2147#endif
2148 __cacheline_group_end(net_device_read_tx);
2149
2150 /* TXRX read-mostly hotpath */
2151 __cacheline_group_begin(net_device_read_txrx);
2152 union {
2153 struct pcpu_lstats __percpu *lstats;
2154 struct pcpu_sw_netstats __percpu *tstats;
2155 struct pcpu_dstats __percpu *dstats;
2156 };
2157 unsigned long state;
2158 unsigned int flags;
2159 unsigned short hard_header_len;
2160 enum netdev_stat_type pcpu_stat_type:8;
2161 netdev_features_t features;
2162 struct inet6_dev __rcu *ip6_ptr;
2163 __cacheline_group_end(net_device_read_txrx);
2164
2165 /* RX read-mostly hotpath */
2166 __cacheline_group_begin(net_device_read_rx);
2167 struct bpf_prog __rcu *xdp_prog;
2168 struct list_head ptype_specific;
2169 int ifindex;
2170 unsigned int real_num_rx_queues;
2171 struct netdev_rx_queue *_rx;
2172 unsigned int gro_max_size;
2173 unsigned int gro_ipv4_max_size;
2174 rx_handler_func_t __rcu *rx_handler;
2175 void __rcu *rx_handler_data;
2176 possible_net_t nd_net;
2177#ifdef CONFIG_NETPOLL
2178 struct netpoll_info __rcu *npinfo;
2179#endif
2180#ifdef CONFIG_NET_XGRESS
2181 struct bpf_mprog_entry __rcu *tcx_ingress;
2182#endif
2183 __cacheline_group_end(net_device_read_rx);
2184
2185 char name[IFNAMSIZ];
2186 struct netdev_name_node *name_node;
2187 struct dev_ifalias __rcu *ifalias;
2188 /*
2189 * I/O specific fields
2190 * FIXME: Merge these and struct ifmap into one
2191 */
2192 unsigned long mem_end;
2193 unsigned long mem_start;
2194 unsigned long base_addr;
2195
2196 /*
2197 * Some hardware also needs these fields (state,dev_list,
2198 * napi_list,unreg_list,close_list) but they are not
2199 * part of the usual set specified in Space.c.
2200 */
2201
2202
2203 struct list_head dev_list;
2204 struct list_head napi_list;
2205 struct list_head unreg_list;
2206 struct list_head close_list;
2207 struct list_head ptype_all;
2208
2209 struct {
2210 struct list_head upper;
2211 struct list_head lower;
2212 } adj_list;
2213
2214 /* Read-mostly cache-line for fast-path access */
2215 xdp_features_t xdp_features;
2216 const struct xdp_metadata_ops *xdp_metadata_ops;
2217 const struct xsk_tx_metadata_ops *xsk_tx_metadata_ops;
2218 unsigned short gflags;
2219
2220 unsigned short needed_tailroom;
2221
2222 netdev_features_t hw_features;
2223 netdev_features_t wanted_features;
2224 netdev_features_t vlan_features;
2225 netdev_features_t hw_enc_features;
2226 netdev_features_t mpls_features;
2227 netdev_features_t mangleid_features;
2228
2229 unsigned int min_mtu;
2230 unsigned int max_mtu;
2231 unsigned short type;
2232 unsigned char min_header_len;
2233 unsigned char name_assign_type;
2234
2235 int group;
2236
2237 struct net_device_stats stats; /* not used by modern drivers */
2238
2239 struct net_device_core_stats __percpu *core_stats;
2240
2241 /* Stats to monitor link on/off, flapping */
2242 atomic_t carrier_up_count;
2243 atomic_t carrier_down_count;
2244
2245#ifdef CONFIG_WIRELESS_EXT
2246 const struct iw_handler_def *wireless_handlers;
2247#endif
2248 const struct ethtool_ops *ethtool_ops;
2249#ifdef CONFIG_NET_L3_MASTER_DEV
2250 const struct l3mdev_ops *l3mdev_ops;
2251#endif
2252#if IS_ENABLED(CONFIG_IPV6)
2253 const struct ndisc_ops *ndisc_ops;
2254#endif
2255
2256#ifdef CONFIG_XFRM_OFFLOAD
2257 const struct xfrmdev_ops *xfrmdev_ops;
2258#endif
2259
2260#if IS_ENABLED(CONFIG_TLS_DEVICE)
2261 const struct tlsdev_ops *tlsdev_ops;
2262#endif
2263
2264 unsigned int operstate;
2265 unsigned char link_mode;
2266
2267 unsigned char if_port;
2268 unsigned char dma;
2269
2270 /* Interface address info. */
2271 unsigned char perm_addr[MAX_ADDR_LEN];
2272 unsigned char addr_assign_type;
2273 unsigned char addr_len;
2274 unsigned char upper_level;
2275 unsigned char lower_level;
2276 u8 threaded;
2277
2278 unsigned short neigh_priv_len;
2279 unsigned short dev_id;
2280 unsigned short dev_port;
2281 int irq;
2282 u32 priv_len;
2283
2284 spinlock_t addr_list_lock;
2285
2286 struct netdev_hw_addr_list uc;
2287 struct netdev_hw_addr_list mc;
2288 struct netdev_hw_addr_list dev_addrs;
2289
2290#ifdef CONFIG_SYSFS
2291 struct kset *queues_kset;
2292#endif
2293#ifdef CONFIG_LOCKDEP
2294 struct list_head unlink_list;
2295#endif
2296 unsigned int promiscuity;
2297 unsigned int allmulti;
2298 bool uc_promisc;
2299#ifdef CONFIG_LOCKDEP
2300 unsigned char nested_level;
2301#endif
2302
2303
2304 /* Protocol-specific pointers */
2305 struct in_device __rcu *ip_ptr;
2306 /** @fib_nh_head: nexthops associated with this netdev */
2307 struct hlist_head fib_nh_head;
2308
2309#if IS_ENABLED(CONFIG_VLAN_8021Q)
2310 struct vlan_info __rcu *vlan_info;
2311#endif
2312#if IS_ENABLED(CONFIG_NET_DSA)
2313 struct dsa_port *dsa_ptr;
2314#endif
2315#if IS_ENABLED(CONFIG_TIPC)
2316 struct tipc_bearer __rcu *tipc_ptr;
2317#endif
2318#if IS_ENABLED(CONFIG_ATALK)
2319 void *atalk_ptr;
2320#endif
2321#if IS_ENABLED(CONFIG_AX25)
2322 struct ax25_dev __rcu *ax25_ptr;
2323#endif
2324#if IS_ENABLED(CONFIG_CFG80211)
2325 struct wireless_dev *ieee80211_ptr;
2326#endif
2327#if IS_ENABLED(CONFIG_IEEE802154) || IS_ENABLED(CONFIG_6LOWPAN)
2328 struct wpan_dev *ieee802154_ptr;
2329#endif
2330#if IS_ENABLED(CONFIG_MPLS_ROUTING)
2331 struct mpls_dev __rcu *mpls_ptr;
2332#endif
2333#if IS_ENABLED(CONFIG_MCTP)
2334 struct mctp_dev __rcu *mctp_ptr;
2335#endif
2336#if IS_ENABLED(CONFIG_INET_PSP)
2337 struct psp_dev __rcu *psp_dev;
2338#endif
2339
2340/*
2341 * Cache lines mostly used on receive path (including eth_type_trans())
2342 */
2343 /* Interface address info used in eth_type_trans() */
2344 const unsigned char *dev_addr;
2345
2346 unsigned int num_rx_queues;
2347#define GRO_LEGACY_MAX_SIZE 65536u
2348/* TCP minimal MSS is 8 (TCP_MIN_GSO_SIZE),
2349 * and shinfo->gso_segs is a 16bit field.
2350 */
2351#define GRO_MAX_SIZE (8 * 65535u)
2352 unsigned int xdp_zc_max_segs;
2353 struct netdev_queue __rcu *ingress_queue;
2354#ifdef CONFIG_NETFILTER_INGRESS
2355 struct nf_hook_entries __rcu *nf_hooks_ingress;
2356#endif
2357
2358 unsigned char broadcast[MAX_ADDR_LEN];
2359#ifdef CONFIG_RFS_ACCEL
2360 struct cpu_rmap *rx_cpu_rmap;
2361#endif
2362 struct hlist_node index_hlist;
2363
2364/*
2365 * Cache lines mostly used on transmit path
2366 */
2367 unsigned int num_tx_queues;
2368 struct Qdisc __rcu *qdisc;
2369 unsigned int tx_queue_len;
2370 spinlock_t tx_global_lock;
2371
2372 struct xdp_dev_bulk_queue __percpu *xdp_bulkq;
2373
2374#ifdef CONFIG_NET_SCHED
2375 DECLARE_HASHTABLE (qdisc_hash, 4);
2376#endif
2377 /* These may be needed for future network-power-down code. */
2378 struct timer_list watchdog_timer;
2379 int watchdog_timeo;
2380
2381 u32 proto_down_reason;
2382
2383 struct list_head todo_list;
2384
2385#ifdef CONFIG_PCPU_DEV_REFCNT
2386 int __percpu *pcpu_refcnt;
2387#else
2388 refcount_t dev_refcnt;
2389#endif
2390 struct ref_tracker_dir refcnt_tracker;
2391
2392 struct list_head link_watch_list;
2393
2394 u8 reg_state;
2395
2396 bool dismantle;
2397
2398 /** @moving_ns: device is changing netns, protected by @lock */
2399 bool moving_ns;
2400 /** @rtnl_link_initializing: Device being created, suppress events */
2401 bool rtnl_link_initializing;
2402
2403 bool needs_free_netdev;
2404 void (*priv_destructor)(struct net_device *dev);
2405
2406 /* mid-layer private */
2407 void *ml_priv;
2408 enum netdev_ml_priv_type ml_priv_type;
2409
2410#if IS_ENABLED(CONFIG_GARP)
2411 struct garp_port __rcu *garp_port;
2412#endif
2413#if IS_ENABLED(CONFIG_MRP)
2414 struct mrp_port __rcu *mrp_port;
2415#endif
2416#if IS_ENABLED(CONFIG_NET_DROP_MONITOR)
2417 struct dm_hw_stat_delta __rcu *dm_private;
2418#endif
2419 struct device dev;
2420 const struct attribute_group *sysfs_groups[5];
2421 const struct attribute_group *sysfs_rx_queue_group;
2422
2423 const struct rtnl_link_ops *rtnl_link_ops;
2424
2425 const struct netdev_stat_ops *stat_ops;
2426
2427 const struct netdev_queue_mgmt_ops *queue_mgmt_ops;
2428
2429 /* for setting kernel sock attribute on TCP connection setup */
2430#define GSO_MAX_SEGS 65535u
2431#define GSO_LEGACY_MAX_SIZE 65536u
2432/* TCP minimal MSS is 8 (TCP_MIN_GSO_SIZE),
2433 * and shinfo->gso_segs is a 16bit field.
2434 */
2435#define GSO_MAX_SIZE (8 * GSO_MAX_SEGS)
2436
2437#define TSO_LEGACY_MAX_SIZE 65536
2438#define TSO_MAX_SIZE UINT_MAX
2439 unsigned int tso_max_size;
2440#define TSO_MAX_SEGS U16_MAX
2441 u16 tso_max_segs;
2442
2443#ifdef CONFIG_DCB
2444 const struct dcbnl_rtnl_ops *dcbnl_ops;
2445#endif
2446 u8 prio_tc_map[TC_BITMASK + 1];
2447
2448#if IS_ENABLED(CONFIG_FCOE)
2449 unsigned int fcoe_ddp_xid;
2450#endif
2451#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
2452 struct netprio_map __rcu *priomap;
2453#endif
2454 struct phy_link_topology *link_topo;
2455 struct phy_device *phydev;
2456 struct sfp_bus *sfp_bus;
2457 struct lock_class_key *qdisc_tx_busylock;
2458 bool proto_down;
2459 bool irq_affinity_auto;
2460 bool rx_cpu_rmap_auto;
2461
2462 /* priv_flags_slow, ungrouped to save space */
2463 unsigned long see_all_hwtstamp_requests:1;
2464 unsigned long change_proto_down:1;
2465 unsigned long netns_immutable:1;
2466 unsigned long fcoe_mtu:1;
2467
2468 struct list_head net_notifier_list;
2469
2470#if IS_ENABLED(CONFIG_MACSEC)
2471 /* MACsec management functions */
2472 const struct macsec_ops *macsec_ops;
2473#endif
2474 const struct udp_tunnel_nic_info *udp_tunnel_nic_info;
2475 struct udp_tunnel_nic *udp_tunnel_nic;
2476
2477 /** @cfg: net_device queue-related configuration */
2478 struct netdev_config *cfg;
2479 /**
2480 * @cfg_pending: same as @cfg but when device is being actively
2481 * reconfigured includes any changes to the configuration
2482 * requested by the user, but which may or may not be rejected.
2483 */
2484 struct netdev_config *cfg_pending;
2485 struct ethtool_netdev_state *ethtool;
2486
2487 /* protected by rtnl_lock */
2488 struct bpf_xdp_entity xdp_state[__MAX_XDP_MODE];
2489
2490 u8 dev_addr_shadow[MAX_ADDR_LEN];
2491 netdevice_tracker linkwatch_dev_tracker;
2492 netdevice_tracker watchdog_dev_tracker;
2493 netdevice_tracker dev_registered_tracker;
2494 struct rtnl_hw_stats64 *offload_xstats_l3;
2495
2496 struct devlink_port *devlink_port;
2497
2498#if IS_ENABLED(CONFIG_DPLL)
2499 struct dpll_pin __rcu *dpll_pin;
2500#endif
2501#if IS_ENABLED(CONFIG_PAGE_POOL)
2502 /** @page_pools: page pools created for this netdevice */
2503 struct hlist_head page_pools;
2504#endif
2505
2506 /** @irq_moder: dim parameters used if IS_ENABLED(CONFIG_DIMLIB). */
2507 struct dim_irq_moder *irq_moder;
2508
2509 u64 max_pacing_offload_horizon;
2510 struct napi_config *napi_config;
2511 u32 num_napi_configs;
2512 u32 napi_defer_hard_irqs;
2513 unsigned long gro_flush_timeout;
2514
2515 /**
2516 * @up: copy of @state's IFF_UP, but safe to read with just @lock.
2517 * May report false negatives while the device is being opened
2518 * or closed (@lock does not protect .ndo_open, or .ndo_close).
2519 */
2520 bool up;
2521
2522 /**
2523 * @request_ops_lock: request the core to run all @netdev_ops and
2524 * @ethtool_ops under the @lock.
2525 */
2526 bool request_ops_lock;
2527
2528 /**
2529 * @lock: netdev-scope lock, protects a small selection of fields.
2530 * Should always be taken using netdev_lock() / netdev_unlock() helpers.
2531 * Drivers are free to use it for other protection.
2532 *
2533 * For the drivers that implement shaper or queue API, the scope
2534 * of this lock is expanded to cover most ndo/queue/ethtool/sysfs
2535 * operations. Drivers may opt-in to this behavior by setting
2536 * @request_ops_lock.
2537 *
2538 * @lock protection mixes with rtnl_lock in multiple ways, fields are
2539 * either:
2540 *
2541 * - simply protected by the instance @lock;
2542 *
2543 * - double protected - writers hold both locks, readers hold either;
2544 *
2545 * - ops protected - protected by the lock held around the NDOs
2546 * and other callbacks, that is the instance lock on devices for
2547 * which netdev_need_ops_lock() returns true, otherwise by rtnl_lock;
2548 *
2549 * - double ops protected - always protected by rtnl_lock but for
2550 * devices for which netdev_need_ops_lock() returns true - also
2551 * the instance lock.
2552 *
2553 * Simply protects:
2554 * @gro_flush_timeout, @napi_defer_hard_irqs, @napi_list,
2555 * @net_shaper_hierarchy, @reg_state, @threaded
2556 *
2557 * Double protects:
2558 * @up, @moving_ns, @nd_net, @xdp_features
2559 *
2560 * Double ops protects:
2561 * @real_num_rx_queues, @real_num_tx_queues
2562 *
2563 * Also protects some fields in:
2564 * struct napi_struct, struct netdev_queue, struct netdev_rx_queue
2565 *
2566 * Ordering: take after rtnl_lock.
2567 */
2568 struct mutex lock;
2569
2570#if IS_ENABLED(CONFIG_NET_SHAPER)
2571 /**
2572 * @net_shaper_hierarchy: data tracking the current shaper status
2573 * see include/net/net_shapers.h
2574 */
2575 struct net_shaper_hierarchy *net_shaper_hierarchy;
2576#endif
2577
2578 struct hlist_head neighbours[NEIGH_NR_TABLES];
2579
2580 struct hwtstamp_provider __rcu *hwprov;
2581
2582 u8 priv[] ____cacheline_aligned
2583 __counted_by(priv_len);
2584} ____cacheline_aligned;
2585#define to_net_dev(d) container_of(d, struct net_device, dev)
2586
2587/*
2588 * Driver should use this to assign devlink port instance to a netdevice
2589 * before it registers the netdevice. Therefore devlink_port is static
2590 * during the netdev lifetime after it is registered.
2591 */
2592#define SET_NETDEV_DEVLINK_PORT(dev, port) \
2593({ \
2594 WARN_ON((dev)->reg_state != NETREG_UNINITIALIZED); \
2595 ((dev)->devlink_port = (port)); \
2596})
2597
2598static inline bool netif_elide_gro(const struct net_device *dev)
2599{
2600 if (!(dev->features & NETIF_F_GRO) || dev->xdp_prog)
2601 return true;
2602 return false;
2603}
2604
2605#define NETDEV_ALIGN 32
2606
2607static inline
2608int netdev_get_prio_tc_map(const struct net_device *dev, u32 prio)
2609{
2610 return dev->prio_tc_map[prio & TC_BITMASK];
2611}
2612
2613static inline
2614int netdev_set_prio_tc_map(struct net_device *dev, u8 prio, u8 tc)
2615{
2616 if (tc >= dev->num_tc)
2617 return -EINVAL;
2618
2619 dev->prio_tc_map[prio & TC_BITMASK] = tc & TC_BITMASK;
2620 return 0;
2621}
2622
2623int netdev_txq_to_tc(struct net_device *dev, unsigned int txq);
2624void netdev_reset_tc(struct net_device *dev);
2625int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset);
2626int netdev_set_num_tc(struct net_device *dev, u8 num_tc);
2627
2628static inline
2629int netdev_get_num_tc(struct net_device *dev)
2630{
2631 return dev->num_tc;
2632}
2633
2634static inline void net_prefetch(void *p)
2635{
2636 prefetch(p);
2637#if L1_CACHE_BYTES < 128
2638 prefetch((u8 *)p + L1_CACHE_BYTES);
2639#endif
2640}
2641
2642static inline void net_prefetchw(void *p)
2643{
2644 prefetchw(p);
2645#if L1_CACHE_BYTES < 128
2646 prefetchw((u8 *)p + L1_CACHE_BYTES);
2647#endif
2648}
2649
2650void netdev_unbind_sb_channel(struct net_device *dev,
2651 struct net_device *sb_dev);
2652int netdev_bind_sb_channel_queue(struct net_device *dev,
2653 struct net_device *sb_dev,
2654 u8 tc, u16 count, u16 offset);
2655int netdev_set_sb_channel(struct net_device *dev, u16 channel);
2656static inline int netdev_get_sb_channel(struct net_device *dev)
2657{
2658 return max_t(int, -dev->num_tc, 0);
2659}
2660
2661static inline
2662struct netdev_queue *netdev_get_tx_queue(const struct net_device *dev,
2663 unsigned int index)
2664{
2665 DEBUG_NET_WARN_ON_ONCE(index >= dev->num_tx_queues);
2666 return &dev->_tx[index];
2667}
2668
2669static inline struct netdev_queue *skb_get_tx_queue(const struct net_device *dev,
2670 const struct sk_buff *skb)
2671{
2672 return netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
2673}
2674
2675static inline void netdev_for_each_tx_queue(struct net_device *dev,
2676 void (*f)(struct net_device *,
2677 struct netdev_queue *,
2678 void *),
2679 void *arg)
2680{
2681 unsigned int i;
2682
2683 for (i = 0; i < dev->num_tx_queues; i++)
2684 f(dev, &dev->_tx[i], arg);
2685}
2686
2687u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
2688 struct net_device *sb_dev);
2689struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
2690 struct sk_buff *skb,
2691 struct net_device *sb_dev);
2692
2693/* returns the headroom that the master device needs to take in account
2694 * when forwarding to this dev
2695 */
2696static inline unsigned netdev_get_fwd_headroom(struct net_device *dev)
2697{
2698 return dev->priv_flags & IFF_PHONY_HEADROOM ? 0 : dev->needed_headroom;
2699}
2700
2701static inline void netdev_set_rx_headroom(struct net_device *dev, int new_hr)
2702{
2703 if (dev->netdev_ops->ndo_set_rx_headroom)
2704 dev->netdev_ops->ndo_set_rx_headroom(dev, new_hr);
2705}
2706
2707/* set the device rx headroom to the dev's default */
2708static inline void netdev_reset_rx_headroom(struct net_device *dev)
2709{
2710 netdev_set_rx_headroom(dev, -1);
2711}
2712
2713static inline void *netdev_get_ml_priv(struct net_device *dev,
2714 enum netdev_ml_priv_type type)
2715{
2716 if (dev->ml_priv_type != type)
2717 return NULL;
2718
2719 return dev->ml_priv;
2720}
2721
2722static inline void netdev_set_ml_priv(struct net_device *dev,
2723 void *ml_priv,
2724 enum netdev_ml_priv_type type)
2725{
2726 WARN(dev->ml_priv_type && dev->ml_priv_type != type,
2727 "Overwriting already set ml_priv_type (%u) with different ml_priv_type (%u)!\n",
2728 dev->ml_priv_type, type);
2729 WARN(!dev->ml_priv_type && dev->ml_priv,
2730 "Overwriting already set ml_priv and ml_priv_type is ML_PRIV_NONE!\n");
2731
2732 dev->ml_priv = ml_priv;
2733 dev->ml_priv_type = type;
2734}
2735
2736/*
2737 * Net namespace inlines
2738 */
2739static inline
2740struct net *dev_net(const struct net_device *dev)
2741{
2742 return read_pnet(&dev->nd_net);
2743}
2744
2745static inline
2746struct net *dev_net_rcu(const struct net_device *dev)
2747{
2748 return read_pnet_rcu(&dev->nd_net);
2749}
2750
2751static inline
2752void dev_net_set(struct net_device *dev, struct net *net)
2753{
2754 write_pnet(&dev->nd_net, net);
2755}
2756
2757/**
2758 * netdev_priv - access network device private data
2759 * @dev: network device
2760 *
2761 * Get network device private data
2762 */
2763static inline void *netdev_priv(const struct net_device *dev)
2764{
2765 return (void *)dev->priv;
2766}
2767
2768/* Set the sysfs physical device reference for the network logical device
2769 * if set prior to registration will cause a symlink during initialization.
2770 */
2771#define SET_NETDEV_DEV(net, pdev) ((net)->dev.parent = (pdev))
2772
2773/* Set the sysfs device type for the network logical device to allow
2774 * fine-grained identification of different network device types. For
2775 * example Ethernet, Wireless LAN, Bluetooth, WiMAX etc.
2776 */
2777#define SET_NETDEV_DEVTYPE(net, devtype) ((net)->dev.type = (devtype))
2778
2779void netif_queue_set_napi(struct net_device *dev, unsigned int queue_index,
2780 enum netdev_queue_type type,
2781 struct napi_struct *napi);
2782
2783static inline void netdev_lock(struct net_device *dev)
2784{
2785 mutex_lock(&dev->lock);
2786}
2787
2788static inline void netdev_unlock(struct net_device *dev)
2789{
2790 mutex_unlock(&dev->lock);
2791}
2792/* Additional netdev_lock()-related helpers are in net/netdev_lock.h */
2793
2794void netif_napi_set_irq_locked(struct napi_struct *napi, int irq);
2795
2796static inline void netif_napi_set_irq(struct napi_struct *napi, int irq)
2797{
2798 netdev_lock(napi->dev);
2799 netif_napi_set_irq_locked(napi, irq);
2800 netdev_unlock(napi->dev);
2801}
2802
2803/* Default NAPI poll() weight
2804 * Device drivers are strongly advised to not use bigger value
2805 */
2806#define NAPI_POLL_WEIGHT 64
2807
2808void netif_napi_add_weight_locked(struct net_device *dev,
2809 struct napi_struct *napi,
2810 int (*poll)(struct napi_struct *, int),
2811 int weight);
2812
2813static inline void
2814netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi,
2815 int (*poll)(struct napi_struct *, int), int weight)
2816{
2817 netdev_lock(dev);
2818 netif_napi_add_weight_locked(dev, napi, poll, weight);
2819 netdev_unlock(dev);
2820}
2821
2822/**
2823 * netif_napi_add() - initialize a NAPI context
2824 * @dev: network device
2825 * @napi: NAPI context
2826 * @poll: polling function
2827 *
2828 * netif_napi_add() must be used to initialize a NAPI context prior to calling
2829 * *any* of the other NAPI-related functions.
2830 */
2831static inline void
2832netif_napi_add(struct net_device *dev, struct napi_struct *napi,
2833 int (*poll)(struct napi_struct *, int))
2834{
2835 netif_napi_add_weight(dev, napi, poll, NAPI_POLL_WEIGHT);
2836}
2837
2838static inline void
2839netif_napi_add_locked(struct net_device *dev, struct napi_struct *napi,
2840 int (*poll)(struct napi_struct *, int))
2841{
2842 netif_napi_add_weight_locked(dev, napi, poll, NAPI_POLL_WEIGHT);
2843}
2844
2845static inline void
2846netif_napi_add_tx_weight(struct net_device *dev,
2847 struct napi_struct *napi,
2848 int (*poll)(struct napi_struct *, int),
2849 int weight)
2850{
2851 set_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state);
2852 netif_napi_add_weight(dev, napi, poll, weight);
2853}
2854
2855static inline void
2856netif_napi_add_config_locked(struct net_device *dev, struct napi_struct *napi,
2857 int (*poll)(struct napi_struct *, int), int index)
2858{
2859 napi->index = index;
2860 napi->config = &dev->napi_config[index];
2861 netif_napi_add_weight_locked(dev, napi, poll, NAPI_POLL_WEIGHT);
2862}
2863
2864/**
2865 * netif_napi_add_config - initialize a NAPI context with persistent config
2866 * @dev: network device
2867 * @napi: NAPI context
2868 * @poll: polling function
2869 * @index: the NAPI index
2870 */
2871static inline void
2872netif_napi_add_config(struct net_device *dev, struct napi_struct *napi,
2873 int (*poll)(struct napi_struct *, int), int index)
2874{
2875 netdev_lock(dev);
2876 netif_napi_add_config_locked(dev, napi, poll, index);
2877 netdev_unlock(dev);
2878}
2879
2880/**
2881 * netif_napi_add_tx() - initialize a NAPI context to be used for Tx only
2882 * @dev: network device
2883 * @napi: NAPI context
2884 * @poll: polling function
2885 *
2886 * This variant of netif_napi_add() should be used from drivers using NAPI
2887 * to exclusively poll a TX queue.
2888 * This will avoid we add it into napi_hash[], thus polluting this hash table.
2889 */
2890static inline void netif_napi_add_tx(struct net_device *dev,
2891 struct napi_struct *napi,
2892 int (*poll)(struct napi_struct *, int))
2893{
2894 netif_napi_add_tx_weight(dev, napi, poll, NAPI_POLL_WEIGHT);
2895}
2896
2897void __netif_napi_del_locked(struct napi_struct *napi);
2898
2899/**
2900 * __netif_napi_del - remove a NAPI context
2901 * @napi: NAPI context
2902 *
2903 * Warning: caller must observe RCU grace period before freeing memory
2904 * containing @napi. Drivers might want to call this helper to combine
2905 * all the needed RCU grace periods into a single one.
2906 */
2907static inline void __netif_napi_del(struct napi_struct *napi)
2908{
2909 netdev_lock(napi->dev);
2910 __netif_napi_del_locked(napi);
2911 netdev_unlock(napi->dev);
2912}
2913
2914static inline void netif_napi_del_locked(struct napi_struct *napi)
2915{
2916 __netif_napi_del_locked(napi);
2917 synchronize_net();
2918}
2919
2920/**
2921 * netif_napi_del - remove a NAPI context
2922 * @napi: NAPI context
2923 *
2924 * netif_napi_del() removes a NAPI context from the network device NAPI list
2925 */
2926static inline void netif_napi_del(struct napi_struct *napi)
2927{
2928 __netif_napi_del(napi);
2929 synchronize_net();
2930}
2931
2932int netif_enable_cpu_rmap(struct net_device *dev, unsigned int num_irqs);
2933void netif_set_affinity_auto(struct net_device *dev);
2934
2935struct packet_type {
2936 __be16 type; /* This is really htons(ether_type). */
2937 bool ignore_outgoing;
2938 struct net_device *dev; /* NULL is wildcarded here */
2939 netdevice_tracker dev_tracker;
2940 int (*func) (struct sk_buff *,
2941 struct net_device *,
2942 struct packet_type *,
2943 struct net_device *);
2944 void (*list_func) (struct list_head *,
2945 struct packet_type *,
2946 struct net_device *);
2947 bool (*id_match)(struct packet_type *ptype,
2948 struct sock *sk);
2949 struct net *af_packet_net;
2950 void *af_packet_priv;
2951 struct list_head list;
2952};
2953
2954struct offload_callbacks {
2955 struct sk_buff *(*gso_segment)(struct sk_buff *skb,
2956 netdev_features_t features);
2957 struct sk_buff *(*gro_receive)(struct list_head *head,
2958 struct sk_buff *skb);
2959 int (*gro_complete)(struct sk_buff *skb, int nhoff);
2960};
2961
2962struct packet_offload {
2963 __be16 type; /* This is really htons(ether_type). */
2964 u16 priority;
2965 struct offload_callbacks callbacks;
2966 struct list_head list;
2967};
2968
2969/* often modified stats are per-CPU, other are shared (netdev->stats) */
2970struct pcpu_sw_netstats {
2971 u64_stats_t rx_packets;
2972 u64_stats_t rx_bytes;
2973 u64_stats_t tx_packets;
2974 u64_stats_t tx_bytes;
2975 struct u64_stats_sync syncp;
2976} __aligned(4 * sizeof(u64));
2977
2978struct pcpu_dstats {
2979 u64_stats_t rx_packets;
2980 u64_stats_t rx_bytes;
2981 u64_stats_t tx_packets;
2982 u64_stats_t tx_bytes;
2983 u64_stats_t rx_drops;
2984 u64_stats_t tx_drops;
2985 struct u64_stats_sync syncp;
2986} __aligned(8 * sizeof(u64));
2987
2988struct pcpu_lstats {
2989 u64_stats_t packets;
2990 u64_stats_t bytes;
2991 struct u64_stats_sync syncp;
2992} __aligned(2 * sizeof(u64));
2993
2994void dev_lstats_read(struct net_device *dev, u64 *packets, u64 *bytes);
2995
2996static inline void dev_sw_netstats_rx_add(struct net_device *dev, unsigned int len)
2997{
2998 struct pcpu_sw_netstats *tstats = this_cpu_ptr(dev->tstats);
2999
3000 u64_stats_update_begin(&tstats->syncp);
3001 u64_stats_add(&tstats->rx_bytes, len);
3002 u64_stats_inc(&tstats->rx_packets);
3003 u64_stats_update_end(&tstats->syncp);
3004}
3005
3006static inline void dev_sw_netstats_tx_add(struct net_device *dev,
3007 unsigned int packets,
3008 unsigned int len)
3009{
3010 struct pcpu_sw_netstats *tstats = this_cpu_ptr(dev->tstats);
3011
3012 u64_stats_update_begin(&tstats->syncp);
3013 u64_stats_add(&tstats->tx_bytes, len);
3014 u64_stats_add(&tstats->tx_packets, packets);
3015 u64_stats_update_end(&tstats->syncp);
3016}
3017
3018static inline void dev_lstats_add(struct net_device *dev, unsigned int len)
3019{
3020 struct pcpu_lstats *lstats = this_cpu_ptr(dev->lstats);
3021
3022 u64_stats_update_begin(&lstats->syncp);
3023 u64_stats_add(&lstats->bytes, len);
3024 u64_stats_inc(&lstats->packets);
3025 u64_stats_update_end(&lstats->syncp);
3026}
3027
3028static inline void dev_dstats_rx_add(struct net_device *dev,
3029 unsigned int len)
3030{
3031 struct pcpu_dstats *dstats = this_cpu_ptr(dev->dstats);
3032
3033 u64_stats_update_begin(&dstats->syncp);
3034 u64_stats_inc(&dstats->rx_packets);
3035 u64_stats_add(&dstats->rx_bytes, len);
3036 u64_stats_update_end(&dstats->syncp);
3037}
3038
3039static inline void dev_dstats_rx_dropped(struct net_device *dev)
3040{
3041 struct pcpu_dstats *dstats = this_cpu_ptr(dev->dstats);
3042
3043 u64_stats_update_begin(&dstats->syncp);
3044 u64_stats_inc(&dstats->rx_drops);
3045 u64_stats_update_end(&dstats->syncp);
3046}
3047
3048static inline void dev_dstats_rx_dropped_add(struct net_device *dev,
3049 unsigned int packets)
3050{
3051 struct pcpu_dstats *dstats = this_cpu_ptr(dev->dstats);
3052
3053 u64_stats_update_begin(&dstats->syncp);
3054 u64_stats_add(&dstats->rx_drops, packets);
3055 u64_stats_update_end(&dstats->syncp);
3056}
3057
3058static inline void dev_dstats_tx_add(struct net_device *dev,
3059 unsigned int len)
3060{
3061 struct pcpu_dstats *dstats = this_cpu_ptr(dev->dstats);
3062
3063 u64_stats_update_begin(&dstats->syncp);
3064 u64_stats_inc(&dstats->tx_packets);
3065 u64_stats_add(&dstats->tx_bytes, len);
3066 u64_stats_update_end(&dstats->syncp);
3067}
3068
3069static inline void dev_dstats_tx_dropped(struct net_device *dev)
3070{
3071 struct pcpu_dstats *dstats = this_cpu_ptr(dev->dstats);
3072
3073 u64_stats_update_begin(&dstats->syncp);
3074 u64_stats_inc(&dstats->tx_drops);
3075 u64_stats_update_end(&dstats->syncp);
3076}
3077
3078#define __netdev_alloc_pcpu_stats(type, gfp) \
3079({ \
3080 typeof(type) __percpu *pcpu_stats = alloc_percpu_gfp(type, gfp);\
3081 if (pcpu_stats) { \
3082 int __cpu; \
3083 for_each_possible_cpu(__cpu) { \
3084 typeof(type) *stat; \
3085 stat = per_cpu_ptr(pcpu_stats, __cpu); \
3086 u64_stats_init(&stat->syncp); \
3087 } \
3088 } \
3089 pcpu_stats; \
3090})
3091
3092#define netdev_alloc_pcpu_stats(type) \
3093 __netdev_alloc_pcpu_stats(type, GFP_KERNEL)
3094
3095#define devm_netdev_alloc_pcpu_stats(dev, type) \
3096({ \
3097 typeof(type) __percpu *pcpu_stats = devm_alloc_percpu(dev, type);\
3098 if (pcpu_stats) { \
3099 int __cpu; \
3100 for_each_possible_cpu(__cpu) { \
3101 typeof(type) *stat; \
3102 stat = per_cpu_ptr(pcpu_stats, __cpu); \
3103 u64_stats_init(&stat->syncp); \
3104 } \
3105 } \
3106 pcpu_stats; \
3107})
3108
3109enum netdev_lag_tx_type {
3110 NETDEV_LAG_TX_TYPE_UNKNOWN,
3111 NETDEV_LAG_TX_TYPE_RANDOM,
3112 NETDEV_LAG_TX_TYPE_BROADCAST,
3113 NETDEV_LAG_TX_TYPE_ROUNDROBIN,
3114 NETDEV_LAG_TX_TYPE_ACTIVEBACKUP,
3115 NETDEV_LAG_TX_TYPE_HASH,
3116};
3117
3118enum netdev_lag_hash {
3119 NETDEV_LAG_HASH_NONE,
3120 NETDEV_LAG_HASH_L2,
3121 NETDEV_LAG_HASH_L34,
3122 NETDEV_LAG_HASH_L23,
3123 NETDEV_LAG_HASH_E23,
3124 NETDEV_LAG_HASH_E34,
3125 NETDEV_LAG_HASH_VLAN_SRCMAC,
3126 NETDEV_LAG_HASH_UNKNOWN,
3127};
3128
3129struct netdev_lag_upper_info {
3130 enum netdev_lag_tx_type tx_type;
3131 enum netdev_lag_hash hash_type;
3132};
3133
3134struct netdev_lag_lower_state_info {
3135 u8 link_up : 1,
3136 tx_enabled : 1;
3137};
3138
3139#include <linux/notifier.h>
3140
3141/* netdevice notifier chain. Please remember to update netdev_cmd_to_name()
3142 * and the rtnetlink notification exclusion list in rtnetlink_event() when
3143 * adding new types.
3144 */
3145enum netdev_cmd {
3146 NETDEV_UP = 1, /* For now you can't veto a device up/down */
3147 NETDEV_DOWN,
3148 NETDEV_REBOOT, /* Tell a protocol stack a network interface
3149 detected a hardware crash and restarted
3150 - we can use this eg to kick tcp sessions
3151 once done */
3152 NETDEV_CHANGE, /* Notify device state change */
3153 NETDEV_REGISTER,
3154 NETDEV_UNREGISTER,
3155 NETDEV_CHANGEMTU, /* notify after mtu change happened */
3156 NETDEV_CHANGEADDR, /* notify after the address change */
3157 NETDEV_PRE_CHANGEADDR, /* notify before the address change */
3158 NETDEV_GOING_DOWN,
3159 NETDEV_CHANGENAME,
3160 NETDEV_FEAT_CHANGE,
3161 NETDEV_BONDING_FAILOVER,
3162 NETDEV_PRE_UP,
3163 NETDEV_PRE_TYPE_CHANGE,
3164 NETDEV_POST_TYPE_CHANGE,
3165 NETDEV_POST_INIT,
3166 NETDEV_PRE_UNINIT,
3167 NETDEV_RELEASE,
3168 NETDEV_NOTIFY_PEERS,
3169 NETDEV_JOIN,
3170 NETDEV_CHANGEUPPER,
3171 NETDEV_RESEND_IGMP,
3172 NETDEV_PRECHANGEMTU, /* notify before mtu change happened */
3173 NETDEV_CHANGEINFODATA,
3174 NETDEV_BONDING_INFO,
3175 NETDEV_PRECHANGEUPPER,
3176 NETDEV_CHANGELOWERSTATE,
3177 NETDEV_UDP_TUNNEL_PUSH_INFO,
3178 NETDEV_UDP_TUNNEL_DROP_INFO,
3179 NETDEV_CHANGE_TX_QUEUE_LEN,
3180 NETDEV_CVLAN_FILTER_PUSH_INFO,
3181 NETDEV_CVLAN_FILTER_DROP_INFO,
3182 NETDEV_SVLAN_FILTER_PUSH_INFO,
3183 NETDEV_SVLAN_FILTER_DROP_INFO,
3184 NETDEV_OFFLOAD_XSTATS_ENABLE,
3185 NETDEV_OFFLOAD_XSTATS_DISABLE,
3186 NETDEV_OFFLOAD_XSTATS_REPORT_USED,
3187 NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
3188 NETDEV_XDP_FEAT_CHANGE,
3189};
3190const char *netdev_cmd_to_name(enum netdev_cmd cmd);
3191
3192int register_netdevice_notifier(struct notifier_block *nb);
3193int unregister_netdevice_notifier(struct notifier_block *nb);
3194int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb);
3195int unregister_netdevice_notifier_net(struct net *net,
3196 struct notifier_block *nb);
3197int register_netdevice_notifier_dev_net(struct net_device *dev,
3198 struct notifier_block *nb,
3199 struct netdev_net_notifier *nn);
3200int unregister_netdevice_notifier_dev_net(struct net_device *dev,
3201 struct notifier_block *nb,
3202 struct netdev_net_notifier *nn);
3203
3204struct netdev_notifier_info {
3205 struct net_device *dev;
3206 struct netlink_ext_ack *extack;
3207};
3208
3209struct netdev_notifier_info_ext {
3210 struct netdev_notifier_info info; /* must be first */
3211 union {
3212 u32 mtu;
3213 } ext;
3214};
3215
3216struct netdev_notifier_change_info {
3217 struct netdev_notifier_info info; /* must be first */
3218 unsigned int flags_changed;
3219};
3220
3221struct netdev_notifier_changeupper_info {
3222 struct netdev_notifier_info info; /* must be first */
3223 struct net_device *upper_dev; /* new upper dev */
3224 bool master; /* is upper dev master */
3225 bool linking; /* is the notification for link or unlink */
3226 void *upper_info; /* upper dev info */
3227};
3228
3229struct netdev_notifier_changelowerstate_info {
3230 struct netdev_notifier_info info; /* must be first */
3231 void *lower_state_info; /* is lower dev state */
3232};
3233
3234struct netdev_notifier_pre_changeaddr_info {
3235 struct netdev_notifier_info info; /* must be first */
3236 const unsigned char *dev_addr;
3237};
3238
3239enum netdev_offload_xstats_type {
3240 NETDEV_OFFLOAD_XSTATS_TYPE_L3 = 1,
3241};
3242
3243struct netdev_notifier_offload_xstats_info {
3244 struct netdev_notifier_info info; /* must be first */
3245 enum netdev_offload_xstats_type type;
3246
3247 union {
3248 /* NETDEV_OFFLOAD_XSTATS_REPORT_DELTA */
3249 struct netdev_notifier_offload_xstats_rd *report_delta;
3250 /* NETDEV_OFFLOAD_XSTATS_REPORT_USED */
3251 struct netdev_notifier_offload_xstats_ru *report_used;
3252 };
3253};
3254
3255int netdev_offload_xstats_enable(struct net_device *dev,
3256 enum netdev_offload_xstats_type type,
3257 struct netlink_ext_ack *extack);
3258int netdev_offload_xstats_disable(struct net_device *dev,
3259 enum netdev_offload_xstats_type type);
3260bool netdev_offload_xstats_enabled(const struct net_device *dev,
3261 enum netdev_offload_xstats_type type);
3262int netdev_offload_xstats_get(struct net_device *dev,
3263 enum netdev_offload_xstats_type type,
3264 struct rtnl_hw_stats64 *stats, bool *used,
3265 struct netlink_ext_ack *extack);
3266void
3267netdev_offload_xstats_report_delta(struct netdev_notifier_offload_xstats_rd *rd,
3268 const struct rtnl_hw_stats64 *stats);
3269void
3270netdev_offload_xstats_report_used(struct netdev_notifier_offload_xstats_ru *ru);
3271void netdev_offload_xstats_push_delta(struct net_device *dev,
3272 enum netdev_offload_xstats_type type,
3273 const struct rtnl_hw_stats64 *stats);
3274
3275static inline void netdev_notifier_info_init(struct netdev_notifier_info *info,
3276 struct net_device *dev)
3277{
3278 info->dev = dev;
3279 info->extack = NULL;
3280}
3281
3282static inline struct net_device *
3283netdev_notifier_info_to_dev(const struct netdev_notifier_info *info)
3284{
3285 return info->dev;
3286}
3287
3288static inline struct netlink_ext_ack *
3289netdev_notifier_info_to_extack(const struct netdev_notifier_info *info)
3290{
3291 return info->extack;
3292}
3293
3294int call_netdevice_notifiers(unsigned long val, struct net_device *dev);
3295int call_netdevice_notifiers_info(unsigned long val,
3296 struct netdev_notifier_info *info);
3297
3298#define for_each_netdev(net, d) \
3299 list_for_each_entry(d, &(net)->dev_base_head, dev_list)
3300#define for_each_netdev_reverse(net, d) \
3301 list_for_each_entry_reverse(d, &(net)->dev_base_head, dev_list)
3302#define for_each_netdev_rcu(net, d) \
3303 list_for_each_entry_rcu(d, &(net)->dev_base_head, dev_list)
3304#define for_each_netdev_safe(net, d, n) \
3305 list_for_each_entry_safe(d, n, &(net)->dev_base_head, dev_list)
3306#define for_each_netdev_continue(net, d) \
3307 list_for_each_entry_continue(d, &(net)->dev_base_head, dev_list)
3308#define for_each_netdev_continue_reverse(net, d) \
3309 list_for_each_entry_continue_reverse(d, &(net)->dev_base_head, \
3310 dev_list)
3311#define for_each_netdev_continue_rcu(net, d) \
3312 list_for_each_entry_continue_rcu(d, &(net)->dev_base_head, dev_list)
3313#define for_each_netdev_in_bond_rcu(bond, slave) \
3314 for_each_netdev_rcu(dev_net_rcu(bond), slave) \
3315 if (netdev_master_upper_dev_get_rcu(slave) == (bond))
3316#define net_device_entry(lh) list_entry(lh, struct net_device, dev_list)
3317
3318#define for_each_netdev_dump(net, d, ifindex) \
3319 for (; (d = xa_find(&(net)->dev_by_index, &ifindex, \
3320 ULONG_MAX, XA_PRESENT)); ifindex++)
3321
3322static inline struct net_device *next_net_device(struct net_device *dev)
3323{
3324 struct list_head *lh;
3325 struct net *net;
3326
3327 net = dev_net(dev);
3328 lh = dev->dev_list.next;
3329 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
3330}
3331
3332static inline struct net_device *next_net_device_rcu(struct net_device *dev)
3333{
3334 struct list_head *lh;
3335 struct net *net;
3336
3337 net = dev_net(dev);
3338 lh = rcu_dereference(list_next_rcu(&dev->dev_list));
3339 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
3340}
3341
3342static inline struct net_device *first_net_device(struct net *net)
3343{
3344 return list_empty(&net->dev_base_head) ? NULL :
3345 net_device_entry(net->dev_base_head.next);
3346}
3347
3348int netdev_boot_setup_check(struct net_device *dev);
3349struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type,
3350 const char *hwaddr);
3351struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
3352 const char *hwaddr);
3353struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type);
3354void dev_add_pack(struct packet_type *pt);
3355void dev_remove_pack(struct packet_type *pt);
3356void __dev_remove_pack(struct packet_type *pt);
3357void dev_add_offload(struct packet_offload *po);
3358void dev_remove_offload(struct packet_offload *po);
3359
3360int dev_get_iflink(const struct net_device *dev);
3361int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb);
3362int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
3363 struct net_device_path_stack *stack);
3364struct net_device *dev_get_by_name(struct net *net, const char *name);
3365struct net_device *dev_get_by_name_rcu(struct net *net, const char *name);
3366struct net_device *__dev_get_by_name(struct net *net, const char *name);
3367bool netdev_name_in_use(struct net *net, const char *name);
3368int dev_alloc_name(struct net_device *dev, const char *name);
3369int netif_open(struct net_device *dev, struct netlink_ext_ack *extack);
3370int dev_open(struct net_device *dev, struct netlink_ext_ack *extack);
3371void netif_close(struct net_device *dev);
3372void dev_close(struct net_device *dev);
3373void netif_close_many(struct list_head *head, bool unlink);
3374void netif_disable_lro(struct net_device *dev);
3375void dev_disable_lro(struct net_device *dev);
3376int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *newskb);
3377u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
3378 struct net_device *sb_dev);
3379
3380int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev);
3381int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id);
3382
3383static inline int dev_queue_xmit(struct sk_buff *skb)
3384{
3385 return __dev_queue_xmit(skb, NULL);
3386}
3387
3388static inline int dev_queue_xmit_accel(struct sk_buff *skb,
3389 struct net_device *sb_dev)
3390{
3391 return __dev_queue_xmit(skb, sb_dev);
3392}
3393
3394static inline int dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
3395{
3396 int ret;
3397
3398 ret = __dev_direct_xmit(skb, queue_id);
3399 if (!dev_xmit_complete(ret))
3400 kfree_skb(skb);
3401 return ret;
3402}
3403
3404int register_netdevice(struct net_device *dev);
3405void unregister_netdevice_queue(struct net_device *dev, struct list_head *head);
3406void unregister_netdevice_many(struct list_head *head);
3407static inline void unregister_netdevice(struct net_device *dev)
3408{
3409 unregister_netdevice_queue(dev, NULL);
3410}
3411
3412int netdev_refcnt_read(const struct net_device *dev);
3413void free_netdev(struct net_device *dev);
3414
3415struct net_device *netdev_get_xmit_slave(struct net_device *dev,
3416 struct sk_buff *skb,
3417 bool all_slaves);
3418struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev,
3419 struct sock *sk);
3420struct net_device *dev_get_by_index(struct net *net, int ifindex);
3421struct net_device *__dev_get_by_index(struct net *net, int ifindex);
3422struct net_device *netdev_get_by_index(struct net *net, int ifindex,
3423 netdevice_tracker *tracker, gfp_t gfp);
3424struct net_device *netdev_get_by_index_lock(struct net *net, int ifindex);
3425struct net_device *netdev_get_by_name(struct net *net, const char *name,
3426 netdevice_tracker *tracker, gfp_t gfp);
3427struct net_device *netdev_get_by_flags_rcu(struct net *net, netdevice_tracker *tracker,
3428 unsigned short flags, unsigned short mask);
3429struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
3430void netdev_copy_name(struct net_device *dev, char *name);
3431
3432static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev,
3433 unsigned short type,
3434 const void *daddr, const void *saddr,
3435 unsigned int len)
3436{
3437 if (!dev->header_ops || !dev->header_ops->create)
3438 return 0;
3439
3440 return dev->header_ops->create(skb, dev, type, daddr, saddr, len);
3441}
3442
3443static inline int dev_parse_header(const struct sk_buff *skb,
3444 unsigned char *haddr)
3445{
3446 const struct net_device *dev = skb->dev;
3447
3448 if (!dev->header_ops || !dev->header_ops->parse)
3449 return 0;
3450 return dev->header_ops->parse(skb, dev, haddr);
3451}
3452
3453static inline __be16 dev_parse_header_protocol(const struct sk_buff *skb)
3454{
3455 const struct net_device *dev = skb->dev;
3456
3457 if (!dev->header_ops || !dev->header_ops->parse_protocol)
3458 return 0;
3459 return dev->header_ops->parse_protocol(skb);
3460}
3461
3462/* ll_header must have at least hard_header_len allocated */
3463static inline bool dev_validate_header(const struct net_device *dev,
3464 char *ll_header, int len)
3465{
3466 if (likely(len >= dev->hard_header_len))
3467 return true;
3468 if (len < dev->min_header_len)
3469 return false;
3470
3471 if (capable(CAP_SYS_RAWIO)) {
3472 memset(ll_header + len, 0, dev->hard_header_len - len);
3473 return true;
3474 }
3475
3476 if (dev->header_ops && dev->header_ops->validate)
3477 return dev->header_ops->validate(ll_header, len);
3478
3479 return false;
3480}
3481
3482static inline bool dev_has_header(const struct net_device *dev)
3483{
3484 return dev->header_ops && dev->header_ops->create;
3485}
3486
3487struct numa_drop_counters {
3488 atomic_t drops0 ____cacheline_aligned_in_smp;
3489 atomic_t drops1 ____cacheline_aligned_in_smp;
3490};
3491
3492static inline int numa_drop_read(const struct numa_drop_counters *ndc)
3493{
3494 return atomic_read(&ndc->drops0) + atomic_read(&ndc->drops1);
3495}
3496
3497static inline void numa_drop_add(struct numa_drop_counters *ndc, int val)
3498{
3499 int n = numa_node_id() % 2;
3500
3501 if (n)
3502 atomic_add(val, &ndc->drops1);
3503 else
3504 atomic_add(val, &ndc->drops0);
3505}
3506
3507static inline void numa_drop_reset(struct numa_drop_counters *ndc)
3508{
3509 atomic_set(&ndc->drops0, 0);
3510 atomic_set(&ndc->drops1, 0);
3511}
3512
3513/*
3514 * Incoming packets are placed on per-CPU queues
3515 */
3516struct softnet_data {
3517 struct list_head poll_list;
3518 struct sk_buff_head process_queue;
3519 local_lock_t process_queue_bh_lock;
3520
3521 /* stats */
3522 unsigned int processed;
3523 unsigned int time_squeeze;
3524#ifdef CONFIG_RPS
3525 struct softnet_data *rps_ipi_list;
3526#endif
3527
3528 unsigned int received_rps;
3529 bool in_net_rx_action;
3530 bool in_napi_threaded_poll;
3531
3532#ifdef CONFIG_NET_FLOW_LIMIT
3533 struct sd_flow_limit __rcu *flow_limit;
3534#endif
3535 struct Qdisc *output_queue;
3536 struct Qdisc **output_queue_tailp;
3537 struct sk_buff *completion_queue;
3538#ifdef CONFIG_XFRM_OFFLOAD
3539 struct sk_buff_head xfrm_backlog;
3540#endif
3541 /* written and read only by owning cpu: */
3542 struct netdev_xmit xmit;
3543#ifdef CONFIG_RPS
3544 /* input_queue_head should be written by cpu owning this struct,
3545 * and only read by other cpus. Worth using a cache line.
3546 */
3547 unsigned int input_queue_head ____cacheline_aligned_in_smp;
3548
3549 /* Elements below can be accessed between CPUs for RPS/RFS */
3550 call_single_data_t csd ____cacheline_aligned_in_smp;
3551 struct softnet_data *rps_ipi_next;
3552 unsigned int cpu;
3553
3554 /* We force a cacheline alignment from here, to hold together
3555 * input_queue_tail, input_pkt_queue and backlog.state.
3556 * We add holes so that backlog.state is the last field
3557 * of this cache line.
3558 */
3559 long pad[3] ____cacheline_aligned_in_smp;
3560 unsigned int input_queue_tail;
3561#endif
3562 struct sk_buff_head input_pkt_queue;
3563
3564 struct napi_struct backlog;
3565
3566 struct numa_drop_counters drop_counters;
3567
3568 int defer_ipi_scheduled ____cacheline_aligned_in_smp;
3569 call_single_data_t defer_csd;
3570};
3571
3572DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
3573
3574struct page_pool_bh {
3575 struct page_pool *pool;
3576 local_lock_t bh_lock;
3577};
3578DECLARE_PER_CPU(struct page_pool_bh, system_page_pool);
3579
3580#define XMIT_RECURSION_LIMIT 8
3581
3582#ifndef CONFIG_PREEMPT_RT
3583static inline int dev_recursion_level(void)
3584{
3585 return this_cpu_read(softnet_data.xmit.recursion);
3586}
3587
3588static inline bool dev_xmit_recursion(void)
3589{
3590 return unlikely(__this_cpu_read(softnet_data.xmit.recursion) >
3591 XMIT_RECURSION_LIMIT);
3592}
3593
3594static inline void dev_xmit_recursion_inc(void)
3595{
3596 __this_cpu_inc(softnet_data.xmit.recursion);
3597}
3598
3599static inline void dev_xmit_recursion_dec(void)
3600{
3601 __this_cpu_dec(softnet_data.xmit.recursion);
3602}
3603#else
3604static inline int dev_recursion_level(void)
3605{
3606 return current->net_xmit.recursion;
3607}
3608
3609static inline bool dev_xmit_recursion(void)
3610{
3611 return unlikely(current->net_xmit.recursion > XMIT_RECURSION_LIMIT);
3612}
3613
3614static inline void dev_xmit_recursion_inc(void)
3615{
3616 current->net_xmit.recursion++;
3617}
3618
3619static inline void dev_xmit_recursion_dec(void)
3620{
3621 current->net_xmit.recursion--;
3622}
3623#endif
3624
3625void __netif_schedule(struct Qdisc *q);
3626void netif_schedule_queue(struct netdev_queue *txq);
3627
3628static inline void netif_tx_schedule_all(struct net_device *dev)
3629{
3630 unsigned int i;
3631
3632 for (i = 0; i < dev->num_tx_queues; i++)
3633 netif_schedule_queue(netdev_get_tx_queue(dev, i));
3634}
3635
3636static __always_inline void netif_tx_start_queue(struct netdev_queue *dev_queue)
3637{
3638 clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3639}
3640
3641/**
3642 * netif_start_queue - allow transmit
3643 * @dev: network device
3644 *
3645 * Allow upper layers to call the device hard_start_xmit routine.
3646 */
3647static inline void netif_start_queue(struct net_device *dev)
3648{
3649 netif_tx_start_queue(netdev_get_tx_queue(dev, 0));
3650}
3651
3652static inline void netif_tx_start_all_queues(struct net_device *dev)
3653{
3654 unsigned int i;
3655
3656 for (i = 0; i < dev->num_tx_queues; i++) {
3657 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3658 netif_tx_start_queue(txq);
3659 }
3660}
3661
3662void netif_tx_wake_queue(struct netdev_queue *dev_queue);
3663
3664/**
3665 * netif_wake_queue - restart transmit
3666 * @dev: network device
3667 *
3668 * Allow upper layers to call the device hard_start_xmit routine.
3669 * Used for flow control when transmit resources are available.
3670 */
3671static inline void netif_wake_queue(struct net_device *dev)
3672{
3673 netif_tx_wake_queue(netdev_get_tx_queue(dev, 0));
3674}
3675
3676static inline void netif_tx_wake_all_queues(struct net_device *dev)
3677{
3678 unsigned int i;
3679
3680 for (i = 0; i < dev->num_tx_queues; i++) {
3681 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3682 netif_tx_wake_queue(txq);
3683 }
3684}
3685
3686static __always_inline void netif_tx_stop_queue(struct netdev_queue *dev_queue)
3687{
3688 /* Paired with READ_ONCE() from dev_watchdog() */
3689 WRITE_ONCE(dev_queue->trans_start, jiffies);
3690
3691 /* This barrier is paired with smp_mb() from dev_watchdog() */
3692 smp_mb__before_atomic();
3693
3694 /* Must be an atomic op see netif_txq_try_stop() */
3695 set_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3696}
3697
3698/**
3699 * netif_stop_queue - stop transmitted packets
3700 * @dev: network device
3701 *
3702 * Stop upper layers calling the device hard_start_xmit routine.
3703 * Used for flow control when transmit resources are unavailable.
3704 */
3705static inline void netif_stop_queue(struct net_device *dev)
3706{
3707 netif_tx_stop_queue(netdev_get_tx_queue(dev, 0));
3708}
3709
3710void netif_tx_stop_all_queues(struct net_device *dev);
3711
3712static inline bool netif_tx_queue_stopped(const struct netdev_queue *dev_queue)
3713{
3714 return test_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3715}
3716
3717/**
3718 * netif_queue_stopped - test if transmit queue is flowblocked
3719 * @dev: network device
3720 *
3721 * Test if transmit queue on device is currently unable to send.
3722 */
3723static inline bool netif_queue_stopped(const struct net_device *dev)
3724{
3725 return netif_tx_queue_stopped(netdev_get_tx_queue(dev, 0));
3726}
3727
3728static inline bool netif_xmit_stopped(const struct netdev_queue *dev_queue)
3729{
3730 return dev_queue->state & QUEUE_STATE_ANY_XOFF;
3731}
3732
3733static inline bool
3734netif_xmit_frozen_or_stopped(const struct netdev_queue *dev_queue)
3735{
3736 return dev_queue->state & QUEUE_STATE_ANY_XOFF_OR_FROZEN;
3737}
3738
3739static inline bool
3740netif_xmit_frozen_or_drv_stopped(const struct netdev_queue *dev_queue)
3741{
3742 return dev_queue->state & QUEUE_STATE_DRV_XOFF_OR_FROZEN;
3743}
3744
3745/**
3746 * netdev_queue_set_dql_min_limit - set dql minimum limit
3747 * @dev_queue: pointer to transmit queue
3748 * @min_limit: dql minimum limit
3749 *
3750 * Forces xmit_more() to return true until the minimum threshold
3751 * defined by @min_limit is reached (or until the tx queue is
3752 * empty). Warning: to be use with care, misuse will impact the
3753 * latency.
3754 */
3755static inline void netdev_queue_set_dql_min_limit(struct netdev_queue *dev_queue,
3756 unsigned int min_limit)
3757{
3758#ifdef CONFIG_BQL
3759 dev_queue->dql.min_limit = min_limit;
3760#endif
3761}
3762
3763static inline int netdev_queue_dql_avail(const struct netdev_queue *txq)
3764{
3765#ifdef CONFIG_BQL
3766 /* Non-BQL migrated drivers will return 0, too. */
3767 return dql_avail(&txq->dql);
3768#else
3769 return 0;
3770#endif
3771}
3772
3773/**
3774 * netdev_txq_bql_enqueue_prefetchw - prefetch bql data for write
3775 * @dev_queue: pointer to transmit queue
3776 *
3777 * BQL enabled drivers might use this helper in their ndo_start_xmit(),
3778 * to give appropriate hint to the CPU.
3779 */
3780static inline void netdev_txq_bql_enqueue_prefetchw(struct netdev_queue *dev_queue)
3781{
3782#ifdef CONFIG_BQL
3783 prefetchw(&dev_queue->dql.num_queued);
3784#endif
3785}
3786
3787/**
3788 * netdev_txq_bql_complete_prefetchw - prefetch bql data for write
3789 * @dev_queue: pointer to transmit queue
3790 *
3791 * BQL enabled drivers might use this helper in their TX completion path,
3792 * to give appropriate hint to the CPU.
3793 */
3794static inline void netdev_txq_bql_complete_prefetchw(struct netdev_queue *dev_queue)
3795{
3796#ifdef CONFIG_BQL
3797 prefetchw(&dev_queue->dql.limit);
3798#endif
3799}
3800
3801/**
3802 * netdev_tx_sent_queue - report the number of bytes queued to a given tx queue
3803 * @dev_queue: network device queue
3804 * @bytes: number of bytes queued to the device queue
3805 *
3806 * Report the number of bytes queued for sending/completion to the network
3807 * device hardware queue. @bytes should be a good approximation and should
3808 * exactly match netdev_completed_queue() @bytes.
3809 * This is typically called once per packet, from ndo_start_xmit().
3810 */
3811static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue,
3812 unsigned int bytes)
3813{
3814#ifdef CONFIG_BQL
3815 dql_queued(&dev_queue->dql, bytes);
3816
3817 if (likely(dql_avail(&dev_queue->dql) >= 0))
3818 return;
3819
3820 /* Paired with READ_ONCE() from dev_watchdog() */
3821 WRITE_ONCE(dev_queue->trans_start, jiffies);
3822
3823 /* This barrier is paired with smp_mb() from dev_watchdog() */
3824 smp_mb__before_atomic();
3825
3826 set_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
3827
3828 /*
3829 * The XOFF flag must be set before checking the dql_avail below,
3830 * because in netdev_tx_completed_queue we update the dql_completed
3831 * before checking the XOFF flag.
3832 */
3833 smp_mb__after_atomic();
3834
3835 /* check again in case another CPU has just made room avail */
3836 if (unlikely(dql_avail(&dev_queue->dql) >= 0))
3837 clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
3838#endif
3839}
3840
3841/* Variant of netdev_tx_sent_queue() for drivers that are aware
3842 * that they should not test BQL status themselves.
3843 * We do want to change __QUEUE_STATE_STACK_XOFF only for the last
3844 * skb of a batch.
3845 * Returns true if the doorbell must be used to kick the NIC.
3846 */
3847static inline bool __netdev_tx_sent_queue(struct netdev_queue *dev_queue,
3848 unsigned int bytes,
3849 bool xmit_more)
3850{
3851 if (xmit_more) {
3852#ifdef CONFIG_BQL
3853 dql_queued(&dev_queue->dql, bytes);
3854#endif
3855 return netif_tx_queue_stopped(dev_queue);
3856 }
3857 netdev_tx_sent_queue(dev_queue, bytes);
3858 return true;
3859}
3860
3861/**
3862 * netdev_sent_queue - report the number of bytes queued to hardware
3863 * @dev: network device
3864 * @bytes: number of bytes queued to the hardware device queue
3865 *
3866 * Report the number of bytes queued for sending/completion to the network
3867 * device hardware queue#0. @bytes should be a good approximation and should
3868 * exactly match netdev_completed_queue() @bytes.
3869 * This is typically called once per packet, from ndo_start_xmit().
3870 */
3871static inline void netdev_sent_queue(struct net_device *dev, unsigned int bytes)
3872{
3873 netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes);
3874}
3875
3876static inline bool __netdev_sent_queue(struct net_device *dev,
3877 unsigned int bytes,
3878 bool xmit_more)
3879{
3880 return __netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes,
3881 xmit_more);
3882}
3883
3884/**
3885 * netdev_tx_completed_queue - report number of packets/bytes at TX completion.
3886 * @dev_queue: network device queue
3887 * @pkts: number of packets (currently ignored)
3888 * @bytes: number of bytes dequeued from the device queue
3889 *
3890 * Must be called at most once per TX completion round (and not per
3891 * individual packet), so that BQL can adjust its limits appropriately.
3892 */
3893static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue,
3894 unsigned int pkts, unsigned int bytes)
3895{
3896#ifdef CONFIG_BQL
3897 if (unlikely(!bytes))
3898 return;
3899
3900 dql_completed(&dev_queue->dql, bytes);
3901
3902 /*
3903 * Without the memory barrier there is a small possibility that
3904 * netdev_tx_sent_queue will miss the update and cause the queue to
3905 * be stopped forever
3906 */
3907 smp_mb(); /* NOTE: netdev_txq_completed_mb() assumes this exists */
3908
3909 if (unlikely(dql_avail(&dev_queue->dql) < 0))
3910 return;
3911
3912 if (test_and_clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state))
3913 netif_schedule_queue(dev_queue);
3914#endif
3915}
3916
3917/**
3918 * netdev_completed_queue - report bytes and packets completed by device
3919 * @dev: network device
3920 * @pkts: actual number of packets sent over the medium
3921 * @bytes: actual number of bytes sent over the medium
3922 *
3923 * Report the number of bytes and packets transmitted by the network device
3924 * hardware queue over the physical medium, @bytes must exactly match the
3925 * @bytes amount passed to netdev_sent_queue()
3926 */
3927static inline void netdev_completed_queue(struct net_device *dev,
3928 unsigned int pkts, unsigned int bytes)
3929{
3930 netdev_tx_completed_queue(netdev_get_tx_queue(dev, 0), pkts, bytes);
3931}
3932
3933static inline void netdev_tx_reset_queue(struct netdev_queue *q)
3934{
3935#ifdef CONFIG_BQL
3936 clear_bit(__QUEUE_STATE_STACK_XOFF, &q->state);
3937 dql_reset(&q->dql);
3938#endif
3939}
3940
3941/**
3942 * netdev_tx_reset_subqueue - reset the BQL stats and state of a netdev queue
3943 * @dev: network device
3944 * @qid: stack index of the queue to reset
3945 */
3946static inline void netdev_tx_reset_subqueue(const struct net_device *dev,
3947 u32 qid)
3948{
3949 netdev_tx_reset_queue(netdev_get_tx_queue(dev, qid));
3950}
3951
3952/**
3953 * netdev_reset_queue - reset the packets and bytes count of a network device
3954 * @dev_queue: network device
3955 *
3956 * Reset the bytes and packet count of a network device and clear the
3957 * software flow control OFF bit for this network device
3958 */
3959static inline void netdev_reset_queue(struct net_device *dev_queue)
3960{
3961 netdev_tx_reset_subqueue(dev_queue, 0);
3962}
3963
3964/**
3965 * netdev_cap_txqueue - check if selected tx queue exceeds device queues
3966 * @dev: network device
3967 * @queue_index: given tx queue index
3968 *
3969 * Returns 0 if given tx queue index >= number of device tx queues,
3970 * otherwise returns the originally passed tx queue index.
3971 */
3972static inline u16 netdev_cap_txqueue(struct net_device *dev, u16 queue_index)
3973{
3974 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
3975 net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
3976 dev->name, queue_index,
3977 dev->real_num_tx_queues);
3978 return 0;
3979 }
3980
3981 return queue_index;
3982}
3983
3984/**
3985 * netif_running - test if up
3986 * @dev: network device
3987 *
3988 * Test if the device has been brought up.
3989 */
3990static inline bool netif_running(const struct net_device *dev)
3991{
3992 return test_bit(__LINK_STATE_START, &dev->state);
3993}
3994
3995/*
3996 * Routines to manage the subqueues on a device. We only need start,
3997 * stop, and a check if it's stopped. All other device management is
3998 * done at the overall netdevice level.
3999 * Also test the device if we're multiqueue.
4000 */
4001
4002/**
4003 * netif_start_subqueue - allow sending packets on subqueue
4004 * @dev: network device
4005 * @queue_index: sub queue index
4006 *
4007 * Start individual transmit queue of a device with multiple transmit queues.
4008 */
4009static inline void netif_start_subqueue(struct net_device *dev, u16 queue_index)
4010{
4011 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
4012
4013 netif_tx_start_queue(txq);
4014}
4015
4016/**
4017 * netif_stop_subqueue - stop sending packets on subqueue
4018 * @dev: network device
4019 * @queue_index: sub queue index
4020 *
4021 * Stop individual transmit queue of a device with multiple transmit queues.
4022 */
4023static inline void netif_stop_subqueue(struct net_device *dev, u16 queue_index)
4024{
4025 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
4026 netif_tx_stop_queue(txq);
4027}
4028
4029/**
4030 * __netif_subqueue_stopped - test status of subqueue
4031 * @dev: network device
4032 * @queue_index: sub queue index
4033 *
4034 * Check individual transmit queue of a device with multiple transmit queues.
4035 */
4036static inline bool __netif_subqueue_stopped(const struct net_device *dev,
4037 u16 queue_index)
4038{
4039 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
4040
4041 return netif_tx_queue_stopped(txq);
4042}
4043
4044/**
4045 * netif_subqueue_stopped - test status of subqueue
4046 * @dev: network device
4047 * @skb: sub queue buffer pointer
4048 *
4049 * Check individual transmit queue of a device with multiple transmit queues.
4050 */
4051static inline bool netif_subqueue_stopped(const struct net_device *dev,
4052 struct sk_buff *skb)
4053{
4054 return __netif_subqueue_stopped(dev, skb_get_queue_mapping(skb));
4055}
4056
4057/**
4058 * netif_wake_subqueue - allow sending packets on subqueue
4059 * @dev: network device
4060 * @queue_index: sub queue index
4061 *
4062 * Resume individual transmit queue of a device with multiple transmit queues.
4063 */
4064static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
4065{
4066 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
4067
4068 netif_tx_wake_queue(txq);
4069}
4070
4071#ifdef CONFIG_XPS
4072int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
4073 u16 index);
4074int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
4075 u16 index, enum xps_map_type type);
4076
4077/**
4078 * netif_attr_test_mask - Test a CPU or Rx queue set in a mask
4079 * @j: CPU/Rx queue index
4080 * @mask: bitmask of all cpus/rx queues
4081 * @nr_bits: number of bits in the bitmask
4082 *
4083 * Test if a CPU or Rx queue index is set in a mask of all CPU/Rx queues.
4084 */
4085static inline bool netif_attr_test_mask(unsigned long j,
4086 const unsigned long *mask,
4087 unsigned int nr_bits)
4088{
4089 cpu_max_bits_warn(j, nr_bits);
4090 return test_bit(j, mask);
4091}
4092
4093/**
4094 * netif_attr_test_online - Test for online CPU/Rx queue
4095 * @j: CPU/Rx queue index
4096 * @online_mask: bitmask for CPUs/Rx queues that are online
4097 * @nr_bits: number of bits in the bitmask
4098 *
4099 * Returns: true if a CPU/Rx queue is online.
4100 */
4101static inline bool netif_attr_test_online(unsigned long j,
4102 const unsigned long *online_mask,
4103 unsigned int nr_bits)
4104{
4105 cpu_max_bits_warn(j, nr_bits);
4106
4107 if (online_mask)
4108 return test_bit(j, online_mask);
4109
4110 return (j < nr_bits);
4111}
4112
4113/**
4114 * netif_attrmask_next - get the next CPU/Rx queue in a cpu/Rx queues mask
4115 * @n: CPU/Rx queue index
4116 * @srcp: the cpumask/Rx queue mask pointer
4117 * @nr_bits: number of bits in the bitmask
4118 *
4119 * Returns: next (after n) CPU/Rx queue index in the mask;
4120 * >= nr_bits if no further CPUs/Rx queues set.
4121 */
4122static inline unsigned int netif_attrmask_next(int n, const unsigned long *srcp,
4123 unsigned int nr_bits)
4124{
4125 /* -1 is a legal arg here. */
4126 if (n != -1)
4127 cpu_max_bits_warn(n, nr_bits);
4128
4129 if (srcp)
4130 return find_next_bit(srcp, nr_bits, n + 1);
4131
4132 return n + 1;
4133}
4134
4135/**
4136 * netif_attrmask_next_and - get the next CPU/Rx queue in \*src1p & \*src2p
4137 * @n: CPU/Rx queue index
4138 * @src1p: the first CPUs/Rx queues mask pointer
4139 * @src2p: the second CPUs/Rx queues mask pointer
4140 * @nr_bits: number of bits in the bitmask
4141 *
4142 * Returns: next (after n) CPU/Rx queue index set in both masks;
4143 * >= nr_bits if no further CPUs/Rx queues set in both.
4144 */
4145static inline int netif_attrmask_next_and(int n, const unsigned long *src1p,
4146 const unsigned long *src2p,
4147 unsigned int nr_bits)
4148{
4149 /* -1 is a legal arg here. */
4150 if (n != -1)
4151 cpu_max_bits_warn(n, nr_bits);
4152
4153 if (src1p && src2p)
4154 return find_next_and_bit(src1p, src2p, nr_bits, n + 1);
4155 else if (src1p)
4156 return find_next_bit(src1p, nr_bits, n + 1);
4157 else if (src2p)
4158 return find_next_bit(src2p, nr_bits, n + 1);
4159
4160 return n + 1;
4161}
4162#else
4163static inline int netif_set_xps_queue(struct net_device *dev,
4164 const struct cpumask *mask,
4165 u16 index)
4166{
4167 return 0;
4168}
4169
4170static inline int __netif_set_xps_queue(struct net_device *dev,
4171 const unsigned long *mask,
4172 u16 index, enum xps_map_type type)
4173{
4174 return 0;
4175}
4176#endif
4177
4178/**
4179 * netif_is_multiqueue - test if device has multiple transmit queues
4180 * @dev: network device
4181 *
4182 * Check if device has multiple transmit queues
4183 */
4184static inline bool netif_is_multiqueue(const struct net_device *dev)
4185{
4186 return dev->num_tx_queues > 1;
4187}
4188
4189int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq);
4190int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq);
4191int netif_set_real_num_queues(struct net_device *dev,
4192 unsigned int txq, unsigned int rxq);
4193
4194int netif_get_num_default_rss_queues(void);
4195
4196void dev_kfree_skb_irq_reason(struct sk_buff *skb, enum skb_drop_reason reason);
4197void dev_kfree_skb_any_reason(struct sk_buff *skb, enum skb_drop_reason reason);
4198
4199/*
4200 * It is not allowed to call kfree_skb() or consume_skb() from hardware
4201 * interrupt context or with hardware interrupts being disabled.
4202 * (in_hardirq() || irqs_disabled())
4203 *
4204 * We provide four helpers that can be used in following contexts :
4205 *
4206 * dev_kfree_skb_irq(skb) when caller drops a packet from irq context,
4207 * replacing kfree_skb(skb)
4208 *
4209 * dev_consume_skb_irq(skb) when caller consumes a packet from irq context.
4210 * Typically used in place of consume_skb(skb) in TX completion path
4211 *
4212 * dev_kfree_skb_any(skb) when caller doesn't know its current irq context,
4213 * replacing kfree_skb(skb)
4214 *
4215 * dev_consume_skb_any(skb) when caller doesn't know its current irq context,
4216 * and consumed a packet. Used in place of consume_skb(skb)
4217 */
4218static inline void dev_kfree_skb_irq(struct sk_buff *skb)
4219{
4220 dev_kfree_skb_irq_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED);
4221}
4222
4223static inline void dev_consume_skb_irq(struct sk_buff *skb)
4224{
4225 dev_kfree_skb_irq_reason(skb, SKB_CONSUMED);
4226}
4227
4228static inline void dev_kfree_skb_any(struct sk_buff *skb)
4229{
4230 dev_kfree_skb_any_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED);
4231}
4232
4233static inline void dev_consume_skb_any(struct sk_buff *skb)
4234{
4235 dev_kfree_skb_any_reason(skb, SKB_CONSUMED);
4236}
4237
4238u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
4239 const struct bpf_prog *xdp_prog);
4240void generic_xdp_tx(struct sk_buff *skb, const struct bpf_prog *xdp_prog);
4241int do_xdp_generic(const struct bpf_prog *xdp_prog, struct sk_buff **pskb);
4242int netif_rx(struct sk_buff *skb);
4243int __netif_rx(struct sk_buff *skb);
4244
4245int netif_receive_skb(struct sk_buff *skb);
4246int netif_receive_skb_core(struct sk_buff *skb);
4247void netif_receive_skb_list_internal(struct list_head *head);
4248void netif_receive_skb_list(struct list_head *head);
4249gro_result_t gro_receive_skb(struct gro_node *gro, struct sk_buff *skb);
4250
4251static inline gro_result_t napi_gro_receive(struct napi_struct *napi,
4252 struct sk_buff *skb)
4253{
4254 return gro_receive_skb(&napi->gro, skb);
4255}
4256
4257struct sk_buff *napi_get_frags(struct napi_struct *napi);
4258gro_result_t napi_gro_frags(struct napi_struct *napi);
4259
4260static inline void napi_free_frags(struct napi_struct *napi)
4261{
4262 kfree_skb(napi->skb);
4263 napi->skb = NULL;
4264}
4265
4266bool netdev_is_rx_handler_busy(struct net_device *dev);
4267int netdev_rx_handler_register(struct net_device *dev,
4268 rx_handler_func_t *rx_handler,
4269 void *rx_handler_data);
4270void netdev_rx_handler_unregister(struct net_device *dev);
4271
4272bool dev_valid_name(const char *name);
4273static inline bool is_socket_ioctl_cmd(unsigned int cmd)
4274{
4275 return _IOC_TYPE(cmd) == SOCK_IOC_TYPE;
4276}
4277int get_user_ifreq(struct ifreq *ifr, void __user **ifrdata, void __user *arg);
4278int put_user_ifreq(struct ifreq *ifr, void __user *arg);
4279int dev_ioctl(struct net *net, unsigned int cmd, struct ifreq *ifr,
4280 void __user *data, bool *need_copyout);
4281int dev_ifconf(struct net *net, struct ifconf __user *ifc);
4282int dev_eth_ioctl(struct net_device *dev,
4283 struct ifreq *ifr, unsigned int cmd);
4284int generic_hwtstamp_get_lower(struct net_device *dev,
4285 struct kernel_hwtstamp_config *kernel_cfg);
4286int generic_hwtstamp_set_lower(struct net_device *dev,
4287 struct kernel_hwtstamp_config *kernel_cfg,
4288 struct netlink_ext_ack *extack);
4289int dev_ethtool(struct net *net, struct ifreq *ifr, void __user *userdata);
4290unsigned int netif_get_flags(const struct net_device *dev);
4291int __dev_change_flags(struct net_device *dev, unsigned int flags,
4292 struct netlink_ext_ack *extack);
4293int netif_change_flags(struct net_device *dev, unsigned int flags,
4294 struct netlink_ext_ack *extack);
4295int dev_change_flags(struct net_device *dev, unsigned int flags,
4296 struct netlink_ext_ack *extack);
4297int netif_set_alias(struct net_device *dev, const char *alias, size_t len);
4298int dev_set_alias(struct net_device *, const char *, size_t);
4299int dev_get_alias(const struct net_device *, char *, size_t);
4300int __dev_change_net_namespace(struct net_device *dev, struct net *net,
4301 const char *pat, int new_ifindex,
4302 struct netlink_ext_ack *extack);
4303int dev_change_net_namespace(struct net_device *dev, struct net *net,
4304 const char *pat);
4305int __netif_set_mtu(struct net_device *dev, int new_mtu);
4306int netif_set_mtu(struct net_device *dev, int new_mtu);
4307int dev_set_mtu(struct net_device *, int);
4308int netif_pre_changeaddr_notify(struct net_device *dev, const char *addr,
4309 struct netlink_ext_ack *extack);
4310int netif_set_mac_address(struct net_device *dev, struct sockaddr_storage *ss,
4311 struct netlink_ext_ack *extack);
4312int dev_set_mac_address(struct net_device *dev, struct sockaddr_storage *ss,
4313 struct netlink_ext_ack *extack);
4314int dev_set_mac_address_user(struct net_device *dev, struct sockaddr_storage *ss,
4315 struct netlink_ext_ack *extack);
4316int netif_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name);
4317int netif_get_port_parent_id(struct net_device *dev,
4318 struct netdev_phys_item_id *ppid, bool recurse);
4319bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b);
4320
4321struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again);
4322struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
4323 struct netdev_queue *txq, int *ret);
4324
4325int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
4326u8 dev_xdp_prog_count(struct net_device *dev);
4327int netif_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf);
4328int dev_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf);
4329u8 dev_xdp_sb_prog_count(struct net_device *dev);
4330u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode);
4331
4332u32 dev_get_min_mp_channel_count(const struct net_device *dev);
4333
4334int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
4335int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
4336int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb);
4337bool is_skb_forwardable(const struct net_device *dev,
4338 const struct sk_buff *skb);
4339
4340static __always_inline bool __is_skb_forwardable(const struct net_device *dev,
4341 const struct sk_buff *skb,
4342 const bool check_mtu)
4343{
4344 const u32 vlan_hdr_len = 4; /* VLAN_HLEN */
4345 unsigned int len;
4346
4347 if (!(dev->flags & IFF_UP))
4348 return false;
4349
4350 if (!check_mtu)
4351 return true;
4352
4353 len = dev->mtu + dev->hard_header_len + vlan_hdr_len;
4354 if (skb->len <= len)
4355 return true;
4356
4357 /* if TSO is enabled, we don't care about the length as the packet
4358 * could be forwarded without being segmented before
4359 */
4360 if (skb_is_gso(skb))
4361 return true;
4362
4363 return false;
4364}
4365
4366void netdev_core_stats_inc(struct net_device *dev, u32 offset);
4367
4368#define DEV_CORE_STATS_INC(FIELD) \
4369static inline void dev_core_stats_##FIELD##_inc(struct net_device *dev) \
4370{ \
4371 netdev_core_stats_inc(dev, \
4372 offsetof(struct net_device_core_stats, FIELD)); \
4373}
4374DEV_CORE_STATS_INC(rx_dropped)
4375DEV_CORE_STATS_INC(tx_dropped)
4376DEV_CORE_STATS_INC(rx_nohandler)
4377DEV_CORE_STATS_INC(rx_otherhost_dropped)
4378#undef DEV_CORE_STATS_INC
4379
4380static __always_inline int ____dev_forward_skb(struct net_device *dev,
4381 struct sk_buff *skb,
4382 const bool check_mtu)
4383{
4384 if (skb_orphan_frags(skb, GFP_ATOMIC) ||
4385 unlikely(!__is_skb_forwardable(dev, skb, check_mtu))) {
4386 dev_core_stats_rx_dropped_inc(dev);
4387 kfree_skb(skb);
4388 return NET_RX_DROP;
4389 }
4390
4391 skb_scrub_packet(skb, !net_eq(dev_net(dev), dev_net(skb->dev)));
4392 skb->priority = 0;
4393 return 0;
4394}
4395
4396bool dev_nit_active_rcu(const struct net_device *dev);
4397static inline bool dev_nit_active(const struct net_device *dev)
4398{
4399 bool ret;
4400
4401 rcu_read_lock();
4402 ret = dev_nit_active_rcu(dev);
4403 rcu_read_unlock();
4404 return ret;
4405}
4406
4407void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev);
4408
4409static inline void __dev_put(struct net_device *dev)
4410{
4411 if (dev) {
4412#ifdef CONFIG_PCPU_DEV_REFCNT
4413 this_cpu_dec(*dev->pcpu_refcnt);
4414#else
4415 refcount_dec(&dev->dev_refcnt);
4416#endif
4417 }
4418}
4419
4420static inline void __dev_hold(struct net_device *dev)
4421{
4422 if (dev) {
4423#ifdef CONFIG_PCPU_DEV_REFCNT
4424 this_cpu_inc(*dev->pcpu_refcnt);
4425#else
4426 refcount_inc(&dev->dev_refcnt);
4427#endif
4428 }
4429}
4430
4431static inline void __netdev_tracker_alloc(struct net_device *dev,
4432 netdevice_tracker *tracker,
4433 gfp_t gfp)
4434{
4435#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
4436 ref_tracker_alloc(&dev->refcnt_tracker, tracker, gfp);
4437#endif
4438}
4439
4440/* netdev_tracker_alloc() can upgrade a prior untracked reference
4441 * taken by dev_get_by_name()/dev_get_by_index() to a tracked one.
4442 */
4443static inline void netdev_tracker_alloc(struct net_device *dev,
4444 netdevice_tracker *tracker, gfp_t gfp)
4445{
4446#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
4447 refcount_dec(&dev->refcnt_tracker.no_tracker);
4448 __netdev_tracker_alloc(dev, tracker, gfp);
4449#endif
4450}
4451
4452static inline void netdev_tracker_free(struct net_device *dev,
4453 netdevice_tracker *tracker)
4454{
4455#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
4456 ref_tracker_free(&dev->refcnt_tracker, tracker);
4457#endif
4458}
4459
4460static inline void netdev_hold(struct net_device *dev,
4461 netdevice_tracker *tracker, gfp_t gfp)
4462{
4463 if (dev) {
4464 __dev_hold(dev);
4465 __netdev_tracker_alloc(dev, tracker, gfp);
4466 }
4467}
4468
4469static inline void netdev_put(struct net_device *dev,
4470 netdevice_tracker *tracker)
4471{
4472 if (dev) {
4473 netdev_tracker_free(dev, tracker);
4474 __dev_put(dev);
4475 }
4476}
4477
4478/**
4479 * dev_hold - get reference to device
4480 * @dev: network device
4481 *
4482 * Hold reference to device to keep it from being freed.
4483 * Try using netdev_hold() instead.
4484 */
4485static inline void dev_hold(struct net_device *dev)
4486{
4487 netdev_hold(dev, NULL, GFP_ATOMIC);
4488}
4489
4490/**
4491 * dev_put - release reference to device
4492 * @dev: network device
4493 *
4494 * Release reference to device to allow it to be freed.
4495 * Try using netdev_put() instead.
4496 */
4497static inline void dev_put(struct net_device *dev)
4498{
4499 netdev_put(dev, NULL);
4500}
4501
4502DEFINE_FREE(dev_put, struct net_device *, if (_T) dev_put(_T))
4503
4504static inline void netdev_ref_replace(struct net_device *odev,
4505 struct net_device *ndev,
4506 netdevice_tracker *tracker,
4507 gfp_t gfp)
4508{
4509 if (odev)
4510 netdev_tracker_free(odev, tracker);
4511
4512 __dev_hold(ndev);
4513 __dev_put(odev);
4514
4515 if (ndev)
4516 __netdev_tracker_alloc(ndev, tracker, gfp);
4517}
4518
4519/* Carrier loss detection, dial on demand. The functions netif_carrier_on
4520 * and _off may be called from IRQ context, but it is caller
4521 * who is responsible for serialization of these calls.
4522 *
4523 * The name carrier is inappropriate, these functions should really be
4524 * called netif_lowerlayer_*() because they represent the state of any
4525 * kind of lower layer not just hardware media.
4526 */
4527void linkwatch_fire_event(struct net_device *dev);
4528
4529/**
4530 * linkwatch_sync_dev - sync linkwatch for the given device
4531 * @dev: network device to sync linkwatch for
4532 *
4533 * Sync linkwatch for the given device, removing it from the
4534 * pending work list (if queued).
4535 */
4536void linkwatch_sync_dev(struct net_device *dev);
4537void __linkwatch_sync_dev(struct net_device *dev);
4538
4539/**
4540 * netif_carrier_ok - test if carrier present
4541 * @dev: network device
4542 *
4543 * Check if carrier is present on device
4544 */
4545static inline bool netif_carrier_ok(const struct net_device *dev)
4546{
4547 return !test_bit(__LINK_STATE_NOCARRIER, &dev->state);
4548}
4549
4550unsigned long dev_trans_start(struct net_device *dev);
4551
4552void netdev_watchdog_up(struct net_device *dev);
4553
4554void netif_carrier_on(struct net_device *dev);
4555void netif_carrier_off(struct net_device *dev);
4556void netif_carrier_event(struct net_device *dev);
4557
4558/**
4559 * netif_dormant_on - mark device as dormant.
4560 * @dev: network device
4561 *
4562 * Mark device as dormant (as per RFC2863).
4563 *
4564 * The dormant state indicates that the relevant interface is not
4565 * actually in a condition to pass packets (i.e., it is not 'up') but is
4566 * in a "pending" state, waiting for some external event. For "on-
4567 * demand" interfaces, this new state identifies the situation where the
4568 * interface is waiting for events to place it in the up state.
4569 */
4570static inline void netif_dormant_on(struct net_device *dev)
4571{
4572 if (!test_and_set_bit(__LINK_STATE_DORMANT, &dev->state))
4573 linkwatch_fire_event(dev);
4574}
4575
4576/**
4577 * netif_dormant_off - set device as not dormant.
4578 * @dev: network device
4579 *
4580 * Device is not in dormant state.
4581 */
4582static inline void netif_dormant_off(struct net_device *dev)
4583{
4584 if (test_and_clear_bit(__LINK_STATE_DORMANT, &dev->state))
4585 linkwatch_fire_event(dev);
4586}
4587
4588/**
4589 * netif_dormant - test if device is dormant
4590 * @dev: network device
4591 *
4592 * Check if device is dormant.
4593 */
4594static inline bool netif_dormant(const struct net_device *dev)
4595{
4596 return test_bit(__LINK_STATE_DORMANT, &dev->state);
4597}
4598
4599
4600/**
4601 * netif_testing_on - mark device as under test.
4602 * @dev: network device
4603 *
4604 * Mark device as under test (as per RFC2863).
4605 *
4606 * The testing state indicates that some test(s) must be performed on
4607 * the interface. After completion, of the test, the interface state
4608 * will change to up, dormant, or down, as appropriate.
4609 */
4610static inline void netif_testing_on(struct net_device *dev)
4611{
4612 if (!test_and_set_bit(__LINK_STATE_TESTING, &dev->state))
4613 linkwatch_fire_event(dev);
4614}
4615
4616/**
4617 * netif_testing_off - set device as not under test.
4618 * @dev: network device
4619 *
4620 * Device is not in testing state.
4621 */
4622static inline void netif_testing_off(struct net_device *dev)
4623{
4624 if (test_and_clear_bit(__LINK_STATE_TESTING, &dev->state))
4625 linkwatch_fire_event(dev);
4626}
4627
4628/**
4629 * netif_testing - test if device is under test
4630 * @dev: network device
4631 *
4632 * Check if device is under test
4633 */
4634static inline bool netif_testing(const struct net_device *dev)
4635{
4636 return test_bit(__LINK_STATE_TESTING, &dev->state);
4637}
4638
4639
4640/**
4641 * netif_oper_up - test if device is operational
4642 * @dev: network device
4643 *
4644 * Check if carrier is operational
4645 */
4646static inline bool netif_oper_up(const struct net_device *dev)
4647{
4648 unsigned int operstate = READ_ONCE(dev->operstate);
4649
4650 return operstate == IF_OPER_UP ||
4651 operstate == IF_OPER_UNKNOWN /* backward compat */;
4652}
4653
4654/**
4655 * netif_device_present - is device available or removed
4656 * @dev: network device
4657 *
4658 * Check if device has not been removed from system.
4659 */
4660static inline bool netif_device_present(const struct net_device *dev)
4661{
4662 return test_bit(__LINK_STATE_PRESENT, &dev->state);
4663}
4664
4665void netif_device_detach(struct net_device *dev);
4666
4667void netif_device_attach(struct net_device *dev);
4668
4669/*
4670 * Network interface message level settings
4671 */
4672
4673enum {
4674 NETIF_MSG_DRV_BIT,
4675 NETIF_MSG_PROBE_BIT,
4676 NETIF_MSG_LINK_BIT,
4677 NETIF_MSG_TIMER_BIT,
4678 NETIF_MSG_IFDOWN_BIT,
4679 NETIF_MSG_IFUP_BIT,
4680 NETIF_MSG_RX_ERR_BIT,
4681 NETIF_MSG_TX_ERR_BIT,
4682 NETIF_MSG_TX_QUEUED_BIT,
4683 NETIF_MSG_INTR_BIT,
4684 NETIF_MSG_TX_DONE_BIT,
4685 NETIF_MSG_RX_STATUS_BIT,
4686 NETIF_MSG_PKTDATA_BIT,
4687 NETIF_MSG_HW_BIT,
4688 NETIF_MSG_WOL_BIT,
4689
4690 /* When you add a new bit above, update netif_msg_class_names array
4691 * in net/ethtool/common.c
4692 */
4693 NETIF_MSG_CLASS_COUNT,
4694};
4695/* Both ethtool_ops interface and internal driver implementation use u32 */
4696static_assert(NETIF_MSG_CLASS_COUNT <= 32);
4697
4698#define __NETIF_MSG_BIT(bit) ((u32)1 << (bit))
4699#define __NETIF_MSG(name) __NETIF_MSG_BIT(NETIF_MSG_ ## name ## _BIT)
4700
4701#define NETIF_MSG_DRV __NETIF_MSG(DRV)
4702#define NETIF_MSG_PROBE __NETIF_MSG(PROBE)
4703#define NETIF_MSG_LINK __NETIF_MSG(LINK)
4704#define NETIF_MSG_TIMER __NETIF_MSG(TIMER)
4705#define NETIF_MSG_IFDOWN __NETIF_MSG(IFDOWN)
4706#define NETIF_MSG_IFUP __NETIF_MSG(IFUP)
4707#define NETIF_MSG_RX_ERR __NETIF_MSG(RX_ERR)
4708#define NETIF_MSG_TX_ERR __NETIF_MSG(TX_ERR)
4709#define NETIF_MSG_TX_QUEUED __NETIF_MSG(TX_QUEUED)
4710#define NETIF_MSG_INTR __NETIF_MSG(INTR)
4711#define NETIF_MSG_TX_DONE __NETIF_MSG(TX_DONE)
4712#define NETIF_MSG_RX_STATUS __NETIF_MSG(RX_STATUS)
4713#define NETIF_MSG_PKTDATA __NETIF_MSG(PKTDATA)
4714#define NETIF_MSG_HW __NETIF_MSG(HW)
4715#define NETIF_MSG_WOL __NETIF_MSG(WOL)
4716
4717#define netif_msg_drv(p) ((p)->msg_enable & NETIF_MSG_DRV)
4718#define netif_msg_probe(p) ((p)->msg_enable & NETIF_MSG_PROBE)
4719#define netif_msg_link(p) ((p)->msg_enable & NETIF_MSG_LINK)
4720#define netif_msg_timer(p) ((p)->msg_enable & NETIF_MSG_TIMER)
4721#define netif_msg_ifdown(p) ((p)->msg_enable & NETIF_MSG_IFDOWN)
4722#define netif_msg_ifup(p) ((p)->msg_enable & NETIF_MSG_IFUP)
4723#define netif_msg_rx_err(p) ((p)->msg_enable & NETIF_MSG_RX_ERR)
4724#define netif_msg_tx_err(p) ((p)->msg_enable & NETIF_MSG_TX_ERR)
4725#define netif_msg_tx_queued(p) ((p)->msg_enable & NETIF_MSG_TX_QUEUED)
4726#define netif_msg_intr(p) ((p)->msg_enable & NETIF_MSG_INTR)
4727#define netif_msg_tx_done(p) ((p)->msg_enable & NETIF_MSG_TX_DONE)
4728#define netif_msg_rx_status(p) ((p)->msg_enable & NETIF_MSG_RX_STATUS)
4729#define netif_msg_pktdata(p) ((p)->msg_enable & NETIF_MSG_PKTDATA)
4730#define netif_msg_hw(p) ((p)->msg_enable & NETIF_MSG_HW)
4731#define netif_msg_wol(p) ((p)->msg_enable & NETIF_MSG_WOL)
4732
4733static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits)
4734{
4735 /* use default */
4736 if (debug_value < 0 || debug_value >= (sizeof(u32) * 8))
4737 return default_msg_enable_bits;
4738 if (debug_value == 0) /* no output */
4739 return 0;
4740 /* set low N bits */
4741 return (1U << debug_value) - 1;
4742}
4743
4744static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu)
4745{
4746 spin_lock(&txq->_xmit_lock);
4747 /* Pairs with READ_ONCE() in netif_tx_owned() */
4748 WRITE_ONCE(txq->xmit_lock_owner, cpu);
4749}
4750
4751static inline bool __netif_tx_acquire(struct netdev_queue *txq)
4752{
4753 __acquire(&txq->_xmit_lock);
4754 return true;
4755}
4756
4757static inline void __netif_tx_release(struct netdev_queue *txq)
4758{
4759 __release(&txq->_xmit_lock);
4760}
4761
4762static inline void __netif_tx_lock_bh(struct netdev_queue *txq)
4763{
4764 spin_lock_bh(&txq->_xmit_lock);
4765 /* Pairs with READ_ONCE() in netif_tx_owned() */
4766 WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id());
4767}
4768
4769static inline bool __netif_tx_trylock(struct netdev_queue *txq)
4770{
4771 bool ok = spin_trylock(&txq->_xmit_lock);
4772
4773 if (likely(ok)) {
4774 /* Pairs with READ_ONCE() in netif_tx_owned() */
4775 WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id());
4776 }
4777 return ok;
4778}
4779
4780static inline void __netif_tx_unlock(struct netdev_queue *txq)
4781{
4782 /* Pairs with READ_ONCE() in netif_tx_owned() */
4783 WRITE_ONCE(txq->xmit_lock_owner, -1);
4784 spin_unlock(&txq->_xmit_lock);
4785}
4786
4787static inline void __netif_tx_unlock_bh(struct netdev_queue *txq)
4788{
4789 /* Pairs with READ_ONCE() in netif_tx_owned() */
4790 WRITE_ONCE(txq->xmit_lock_owner, -1);
4791 spin_unlock_bh(&txq->_xmit_lock);
4792}
4793
4794/*
4795 * txq->trans_start can be read locklessly from dev_watchdog()
4796 */
4797static inline void txq_trans_update(const struct net_device *dev,
4798 struct netdev_queue *txq)
4799{
4800 if (!dev->lltx)
4801 WRITE_ONCE(txq->trans_start, jiffies);
4802}
4803
4804static inline void txq_trans_cond_update(struct netdev_queue *txq)
4805{
4806 unsigned long now = jiffies;
4807
4808 if (READ_ONCE(txq->trans_start) != now)
4809 WRITE_ONCE(txq->trans_start, now);
4810}
4811
4812/* legacy drivers only, netdev_start_xmit() sets txq->trans_start */
4813static inline void netif_trans_update(struct net_device *dev)
4814{
4815 struct netdev_queue *txq = netdev_get_tx_queue(dev, 0);
4816
4817 txq_trans_cond_update(txq);
4818}
4819
4820/**
4821 * netif_tx_lock - grab network device transmit lock
4822 * @dev: network device
4823 *
4824 * Get network device transmit lock
4825 */
4826void netif_tx_lock(struct net_device *dev);
4827
4828static inline void netif_tx_lock_bh(struct net_device *dev)
4829{
4830 local_bh_disable();
4831 netif_tx_lock(dev);
4832}
4833
4834void netif_tx_unlock(struct net_device *dev);
4835
4836static inline void netif_tx_unlock_bh(struct net_device *dev)
4837{
4838 netif_tx_unlock(dev);
4839 local_bh_enable();
4840}
4841
4842#define HARD_TX_LOCK(dev, txq, cpu) { \
4843 if (!(dev)->lltx) { \
4844 __netif_tx_lock(txq, cpu); \
4845 } else { \
4846 __netif_tx_acquire(txq); \
4847 } \
4848}
4849
4850#define HARD_TX_TRYLOCK(dev, txq) \
4851 (!(dev)->lltx ? \
4852 __netif_tx_trylock(txq) : \
4853 __netif_tx_acquire(txq))
4854
4855#define HARD_TX_UNLOCK(dev, txq) { \
4856 if (!(dev)->lltx) { \
4857 __netif_tx_unlock(txq); \
4858 } else { \
4859 __netif_tx_release(txq); \
4860 } \
4861}
4862
4863static inline void netif_tx_disable(struct net_device *dev)
4864{
4865 unsigned int i;
4866 int cpu;
4867
4868 local_bh_disable();
4869 cpu = smp_processor_id();
4870 spin_lock(&dev->tx_global_lock);
4871 for (i = 0; i < dev->num_tx_queues; i++) {
4872 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
4873
4874 __netif_tx_lock(txq, cpu);
4875 netif_tx_stop_queue(txq);
4876 __netif_tx_unlock(txq);
4877 }
4878 spin_unlock(&dev->tx_global_lock);
4879 local_bh_enable();
4880}
4881
4882#ifndef CONFIG_PREEMPT_RT
4883static inline bool netif_tx_owned(struct netdev_queue *txq, unsigned int cpu)
4884{
4885 /* Other cpus might concurrently change txq->xmit_lock_owner
4886 * to -1 or to their cpu id, but not to our id.
4887 */
4888 return READ_ONCE(txq->xmit_lock_owner) == cpu;
4889}
4890
4891#else
4892static inline bool netif_tx_owned(struct netdev_queue *txq, unsigned int cpu)
4893{
4894 return rt_mutex_owner(&txq->_xmit_lock.lock) == current;
4895}
4896
4897#endif
4898
4899static inline void netif_addr_lock(struct net_device *dev)
4900{
4901 unsigned char nest_level = 0;
4902
4903#ifdef CONFIG_LOCKDEP
4904 nest_level = dev->nested_level;
4905#endif
4906 spin_lock_nested(&dev->addr_list_lock, nest_level);
4907}
4908
4909static inline void netif_addr_lock_bh(struct net_device *dev)
4910{
4911 unsigned char nest_level = 0;
4912
4913#ifdef CONFIG_LOCKDEP
4914 nest_level = dev->nested_level;
4915#endif
4916 local_bh_disable();
4917 spin_lock_nested(&dev->addr_list_lock, nest_level);
4918}
4919
4920static inline void netif_addr_unlock(struct net_device *dev)
4921{
4922 spin_unlock(&dev->addr_list_lock);
4923}
4924
4925static inline void netif_addr_unlock_bh(struct net_device *dev)
4926{
4927 spin_unlock_bh(&dev->addr_list_lock);
4928}
4929
4930/*
4931 * dev_addrs walker. Should be used only for read access. Call with
4932 * rcu_read_lock held.
4933 */
4934#define for_each_dev_addr(dev, ha) \
4935 list_for_each_entry_rcu(ha, &dev->dev_addrs.list, list)
4936
4937/* These functions live elsewhere (drivers/net/net_init.c, but related) */
4938
4939void ether_setup(struct net_device *dev);
4940
4941/* Allocate dummy net_device */
4942struct net_device *alloc_netdev_dummy(int sizeof_priv);
4943
4944/* Support for loadable net-drivers */
4945struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
4946 unsigned char name_assign_type,
4947 void (*setup)(struct net_device *),
4948 unsigned int txqs, unsigned int rxqs);
4949#define alloc_netdev(sizeof_priv, name, name_assign_type, setup) \
4950 alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, 1, 1)
4951
4952#define alloc_netdev_mq(sizeof_priv, name, name_assign_type, setup, count) \
4953 alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, count, \
4954 count)
4955
4956int register_netdev(struct net_device *dev);
4957void unregister_netdev(struct net_device *dev);
4958
4959int devm_register_netdev(struct device *dev, struct net_device *ndev);
4960
4961/* General hardware address lists handling functions */
4962int __hw_addr_sync(struct netdev_hw_addr_list *to_list,
4963 struct netdev_hw_addr_list *from_list, int addr_len);
4964int __hw_addr_sync_multiple(struct netdev_hw_addr_list *to_list,
4965 struct netdev_hw_addr_list *from_list,
4966 int addr_len);
4967void __hw_addr_unsync(struct netdev_hw_addr_list *to_list,
4968 struct netdev_hw_addr_list *from_list, int addr_len);
4969int __hw_addr_sync_dev(struct netdev_hw_addr_list *list,
4970 struct net_device *dev,
4971 int (*sync)(struct net_device *, const unsigned char *),
4972 int (*unsync)(struct net_device *,
4973 const unsigned char *));
4974int __hw_addr_ref_sync_dev(struct netdev_hw_addr_list *list,
4975 struct net_device *dev,
4976 int (*sync)(struct net_device *,
4977 const unsigned char *, int),
4978 int (*unsync)(struct net_device *,
4979 const unsigned char *, int));
4980void __hw_addr_ref_unsync_dev(struct netdev_hw_addr_list *list,
4981 struct net_device *dev,
4982 int (*unsync)(struct net_device *,
4983 const unsigned char *, int));
4984void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list,
4985 struct net_device *dev,
4986 int (*unsync)(struct net_device *,
4987 const unsigned char *));
4988void __hw_addr_init(struct netdev_hw_addr_list *list);
4989
4990/* Functions used for device addresses handling */
4991void dev_addr_mod(struct net_device *dev, unsigned int offset,
4992 const void *addr, size_t len);
4993
4994static inline void
4995__dev_addr_set(struct net_device *dev, const void *addr, size_t len)
4996{
4997 dev_addr_mod(dev, 0, addr, len);
4998}
4999
5000static inline void dev_addr_set(struct net_device *dev, const u8 *addr)
5001{
5002 __dev_addr_set(dev, addr, dev->addr_len);
5003}
5004
5005int dev_addr_add(struct net_device *dev, const unsigned char *addr,
5006 unsigned char addr_type);
5007int dev_addr_del(struct net_device *dev, const unsigned char *addr,
5008 unsigned char addr_type);
5009
5010/* Functions used for unicast addresses handling */
5011int dev_uc_add(struct net_device *dev, const unsigned char *addr);
5012int dev_uc_add_excl(struct net_device *dev, const unsigned char *addr);
5013int dev_uc_del(struct net_device *dev, const unsigned char *addr);
5014int dev_uc_sync(struct net_device *to, struct net_device *from);
5015int dev_uc_sync_multiple(struct net_device *to, struct net_device *from);
5016void dev_uc_unsync(struct net_device *to, struct net_device *from);
5017void dev_uc_flush(struct net_device *dev);
5018void dev_uc_init(struct net_device *dev);
5019
5020/**
5021 * __dev_uc_sync - Synchronize device's unicast list
5022 * @dev: device to sync
5023 * @sync: function to call if address should be added
5024 * @unsync: function to call if address should be removed
5025 *
5026 * Add newly added addresses to the interface, and release
5027 * addresses that have been deleted.
5028 */
5029static inline int __dev_uc_sync(struct net_device *dev,
5030 int (*sync)(struct net_device *,
5031 const unsigned char *),
5032 int (*unsync)(struct net_device *,
5033 const unsigned char *))
5034{
5035 return __hw_addr_sync_dev(&dev->uc, dev, sync, unsync);
5036}
5037
5038/**
5039 * __dev_uc_unsync - Remove synchronized addresses from device
5040 * @dev: device to sync
5041 * @unsync: function to call if address should be removed
5042 *
5043 * Remove all addresses that were added to the device by dev_uc_sync().
5044 */
5045static inline void __dev_uc_unsync(struct net_device *dev,
5046 int (*unsync)(struct net_device *,
5047 const unsigned char *))
5048{
5049 __hw_addr_unsync_dev(&dev->uc, dev, unsync);
5050}
5051
5052/* Functions used for multicast addresses handling */
5053int dev_mc_add(struct net_device *dev, const unsigned char *addr);
5054int dev_mc_add_global(struct net_device *dev, const unsigned char *addr);
5055int dev_mc_add_excl(struct net_device *dev, const unsigned char *addr);
5056int dev_mc_del(struct net_device *dev, const unsigned char *addr);
5057int dev_mc_del_global(struct net_device *dev, const unsigned char *addr);
5058int dev_mc_sync(struct net_device *to, struct net_device *from);
5059int dev_mc_sync_multiple(struct net_device *to, struct net_device *from);
5060void dev_mc_unsync(struct net_device *to, struct net_device *from);
5061void dev_mc_flush(struct net_device *dev);
5062void dev_mc_init(struct net_device *dev);
5063
5064/**
5065 * __dev_mc_sync - Synchronize device's multicast list
5066 * @dev: device to sync
5067 * @sync: function to call if address should be added
5068 * @unsync: function to call if address should be removed
5069 *
5070 * Add newly added addresses to the interface, and release
5071 * addresses that have been deleted.
5072 */
5073static inline int __dev_mc_sync(struct net_device *dev,
5074 int (*sync)(struct net_device *,
5075 const unsigned char *),
5076 int (*unsync)(struct net_device *,
5077 const unsigned char *))
5078{
5079 return __hw_addr_sync_dev(&dev->mc, dev, sync, unsync);
5080}
5081
5082/**
5083 * __dev_mc_unsync - Remove synchronized addresses from device
5084 * @dev: device to sync
5085 * @unsync: function to call if address should be removed
5086 *
5087 * Remove all addresses that were added to the device by dev_mc_sync().
5088 */
5089static inline void __dev_mc_unsync(struct net_device *dev,
5090 int (*unsync)(struct net_device *,
5091 const unsigned char *))
5092{
5093 __hw_addr_unsync_dev(&dev->mc, dev, unsync);
5094}
5095
5096/* Functions used for secondary unicast and multicast support */
5097void dev_set_rx_mode(struct net_device *dev);
5098int netif_set_promiscuity(struct net_device *dev, int inc);
5099int dev_set_promiscuity(struct net_device *dev, int inc);
5100int netif_set_allmulti(struct net_device *dev, int inc, bool notify);
5101int dev_set_allmulti(struct net_device *dev, int inc);
5102void netif_state_change(struct net_device *dev);
5103void netdev_state_change(struct net_device *dev);
5104void __netdev_notify_peers(struct net_device *dev);
5105void netdev_notify_peers(struct net_device *dev);
5106void netdev_features_change(struct net_device *dev);
5107/* Load a device via the kmod */
5108void dev_load(struct net *net, const char *name);
5109struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
5110 struct rtnl_link_stats64 *storage);
5111void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
5112 const struct net_device_stats *netdev_stats);
5113void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
5114 const struct pcpu_sw_netstats __percpu *netstats);
5115void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s);
5116
5117enum {
5118 NESTED_SYNC_IMM_BIT,
5119 NESTED_SYNC_TODO_BIT,
5120};
5121
5122#define __NESTED_SYNC_BIT(bit) ((u32)1 << (bit))
5123#define __NESTED_SYNC(name) __NESTED_SYNC_BIT(NESTED_SYNC_ ## name ## _BIT)
5124
5125#define NESTED_SYNC_IMM __NESTED_SYNC(IMM)
5126#define NESTED_SYNC_TODO __NESTED_SYNC(TODO)
5127
5128struct netdev_nested_priv {
5129 unsigned char flags;
5130 void *data;
5131};
5132
5133bool netdev_has_upper_dev(struct net_device *dev, struct net_device *upper_dev);
5134struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
5135 struct list_head **iter);
5136
5137/* iterate through upper list, must be called under RCU read lock */
5138#define netdev_for_each_upper_dev_rcu(dev, updev, iter) \
5139 for (iter = &(dev)->adj_list.upper, \
5140 updev = netdev_upper_get_next_dev_rcu(dev, &(iter)); \
5141 updev; \
5142 updev = netdev_upper_get_next_dev_rcu(dev, &(iter)))
5143
5144int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
5145 int (*fn)(struct net_device *upper_dev,
5146 struct netdev_nested_priv *priv),
5147 struct netdev_nested_priv *priv);
5148
5149bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
5150 struct net_device *upper_dev);
5151
5152bool netdev_has_any_upper_dev(struct net_device *dev);
5153
5154void *netdev_lower_get_next_private(struct net_device *dev,
5155 struct list_head **iter);
5156void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5157 struct list_head **iter);
5158
5159#define netdev_for_each_lower_private(dev, priv, iter) \
5160 for (iter = (dev)->adj_list.lower.next, \
5161 priv = netdev_lower_get_next_private(dev, &(iter)); \
5162 priv; \
5163 priv = netdev_lower_get_next_private(dev, &(iter)))
5164
5165#define netdev_for_each_lower_private_rcu(dev, priv, iter) \
5166 for (iter = &(dev)->adj_list.lower, \
5167 priv = netdev_lower_get_next_private_rcu(dev, &(iter)); \
5168 priv; \
5169 priv = netdev_lower_get_next_private_rcu(dev, &(iter)))
5170
5171void *netdev_lower_get_next(struct net_device *dev,
5172 struct list_head **iter);
5173
5174#define netdev_for_each_lower_dev(dev, ldev, iter) \
5175 for (iter = (dev)->adj_list.lower.next, \
5176 ldev = netdev_lower_get_next(dev, &(iter)); \
5177 ldev; \
5178 ldev = netdev_lower_get_next(dev, &(iter)))
5179
5180struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
5181 struct list_head **iter);
5182int netdev_walk_all_lower_dev(struct net_device *dev,
5183 int (*fn)(struct net_device *lower_dev,
5184 struct netdev_nested_priv *priv),
5185 struct netdev_nested_priv *priv);
5186int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
5187 int (*fn)(struct net_device *lower_dev,
5188 struct netdev_nested_priv *priv),
5189 struct netdev_nested_priv *priv);
5190
5191void *netdev_adjacent_get_private(struct list_head *adj_list);
5192void *netdev_lower_get_first_private_rcu(struct net_device *dev);
5193struct net_device *netdev_master_upper_dev_get(struct net_device *dev);
5194struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev);
5195int netdev_upper_dev_link(struct net_device *dev, struct net_device *upper_dev,
5196 struct netlink_ext_ack *extack);
5197int netdev_master_upper_dev_link(struct net_device *dev,
5198 struct net_device *upper_dev,
5199 void *upper_priv, void *upper_info,
5200 struct netlink_ext_ack *extack);
5201void netdev_upper_dev_unlink(struct net_device *dev,
5202 struct net_device *upper_dev);
5203int netdev_adjacent_change_prepare(struct net_device *old_dev,
5204 struct net_device *new_dev,
5205 struct net_device *dev,
5206 struct netlink_ext_ack *extack);
5207void netdev_adjacent_change_commit(struct net_device *old_dev,
5208 struct net_device *new_dev,
5209 struct net_device *dev);
5210void netdev_adjacent_change_abort(struct net_device *old_dev,
5211 struct net_device *new_dev,
5212 struct net_device *dev);
5213void netdev_adjacent_rename_links(struct net_device *dev, char *oldname);
5214void *netdev_lower_dev_get_private(struct net_device *dev,
5215 struct net_device *lower_dev);
5216void netdev_lower_state_changed(struct net_device *lower_dev,
5217 void *lower_state_info);
5218
5219#define NETDEV_RSS_KEY_LEN 256
5220extern u8 netdev_rss_key[NETDEV_RSS_KEY_LEN] __read_mostly;
5221void netdev_rss_key_fill(void *buffer, size_t len);
5222
5223int skb_checksum_help(struct sk_buff *skb);
5224int skb_crc32c_csum_help(struct sk_buff *skb);
5225int skb_csum_hwoffload_help(struct sk_buff *skb,
5226 const netdev_features_t features);
5227
5228struct netdev_bonding_info {
5229 ifslave slave;
5230 ifbond master;
5231};
5232
5233struct netdev_notifier_bonding_info {
5234 struct netdev_notifier_info info; /* must be first */
5235 struct netdev_bonding_info bonding_info;
5236};
5237
5238void netdev_bonding_info_change(struct net_device *dev,
5239 struct netdev_bonding_info *bonding_info);
5240
5241#if IS_ENABLED(CONFIG_ETHTOOL_NETLINK)
5242void ethtool_notify(struct net_device *dev, unsigned int cmd);
5243#else
5244static inline void ethtool_notify(struct net_device *dev, unsigned int cmd)
5245{
5246}
5247#endif
5248
5249__be16 skb_network_protocol(struct sk_buff *skb, int *depth);
5250
5251static inline bool can_checksum_protocol(netdev_features_t features,
5252 __be16 protocol)
5253{
5254 if (protocol == htons(ETH_P_FCOE))
5255 return !!(features & NETIF_F_FCOE_CRC);
5256
5257 /* Assume this is an IP checksum (not SCTP CRC) */
5258
5259 if (features & NETIF_F_HW_CSUM) {
5260 /* Can checksum everything */
5261 return true;
5262 }
5263
5264 switch (protocol) {
5265 case htons(ETH_P_IP):
5266 return !!(features & NETIF_F_IP_CSUM);
5267 case htons(ETH_P_IPV6):
5268 return !!(features & NETIF_F_IPV6_CSUM);
5269 default:
5270 return false;
5271 }
5272}
5273
5274#ifdef CONFIG_BUG
5275void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb);
5276#else
5277static inline void netdev_rx_csum_fault(struct net_device *dev,
5278 struct sk_buff *skb)
5279{
5280}
5281#endif
5282/* rx skb timestamps */
5283void net_enable_timestamp(void);
5284void net_disable_timestamp(void);
5285
5286static inline ktime_t netdev_get_tstamp(struct net_device *dev,
5287 const struct skb_shared_hwtstamps *hwtstamps,
5288 bool cycles)
5289{
5290 const struct net_device_ops *ops = dev->netdev_ops;
5291
5292 if (ops->ndo_get_tstamp)
5293 return ops->ndo_get_tstamp(dev, hwtstamps, cycles);
5294
5295 return hwtstamps->hwtstamp;
5296}
5297
5298#ifndef CONFIG_PREEMPT_RT
5299static inline void netdev_xmit_set_more(bool more)
5300{
5301 __this_cpu_write(softnet_data.xmit.more, more);
5302}
5303
5304static inline bool netdev_xmit_more(void)
5305{
5306 return __this_cpu_read(softnet_data.xmit.more);
5307}
5308#else
5309static inline void netdev_xmit_set_more(bool more)
5310{
5311 current->net_xmit.more = more;
5312}
5313
5314static inline bool netdev_xmit_more(void)
5315{
5316 return current->net_xmit.more;
5317}
5318#endif
5319
5320static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
5321 struct sk_buff *skb, struct net_device *dev,
5322 bool more)
5323{
5324 netdev_xmit_set_more(more);
5325 return ops->ndo_start_xmit(skb, dev);
5326}
5327
5328static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
5329 struct netdev_queue *txq, bool more)
5330{
5331 const struct net_device_ops *ops = dev->netdev_ops;
5332 netdev_tx_t rc;
5333
5334 rc = __netdev_start_xmit(ops, skb, dev, more);
5335 if (rc == NETDEV_TX_OK)
5336 txq_trans_update(dev, txq);
5337
5338 return rc;
5339}
5340
5341int netdev_class_create_file_ns(const struct class_attribute *class_attr,
5342 const void *ns);
5343void netdev_class_remove_file_ns(const struct class_attribute *class_attr,
5344 const void *ns);
5345
5346extern const struct kobj_ns_type_operations net_ns_type_operations;
5347
5348const char *netdev_drivername(const struct net_device *dev);
5349
5350static inline netdev_features_t netdev_intersect_features(netdev_features_t f1,
5351 netdev_features_t f2)
5352{
5353 if ((f1 ^ f2) & NETIF_F_HW_CSUM) {
5354 if (f1 & NETIF_F_HW_CSUM)
5355 f1 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5356 else
5357 f2 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5358 }
5359
5360 return f1 & f2;
5361}
5362
5363static inline netdev_features_t netdev_get_wanted_features(
5364 struct net_device *dev)
5365{
5366 return (dev->features & ~dev->hw_features) | dev->wanted_features;
5367}
5368netdev_features_t netdev_increment_features(netdev_features_t all,
5369 netdev_features_t one, netdev_features_t mask);
5370
5371/* Allow TSO being used on stacked device :
5372 * Performing the GSO segmentation before last device
5373 * is a performance improvement.
5374 */
5375static inline netdev_features_t netdev_add_tso_features(netdev_features_t features,
5376 netdev_features_t mask)
5377{
5378 return netdev_increment_features(features, NETIF_F_ALL_TSO |
5379 NETIF_F_ALL_FOR_ALL, mask);
5380}
5381
5382int __netdev_update_features(struct net_device *dev);
5383void netdev_update_features(struct net_device *dev);
5384void netdev_change_features(struct net_device *dev);
5385void netdev_compute_master_upper_features(struct net_device *dev, bool update_header);
5386
5387void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5388 struct net_device *dev);
5389
5390netdev_features_t passthru_features_check(struct sk_buff *skb,
5391 struct net_device *dev,
5392 netdev_features_t features);
5393netdev_features_t netif_skb_features(struct sk_buff *skb);
5394void skb_warn_bad_offload(const struct sk_buff *skb);
5395
5396static inline bool net_gso_ok(netdev_features_t features, int gso_type)
5397{
5398 netdev_features_t feature;
5399
5400 if (gso_type & (SKB_GSO_TCP_FIXEDID | SKB_GSO_TCP_FIXEDID_INNER))
5401 gso_type |= __SKB_GSO_TCP_FIXEDID;
5402
5403 feature = ((netdev_features_t)gso_type << NETIF_F_GSO_SHIFT) & NETIF_F_GSO_MASK;
5404
5405 /* check flags correspondence */
5406 BUILD_BUG_ON(SKB_GSO_TCPV4 != (NETIF_F_TSO >> NETIF_F_GSO_SHIFT));
5407 BUILD_BUG_ON(SKB_GSO_DODGY != (NETIF_F_GSO_ROBUST >> NETIF_F_GSO_SHIFT));
5408 BUILD_BUG_ON(SKB_GSO_TCP_ECN != (NETIF_F_TSO_ECN >> NETIF_F_GSO_SHIFT));
5409 BUILD_BUG_ON(__SKB_GSO_TCP_FIXEDID != (NETIF_F_TSO_MANGLEID >> NETIF_F_GSO_SHIFT));
5410 BUILD_BUG_ON(SKB_GSO_TCPV6 != (NETIF_F_TSO6 >> NETIF_F_GSO_SHIFT));
5411 BUILD_BUG_ON(SKB_GSO_FCOE != (NETIF_F_FSO >> NETIF_F_GSO_SHIFT));
5412 BUILD_BUG_ON(SKB_GSO_GRE != (NETIF_F_GSO_GRE >> NETIF_F_GSO_SHIFT));
5413 BUILD_BUG_ON(SKB_GSO_GRE_CSUM != (NETIF_F_GSO_GRE_CSUM >> NETIF_F_GSO_SHIFT));
5414 BUILD_BUG_ON(SKB_GSO_IPXIP4 != (NETIF_F_GSO_IPXIP4 >> NETIF_F_GSO_SHIFT));
5415 BUILD_BUG_ON(SKB_GSO_IPXIP6 != (NETIF_F_GSO_IPXIP6 >> NETIF_F_GSO_SHIFT));
5416 BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL != (NETIF_F_GSO_UDP_TUNNEL >> NETIF_F_GSO_SHIFT));
5417 BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL_CSUM != (NETIF_F_GSO_UDP_TUNNEL_CSUM >> NETIF_F_GSO_SHIFT));
5418 BUILD_BUG_ON(SKB_GSO_PARTIAL != (NETIF_F_GSO_PARTIAL >> NETIF_F_GSO_SHIFT));
5419 BUILD_BUG_ON(SKB_GSO_TUNNEL_REMCSUM != (NETIF_F_GSO_TUNNEL_REMCSUM >> NETIF_F_GSO_SHIFT));
5420 BUILD_BUG_ON(SKB_GSO_SCTP != (NETIF_F_GSO_SCTP >> NETIF_F_GSO_SHIFT));
5421 BUILD_BUG_ON(SKB_GSO_ESP != (NETIF_F_GSO_ESP >> NETIF_F_GSO_SHIFT));
5422 BUILD_BUG_ON(SKB_GSO_UDP != (NETIF_F_GSO_UDP >> NETIF_F_GSO_SHIFT));
5423 BUILD_BUG_ON(SKB_GSO_UDP_L4 != (NETIF_F_GSO_UDP_L4 >> NETIF_F_GSO_SHIFT));
5424 BUILD_BUG_ON(SKB_GSO_FRAGLIST != (NETIF_F_GSO_FRAGLIST >> NETIF_F_GSO_SHIFT));
5425 BUILD_BUG_ON(SKB_GSO_TCP_ACCECN !=
5426 (NETIF_F_GSO_ACCECN >> NETIF_F_GSO_SHIFT));
5427
5428 return (features & feature) == feature;
5429}
5430
5431static inline bool skb_gso_ok(struct sk_buff *skb, netdev_features_t features)
5432{
5433 return net_gso_ok(features, skb_shinfo(skb)->gso_type) &&
5434 (!skb_has_frag_list(skb) || (features & NETIF_F_FRAGLIST));
5435}
5436
5437static inline bool netif_needs_gso(struct sk_buff *skb,
5438 netdev_features_t features)
5439{
5440 return skb_is_gso(skb) && (!skb_gso_ok(skb, features) ||
5441 unlikely((skb->ip_summed != CHECKSUM_PARTIAL) &&
5442 (skb->ip_summed != CHECKSUM_UNNECESSARY)));
5443}
5444
5445void netif_set_tso_max_size(struct net_device *dev, unsigned int size);
5446void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs);
5447void netif_inherit_tso_max(struct net_device *to,
5448 const struct net_device *from);
5449
5450static inline unsigned int
5451netif_get_gro_max_size(const struct net_device *dev, const struct sk_buff *skb)
5452{
5453 /* pairs with WRITE_ONCE() in netif_set_gro(_ipv4)_max_size() */
5454 return skb->protocol == htons(ETH_P_IPV6) ?
5455 READ_ONCE(dev->gro_max_size) :
5456 READ_ONCE(dev->gro_ipv4_max_size);
5457}
5458
5459static inline unsigned int
5460netif_get_gso_max_size(const struct net_device *dev, const struct sk_buff *skb)
5461{
5462 /* pairs with WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */
5463 return skb->protocol == htons(ETH_P_IPV6) ?
5464 READ_ONCE(dev->gso_max_size) :
5465 READ_ONCE(dev->gso_ipv4_max_size);
5466}
5467
5468static inline bool netif_is_macsec(const struct net_device *dev)
5469{
5470 return dev->priv_flags & IFF_MACSEC;
5471}
5472
5473static inline bool netif_is_macvlan(const struct net_device *dev)
5474{
5475 return dev->priv_flags & IFF_MACVLAN;
5476}
5477
5478static inline bool netif_is_macvlan_port(const struct net_device *dev)
5479{
5480 return dev->priv_flags & IFF_MACVLAN_PORT;
5481}
5482
5483static inline bool netif_is_bond_master(const struct net_device *dev)
5484{
5485 return dev->flags & IFF_MASTER && dev->priv_flags & IFF_BONDING;
5486}
5487
5488static inline bool netif_is_bond_slave(const struct net_device *dev)
5489{
5490 return dev->flags & IFF_SLAVE && dev->priv_flags & IFF_BONDING;
5491}
5492
5493static inline bool netif_supports_nofcs(struct net_device *dev)
5494{
5495 return dev->priv_flags & IFF_SUPP_NOFCS;
5496}
5497
5498static inline bool netif_has_l3_rx_handler(const struct net_device *dev)
5499{
5500 return dev->priv_flags & IFF_L3MDEV_RX_HANDLER;
5501}
5502
5503static inline bool netif_is_l3_master(const struct net_device *dev)
5504{
5505 return dev->priv_flags & IFF_L3MDEV_MASTER;
5506}
5507
5508static inline bool netif_is_l3_slave(const struct net_device *dev)
5509{
5510 return dev->priv_flags & IFF_L3MDEV_SLAVE;
5511}
5512
5513static inline int dev_sdif(const struct net_device *dev)
5514{
5515#ifdef CONFIG_NET_L3_MASTER_DEV
5516 if (netif_is_l3_slave(dev))
5517 return dev->ifindex;
5518#endif
5519 return 0;
5520}
5521
5522static inline bool netif_is_bridge_master(const struct net_device *dev)
5523{
5524 return dev->priv_flags & IFF_EBRIDGE;
5525}
5526
5527static inline bool netif_is_bridge_port(const struct net_device *dev)
5528{
5529 return dev->priv_flags & IFF_BRIDGE_PORT;
5530}
5531
5532static inline bool netif_is_ovs_master(const struct net_device *dev)
5533{
5534 return dev->priv_flags & IFF_OPENVSWITCH;
5535}
5536
5537static inline bool netif_is_ovs_port(const struct net_device *dev)
5538{
5539 return dev->priv_flags & IFF_OVS_DATAPATH;
5540}
5541
5542static inline bool netif_is_any_bridge_master(const struct net_device *dev)
5543{
5544 return netif_is_bridge_master(dev) || netif_is_ovs_master(dev);
5545}
5546
5547static inline bool netif_is_any_bridge_port(const struct net_device *dev)
5548{
5549 return netif_is_bridge_port(dev) || netif_is_ovs_port(dev);
5550}
5551
5552static inline bool netif_is_team_master(const struct net_device *dev)
5553{
5554 return dev->priv_flags & IFF_TEAM;
5555}
5556
5557static inline bool netif_is_team_port(const struct net_device *dev)
5558{
5559 return dev->priv_flags & IFF_TEAM_PORT;
5560}
5561
5562static inline bool netif_is_lag_master(const struct net_device *dev)
5563{
5564 return netif_is_bond_master(dev) || netif_is_team_master(dev);
5565}
5566
5567static inline bool netif_is_lag_port(const struct net_device *dev)
5568{
5569 return netif_is_bond_slave(dev) || netif_is_team_port(dev);
5570}
5571
5572static inline bool netif_is_rxfh_configured(const struct net_device *dev)
5573{
5574 return dev->priv_flags & IFF_RXFH_CONFIGURED;
5575}
5576
5577static inline bool netif_is_failover(const struct net_device *dev)
5578{
5579 return dev->priv_flags & IFF_FAILOVER;
5580}
5581
5582static inline bool netif_is_failover_slave(const struct net_device *dev)
5583{
5584 return dev->priv_flags & IFF_FAILOVER_SLAVE;
5585}
5586
5587/* This device needs to keep skb dst for qdisc enqueue or ndo_start_xmit() */
5588static inline void netif_keep_dst(struct net_device *dev)
5589{
5590 dev->priv_flags &= ~(IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM);
5591}
5592
5593/* return true if dev can't cope with mtu frames that need vlan tag insertion */
5594static inline bool netif_reduces_vlan_mtu(struct net_device *dev)
5595{
5596 /* TODO: reserve and use an additional IFF bit, if we get more users */
5597 return netif_is_macsec(dev);
5598}
5599
5600extern struct pernet_operations __net_initdata loopback_net_ops;
5601
5602/* Logging, debugging and troubleshooting/diagnostic helpers. */
5603
5604/* netdev_printk helpers, similar to dev_printk */
5605
5606static inline const char *netdev_name(const struct net_device *dev)
5607{
5608 if (!dev->name[0] || strchr(dev->name, '%'))
5609 return "(unnamed net_device)";
5610 return dev->name;
5611}
5612
5613static inline const char *netdev_reg_state(const struct net_device *dev)
5614{
5615 u8 reg_state = READ_ONCE(dev->reg_state);
5616
5617 switch (reg_state) {
5618 case NETREG_UNINITIALIZED: return " (uninitialized)";
5619 case NETREG_REGISTERED: return "";
5620 case NETREG_UNREGISTERING: return " (unregistering)";
5621 case NETREG_UNREGISTERED: return " (unregistered)";
5622 case NETREG_RELEASED: return " (released)";
5623 case NETREG_DUMMY: return " (dummy)";
5624 }
5625
5626 WARN_ONCE(1, "%s: unknown reg_state %d\n", dev->name, reg_state);
5627 return " (unknown)";
5628}
5629
5630#define MODULE_ALIAS_NETDEV(device) \
5631 MODULE_ALIAS("netdev-" device)
5632
5633/*
5634 * netdev_WARN() acts like dev_printk(), but with the key difference
5635 * of using a WARN/WARN_ON to get the message out, including the
5636 * file/line information and a backtrace.
5637 */
5638#define netdev_WARN(dev, format, args...) \
5639 WARN(1, "netdevice: %s%s: " format, netdev_name(dev), \
5640 netdev_reg_state(dev), ##args)
5641
5642#define netdev_WARN_ONCE(dev, format, args...) \
5643 WARN_ONCE(1, "netdevice: %s%s: " format, netdev_name(dev), \
5644 netdev_reg_state(dev), ##args)
5645
5646/*
5647 * The list of packet types we will receive (as opposed to discard)
5648 * and the routines to invoke.
5649 *
5650 * Why 16. Because with 16 the only overlap we get on a hash of the
5651 * low nibble of the protocol value is RARP/SNAP/X.25.
5652 *
5653 * 0800 IP
5654 * 0001 802.3
5655 * 0002 AX.25
5656 * 0004 802.2
5657 * 8035 RARP
5658 * 0005 SNAP
5659 * 0805 X.25
5660 * 0806 ARP
5661 * 8137 IPX
5662 * 0009 Localtalk
5663 * 86DD IPv6
5664 */
5665#define PTYPE_HASH_SIZE (16)
5666#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
5667
5668extern struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
5669
5670extern struct net_device *blackhole_netdev;
5671
5672/* Note: Avoid these macros in fast path, prefer per-cpu or per-queue counters. */
5673#define DEV_STATS_INC(DEV, FIELD) atomic_long_inc(&(DEV)->stats.__##FIELD)
5674#define DEV_STATS_ADD(DEV, FIELD, VAL) \
5675 atomic_long_add((VAL), &(DEV)->stats.__##FIELD)
5676#define DEV_STATS_READ(DEV, FIELD) atomic_long_read(&(DEV)->stats.__##FIELD)
5677
5678#endif /* _LINUX_NETDEVICE_H */